diff options
114 files changed, 5748 insertions, 3362 deletions
diff --git a/Documentation/devicetree/bindings/arm/pmu.txt b/Documentation/devicetree/bindings/arm/pmu.txt index 3e1e498fea96..fe5cef8976cb 100644 --- a/Documentation/devicetree/bindings/arm/pmu.txt +++ b/Documentation/devicetree/bindings/arm/pmu.txt @@ -9,6 +9,7 @@ Required properties: - compatible : should be one of "arm,armv8-pmuv3" "arm,cortex-a15-pmu" + "arm,cortex-a12-pmu" "arm,cortex-a9-pmu" "arm,cortex-a8-pmu" "arm,cortex-a7-pmu" @@ -16,7 +17,14 @@ Required properties: "arm,arm11mpcore-pmu" "arm,arm1176-pmu" "arm,arm1136-pmu" -- interrupts : 1 combined interrupt or 1 per core. + "qcom,krait-pmu" +- interrupts : 1 combined interrupt or 1 per core. If the interrupt is a per-cpu + interrupt (PPI) then 1 interrupt should be specified. + +Optional properties: + +- qcom,no-pc-write : Indicates that this PMU doesn't support the 0xc and 0xd + events. Example: diff --git a/arch/Kconfig b/arch/Kconfig index 80bbb8ccd0d1..97ff872c7acc 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -86,9 +86,7 @@ config KPROBES_ON_FTRACE optimize on top of function tracing. config UPROBES - bool "Transparent user-space probes (EXPERIMENTAL)" - depends on UPROBE_EVENT && PERF_EVENTS - default n + def_bool n select PERCPU_RWSEM help Uprobes is the user-space counterpart to kprobes: they @@ -101,8 +99,6 @@ config UPROBES managed by the kernel and kept transparent to the probed application. ) - If in doubt, say "N". - config HAVE_64BIT_ALIGNED_ACCESS def_bool 64BIT && !HAVE_EFFICIENT_UNALIGNED_ACCESS help diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 503da0a2a8ea..6b4ac5de6dec 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -207,6 +207,9 @@ config ZONE_DMA config NEED_DMA_MAP_STATE def_bool y +config ARCH_SUPPORTS_UPROBES + def_bool y + config ARCH_HAS_DMA_SET_COHERENT_MASK bool @@ -2271,7 +2274,7 @@ source "kernel/power/Kconfig" config ARCH_SUSPEND_POSSIBLE depends on !ARCH_S5PC100 depends on CPU_ARM920T || CPU_ARM926T || CPU_FEROCEON || CPU_SA1100 || \ - CPU_V6 || CPU_V6K || CPU_V7 || CPU_XSC3 || CPU_XSCALE || CPU_MOHAWK + CPU_V6 || CPU_V6K || CPU_V7 || CPU_V7M || CPU_XSC3 || CPU_XSCALE || CPU_MOHAWK def_bool y config ARM_CPU_SUSPEND diff --git a/arch/arm/common/Makefile b/arch/arm/common/Makefile index 4bdc41622c36..70b1eff477b3 100644 --- a/arch/arm/common/Makefile +++ b/arch/arm/common/Makefile @@ -13,6 +13,7 @@ obj-$(CONFIG_SHARP_SCOOP) += scoop.o obj-$(CONFIG_PCI_HOST_ITE8152) += it8152.o obj-$(CONFIG_ARM_TIMER_SP804) += timer-sp.o obj-$(CONFIG_MCPM) += mcpm_head.o mcpm_entry.o mcpm_platsmp.o vlock.o +CFLAGS_REMOVE_mcpm_entry.o = -pg AFLAGS_mcpm_head.o := -march=armv7-a AFLAGS_vlock.o := -march=armv7-a obj-$(CONFIG_TI_PRIV_EDMA) += edma.o diff --git a/arch/arm/common/scoop.c b/arch/arm/common/scoop.c index a5c3dc38aa18..6ef146edd0cd 100644 --- a/arch/arm/common/scoop.c +++ b/arch/arm/common/scoop.c @@ -232,8 +232,6 @@ static int scoop_probe(struct platform_device *pdev) return 0; - if (devptr->gpio.base != -1) - temp = gpiochip_remove(&devptr->gpio); err_gpio: platform_set_drvdata(pdev, NULL); err_ioremap: diff --git a/arch/arm/include/asm/assembler.h b/arch/arm/include/asm/assembler.h index 5c2285160575..380ac4f20000 100644 --- a/arch/arm/include/asm/assembler.h +++ b/arch/arm/include/asm/assembler.h @@ -30,8 +30,8 @@ * Endian independent macros for shifting bytes within registers. */ #ifndef __ARMEB__ -#define pull lsr -#define push lsl +#define lspull lsr +#define lspush lsl #define get_byte_0 lsl #0 #define get_byte_1 lsr #8 #define get_byte_2 lsr #16 @@ -41,8 +41,8 @@ #define put_byte_2 lsl #16 #define put_byte_3 lsl #24 #else -#define pull lsl -#define push lsr +#define lspull lsl +#define lspush lsr #define get_byte_0 lsr #24 #define get_byte_1 lsr #16 #define get_byte_2 lsr #8 diff --git a/arch/arm/include/asm/atomic.h b/arch/arm/include/asm/atomic.h index 62d2cb53b069..9a92fd7864a8 100644 --- a/arch/arm/include/asm/atomic.h +++ b/arch/arm/include/asm/atomic.h @@ -60,6 +60,7 @@ static inline int atomic_add_return(int i, atomic_t *v) int result; smp_mb(); + prefetchw(&v->counter); __asm__ __volatile__("@ atomic_add_return\n" "1: ldrex %0, [%3]\n" @@ -99,6 +100,7 @@ static inline int atomic_sub_return(int i, atomic_t *v) int result; smp_mb(); + prefetchw(&v->counter); __asm__ __volatile__("@ atomic_sub_return\n" "1: ldrex %0, [%3]\n" @@ -121,6 +123,7 @@ static inline int atomic_cmpxchg(atomic_t *ptr, int old, int new) unsigned long res; smp_mb(); + prefetchw(&ptr->counter); do { __asm__ __volatile__("@ atomic_cmpxchg\n" @@ -138,6 +141,33 @@ static inline int atomic_cmpxchg(atomic_t *ptr, int old, int new) return oldval; } +static inline int __atomic_add_unless(atomic_t *v, int a, int u) +{ + int oldval, newval; + unsigned long tmp; + + smp_mb(); + prefetchw(&v->counter); + + __asm__ __volatile__ ("@ atomic_add_unless\n" +"1: ldrex %0, [%4]\n" +" teq %0, %5\n" +" beq 2f\n" +" add %1, %0, %6\n" +" strex %2, %1, [%4]\n" +" teq %2, #0\n" +" bne 1b\n" +"2:" + : "=&r" (oldval), "=&r" (newval), "=&r" (tmp), "+Qo" (v->counter) + : "r" (&v->counter), "r" (u), "r" (a) + : "cc"); + + if (oldval != u) + smp_mb(); + + return oldval; +} + #else /* ARM_ARCH_6 */ #ifdef CONFIG_SMP @@ -186,10 +216,6 @@ static inline int atomic_cmpxchg(atomic_t *v, int old, int new) return ret; } -#endif /* __LINUX_ARM_ARCH__ */ - -#define atomic_xchg(v, new) (xchg(&((v)->counter), new)) - static inline int __atomic_add_unless(atomic_t *v, int a, int u) { int c, old; @@ -200,6 +226,10 @@ static inline int __atomic_add_unless(atomic_t *v, int a, int u) return c; } +#endif /* __LINUX_ARM_ARCH__ */ + +#define atomic_xchg(v, new) (xchg(&((v)->counter), new)) + #define atomic_inc(v) atomic_add(1, v) #define atomic_dec(v) atomic_sub(1, v) @@ -299,6 +329,7 @@ static inline long long atomic64_add_return(long long i, atomic64_t *v) unsigned long tmp; smp_mb(); + prefetchw(&v->counter); __asm__ __volatile__("@ atomic64_add_return\n" "1: ldrexd %0, %H0, [%3]\n" @@ -340,6 +371,7 @@ static inline long long atomic64_sub_return(long long i, atomic64_t *v) unsigned long tmp; smp_mb(); + prefetchw(&v->counter); __asm__ __volatile__("@ atomic64_sub_return\n" "1: ldrexd %0, %H0, [%3]\n" @@ -364,6 +396,7 @@ static inline long long atomic64_cmpxchg(atomic64_t *ptr, long long old, unsigned long res; smp_mb(); + prefetchw(&ptr->counter); do { __asm__ __volatile__("@ atomic64_cmpxchg\n" @@ -388,6 +421,7 @@ static inline long long atomic64_xchg(atomic64_t *ptr, long long new) unsigned long tmp; smp_mb(); + prefetchw(&ptr->counter); __asm__ __volatile__("@ atomic64_xchg\n" "1: ldrexd %0, %H0, [%3]\n" @@ -409,6 +443,7 @@ static inline long long atomic64_dec_if_positive(atomic64_t *v) unsigned long tmp; smp_mb(); + prefetchw(&v->counter); __asm__ __volatile__("@ atomic64_dec_if_positive\n" "1: ldrexd %0, %H0, [%3]\n" @@ -436,6 +471,7 @@ static inline int atomic64_add_unless(atomic64_t *v, long long a, long long u) int ret = 1; smp_mb(); + prefetchw(&v->counter); __asm__ __volatile__("@ atomic64_add_unless\n" "1: ldrexd %0, %H0, [%4]\n" diff --git a/arch/arm/include/asm/cmpxchg.h b/arch/arm/include/asm/cmpxchg.h index df2fbba7efc8..abb2c3769b01 100644 --- a/arch/arm/include/asm/cmpxchg.h +++ b/arch/arm/include/asm/cmpxchg.h @@ -2,6 +2,7 @@ #define __ASM_ARM_CMPXCHG_H #include <linux/irqflags.h> +#include <linux/prefetch.h> #include <asm/barrier.h> #if defined(CONFIG_CPU_SA1100) || defined(CONFIG_CPU_SA110) @@ -35,6 +36,7 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size #endif smp_mb(); + prefetchw((const void *)ptr); switch (size) { #if __LINUX_ARM_ARCH__ >= 6 @@ -138,6 +140,8 @@ static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old, { unsigned long oldval, res; + prefetchw((const void *)ptr); + switch (size) { #ifndef CONFIG_CPU_V6 /* min ARCH >= ARMv6K */ case 1: @@ -230,6 +234,8 @@ static inline unsigned long long __cmpxchg64(unsigned long long *ptr, unsigned long long oldval; unsigned long res; + prefetchw(ptr); + __asm__ __volatile__( "1: ldrexd %1, %H1, [%3]\n" " teq %1, %4\n" diff --git a/arch/arm/include/asm/cputype.h b/arch/arm/include/asm/cputype.h index acdde76b39bb..42f0889f0584 100644 --- a/arch/arm/include/asm/cputype.h +++ b/arch/arm/include/asm/cputype.h @@ -71,6 +71,7 @@ #define ARM_CPU_PART_CORTEX_A5 0xC050 #define ARM_CPU_PART_CORTEX_A15 0xC0F0 #define ARM_CPU_PART_CORTEX_A7 0xC070 +#define ARM_CPU_PART_CORTEX_A12 0xC0D0 #define ARM_CPU_XSCALE_ARCH_MASK 0xe000 #define ARM_CPU_XSCALE_ARCH_V1 0x2000 diff --git a/arch/arm/include/asm/floppy.h b/arch/arm/include/asm/floppy.h index c9f03eccc9d8..f4882553fbb0 100644 --- a/arch/arm/include/asm/floppy.h +++ b/arch/arm/include/asm/floppy.h @@ -25,7 +25,7 @@ #define fd_inb(port) inb((port)) #define fd_request_irq() request_irq(IRQ_FLOPPYDISK,floppy_interrupt,\ - IRQF_DISABLED,"floppy",NULL) + 0,"floppy",NULL) #define fd_free_irq() free_irq(IRQ_FLOPPYDISK,NULL) #define fd_disable_irq() disable_irq(IRQ_FLOPPYDISK) #define fd_enable_irq() enable_irq(IRQ_FLOPPYDISK) diff --git a/arch/arm/include/asm/futex.h b/arch/arm/include/asm/futex.h index e42cf597f6e6..53e69dae796f 100644 --- a/arch/arm/include/asm/futex.h +++ b/arch/arm/include/asm/futex.h @@ -3,11 +3,6 @@ #ifdef __KERNEL__ -#if defined(CONFIG_CPU_USE_DOMAINS) && defined(CONFIG_SMP) -/* ARM doesn't provide unprivileged exclusive memory accessors */ -#include <asm-generic/futex.h> -#else - #include <linux/futex.h> #include <linux/uaccess.h> #include <asm/errno.h> @@ -28,6 +23,7 @@ #define __futex_atomic_op(insn, ret, oldval, tmp, uaddr, oparg) \ smp_mb(); \ + prefetchw(uaddr); \ __asm__ __volatile__( \ "1: ldrex %1, [%3]\n" \ " " insn "\n" \ @@ -51,6 +47,8 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, return -EFAULT; smp_mb(); + /* Prefetching cannot fault */ + prefetchw(uaddr); __asm__ __volatile__("@futex_atomic_cmpxchg_inatomic\n" "1: ldrex %1, [%4]\n" " teq %1, %2\n" @@ -164,6 +162,5 @@ futex_atomic_op_inuser (int encoded_op, u32 __user *uaddr) return ret; } -#endif /* !(CPU_USE_DOMAINS && SMP) */ #endif /* __KERNEL__ */ #endif /* _ASM_ARM_FUTEX_H */ diff --git a/arch/arm/include/asm/hw_breakpoint.h b/arch/arm/include/asm/hw_breakpoint.h index eef55ea9ef00..8e427c7b4425 100644 --- a/arch/arm/include/asm/hw_breakpoint.h +++ b/arch/arm/include/asm/hw_breakpoint.h @@ -51,6 +51,7 @@ static inline void decode_ctrl_reg(u32 reg, #define ARM_DEBUG_ARCH_V7_ECP14 3 #define ARM_DEBUG_ARCH_V7_MM 4 #define ARM_DEBUG_ARCH_V7_1 5 +#define ARM_DEBUG_ARCH_V8 6 /* Breakpoint */ #define ARM_BREAKPOINT_EXECUTE 0 diff --git a/arch/arm/include/asm/hwcap.h b/arch/arm/include/asm/hwcap.h index 6ff56eca3f1f..6e183fd269fb 100644 --- a/arch/arm/include/asm/hwcap.h +++ b/arch/arm/include/asm/hwcap.h @@ -9,6 +9,7 @@ * instruction set this cpu supports. */ #define ELF_HWCAP (elf_hwcap) -extern unsigned int elf_hwcap; +#define ELF_HWCAP2 (elf_hwcap2) +extern unsigned int elf_hwcap, elf_hwcap2; #endif #endif diff --git a/arch/arm/include/asm/jump_label.h b/arch/arm/include/asm/jump_label.h index 863c892b4aaa..70f9b9bfb1f9 100644 --- a/arch/arm/include/asm/jump_label.h +++ b/arch/arm/include/asm/jump_label.h @@ -4,7 +4,6 @@ #ifdef __KERNEL__ #include <linux/types.h> -#include <asm/system.h> #define JUMP_LABEL_NOP_SIZE 4 diff --git a/arch/arm/include/asm/kprobes.h b/arch/arm/include/asm/kprobes.h index f82ec22eeb11..49fa0dfaad33 100644 --- a/arch/arm/include/asm/kprobes.h +++ b/arch/arm/include/asm/kprobes.h @@ -18,7 +18,7 @@ #include <linux/types.h> #include <linux/ptrace.h> -#include <linux/percpu.h> +#include <linux/notifier.h> #define __ARCH_WANT_KPROBES_INSN_SLOT #define MAX_INSN_SIZE 2 @@ -28,21 +28,10 @@ #define kretprobe_blacklist_size 0 typedef u32 kprobe_opcode_t; - struct kprobe; -typedef void (kprobe_insn_handler_t)(struct kprobe *, struct pt_regs *); -typedef unsigned long (kprobe_check_cc)(unsigned long); -typedef void (kprobe_insn_singlestep_t)(struct kprobe *, struct pt_regs *); -typedef void (kprobe_insn_fn_t)(void); +#include <asm/probes.h> -/* Architecture specific copy of original instruction. */ -struct arch_specific_insn { - kprobe_opcode_t *insn; - kprobe_insn_handler_t *insn_handler; - kprobe_check_cc *insn_check_cc; - kprobe_insn_singlestep_t *insn_singlestep; - kprobe_insn_fn_t *insn_fn; -}; +#define arch_specific_insn arch_probes_insn struct prev_kprobe { struct kprobe *kp; diff --git a/arch/arm/include/asm/memory.h b/arch/arm/include/asm/memory.h index 4afb376d9c7c..02fa2558f662 100644 --- a/arch/arm/include/asm/memory.h +++ b/arch/arm/include/asm/memory.h @@ -166,9 +166,17 @@ * Physical vs virtual RAM address space conversion. These are * private definitions which should NOT be used outside memory.h * files. Use virt_to_phys/phys_to_virt/__pa/__va instead. + * + * PFNs are used to describe any physical page; this means + * PFN 0 == physical address 0. */ -#ifndef __virt_to_phys -#ifdef CONFIG_ARM_PATCH_PHYS_VIRT +#if defined(__virt_to_phys) +#define PHYS_OFFSET PLAT_PHYS_OFFSET +#define PHYS_PFN_OFFSET ((unsigned long)(PHYS_OFFSET >> PAGE_SHIFT)) + +#define virt_to_pfn(kaddr) (__pa(kaddr) >> PAGE_SHIFT) + +#elif defined(CONFIG_ARM_PATCH_PHYS_VIRT) /* * Constants used to force the right instruction encodings and shifts @@ -177,12 +185,17 @@ #define __PV_BITS_31_24 0x81000000 #define __PV_BITS_7_0 0x81 -extern u64 __pv_phys_offset; +extern unsigned long __pv_phys_pfn_offset; extern u64 __pv_offset; extern void fixup_pv_table(const void *, unsigned long); extern const void *__pv_table_begin, *__pv_table_end; -#define PHYS_OFFSET __pv_phys_offset +#define PHYS_OFFSET ((phys_addr_t)__pv_phys_pfn_offset << PAGE_SHIFT) +#define PHYS_PFN_OFFSET (__pv_phys_pfn_offset) + +#define virt_to_pfn(kaddr) \ + ((((unsigned long)(kaddr) - PAGE_OFFSET) >> PAGE_SHIFT) + \ + PHYS_PFN_OFFSET) #define __pv_stub(from,to,instr,type) \ __asm__("@ __pv_stub\n" \ @@ -243,6 +256,7 @@ static inline unsigned long __phys_to_virt(phys_addr_t x) #else #define PHYS_OFFSET PLAT_PHYS_OFFSET +#define PHYS_PFN_OFFSET ((unsigned long)(PHYS_OFFSET >> PAGE_SHIFT)) static inline phys_addr_t __virt_to_phys(unsigned long x) { @@ -254,18 +268,11 @@ static inline unsigned long __phys_to_virt(phys_addr_t x) return x - PHYS_OFFSET + PAGE_OFFSET; } -#endif -#endif +#define virt_to_pfn(kaddr) \ + ((((unsigned long)(kaddr) - PAGE_OFFSET) >> PAGE_SHIFT) + \ + PHYS_PFN_OFFSET) -/* - * PFNs are used to describe any physical page; this means - * PFN 0 == physical address 0. - * - * This is the PFN of the first RAM page in the kernel - * direct-mapped view. We assume this is the first page - * of RAM in the mem_map as well. - */ -#define PHYS_PFN_OFFSET ((unsigned long)(PHYS_OFFSET >> PAGE_SHIFT)) +#endif /* * These are *only* valid on the kernel direct mapped RAM memory. @@ -343,9 +350,9 @@ static inline __deprecated void *bus_to_virt(unsigned long x) */ #define ARCH_PFN_OFFSET PHYS_PFN_OFFSET -#define virt_to_page(kaddr) pfn_to_page(__pa(kaddr) >> PAGE_SHIFT) +#define virt_to_page(kaddr) pfn_to_page(virt_to_pfn(kaddr)) #define virt_addr_valid(kaddr) (((unsigned long)(kaddr) >= PAGE_OFFSET && (unsigned long)(kaddr) < (unsigned long)high_memory) \ - && pfn_valid(__pa(kaddr) >> PAGE_SHIFT) ) + && pfn_valid(virt_to_pfn(kaddr))) #endif diff --git a/arch/arm/include/asm/pgtable-2level.h b/arch/arm/include/asm/pgtable-2level.h index dfff709fda3c..219ac88a9542 100644 --- a/arch/arm/include/asm/pgtable-2level.h +++ b/arch/arm/include/asm/pgtable-2level.h @@ -140,6 +140,7 @@ #define L_PTE_MT_DEV_NONSHARED (_AT(pteval_t, 0x0c) << 2) /* 1100 */ #define L_PTE_MT_DEV_WC (_AT(pteval_t, 0x09) << 2) /* 1001 */ #define L_PTE_MT_DEV_CACHED (_AT(pteval_t, 0x0b) << 2) /* 1011 */ +#define L_PTE_MT_VECTORS (_AT(pteval_t, 0x0f) << 2) /* 1111 */ #define L_PTE_MT_MASK (_AT(pteval_t, 0x0f) << 2) #ifndef __ASSEMBLY__ diff --git a/arch/arm/include/asm/pgtable.h b/arch/arm/include/asm/pgtable.h index 7d59b524f2af..5478e5d6ad89 100644 --- a/arch/arm/include/asm/pgtable.h +++ b/arch/arm/include/asm/pgtable.h @@ -216,13 +216,16 @@ static inline pte_t *pmd_page_vaddr(pmd_t pmd) #define pte_none(pte) (!pte_val(pte)) #define pte_present(pte) (pte_val(pte) & L_PTE_PRESENT) +#define pte_valid(pte) (pte_val(pte) & L_PTE_VALID) +#define pte_accessible(mm, pte) (mm_tlb_flush_pending(mm) ? pte_present(pte) : pte_valid(pte)) #define pte_write(pte) (!(pte_val(pte) & L_PTE_RDONLY)) #define pte_dirty(pte) (pte_val(pte) & L_PTE_DIRTY) #define pte_young(pte) (pte_val(pte) & L_PTE_YOUNG) #define pte_exec(pte) (!(pte_val(pte) & L_PTE_XN)) #define pte_special(pte) (0) -#define pte_present_user(pte) (pte_present(pte) && (pte_val(pte) & L_PTE_USER)) +#define pte_valid_user(pte) \ + (pte_valid(pte) && (pte_val(pte) & L_PTE_USER) && pte_young(pte)) #if __LINUX_ARM_ARCH__ < 6 static inline void __sync_icache_dcache(pte_t pteval) @@ -237,7 +240,7 @@ static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, { unsigned long ext = 0; - if (addr < TASK_SIZE && pte_present_user(pteval)) { + if (addr < TASK_SIZE && pte_valid_user(pteval)) { __sync_icache_dcache(pteval); ext |= PTE_EXT_NG; } diff --git a/arch/arm/include/asm/pmu.h b/arch/arm/include/asm/pmu.h index f24edad26c70..ae1919be8f98 100644 --- a/arch/arm/include/asm/pmu.h +++ b/arch/arm/include/asm/pmu.h @@ -71,6 +71,8 @@ struct arm_pmu { void (*disable)(struct perf_event *event); int (*get_event_idx)(struct pmu_hw_events *hw_events, struct perf_event *event); + void (*clear_event_idx)(struct pmu_hw_events *hw_events, + struct perf_event *event); int (*set_event_filter)(struct hw_perf_event *evt, struct perf_event_attr *attr); u32 (*read_counter)(struct perf_event *event); diff --git a/arch/arm/include/asm/probes.h b/arch/arm/include/asm/probes.h new file mode 100644 index 000000000000..806cfe622a9e --- /dev/null +++ b/arch/arm/include/asm/probes.h @@ -0,0 +1,43 @@ +/* + * arch/arm/include/asm/probes.h + * + * Original contents copied from arch/arm/include/asm/kprobes.h + * which contains the following notice... + * + * Copyright (C) 2006, 2007 Motorola Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ + +#ifndef _ASM_PROBES_H +#define _ASM_PROBES_H + +typedef u32 probes_opcode_t; + +struct arch_probes_insn; +typedef void (probes_insn_handler_t)(probes_opcode_t, + struct arch_probes_insn *, + struct pt_regs *); +typedef unsigned long (probes_check_cc)(unsigned long); +typedef void (probes_insn_singlestep_t)(probes_opcode_t, + struct arch_probes_insn *, + struct pt_regs *); +typedef void (probes_insn_fn_t)(void); + +/* Architecture specific copy of original instruction. */ +struct arch_probes_insn { + probes_opcode_t *insn; + probes_insn_handler_t *insn_handler; + probes_check_cc *insn_check_cc; + probes_insn_singlestep_t *insn_singlestep; + probes_insn_fn_t *insn_fn; +}; + +#endif diff --git a/arch/arm/include/asm/ptrace.h b/arch/arm/include/asm/ptrace.h index 04c99f36ff7f..c877654fe3bf 100644 --- a/arch/arm/include/asm/ptrace.h +++ b/arch/arm/include/asm/ptrace.h @@ -27,9 +27,13 @@ struct pt_regs { #define thumb_mode(regs) (0) #endif +#ifndef CONFIG_CPU_V7M #define isa_mode(regs) \ - ((((regs)->ARM_cpsr & PSR_J_BIT) >> 23) | \ - (((regs)->ARM_cpsr & PSR_T_BIT) >> 5)) + ((((regs)->ARM_cpsr & PSR_J_BIT) >> (__ffs(PSR_J_BIT) - 1)) | \ + (((regs)->ARM_cpsr & PSR_T_BIT) >> (__ffs(PSR_T_BIT)))) +#else +#define isa_mode(regs) 1 /* Thumb */ +#endif #define processor_mode(regs) \ ((regs)->ARM_cpsr & MODE_MASK) @@ -80,6 +84,12 @@ static inline long regs_return_value(struct pt_regs *regs) #define instruction_pointer(regs) (regs)->ARM_pc +static inline void instruction_pointer_set(struct pt_regs *regs, + unsigned long val) +{ + instruction_pointer(regs) = val; +} + #ifdef CONFIG_SMP extern unsigned long profile_pc(struct pt_regs *regs); #else diff --git a/arch/arm/include/asm/sync_bitops.h b/arch/arm/include/asm/sync_bitops.h index 63479eecbf76..9732b8e11e63 100644 --- a/arch/arm/include/asm/sync_bitops.h +++ b/arch/arm/include/asm/sync_bitops.h @@ -2,7 +2,6 @@ #define __ASM_SYNC_BITOPS_H__ #include <asm/bitops.h> -#include <asm/system.h> /* sync_bitops functions are equivalent to the SMP implementation of the * original functions, independently from CONFIG_SMP being defined. diff --git a/arch/arm/include/asm/system.h b/arch/arm/include/asm/system.h deleted file mode 100644 index 368165e33c1c..000000000000 --- a/arch/arm/include/asm/system.h +++ /dev/null @@ -1,7 +0,0 @@ -/* FILE TO BE DELETED. DO NOT ADD STUFF HERE! */ -#include <asm/barrier.h> -#include <asm/compiler.h> -#include <asm/cmpxchg.h> -#include <asm/switch_to.h> -#include <asm/system_info.h> -#include <asm/system_misc.h> diff --git a/arch/arm/include/asm/thread_info.h b/arch/arm/include/asm/thread_info.h index 71a06b293489..f989d7c22dc5 100644 --- a/arch/arm/include/asm/thread_info.h +++ b/arch/arm/include/asm/thread_info.h @@ -153,6 +153,7 @@ extern int vfp_restore_user_hwstate(struct user_vfp __user *, #define TIF_SIGPENDING 0 #define TIF_NEED_RESCHED 1 #define TIF_NOTIFY_RESUME 2 /* callback before returning to user */ +#define TIF_UPROBE 7 #define TIF_SYSCALL_TRACE 8 #define TIF_SYSCALL_AUDIT 9 #define TIF_SYSCALL_TRACEPOINT 10 @@ -165,6 +166,7 @@ extern int vfp_restore_user_hwstate(struct user_vfp __user *, #define _TIF_SIGPENDING (1 << TIF_SIGPENDING) #define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED) #define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME) +#define _TIF_UPROBE (1 << TIF_UPROBE) #define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE) #define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT) #define _TIF_SYSCALL_TRACEPOINT (1 << TIF_SYSCALL_TRACEPOINT) @@ -178,7 +180,8 @@ extern int vfp_restore_user_hwstate(struct user_vfp __user *, /* * Change these and you break ASM code in entry-common.S */ -#define _TIF_WORK_MASK (_TIF_NEED_RESCHED | _TIF_SIGPENDING | _TIF_NOTIFY_RESUME) +#define _TIF_WORK_MASK (_TIF_NEED_RESCHED | _TIF_SIGPENDING | \ + _TIF_NOTIFY_RESUME | _TIF_UPROBE) #endif /* __KERNEL__ */ #endif /* __ASM_ARM_THREAD_INFO_H */ diff --git a/arch/arm/include/asm/uaccess.h b/arch/arm/include/asm/uaccess.h index 72abdc541f38..12c3a5decc60 100644 --- a/arch/arm/include/asm/uaccess.h +++ b/arch/arm/include/asm/uaccess.h @@ -19,7 +19,7 @@ #include <asm/unified.h> #include <asm/compiler.h> -#if __LINUX_ARM_ARCH__ < 6 +#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS #include <asm-generic/uaccess-unaligned.h> #else #define __get_user_unaligned __get_user diff --git a/arch/arm/include/asm/unistd.h b/arch/arm/include/asm/unistd.h index acabef1a75df..43876245fc57 100644 --- a/arch/arm/include/asm/unistd.h +++ b/arch/arm/include/asm/unistd.h @@ -48,6 +48,5 @@ */ #define __IGNORE_fadvise64_64 #define __IGNORE_migrate_pages -#define __IGNORE_kcmp #endif /* __ASM_ARM_UNISTD_H */ diff --git a/arch/arm/include/asm/uprobes.h b/arch/arm/include/asm/uprobes.h new file mode 100644 index 000000000000..9472c20b7d49 --- /dev/null +++ b/arch/arm/include/asm/uprobes.h @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2012 Rabin Vincent <rabin at rab.in> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#ifndef _ASM_UPROBES_H +#define _ASM_UPROBES_H + +#include <asm/probes.h> +#include <asm/opcodes.h> + +typedef u32 uprobe_opcode_t; + +#define MAX_UINSN_BYTES 4 +#define UPROBE_XOL_SLOT_BYTES 64 + +#define UPROBE_SWBP_ARM_INSN 0xe7f001f9 +#define UPROBE_SS_ARM_INSN 0xe7f001fa +#define UPROBE_SWBP_INSN __opcode_to_mem_arm(UPROBE_SWBP_ARM_INSN) +#define UPROBE_SWBP_INSN_SIZE 4 + +struct arch_uprobe_task { + u32 backup; + unsigned long saved_trap_no; +}; + +struct arch_uprobe { + u8 insn[MAX_UINSN_BYTES]; + unsigned long ixol[2]; + uprobe_opcode_t bpinsn; + bool simulate; + u32 pcreg; + void (*prehandler)(struct arch_uprobe *auprobe, + struct arch_uprobe_task *autask, + struct pt_regs *regs); + void (*posthandler)(struct arch_uprobe *auprobe, + struct arch_uprobe_task *autask, + struct pt_regs *regs); + struct arch_probes_insn asi; +}; + +#endif diff --git a/arch/arm/include/uapi/asm/hwcap.h b/arch/arm/include/uapi/asm/hwcap.h index 7dcc10d67253..20d12f230a2f 100644 --- a/arch/arm/include/uapi/asm/hwcap.h +++ b/arch/arm/include/uapi/asm/hwcap.h @@ -28,4 +28,13 @@ #define HWCAP_LPAE (1 << 20) #define HWCAP_EVTSTRM (1 << 21) +/* + * HWCAP2 flags - for elf_hwcap2 (in kernel) and AT_HWCAP2 + */ +#define HWCAP2_AES (1 << 0) +#define HWCAP2_PMULL (1 << 1) +#define HWCAP2_SHA1 (1 << 2) +#define HWCAP2_SHA2 (1 << 3) +#define HWCAP2_CRC32 (1 << 4) + #endif /* _UAPI__ASMARM_HWCAP_H */ diff --git a/arch/arm/kernel/Makefile b/arch/arm/kernel/Makefile index a30fc9be9e9e..a766bcbaf8ad 100644 --- a/arch/arm/kernel/Makefile +++ b/arch/arm/kernel/Makefile @@ -50,11 +50,12 @@ obj-$(CONFIG_DYNAMIC_FTRACE) += ftrace.o insn.o obj-$(CONFIG_FUNCTION_GRAPH_TRACER) += ftrace.o insn.o obj-$(CONFIG_JUMP_LABEL) += jump_label.o insn.o patch.o obj-$(CONFIG_KEXEC) += machine_kexec.o relocate_kernel.o -obj-$(CONFIG_KPROBES) += kprobes.o kprobes-common.o patch.o +obj-$(CONFIG_UPROBES) += probes.o probes-arm.o uprobes.o uprobes-arm.o +obj-$(CONFIG_KPROBES) += probes.o kprobes.o kprobes-common.o patch.o ifdef CONFIG_THUMB2_KERNEL -obj-$(CONFIG_KPROBES) += kprobes-thumb.o +obj-$(CONFIG_KPROBES) += kprobes-thumb.o probes-thumb.o else -obj-$(CONFIG_KPROBES) += kprobes-arm.o +obj-$(CONFIG_KPROBES) += kprobes-arm.o probes-arm.o endif obj-$(CONFIG_ARM_KPROBES_TEST) += test-kprobes.o test-kprobes-objs := kprobes-test.o diff --git a/arch/arm/kernel/armksyms.c b/arch/arm/kernel/armksyms.c index 85e664b6a5f1..f7b450f97e68 100644 --- a/arch/arm/kernel/armksyms.c +++ b/arch/arm/kernel/armksyms.c @@ -158,6 +158,6 @@ EXPORT_SYMBOL(__gnu_mcount_nc); #endif #ifdef CONFIG_ARM_PATCH_PHYS_VIRT -EXPORT_SYMBOL(__pv_phys_offset); +EXPORT_SYMBOL(__pv_phys_pfn_offset); EXPORT_SYMBOL(__pv_offset); #endif diff --git a/arch/arm/kernel/bios32.c b/arch/arm/kernel/bios32.c index d0d46786892c..16d43cd45619 100644 --- a/arch/arm/kernel/bios32.c +++ b/arch/arm/kernel/bios32.c @@ -605,41 +605,10 @@ resource_size_t pcibios_align_resource(void *data, const struct resource *res, */ int pcibios_enable_device(struct pci_dev *dev, int mask) { - u16 cmd, old_cmd; - int idx; - struct resource *r; - - pci_read_config_word(dev, PCI_COMMAND, &cmd); - old_cmd = cmd; - for (idx = 0; idx < 6; idx++) { - /* Only set up the requested stuff */ - if (!(mask & (1 << idx))) - continue; - - r = dev->resource + idx; - if (!r->start && r->end) { - printk(KERN_ERR "PCI: Device %s not available because" - " of resource collisions\n", pci_name(dev)); - return -EINVAL; - } - if (r->flags & IORESOURCE_IO) - cmd |= PCI_COMMAND_IO; - if (r->flags & IORESOURCE_MEM) - cmd |= PCI_COMMAND_MEMORY; - } + if (pci_has_flag(PCI_PROBE_ONLY)) + return 0; - /* - * Bridges (eg, cardbus bridges) need to be fully enabled - */ - if ((dev->class >> 16) == PCI_BASE_CLASS_BRIDGE) - cmd |= PCI_COMMAND_IO | PCI_COMMAND_MEMORY; - - if (cmd != old_cmd) { - printk("PCI: enabling device %s (%04x -> %04x)\n", - pci_name(dev), old_cmd, cmd); - pci_write_config_word(dev, PCI_COMMAND, cmd); - } - return 0; + return pci_enable_resources(dev, mask); } int pci_mmap_page_range(struct pci_dev *dev, struct vm_area_struct *vma, diff --git a/arch/arm/kernel/head.S b/arch/arm/kernel/head.S index f5f381d91556..f8c08839edf3 100644 --- a/arch/arm/kernel/head.S +++ b/arch/arm/kernel/head.S @@ -584,9 +584,10 @@ __fixup_pv_table: subs r3, r0, r3 @ PHYS_OFFSET - PAGE_OFFSET add r4, r4, r3 @ adjust table start address add r5, r5, r3 @ adjust table end address - add r6, r6, r3 @ adjust __pv_phys_offset address + add r6, r6, r3 @ adjust __pv_phys_pfn_offset address add r7, r7, r3 @ adjust __pv_offset address - str r8, [r6, #LOW_OFFSET] @ save computed PHYS_OFFSET to __pv_phys_offset + mov r0, r8, lsr #12 @ convert to PFN + str r0, [r6, #LOW_OFFSET] @ save computed PHYS_OFFSET to __pv_phys_pfn_offset strcc ip, [r7, #HIGH_OFFSET] @ save to __pv_offset high bits mov r6, r3, lsr #24 @ constant for add/sub instructions teq r3, r6, lsl #24 @ must be 16MiB aligned @@ -600,7 +601,7 @@ ENDPROC(__fixup_pv_table) 1: .long . .long __pv_table_begin .long __pv_table_end -2: .long __pv_phys_offset +2: .long __pv_phys_pfn_offset .long __pv_offset .text @@ -688,11 +689,11 @@ ENTRY(fixup_pv_table) ENDPROC(fixup_pv_table) .data - .globl __pv_phys_offset - .type __pv_phys_offset, %object -__pv_phys_offset: - .quad 0 - .size __pv_phys_offset, . -__pv_phys_offset + .globl __pv_phys_pfn_offset + .type __pv_phys_pfn_offset, %object +__pv_phys_pfn_offset: + .word 0 + .size __pv_phys_pfn_offset, . -__pv_phys_pfn_offset .globl __pv_offset .type __pv_offset, %object diff --git a/arch/arm/kernel/hw_breakpoint.c b/arch/arm/kernel/hw_breakpoint.c index 3d446605cbf8..9da35c6d3411 100644 --- a/arch/arm/kernel/hw_breakpoint.c +++ b/arch/arm/kernel/hw_breakpoint.c @@ -167,7 +167,7 @@ static int debug_arch_supported(void) /* Can we determine the watchpoint access type from the fsr? */ static int debug_exception_updates_fsr(void) { - return 0; + return get_debug_arch() >= ARM_DEBUG_ARCH_V8; } /* Determine number of WRP registers available. */ @@ -257,6 +257,7 @@ static int enable_monitor_mode(void) break; case ARM_DEBUG_ARCH_V7_ECP14: case ARM_DEBUG_ARCH_V7_1: + case ARM_DEBUG_ARCH_V8: ARM_DBG_WRITE(c0, c2, 2, (dscr | ARM_DSCR_MDBGEN)); isb(); break; diff --git a/arch/arm/kernel/kprobes-arm.c b/arch/arm/kernel/kprobes-arm.c index 8a30c89da70e..ac300c60d656 100644 --- a/arch/arm/kernel/kprobes-arm.c +++ b/arch/arm/kernel/kprobes-arm.c @@ -60,13 +60,10 @@ #include <linux/kernel.h> #include <linux/kprobes.h> -#include <linux/module.h> +#include <linux/ptrace.h> #include "kprobes.h" - -#define sign_extend(x, signbit) ((x) | (0 - ((x) & (1 << (signbit))))) - -#define branch_displacement(insn) sign_extend(((insn) & 0xffffff) << 2, 25) +#include "probes-arm.h" #if __LINUX_ARM_ARCH__ >= 6 #define BLX(reg) "blx "reg" \n\t" @@ -75,92 +72,11 @@ "mov pc, "reg" \n\t" #endif -/* - * To avoid the complications of mimicing single-stepping on a - * processor without a Next-PC or a single-step mode, and to - * avoid having to deal with the side-effects of boosting, we - * simulate or emulate (almost) all ARM instructions. - * - * "Simulation" is where the instruction's behavior is duplicated in - * C code. "Emulation" is where the original instruction is rewritten - * and executed, often by altering its registers. - * - * By having all behavior of the kprobe'd instruction completed before - * returning from the kprobe_handler(), all locks (scheduler and - * interrupt) can safely be released. There is no need for secondary - * breakpoints, no race with MP or preemptable kernels, nor having to - * clean up resources counts at a later time impacting overall system - * performance. By rewriting the instruction, only the minimum registers - * need to be loaded and saved back optimizing performance. - * - * Calling the insnslot_*_rwflags version of a function doesn't hurt - * anything even when the CPSR flags aren't updated by the - * instruction. It's just a little slower in return for saving - * a little space by not having a duplicate function that doesn't - * update the flags. (The same optimization can be said for - * instructions that do or don't perform register writeback) - * Also, instructions can either read the flags, only write the - * flags, or read and write the flags. To save combinations - * rather than for sheer performance, flag functions just assume - * read and write of flags. - */ - -static void __kprobes simulate_bbl(struct kprobe *p, struct pt_regs *regs) -{ - kprobe_opcode_t insn = p->opcode; - long iaddr = (long)p->addr; - int disp = branch_displacement(insn); - - if (insn & (1 << 24)) - regs->ARM_lr = iaddr + 4; - - regs->ARM_pc = iaddr + 8 + disp; -} - -static void __kprobes simulate_blx1(struct kprobe *p, struct pt_regs *regs) -{ - kprobe_opcode_t insn = p->opcode; - long iaddr = (long)p->addr; - int disp = branch_displacement(insn); - - regs->ARM_lr = iaddr + 4; - regs->ARM_pc = iaddr + 8 + disp + ((insn >> 23) & 0x2); - regs->ARM_cpsr |= PSR_T_BIT; -} - -static void __kprobes simulate_blx2bx(struct kprobe *p, struct pt_regs *regs) -{ - kprobe_opcode_t insn = p->opcode; - int rm = insn & 0xf; - long rmv = regs->uregs[rm]; - - if (insn & (1 << 5)) - regs->ARM_lr = (long)p->addr + 4; - - regs->ARM_pc = rmv & ~0x1; - regs->ARM_cpsr &= ~PSR_T_BIT; - if (rmv & 0x1) - regs->ARM_cpsr |= PSR_T_BIT; -} - -static void __kprobes simulate_mrs(struct kprobe *p, struct pt_regs *regs) -{ - kprobe_opcode_t insn = p->opcode; - int rd = (insn >> 12) & 0xf; - unsigned long mask = 0xf8ff03df; /* Mask out execution state */ - regs->uregs[rd] = regs->ARM_cpsr & mask; -} - -static void __kprobes simulate_mov_ipsp(struct kprobe *p, struct pt_regs *regs) -{ - regs->uregs[12] = regs->uregs[13]; -} - static void __kprobes -emulate_ldrdstrd(struct kprobe *p, struct pt_regs *regs) +emulate_ldrdstrd(probes_opcode_t insn, + struct arch_probes_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; - unsigned long pc = (unsigned long)p->addr + 8; + unsigned long pc = regs->ARM_pc + 4; int rt = (insn >> 12) & 0xf; int rn = (insn >> 16) & 0xf; int rm = insn & 0xf; @@ -175,7 +91,7 @@ emulate_ldrdstrd(struct kprobe *p, struct pt_regs *regs) BLX("%[fn]") : "=r" (rtv), "=r" (rt2v), "=r" (rnv) : "0" (rtv), "1" (rt2v), "2" (rnv), "r" (rmv), - [fn] "r" (p->ainsn.insn_fn) + [fn] "r" (asi->insn_fn) : "lr", "memory", "cc" ); @@ -186,10 +102,10 @@ emulate_ldrdstrd(struct kprobe *p, struct pt_regs *regs) } static void __kprobes -emulate_ldr(struct kprobe *p, struct pt_regs *regs) +emulate_ldr(probes_opcode_t insn, + struct arch_probes_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; - unsigned long pc = (unsigned long)p->addr + 8; + unsigned long pc = regs->ARM_pc + 4; int rt = (insn >> 12) & 0xf; int rn = (insn >> 16) & 0xf; int rm = insn & 0xf; @@ -202,7 +118,7 @@ emulate_ldr(struct kprobe *p, struct pt_regs *regs) __asm__ __volatile__ ( BLX("%[fn]") : "=r" (rtv), "=r" (rnv) - : "1" (rnv), "r" (rmv), [fn] "r" (p->ainsn.insn_fn) + : "1" (rnv), "r" (rmv), [fn] "r" (asi->insn_fn) : "lr", "memory", "cc" ); @@ -216,11 +132,11 @@ emulate_ldr(struct kprobe *p, struct pt_regs *regs) } static void __kprobes -emulate_str(struct kprobe *p, struct pt_regs *regs) +emulate_str(probes_opcode_t insn, + struct arch_probes_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; - unsigned long rtpc = (unsigned long)p->addr + str_pc_offset; - unsigned long rnpc = (unsigned long)p->addr + 8; + unsigned long rtpc = regs->ARM_pc - 4 + str_pc_offset; + unsigned long rnpc = regs->ARM_pc + 4; int rt = (insn >> 12) & 0xf; int rn = (insn >> 16) & 0xf; int rm = insn & 0xf; @@ -234,7 +150,7 @@ emulate_str(struct kprobe *p, struct pt_regs *regs) __asm__ __volatile__ ( BLX("%[fn]") : "=r" (rnv) - : "r" (rtv), "0" (rnv), "r" (rmv), [fn] "r" (p->ainsn.insn_fn) + : "r" (rtv), "0" (rnv), "r" (rmv), [fn] "r" (asi->insn_fn) : "lr", "memory", "cc" ); @@ -243,10 +159,10 @@ emulate_str(struct kprobe *p, struct pt_regs *regs) } static void __kprobes -emulate_rd12rn16rm0rs8_rwflags(struct kprobe *p, struct pt_regs *regs) +emulate_rd12rn16rm0rs8_rwflags(probes_opcode_t insn, + struct arch_probes_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; - unsigned long pc = (unsigned long)p->addr + 8; + unsigned long pc = regs->ARM_pc + 4; int rd = (insn >> 12) & 0xf; int rn = (insn >> 16) & 0xf; int rm = insn & 0xf; @@ -266,7 +182,7 @@ emulate_rd12rn16rm0rs8_rwflags(struct kprobe *p, struct pt_regs *regs) "mrs %[cpsr], cpsr \n\t" : "=r" (rdv), [cpsr] "=r" (cpsr) : "0" (rdv), "r" (rnv), "r" (rmv), "r" (rsv), - "1" (cpsr), [fn] "r" (p->ainsn.insn_fn) + "1" (cpsr), [fn] "r" (asi->insn_fn) : "lr", "memory", "cc" ); @@ -278,9 +194,9 @@ emulate_rd12rn16rm0rs8_rwflags(struct kprobe *p, struct pt_regs *regs) } static void __kprobes -emulate_rd12rn16rm0_rwflags_nopc(struct kprobe *p, struct pt_regs *regs) +emulate_rd12rn16rm0_rwflags_nopc(probes_opcode_t insn, + struct arch_probes_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; int rd = (insn >> 12) & 0xf; int rn = (insn >> 16) & 0xf; int rm = insn & 0xf; @@ -296,7 +212,7 @@ emulate_rd12rn16rm0_rwflags_nopc(struct kprobe *p, struct pt_regs *regs) "mrs %[cpsr], cpsr \n\t" : "=r" (rdv), [cpsr] "=r" (cpsr) : "0" (rdv), "r" (rnv), "r" (rmv), - "1" (cpsr), [fn] "r" (p->ainsn.insn_fn) + "1" (cpsr), [fn] "r" (asi->insn_fn) : "lr", "memory", "cc" ); @@ -305,9 +221,10 @@ emulate_rd12rn16rm0_rwflags_nopc(struct kprobe *p, struct pt_regs *regs) } static void __kprobes -emulate_rd16rn12rm0rs8_rwflags_nopc(struct kprobe *p, struct pt_regs *regs) +emulate_rd16rn12rm0rs8_rwflags_nopc(probes_opcode_t insn, + struct arch_probes_insn *asi, + struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; int rd = (insn >> 16) & 0xf; int rn = (insn >> 12) & 0xf; int rm = insn & 0xf; @@ -325,7 +242,7 @@ emulate_rd16rn12rm0rs8_rwflags_nopc(struct kprobe *p, struct pt_regs *regs) "mrs %[cpsr], cpsr \n\t" : "=r" (rdv), [cpsr] "=r" (cpsr) : "0" (rdv), "r" (rnv), "r" (rmv), "r" (rsv), - "1" (cpsr), [fn] "r" (p->ainsn.insn_fn) + "1" (cpsr), [fn] "r" (asi->insn_fn) : "lr", "memory", "cc" ); @@ -334,9 +251,9 @@ emulate_rd16rn12rm0rs8_rwflags_nopc(struct kprobe *p, struct pt_regs *regs) } static void __kprobes -emulate_rd12rm0_noflags_nopc(struct kprobe *p, struct pt_regs *regs) +emulate_rd12rm0_noflags_nopc(probes_opcode_t insn, + struct arch_probes_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; int rd = (insn >> 12) & 0xf; int rm = insn & 0xf; @@ -346,7 +263,7 @@ emulate_rd12rm0_noflags_nopc(struct kprobe *p, struct pt_regs *regs) __asm__ __volatile__ ( BLX("%[fn]") : "=r" (rdv) - : "0" (rdv), "r" (rmv), [fn] "r" (p->ainsn.insn_fn) + : "0" (rdv), "r" (rmv), [fn] "r" (asi->insn_fn) : "lr", "memory", "cc" ); @@ -354,9 +271,10 @@ emulate_rd12rm0_noflags_nopc(struct kprobe *p, struct pt_regs *regs) } static void __kprobes -emulate_rdlo12rdhi16rn0rm8_rwflags_nopc(struct kprobe *p, struct pt_regs *regs) +emulate_rdlo12rdhi16rn0rm8_rwflags_nopc(probes_opcode_t insn, + struct arch_probes_insn *asi, + struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; int rdlo = (insn >> 12) & 0xf; int rdhi = (insn >> 16) & 0xf; int rn = insn & 0xf; @@ -374,7 +292,7 @@ emulate_rdlo12rdhi16rn0rm8_rwflags_nopc(struct kprobe *p, struct pt_regs *regs) "mrs %[cpsr], cpsr \n\t" : "=r" (rdlov), "=r" (rdhiv), [cpsr] "=r" (cpsr) : "0" (rdlov), "1" (rdhiv), "r" (rnv), "r" (rmv), - "2" (cpsr), [fn] "r" (p->ainsn.insn_fn) + "2" (cpsr), [fn] "r" (asi->insn_fn) : "lr", "memory", "cc" ); @@ -383,623 +301,43 @@ emulate_rdlo12rdhi16rn0rm8_rwflags_nopc(struct kprobe *p, struct pt_regs *regs) regs->ARM_cpsr = (regs->ARM_cpsr & ~APSR_MASK) | (cpsr & APSR_MASK); } -/* - * For the instruction masking and comparisons in all the "space_*" - * functions below, Do _not_ rearrange the order of tests unless - * you're very, very sure of what you are doing. For the sake of - * efficiency, the masks for some tests sometimes assume other test - * have been done prior to them so the number of patterns to test - * for an instruction set can be as broad as possible to reduce the - * number of tests needed. - */ - -static const union decode_item arm_1111_table[] = { - /* Unconditional instructions */ - - /* memory hint 1111 0100 x001 xxxx xxxx xxxx xxxx xxxx */ - /* PLDI (immediate) 1111 0100 x101 xxxx xxxx xxxx xxxx xxxx */ - /* PLDW (immediate) 1111 0101 x001 xxxx xxxx xxxx xxxx xxxx */ - /* PLD (immediate) 1111 0101 x101 xxxx xxxx xxxx xxxx xxxx */ - DECODE_SIMULATE (0xfe300000, 0xf4100000, kprobe_simulate_nop), - - /* memory hint 1111 0110 x001 xxxx xxxx xxxx xxx0 xxxx */ - /* PLDI (register) 1111 0110 x101 xxxx xxxx xxxx xxx0 xxxx */ - /* PLDW (register) 1111 0111 x001 xxxx xxxx xxxx xxx0 xxxx */ - /* PLD (register) 1111 0111 x101 xxxx xxxx xxxx xxx0 xxxx */ - DECODE_SIMULATE (0xfe300010, 0xf6100000, kprobe_simulate_nop), - - /* BLX (immediate) 1111 101x xxxx xxxx xxxx xxxx xxxx xxxx */ - DECODE_SIMULATE (0xfe000000, 0xfa000000, simulate_blx1), - - /* CPS 1111 0001 0000 xxx0 xxxx xxxx xx0x xxxx */ - /* SETEND 1111 0001 0000 0001 xxxx xxxx 0000 xxxx */ - /* SRS 1111 100x x1x0 xxxx xxxx xxxx xxxx xxxx */ - /* RFE 1111 100x x0x1 xxxx xxxx xxxx xxxx xxxx */ - - /* Coprocessor instructions... */ - /* MCRR2 1111 1100 0100 xxxx xxxx xxxx xxxx xxxx */ - /* MRRC2 1111 1100 0101 xxxx xxxx xxxx xxxx xxxx */ - /* LDC2 1111 110x xxx1 xxxx xxxx xxxx xxxx xxxx */ - /* STC2 1111 110x xxx0 xxxx xxxx xxxx xxxx xxxx */ - /* CDP2 1111 1110 xxxx xxxx xxxx xxxx xxx0 xxxx */ - /* MCR2 1111 1110 xxx0 xxxx xxxx xxxx xxx1 xxxx */ - /* MRC2 1111 1110 xxx1 xxxx xxxx xxxx xxx1 xxxx */ - - /* Other unallocated instructions... */ - DECODE_END -}; - -static const union decode_item arm_cccc_0001_0xx0____0xxx_table[] = { - /* Miscellaneous instructions */ - - /* MRS cpsr cccc 0001 0000 xxxx xxxx xxxx 0000 xxxx */ - DECODE_SIMULATEX(0x0ff000f0, 0x01000000, simulate_mrs, - REGS(0, NOPC, 0, 0, 0)), - - /* BX cccc 0001 0010 xxxx xxxx xxxx 0001 xxxx */ - DECODE_SIMULATE (0x0ff000f0, 0x01200010, simulate_blx2bx), - - /* BLX (register) cccc 0001 0010 xxxx xxxx xxxx 0011 xxxx */ - DECODE_SIMULATEX(0x0ff000f0, 0x01200030, simulate_blx2bx, - REGS(0, 0, 0, 0, NOPC)), - - /* CLZ cccc 0001 0110 xxxx xxxx xxxx 0001 xxxx */ - DECODE_EMULATEX (0x0ff000f0, 0x01600010, emulate_rd12rm0_noflags_nopc, - REGS(0, NOPC, 0, 0, NOPC)), - - /* QADD cccc 0001 0000 xxxx xxxx xxxx 0101 xxxx */ - /* QSUB cccc 0001 0010 xxxx xxxx xxxx 0101 xxxx */ - /* QDADD cccc 0001 0100 xxxx xxxx xxxx 0101 xxxx */ - /* QDSUB cccc 0001 0110 xxxx xxxx xxxx 0101 xxxx */ - DECODE_EMULATEX (0x0f9000f0, 0x01000050, emulate_rd12rn16rm0_rwflags_nopc, - REGS(NOPC, NOPC, 0, 0, NOPC)), - - /* BXJ cccc 0001 0010 xxxx xxxx xxxx 0010 xxxx */ - /* MSR cccc 0001 0x10 xxxx xxxx xxxx 0000 xxxx */ - /* MRS spsr cccc 0001 0100 xxxx xxxx xxxx 0000 xxxx */ - /* BKPT 1110 0001 0010 xxxx xxxx xxxx 0111 xxxx */ - /* SMC cccc 0001 0110 xxxx xxxx xxxx 0111 xxxx */ - /* And unallocated instructions... */ - DECODE_END -}; - -static const union decode_item arm_cccc_0001_0xx0____1xx0_table[] = { - /* Halfword multiply and multiply-accumulate */ - - /* SMLALxy cccc 0001 0100 xxxx xxxx xxxx 1xx0 xxxx */ - DECODE_EMULATEX (0x0ff00090, 0x01400080, emulate_rdlo12rdhi16rn0rm8_rwflags_nopc, - REGS(NOPC, NOPC, NOPC, 0, NOPC)), - - /* SMULWy cccc 0001 0010 xxxx xxxx xxxx 1x10 xxxx */ - DECODE_OR (0x0ff000b0, 0x012000a0), - /* SMULxy cccc 0001 0110 xxxx xxxx xxxx 1xx0 xxxx */ - DECODE_EMULATEX (0x0ff00090, 0x01600080, emulate_rd16rn12rm0rs8_rwflags_nopc, - REGS(NOPC, 0, NOPC, 0, NOPC)), - - /* SMLAxy cccc 0001 0000 xxxx xxxx xxxx 1xx0 xxxx */ - DECODE_OR (0x0ff00090, 0x01000080), - /* SMLAWy cccc 0001 0010 xxxx xxxx xxxx 1x00 xxxx */ - DECODE_EMULATEX (0x0ff000b0, 0x01200080, emulate_rd16rn12rm0rs8_rwflags_nopc, - REGS(NOPC, NOPC, NOPC, 0, NOPC)), - - DECODE_END +const union decode_action kprobes_arm_actions[NUM_PROBES_ARM_ACTIONS] = { + [PROBES_EMULATE_NONE] = {.handler = probes_emulate_none}, + [PROBES_SIMULATE_NOP] = {.handler = probes_simulate_nop}, + [PROBES_PRELOAD_IMM] = {.handler = probes_simulate_nop}, + [PROBES_PRELOAD_REG] = {.handler = probes_simulate_nop}, + [PROBES_BRANCH_IMM] = {.handler = simulate_blx1}, + [PROBES_MRS] = {.handler = simulate_mrs}, + [PROBES_BRANCH_REG] = {.handler = simulate_blx2bx}, + [PROBES_CLZ] = {.handler = emulate_rd12rm0_noflags_nopc}, + [PROBES_SATURATING_ARITHMETIC] = { + .handler = emulate_rd12rn16rm0_rwflags_nopc}, + [PROBES_MUL1] = {.handler = emulate_rdlo12rdhi16rn0rm8_rwflags_nopc}, + [PROBES_MUL2] = {.handler = emulate_rd16rn12rm0rs8_rwflags_nopc}, + [PROBES_SWP] = {.handler = emulate_rd12rn16rm0_rwflags_nopc}, + [PROBES_LDRSTRD] = {.handler = emulate_ldrdstrd}, + [PROBES_LOAD_EXTRA] = {.handler = emulate_ldr}, + [PROBES_LOAD] = {.handler = emulate_ldr}, + [PROBES_STORE_EXTRA] = {.handler = emulate_str}, + [PROBES_STORE] = {.handler = emulate_str}, + [PROBES_MOV_IP_SP] = {.handler = simulate_mov_ipsp}, + [PROBES_DATA_PROCESSING_REG] = { + .handler = emulate_rd12rn16rm0rs8_rwflags}, + [PROBES_DATA_PROCESSING_IMM] = { + .handler = emulate_rd12rn16rm0rs8_rwflags}, + [PROBES_MOV_HALFWORD] = {.handler = emulate_rd12rm0_noflags_nopc}, + [PROBES_SEV] = {.handler = probes_emulate_none}, + [PROBES_WFE] = {.handler = probes_simulate_nop}, + [PROBES_SATURATE] = {.handler = emulate_rd12rn16rm0_rwflags_nopc}, + [PROBES_REV] = {.handler = emulate_rd12rm0_noflags_nopc}, + [PROBES_MMI] = {.handler = emulate_rd12rn16rm0_rwflags_nopc}, + [PROBES_PACK] = {.handler = emulate_rd12rn16rm0_rwflags_nopc}, + [PROBES_EXTEND] = {.handler = emulate_rd12rm0_noflags_nopc}, + [PROBES_EXTEND_ADD] = {.handler = emulate_rd12rn16rm0_rwflags_nopc}, + [PROBES_MUL_ADD_LONG] = { + .handler = emulate_rdlo12rdhi16rn0rm8_rwflags_nopc}, + [PROBES_MUL_ADD] = {.handler = emulate_rd16rn12rm0rs8_rwflags_nopc}, + [PROBES_BITFIELD] = {.handler = emulate_rd12rm0_noflags_nopc}, + [PROBES_BRANCH] = {.handler = simulate_bbl}, + [PROBES_LDMSTM] = {.decoder = kprobe_decode_ldmstm} }; - -static const union decode_item arm_cccc_0000_____1001_table[] = { - /* Multiply and multiply-accumulate */ - - /* MUL cccc 0000 0000 xxxx xxxx xxxx 1001 xxxx */ - /* MULS cccc 0000 0001 xxxx xxxx xxxx 1001 xxxx */ - DECODE_EMULATEX (0x0fe000f0, 0x00000090, emulate_rd16rn12rm0rs8_rwflags_nopc, - REGS(NOPC, 0, NOPC, 0, NOPC)), - - /* MLA cccc 0000 0010 xxxx xxxx xxxx 1001 xxxx */ - /* MLAS cccc 0000 0011 xxxx xxxx xxxx 1001 xxxx */ - DECODE_OR (0x0fe000f0, 0x00200090), - /* MLS cccc 0000 0110 xxxx xxxx xxxx 1001 xxxx */ - DECODE_EMULATEX (0x0ff000f0, 0x00600090, emulate_rd16rn12rm0rs8_rwflags_nopc, - REGS(NOPC, NOPC, NOPC, 0, NOPC)), - - /* UMAAL cccc 0000 0100 xxxx xxxx xxxx 1001 xxxx */ - DECODE_OR (0x0ff000f0, 0x00400090), - /* UMULL cccc 0000 1000 xxxx xxxx xxxx 1001 xxxx */ - /* UMULLS cccc 0000 1001 xxxx xxxx xxxx 1001 xxxx */ - /* UMLAL cccc 0000 1010 xxxx xxxx xxxx 1001 xxxx */ - /* UMLALS cccc 0000 1011 xxxx xxxx xxxx 1001 xxxx */ - /* SMULL cccc 0000 1100 xxxx xxxx xxxx 1001 xxxx */ - /* SMULLS cccc 0000 1101 xxxx xxxx xxxx 1001 xxxx */ - /* SMLAL cccc 0000 1110 xxxx xxxx xxxx 1001 xxxx */ - /* SMLALS cccc 0000 1111 xxxx xxxx xxxx 1001 xxxx */ - DECODE_EMULATEX (0x0f8000f0, 0x00800090, emulate_rdlo12rdhi16rn0rm8_rwflags_nopc, - REGS(NOPC, NOPC, NOPC, 0, NOPC)), - - DECODE_END -}; - -static const union decode_item arm_cccc_0001_____1001_table[] = { - /* Synchronization primitives */ - -#if __LINUX_ARM_ARCH__ < 6 - /* Deprecated on ARMv6 and may be UNDEFINED on v7 */ - /* SMP/SWPB cccc 0001 0x00 xxxx xxxx xxxx 1001 xxxx */ - DECODE_EMULATEX (0x0fb000f0, 0x01000090, emulate_rd12rn16rm0_rwflags_nopc, - REGS(NOPC, NOPC, 0, 0, NOPC)), -#endif - /* LDREX/STREX{,D,B,H} cccc 0001 1xxx xxxx xxxx xxxx 1001 xxxx */ - /* And unallocated instructions... */ - DECODE_END -}; - -static const union decode_item arm_cccc_000x_____1xx1_table[] = { - /* Extra load/store instructions */ - - /* STRHT cccc 0000 xx10 xxxx xxxx xxxx 1011 xxxx */ - /* ??? cccc 0000 xx10 xxxx xxxx xxxx 11x1 xxxx */ - /* LDRHT cccc 0000 xx11 xxxx xxxx xxxx 1011 xxxx */ - /* LDRSBT cccc 0000 xx11 xxxx xxxx xxxx 1101 xxxx */ - /* LDRSHT cccc 0000 xx11 xxxx xxxx xxxx 1111 xxxx */ - DECODE_REJECT (0x0f200090, 0x00200090), - - /* LDRD/STRD lr,pc,{... cccc 000x x0x0 xxxx 111x xxxx 1101 xxxx */ - DECODE_REJECT (0x0e10e0d0, 0x0000e0d0), - - /* LDRD (register) cccc 000x x0x0 xxxx xxxx xxxx 1101 xxxx */ - /* STRD (register) cccc 000x x0x0 xxxx xxxx xxxx 1111 xxxx */ - DECODE_EMULATEX (0x0e5000d0, 0x000000d0, emulate_ldrdstrd, - REGS(NOPCWB, NOPCX, 0, 0, NOPC)), - - /* LDRD (immediate) cccc 000x x1x0 xxxx xxxx xxxx 1101 xxxx */ - /* STRD (immediate) cccc 000x x1x0 xxxx xxxx xxxx 1111 xxxx */ - DECODE_EMULATEX (0x0e5000d0, 0x004000d0, emulate_ldrdstrd, - REGS(NOPCWB, NOPCX, 0, 0, 0)), - - /* STRH (register) cccc 000x x0x0 xxxx xxxx xxxx 1011 xxxx */ - DECODE_EMULATEX (0x0e5000f0, 0x000000b0, emulate_str, - REGS(NOPCWB, NOPC, 0, 0, NOPC)), - - /* LDRH (register) cccc 000x x0x1 xxxx xxxx xxxx 1011 xxxx */ - /* LDRSB (register) cccc 000x x0x1 xxxx xxxx xxxx 1101 xxxx */ - /* LDRSH (register) cccc 000x x0x1 xxxx xxxx xxxx 1111 xxxx */ - DECODE_EMULATEX (0x0e500090, 0x00100090, emulate_ldr, - REGS(NOPCWB, NOPC, 0, 0, NOPC)), - - /* STRH (immediate) cccc 000x x1x0 xxxx xxxx xxxx 1011 xxxx */ - DECODE_EMULATEX (0x0e5000f0, 0x004000b0, emulate_str, - REGS(NOPCWB, NOPC, 0, 0, 0)), - - /* LDRH (immediate) cccc 000x x1x1 xxxx xxxx xxxx 1011 xxxx */ - /* LDRSB (immediate) cccc 000x x1x1 xxxx xxxx xxxx 1101 xxxx */ - /* LDRSH (immediate) cccc 000x x1x1 xxxx xxxx xxxx 1111 xxxx */ - DECODE_EMULATEX (0x0e500090, 0x00500090, emulate_ldr, - REGS(NOPCWB, NOPC, 0, 0, 0)), - - DECODE_END -}; - -static const union decode_item arm_cccc_000x_table[] = { - /* Data-processing (register) */ - - /* <op>S PC, ... cccc 000x xxx1 xxxx 1111 xxxx xxxx xxxx */ - DECODE_REJECT (0x0e10f000, 0x0010f000), - - /* MOV IP, SP 1110 0001 1010 0000 1100 0000 0000 1101 */ - DECODE_SIMULATE (0xffffffff, 0xe1a0c00d, simulate_mov_ipsp), - - /* TST (register) cccc 0001 0001 xxxx xxxx xxxx xxx0 xxxx */ - /* TEQ (register) cccc 0001 0011 xxxx xxxx xxxx xxx0 xxxx */ - /* CMP (register) cccc 0001 0101 xxxx xxxx xxxx xxx0 xxxx */ - /* CMN (register) cccc 0001 0111 xxxx xxxx xxxx xxx0 xxxx */ - DECODE_EMULATEX (0x0f900010, 0x01100000, emulate_rd12rn16rm0rs8_rwflags, - REGS(ANY, 0, 0, 0, ANY)), - - /* MOV (register) cccc 0001 101x xxxx xxxx xxxx xxx0 xxxx */ - /* MVN (register) cccc 0001 111x xxxx xxxx xxxx xxx0 xxxx */ - DECODE_EMULATEX (0x0fa00010, 0x01a00000, emulate_rd12rn16rm0rs8_rwflags, - REGS(0, ANY, 0, 0, ANY)), - - /* AND (register) cccc 0000 000x xxxx xxxx xxxx xxx0 xxxx */ - /* EOR (register) cccc 0000 001x xxxx xxxx xxxx xxx0 xxxx */ - /* SUB (register) cccc 0000 010x xxxx xxxx xxxx xxx0 xxxx */ - /* RSB (register) cccc 0000 011x xxxx xxxx xxxx xxx0 xxxx */ - /* ADD (register) cccc 0000 100x xxxx xxxx xxxx xxx0 xxxx */ - /* ADC (register) cccc 0000 101x xxxx xxxx xxxx xxx0 xxxx */ - /* SBC (register) cccc 0000 110x xxxx xxxx xxxx xxx0 xxxx */ - /* RSC (register) cccc 0000 111x xxxx xxxx xxxx xxx0 xxxx */ - /* ORR (register) cccc 0001 100x xxxx xxxx xxxx xxx0 xxxx */ - /* BIC (register) cccc 0001 110x xxxx xxxx xxxx xxx0 xxxx */ - DECODE_EMULATEX (0x0e000010, 0x00000000, emulate_rd12rn16rm0rs8_rwflags, - REGS(ANY, ANY, 0, 0, ANY)), - - /* TST (reg-shift reg) cccc 0001 0001 xxxx xxxx xxxx 0xx1 xxxx */ - /* TEQ (reg-shift reg) cccc 0001 0011 xxxx xxxx xxxx 0xx1 xxxx */ - /* CMP (reg-shift reg) cccc 0001 0101 xxxx xxxx xxxx 0xx1 xxxx */ - /* CMN (reg-shift reg) cccc 0001 0111 xxxx xxxx xxxx 0xx1 xxxx */ - DECODE_EMULATEX (0x0f900090, 0x01100010, emulate_rd12rn16rm0rs8_rwflags, - REGS(ANY, 0, NOPC, 0, ANY)), - - /* MOV (reg-shift reg) cccc 0001 101x xxxx xxxx xxxx 0xx1 xxxx */ - /* MVN (reg-shift reg) cccc 0001 111x xxxx xxxx xxxx 0xx1 xxxx */ - DECODE_EMULATEX (0x0fa00090, 0x01a00010, emulate_rd12rn16rm0rs8_rwflags, - REGS(0, ANY, NOPC, 0, ANY)), - - /* AND (reg-shift reg) cccc 0000 000x xxxx xxxx xxxx 0xx1 xxxx */ - /* EOR (reg-shift reg) cccc 0000 001x xxxx xxxx xxxx 0xx1 xxxx */ - /* SUB (reg-shift reg) cccc 0000 010x xxxx xxxx xxxx 0xx1 xxxx */ - /* RSB (reg-shift reg) cccc 0000 011x xxxx xxxx xxxx 0xx1 xxxx */ - /* ADD (reg-shift reg) cccc 0000 100x xxxx xxxx xxxx 0xx1 xxxx */ - /* ADC (reg-shift reg) cccc 0000 101x xxxx xxxx xxxx 0xx1 xxxx */ - /* SBC (reg-shift reg) cccc 0000 110x xxxx xxxx xxxx 0xx1 xxxx */ - /* RSC (reg-shift reg) cccc 0000 111x xxxx xxxx xxxx 0xx1 xxxx */ - /* ORR (reg-shift reg) cccc 0001 100x xxxx xxxx xxxx 0xx1 xxxx */ - /* BIC (reg-shift reg) cccc 0001 110x xxxx xxxx xxxx 0xx1 xxxx */ - DECODE_EMULATEX (0x0e000090, 0x00000010, emulate_rd12rn16rm0rs8_rwflags, - REGS(ANY, ANY, NOPC, 0, ANY)), - - DECODE_END -}; - -static const union decode_item arm_cccc_001x_table[] = { - /* Data-processing (immediate) */ - - /* MOVW cccc 0011 0000 xxxx xxxx xxxx xxxx xxxx */ - /* MOVT cccc 0011 0100 xxxx xxxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0x0fb00000, 0x03000000, emulate_rd12rm0_noflags_nopc, - REGS(0, NOPC, 0, 0, 0)), - - /* YIELD cccc 0011 0010 0000 xxxx xxxx 0000 0001 */ - DECODE_OR (0x0fff00ff, 0x03200001), - /* SEV cccc 0011 0010 0000 xxxx xxxx 0000 0100 */ - DECODE_EMULATE (0x0fff00ff, 0x03200004, kprobe_emulate_none), - /* NOP cccc 0011 0010 0000 xxxx xxxx 0000 0000 */ - /* WFE cccc 0011 0010 0000 xxxx xxxx 0000 0010 */ - /* WFI cccc 0011 0010 0000 xxxx xxxx 0000 0011 */ - DECODE_SIMULATE (0x0fff00fc, 0x03200000, kprobe_simulate_nop), - /* DBG cccc 0011 0010 0000 xxxx xxxx ffff xxxx */ - /* unallocated hints cccc 0011 0010 0000 xxxx xxxx xxxx xxxx */ - /* MSR (immediate) cccc 0011 0x10 xxxx xxxx xxxx xxxx xxxx */ - DECODE_REJECT (0x0fb00000, 0x03200000), - - /* <op>S PC, ... cccc 001x xxx1 xxxx 1111 xxxx xxxx xxxx */ - DECODE_REJECT (0x0e10f000, 0x0210f000), - - /* TST (immediate) cccc 0011 0001 xxxx xxxx xxxx xxxx xxxx */ - /* TEQ (immediate) cccc 0011 0011 xxxx xxxx xxxx xxxx xxxx */ - /* CMP (immediate) cccc 0011 0101 xxxx xxxx xxxx xxxx xxxx */ - /* CMN (immediate) cccc 0011 0111 xxxx xxxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0x0f900000, 0x03100000, emulate_rd12rn16rm0rs8_rwflags, - REGS(ANY, 0, 0, 0, 0)), - - /* MOV (immediate) cccc 0011 101x xxxx xxxx xxxx xxxx xxxx */ - /* MVN (immediate) cccc 0011 111x xxxx xxxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0x0fa00000, 0x03a00000, emulate_rd12rn16rm0rs8_rwflags, - REGS(0, ANY, 0, 0, 0)), - - /* AND (immediate) cccc 0010 000x xxxx xxxx xxxx xxxx xxxx */ - /* EOR (immediate) cccc 0010 001x xxxx xxxx xxxx xxxx xxxx */ - /* SUB (immediate) cccc 0010 010x xxxx xxxx xxxx xxxx xxxx */ - /* RSB (immediate) cccc 0010 011x xxxx xxxx xxxx xxxx xxxx */ - /* ADD (immediate) cccc 0010 100x xxxx xxxx xxxx xxxx xxxx */ - /* ADC (immediate) cccc 0010 101x xxxx xxxx xxxx xxxx xxxx */ - /* SBC (immediate) cccc 0010 110x xxxx xxxx xxxx xxxx xxxx */ - /* RSC (immediate) cccc 0010 111x xxxx xxxx xxxx xxxx xxxx */ - /* ORR (immediate) cccc 0011 100x xxxx xxxx xxxx xxxx xxxx */ - /* BIC (immediate) cccc 0011 110x xxxx xxxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0x0e000000, 0x02000000, emulate_rd12rn16rm0rs8_rwflags, - REGS(ANY, ANY, 0, 0, 0)), - - DECODE_END -}; - -static const union decode_item arm_cccc_0110_____xxx1_table[] = { - /* Media instructions */ - - /* SEL cccc 0110 1000 xxxx xxxx xxxx 1011 xxxx */ - DECODE_EMULATEX (0x0ff000f0, 0x068000b0, emulate_rd12rn16rm0_rwflags_nopc, - REGS(NOPC, NOPC, 0, 0, NOPC)), - - /* SSAT cccc 0110 101x xxxx xxxx xxxx xx01 xxxx */ - /* USAT cccc 0110 111x xxxx xxxx xxxx xx01 xxxx */ - DECODE_OR(0x0fa00030, 0x06a00010), - /* SSAT16 cccc 0110 1010 xxxx xxxx xxxx 0011 xxxx */ - /* USAT16 cccc 0110 1110 xxxx xxxx xxxx 0011 xxxx */ - DECODE_EMULATEX (0x0fb000f0, 0x06a00030, emulate_rd12rn16rm0_rwflags_nopc, - REGS(0, NOPC, 0, 0, NOPC)), - - /* REV cccc 0110 1011 xxxx xxxx xxxx 0011 xxxx */ - /* REV16 cccc 0110 1011 xxxx xxxx xxxx 1011 xxxx */ - /* RBIT cccc 0110 1111 xxxx xxxx xxxx 0011 xxxx */ - /* REVSH cccc 0110 1111 xxxx xxxx xxxx 1011 xxxx */ - DECODE_EMULATEX (0x0fb00070, 0x06b00030, emulate_rd12rm0_noflags_nopc, - REGS(0, NOPC, 0, 0, NOPC)), - - /* ??? cccc 0110 0x00 xxxx xxxx xxxx xxx1 xxxx */ - DECODE_REJECT (0x0fb00010, 0x06000010), - /* ??? cccc 0110 0xxx xxxx xxxx xxxx 1011 xxxx */ - DECODE_REJECT (0x0f8000f0, 0x060000b0), - /* ??? cccc 0110 0xxx xxxx xxxx xxxx 1101 xxxx */ - DECODE_REJECT (0x0f8000f0, 0x060000d0), - /* SADD16 cccc 0110 0001 xxxx xxxx xxxx 0001 xxxx */ - /* SADDSUBX cccc 0110 0001 xxxx xxxx xxxx 0011 xxxx */ - /* SSUBADDX cccc 0110 0001 xxxx xxxx xxxx 0101 xxxx */ - /* SSUB16 cccc 0110 0001 xxxx xxxx xxxx 0111 xxxx */ - /* SADD8 cccc 0110 0001 xxxx xxxx xxxx 1001 xxxx */ - /* SSUB8 cccc 0110 0001 xxxx xxxx xxxx 1111 xxxx */ - /* QADD16 cccc 0110 0010 xxxx xxxx xxxx 0001 xxxx */ - /* QADDSUBX cccc 0110 0010 xxxx xxxx xxxx 0011 xxxx */ - /* QSUBADDX cccc 0110 0010 xxxx xxxx xxxx 0101 xxxx */ - /* QSUB16 cccc 0110 0010 xxxx xxxx xxxx 0111 xxxx */ - /* QADD8 cccc 0110 0010 xxxx xxxx xxxx 1001 xxxx */ - /* QSUB8 cccc 0110 0010 xxxx xxxx xxxx 1111 xxxx */ - /* SHADD16 cccc 0110 0011 xxxx xxxx xxxx 0001 xxxx */ - /* SHADDSUBX cccc 0110 0011 xxxx xxxx xxxx 0011 xxxx */ - /* SHSUBADDX cccc 0110 0011 xxxx xxxx xxxx 0101 xxxx */ - /* SHSUB16 cccc 0110 0011 xxxx xxxx xxxx 0111 xxxx */ - /* SHADD8 cccc 0110 0011 xxxx xxxx xxxx 1001 xxxx */ - /* SHSUB8 cccc 0110 0011 xxxx xxxx xxxx 1111 xxxx */ - /* UADD16 cccc 0110 0101 xxxx xxxx xxxx 0001 xxxx */ - /* UADDSUBX cccc 0110 0101 xxxx xxxx xxxx 0011 xxxx */ - /* USUBADDX cccc 0110 0101 xxxx xxxx xxxx 0101 xxxx */ - /* USUB16 cccc 0110 0101 xxxx xxxx xxxx 0111 xxxx */ - /* UADD8 cccc 0110 0101 xxxx xxxx xxxx 1001 xxxx */ - /* USUB8 cccc 0110 0101 xxxx xxxx xxxx 1111 xxxx */ - /* UQADD16 cccc 0110 0110 xxxx xxxx xxxx 0001 xxxx */ - /* UQADDSUBX cccc 0110 0110 xxxx xxxx xxxx 0011 xxxx */ - /* UQSUBADDX cccc 0110 0110 xxxx xxxx xxxx 0101 xxxx */ - /* UQSUB16 cccc 0110 0110 xxxx xxxx xxxx 0111 xxxx */ - /* UQADD8 cccc 0110 0110 xxxx xxxx xxxx 1001 xxxx */ - /* UQSUB8 cccc 0110 0110 xxxx xxxx xxxx 1111 xxxx */ - /* UHADD16 cccc 0110 0111 xxxx xxxx xxxx 0001 xxxx */ - /* UHADDSUBX cccc 0110 0111 xxxx xxxx xxxx 0011 xxxx */ - /* UHSUBADDX cccc 0110 0111 xxxx xxxx xxxx 0101 xxxx */ - /* UHSUB16 cccc 0110 0111 xxxx xxxx xxxx 0111 xxxx */ - /* UHADD8 cccc 0110 0111 xxxx xxxx xxxx 1001 xxxx */ - /* UHSUB8 cccc 0110 0111 xxxx xxxx xxxx 1111 xxxx */ - DECODE_EMULATEX (0x0f800010, 0x06000010, emulate_rd12rn16rm0_rwflags_nopc, - REGS(NOPC, NOPC, 0, 0, NOPC)), - - /* PKHBT cccc 0110 1000 xxxx xxxx xxxx x001 xxxx */ - /* PKHTB cccc 0110 1000 xxxx xxxx xxxx x101 xxxx */ - DECODE_EMULATEX (0x0ff00030, 0x06800010, emulate_rd12rn16rm0_rwflags_nopc, - REGS(NOPC, NOPC, 0, 0, NOPC)), - - /* ??? cccc 0110 1001 xxxx xxxx xxxx 0111 xxxx */ - /* ??? cccc 0110 1101 xxxx xxxx xxxx 0111 xxxx */ - DECODE_REJECT (0x0fb000f0, 0x06900070), - - /* SXTB16 cccc 0110 1000 1111 xxxx xxxx 0111 xxxx */ - /* SXTB cccc 0110 1010 1111 xxxx xxxx 0111 xxxx */ - /* SXTH cccc 0110 1011 1111 xxxx xxxx 0111 xxxx */ - /* UXTB16 cccc 0110 1100 1111 xxxx xxxx 0111 xxxx */ - /* UXTB cccc 0110 1110 1111 xxxx xxxx 0111 xxxx */ - /* UXTH cccc 0110 1111 1111 xxxx xxxx 0111 xxxx */ - DECODE_EMULATEX (0x0f8f00f0, 0x068f0070, emulate_rd12rm0_noflags_nopc, - REGS(0, NOPC, 0, 0, NOPC)), - - /* SXTAB16 cccc 0110 1000 xxxx xxxx xxxx 0111 xxxx */ - /* SXTAB cccc 0110 1010 xxxx xxxx xxxx 0111 xxxx */ - /* SXTAH cccc 0110 1011 xxxx xxxx xxxx 0111 xxxx */ - /* UXTAB16 cccc 0110 1100 xxxx xxxx xxxx 0111 xxxx */ - /* UXTAB cccc 0110 1110 xxxx xxxx xxxx 0111 xxxx */ - /* UXTAH cccc 0110 1111 xxxx xxxx xxxx 0111 xxxx */ - DECODE_EMULATEX (0x0f8000f0, 0x06800070, emulate_rd12rn16rm0_rwflags_nopc, - REGS(NOPCX, NOPC, 0, 0, NOPC)), - - DECODE_END -}; - -static const union decode_item arm_cccc_0111_____xxx1_table[] = { - /* Media instructions */ - - /* UNDEFINED cccc 0111 1111 xxxx xxxx xxxx 1111 xxxx */ - DECODE_REJECT (0x0ff000f0, 0x07f000f0), - - /* SMLALD cccc 0111 0100 xxxx xxxx xxxx 00x1 xxxx */ - /* SMLSLD cccc 0111 0100 xxxx xxxx xxxx 01x1 xxxx */ - DECODE_EMULATEX (0x0ff00090, 0x07400010, emulate_rdlo12rdhi16rn0rm8_rwflags_nopc, - REGS(NOPC, NOPC, NOPC, 0, NOPC)), - - /* SMUAD cccc 0111 0000 xxxx 1111 xxxx 00x1 xxxx */ - /* SMUSD cccc 0111 0000 xxxx 1111 xxxx 01x1 xxxx */ - DECODE_OR (0x0ff0f090, 0x0700f010), - /* SMMUL cccc 0111 0101 xxxx 1111 xxxx 00x1 xxxx */ - DECODE_OR (0x0ff0f0d0, 0x0750f010), - /* USAD8 cccc 0111 1000 xxxx 1111 xxxx 0001 xxxx */ - DECODE_EMULATEX (0x0ff0f0f0, 0x0780f010, emulate_rd16rn12rm0rs8_rwflags_nopc, - REGS(NOPC, 0, NOPC, 0, NOPC)), - - /* SMLAD cccc 0111 0000 xxxx xxxx xxxx 00x1 xxxx */ - /* SMLSD cccc 0111 0000 xxxx xxxx xxxx 01x1 xxxx */ - DECODE_OR (0x0ff00090, 0x07000010), - /* SMMLA cccc 0111 0101 xxxx xxxx xxxx 00x1 xxxx */ - DECODE_OR (0x0ff000d0, 0x07500010), - /* USADA8 cccc 0111 1000 xxxx xxxx xxxx 0001 xxxx */ - DECODE_EMULATEX (0x0ff000f0, 0x07800010, emulate_rd16rn12rm0rs8_rwflags_nopc, - REGS(NOPC, NOPCX, NOPC, 0, NOPC)), - - /* SMMLS cccc 0111 0101 xxxx xxxx xxxx 11x1 xxxx */ - DECODE_EMULATEX (0x0ff000d0, 0x075000d0, emulate_rd16rn12rm0rs8_rwflags_nopc, - REGS(NOPC, NOPC, NOPC, 0, NOPC)), - - /* SBFX cccc 0111 101x xxxx xxxx xxxx x101 xxxx */ - /* UBFX cccc 0111 111x xxxx xxxx xxxx x101 xxxx */ - DECODE_EMULATEX (0x0fa00070, 0x07a00050, emulate_rd12rm0_noflags_nopc, - REGS(0, NOPC, 0, 0, NOPC)), - - /* BFC cccc 0111 110x xxxx xxxx xxxx x001 1111 */ - DECODE_EMULATEX (0x0fe0007f, 0x07c0001f, emulate_rd12rm0_noflags_nopc, - REGS(0, NOPC, 0, 0, 0)), - - /* BFI cccc 0111 110x xxxx xxxx xxxx x001 xxxx */ - DECODE_EMULATEX (0x0fe00070, 0x07c00010, emulate_rd12rm0_noflags_nopc, - REGS(0, NOPC, 0, 0, NOPCX)), - - DECODE_END -}; - -static const union decode_item arm_cccc_01xx_table[] = { - /* Load/store word and unsigned byte */ - - /* LDRB/STRB pc,[...] cccc 01xx x0xx xxxx xxxx xxxx xxxx xxxx */ - DECODE_REJECT (0x0c40f000, 0x0440f000), - - /* STRT cccc 01x0 x010 xxxx xxxx xxxx xxxx xxxx */ - /* LDRT cccc 01x0 x011 xxxx xxxx xxxx xxxx xxxx */ - /* STRBT cccc 01x0 x110 xxxx xxxx xxxx xxxx xxxx */ - /* LDRBT cccc 01x0 x111 xxxx xxxx xxxx xxxx xxxx */ - DECODE_REJECT (0x0d200000, 0x04200000), - - /* STR (immediate) cccc 010x x0x0 xxxx xxxx xxxx xxxx xxxx */ - /* STRB (immediate) cccc 010x x1x0 xxxx xxxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0x0e100000, 0x04000000, emulate_str, - REGS(NOPCWB, ANY, 0, 0, 0)), - - /* LDR (immediate) cccc 010x x0x1 xxxx xxxx xxxx xxxx xxxx */ - /* LDRB (immediate) cccc 010x x1x1 xxxx xxxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0x0e100000, 0x04100000, emulate_ldr, - REGS(NOPCWB, ANY, 0, 0, 0)), - - /* STR (register) cccc 011x x0x0 xxxx xxxx xxxx xxxx xxxx */ - /* STRB (register) cccc 011x x1x0 xxxx xxxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0x0e100000, 0x06000000, emulate_str, - REGS(NOPCWB, ANY, 0, 0, NOPC)), - - /* LDR (register) cccc 011x x0x1 xxxx xxxx xxxx xxxx xxxx */ - /* LDRB (register) cccc 011x x1x1 xxxx xxxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0x0e100000, 0x06100000, emulate_ldr, - REGS(NOPCWB, ANY, 0, 0, NOPC)), - - DECODE_END -}; - -static const union decode_item arm_cccc_100x_table[] = { - /* Block data transfer instructions */ - - /* LDM cccc 100x x0x1 xxxx xxxx xxxx xxxx xxxx */ - /* STM cccc 100x x0x0 xxxx xxxx xxxx xxxx xxxx */ - DECODE_CUSTOM (0x0e400000, 0x08000000, kprobe_decode_ldmstm), - - /* STM (user registers) cccc 100x x1x0 xxxx xxxx xxxx xxxx xxxx */ - /* LDM (user registers) cccc 100x x1x1 xxxx 0xxx xxxx xxxx xxxx */ - /* LDM (exception ret) cccc 100x x1x1 xxxx 1xxx xxxx xxxx xxxx */ - DECODE_END -}; - -const union decode_item kprobe_decode_arm_table[] = { - /* - * Unconditional instructions - * 1111 xxxx xxxx xxxx xxxx xxxx xxxx xxxx - */ - DECODE_TABLE (0xf0000000, 0xf0000000, arm_1111_table), - - /* - * Miscellaneous instructions - * cccc 0001 0xx0 xxxx xxxx xxxx 0xxx xxxx - */ - DECODE_TABLE (0x0f900080, 0x01000000, arm_cccc_0001_0xx0____0xxx_table), - - /* - * Halfword multiply and multiply-accumulate - * cccc 0001 0xx0 xxxx xxxx xxxx 1xx0 xxxx - */ - DECODE_TABLE (0x0f900090, 0x01000080, arm_cccc_0001_0xx0____1xx0_table), - - /* - * Multiply and multiply-accumulate - * cccc 0000 xxxx xxxx xxxx xxxx 1001 xxxx - */ - DECODE_TABLE (0x0f0000f0, 0x00000090, arm_cccc_0000_____1001_table), - - /* - * Synchronization primitives - * cccc 0001 xxxx xxxx xxxx xxxx 1001 xxxx - */ - DECODE_TABLE (0x0f0000f0, 0x01000090, arm_cccc_0001_____1001_table), - - /* - * Extra load/store instructions - * cccc 000x xxxx xxxx xxxx xxxx 1xx1 xxxx - */ - DECODE_TABLE (0x0e000090, 0x00000090, arm_cccc_000x_____1xx1_table), - - /* - * Data-processing (register) - * cccc 000x xxxx xxxx xxxx xxxx xxx0 xxxx - * Data-processing (register-shifted register) - * cccc 000x xxxx xxxx xxxx xxxx 0xx1 xxxx - */ - DECODE_TABLE (0x0e000000, 0x00000000, arm_cccc_000x_table), - - /* - * Data-processing (immediate) - * cccc 001x xxxx xxxx xxxx xxxx xxxx xxxx - */ - DECODE_TABLE (0x0e000000, 0x02000000, arm_cccc_001x_table), - - /* - * Media instructions - * cccc 011x xxxx xxxx xxxx xxxx xxx1 xxxx - */ - DECODE_TABLE (0x0f000010, 0x06000010, arm_cccc_0110_____xxx1_table), - DECODE_TABLE (0x0f000010, 0x07000010, arm_cccc_0111_____xxx1_table), - - /* - * Load/store word and unsigned byte - * cccc 01xx xxxx xxxx xxxx xxxx xxxx xxxx - */ - DECODE_TABLE (0x0c000000, 0x04000000, arm_cccc_01xx_table), - - /* - * Block data transfer instructions - * cccc 100x xxxx xxxx xxxx xxxx xxxx xxxx - */ - DECODE_TABLE (0x0e000000, 0x08000000, arm_cccc_100x_table), - - /* B cccc 1010 xxxx xxxx xxxx xxxx xxxx xxxx */ - /* BL cccc 1011 xxxx xxxx xxxx xxxx xxxx xxxx */ - DECODE_SIMULATE (0x0e000000, 0x0a000000, simulate_bbl), - - /* - * Supervisor Call, and coprocessor instructions - */ - - /* MCRR cccc 1100 0100 xxxx xxxx xxxx xxxx xxxx */ - /* MRRC cccc 1100 0101 xxxx xxxx xxxx xxxx xxxx */ - /* LDC cccc 110x xxx1 xxxx xxxx xxxx xxxx xxxx */ - /* STC cccc 110x xxx0 xxxx xxxx xxxx xxxx xxxx */ - /* CDP cccc 1110 xxxx xxxx xxxx xxxx xxx0 xxxx */ - /* MCR cccc 1110 xxx0 xxxx xxxx xxxx xxx1 xxxx */ - /* MRC cccc 1110 xxx1 xxxx xxxx xxxx xxx1 xxxx */ - /* SVC cccc 1111 xxxx xxxx xxxx xxxx xxxx xxxx */ - DECODE_REJECT (0x0c000000, 0x0c000000), - - DECODE_END -}; -#ifdef CONFIG_ARM_KPROBES_TEST_MODULE -EXPORT_SYMBOL_GPL(kprobe_decode_arm_table); -#endif - -static void __kprobes arm_singlestep(struct kprobe *p, struct pt_regs *regs) -{ - regs->ARM_pc += 4; - p->ainsn.insn_handler(p, regs); -} - -/* Return: - * INSN_REJECTED If instruction is one not allowed to kprobe, - * INSN_GOOD If instruction is supported and uses instruction slot, - * INSN_GOOD_NO_SLOT If instruction is supported but doesn't use its slot. - * - * For instructions we don't want to kprobe (INSN_REJECTED return result): - * These are generally ones that modify the processor state making - * them "hard" to simulate such as switches processor modes or - * make accesses in alternate modes. Any of these could be simulated - * if the work was put into it, but low return considering they - * should also be very rare. - */ -enum kprobe_insn __kprobes -arm_kprobe_decode_insn(kprobe_opcode_t insn, struct arch_specific_insn *asi) -{ - asi->insn_singlestep = arm_singlestep; - asi->insn_check_cc = kprobe_condition_checks[insn>>28]; - return kprobe_decode_insn(insn, asi, kprobe_decode_arm_table, false); -} diff --git a/arch/arm/kernel/kprobes-common.c b/arch/arm/kernel/kprobes-common.c index 18a76282970e..c311ed94ff1c 100644 --- a/arch/arm/kernel/kprobes-common.c +++ b/arch/arm/kernel/kprobes-common.c @@ -13,178 +13,14 @@ #include <linux/kernel.h> #include <linux/kprobes.h> -#include <asm/system_info.h> #include "kprobes.h" -#ifndef find_str_pc_offset - -/* - * For STR and STM instructions, an ARM core may choose to use either - * a +8 or a +12 displacement from the current instruction's address. - * Whichever value is chosen for a given core, it must be the same for - * both instructions and may not change. This function measures it. - */ - -int str_pc_offset; - -void __init find_str_pc_offset(void) -{ - int addr, scratch, ret; - - __asm__ ( - "sub %[ret], pc, #4 \n\t" - "str pc, %[addr] \n\t" - "ldr %[scr], %[addr] \n\t" - "sub %[ret], %[scr], %[ret] \n\t" - : [ret] "=r" (ret), [scr] "=r" (scratch), [addr] "+m" (addr)); - - str_pc_offset = ret; -} - -#endif /* !find_str_pc_offset */ - - -#ifndef test_load_write_pc_interworking - -bool load_write_pc_interworks; - -void __init test_load_write_pc_interworking(void) -{ - int arch = cpu_architecture(); - BUG_ON(arch == CPU_ARCH_UNKNOWN); - load_write_pc_interworks = arch >= CPU_ARCH_ARMv5T; -} - -#endif /* !test_load_write_pc_interworking */ - - -#ifndef test_alu_write_pc_interworking - -bool alu_write_pc_interworks; - -void __init test_alu_write_pc_interworking(void) -{ - int arch = cpu_architecture(); - BUG_ON(arch == CPU_ARCH_UNKNOWN); - alu_write_pc_interworks = arch >= CPU_ARCH_ARMv7; -} - -#endif /* !test_alu_write_pc_interworking */ - - -void __init arm_kprobe_decode_init(void) -{ - find_str_pc_offset(); - test_load_write_pc_interworking(); - test_alu_write_pc_interworking(); -} - - -static unsigned long __kprobes __check_eq(unsigned long cpsr) -{ - return cpsr & PSR_Z_BIT; -} - -static unsigned long __kprobes __check_ne(unsigned long cpsr) -{ - return (~cpsr) & PSR_Z_BIT; -} - -static unsigned long __kprobes __check_cs(unsigned long cpsr) -{ - return cpsr & PSR_C_BIT; -} - -static unsigned long __kprobes __check_cc(unsigned long cpsr) -{ - return (~cpsr) & PSR_C_BIT; -} - -static unsigned long __kprobes __check_mi(unsigned long cpsr) -{ - return cpsr & PSR_N_BIT; -} - -static unsigned long __kprobes __check_pl(unsigned long cpsr) -{ - return (~cpsr) & PSR_N_BIT; -} - -static unsigned long __kprobes __check_vs(unsigned long cpsr) -{ - return cpsr & PSR_V_BIT; -} - -static unsigned long __kprobes __check_vc(unsigned long cpsr) -{ - return (~cpsr) & PSR_V_BIT; -} - -static unsigned long __kprobes __check_hi(unsigned long cpsr) -{ - cpsr &= ~(cpsr >> 1); /* PSR_C_BIT &= ~PSR_Z_BIT */ - return cpsr & PSR_C_BIT; -} - -static unsigned long __kprobes __check_ls(unsigned long cpsr) -{ - cpsr &= ~(cpsr >> 1); /* PSR_C_BIT &= ~PSR_Z_BIT */ - return (~cpsr) & PSR_C_BIT; -} - -static unsigned long __kprobes __check_ge(unsigned long cpsr) -{ - cpsr ^= (cpsr << 3); /* PSR_N_BIT ^= PSR_V_BIT */ - return (~cpsr) & PSR_N_BIT; -} - -static unsigned long __kprobes __check_lt(unsigned long cpsr) -{ - cpsr ^= (cpsr << 3); /* PSR_N_BIT ^= PSR_V_BIT */ - return cpsr & PSR_N_BIT; -} - -static unsigned long __kprobes __check_gt(unsigned long cpsr) -{ - unsigned long temp = cpsr ^ (cpsr << 3); /* PSR_N_BIT ^= PSR_V_BIT */ - temp |= (cpsr << 1); /* PSR_N_BIT |= PSR_Z_BIT */ - return (~temp) & PSR_N_BIT; -} - -static unsigned long __kprobes __check_le(unsigned long cpsr) -{ - unsigned long temp = cpsr ^ (cpsr << 3); /* PSR_N_BIT ^= PSR_V_BIT */ - temp |= (cpsr << 1); /* PSR_N_BIT |= PSR_Z_BIT */ - return temp & PSR_N_BIT; -} - -static unsigned long __kprobes __check_al(unsigned long cpsr) -{ - return true; -} - -kprobe_check_cc * const kprobe_condition_checks[16] = { - &__check_eq, &__check_ne, &__check_cs, &__check_cc, - &__check_mi, &__check_pl, &__check_vs, &__check_vc, - &__check_hi, &__check_ls, &__check_ge, &__check_lt, - &__check_gt, &__check_le, &__check_al, &__check_al -}; - - -void __kprobes kprobe_simulate_nop(struct kprobe *p, struct pt_regs *regs) -{ -} - -void __kprobes kprobe_emulate_none(struct kprobe *p, struct pt_regs *regs) -{ - p->ainsn.insn_fn(); -} - -static void __kprobes simulate_ldm1stm1(struct kprobe *p, struct pt_regs *regs) +static void __kprobes simulate_ldm1stm1(probes_opcode_t insn, + struct arch_probes_insn *asi, + struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; int rn = (insn >> 16) & 0xf; int lbit = insn & (1 << 20); int wbit = insn & (1 << 21); @@ -223,24 +59,31 @@ static void __kprobes simulate_ldm1stm1(struct kprobe *p, struct pt_regs *regs) } } -static void __kprobes simulate_stm1_pc(struct kprobe *p, struct pt_regs *regs) +static void __kprobes simulate_stm1_pc(probes_opcode_t insn, + struct arch_probes_insn *asi, + struct pt_regs *regs) { - regs->ARM_pc = (long)p->addr + str_pc_offset; - simulate_ldm1stm1(p, regs); - regs->ARM_pc = (long)p->addr + 4; + unsigned long addr = regs->ARM_pc - 4; + + regs->ARM_pc = (long)addr + str_pc_offset; + simulate_ldm1stm1(insn, asi, regs); + regs->ARM_pc = (long)addr + 4; } -static void __kprobes simulate_ldm1_pc(struct kprobe *p, struct pt_regs *regs) +static void __kprobes simulate_ldm1_pc(probes_opcode_t insn, + struct arch_probes_insn *asi, + struct pt_regs *regs) { - simulate_ldm1stm1(p, regs); + simulate_ldm1stm1(insn, asi, regs); load_write_pc(regs->ARM_pc, regs); } static void __kprobes -emulate_generic_r0_12_noflags(struct kprobe *p, struct pt_regs *regs) +emulate_generic_r0_12_noflags(probes_opcode_t insn, + struct arch_probes_insn *asi, struct pt_regs *regs) { register void *rregs asm("r1") = regs; - register void *rfn asm("lr") = p->ainsn.insn_fn; + register void *rfn asm("lr") = asi->insn_fn; __asm__ __volatile__ ( "stmdb sp!, {%[regs], r11} \n\t" @@ -264,22 +107,27 @@ emulate_generic_r0_12_noflags(struct kprobe *p, struct pt_regs *regs) } static void __kprobes -emulate_generic_r2_14_noflags(struct kprobe *p, struct pt_regs *regs) +emulate_generic_r2_14_noflags(probes_opcode_t insn, + struct arch_probes_insn *asi, struct pt_regs *regs) { - emulate_generic_r0_12_noflags(p, (struct pt_regs *)(regs->uregs+2)); + emulate_generic_r0_12_noflags(insn, asi, + (struct pt_regs *)(regs->uregs+2)); } static void __kprobes -emulate_ldm_r3_15(struct kprobe *p, struct pt_regs *regs) +emulate_ldm_r3_15(probes_opcode_t insn, + struct arch_probes_insn *asi, struct pt_regs *regs) { - emulate_generic_r0_12_noflags(p, (struct pt_regs *)(regs->uregs+3)); + emulate_generic_r0_12_noflags(insn, asi, + (struct pt_regs *)(regs->uregs+3)); load_write_pc(regs->ARM_pc, regs); } -enum kprobe_insn __kprobes -kprobe_decode_ldmstm(kprobe_opcode_t insn, struct arch_specific_insn *asi) +enum probes_insn __kprobes +kprobe_decode_ldmstm(probes_opcode_t insn, struct arch_probes_insn *asi, + const struct decode_header *h) { - kprobe_insn_handler_t *handler = 0; + probes_insn_handler_t *handler = 0; unsigned reglist = insn & 0xffff; int is_ldm = insn & 0x100000; int rn = (insn >> 16) & 0xf; @@ -319,260 +167,3 @@ kprobe_decode_ldmstm(kprobe_opcode_t insn, struct arch_specific_insn *asi) return INSN_GOOD_NO_SLOT; } - -/* - * Prepare an instruction slot to receive an instruction for emulating. - * This is done by placing a subroutine return after the location where the - * instruction will be placed. We also modify ARM instructions to be - * unconditional as the condition code will already be checked before any - * emulation handler is called. - */ -static kprobe_opcode_t __kprobes -prepare_emulated_insn(kprobe_opcode_t insn, struct arch_specific_insn *asi, - bool thumb) -{ -#ifdef CONFIG_THUMB2_KERNEL - if (thumb) { - u16 *thumb_insn = (u16 *)asi->insn; - thumb_insn[1] = 0x4770; /* Thumb bx lr */ - thumb_insn[2] = 0x4770; /* Thumb bx lr */ - return insn; - } - asi->insn[1] = 0xe12fff1e; /* ARM bx lr */ -#else - asi->insn[1] = 0xe1a0f00e; /* mov pc, lr */ -#endif - /* Make an ARM instruction unconditional */ - if (insn < 0xe0000000) - insn = (insn | 0xe0000000) & ~0x10000000; - return insn; -} - -/* - * Write a (probably modified) instruction into the slot previously prepared by - * prepare_emulated_insn - */ -static void __kprobes -set_emulated_insn(kprobe_opcode_t insn, struct arch_specific_insn *asi, - bool thumb) -{ -#ifdef CONFIG_THUMB2_KERNEL - if (thumb) { - u16 *ip = (u16 *)asi->insn; - if (is_wide_instruction(insn)) - *ip++ = insn >> 16; - *ip++ = insn; - return; - } -#endif - asi->insn[0] = insn; -} - -/* - * When we modify the register numbers encoded in an instruction to be emulated, - * the new values come from this define. For ARM and 32-bit Thumb instructions - * this gives... - * - * bit position 16 12 8 4 0 - * ---------------+---+---+---+---+---+ - * register r2 r0 r1 -- r3 - */ -#define INSN_NEW_BITS 0x00020103 - -/* Each nibble has same value as that at INSN_NEW_BITS bit 16 */ -#define INSN_SAMEAS16_BITS 0x22222222 - -/* - * Validate and modify each of the registers encoded in an instruction. - * - * Each nibble in regs contains a value from enum decode_reg_type. For each - * non-zero value, the corresponding nibble in pinsn is validated and modified - * according to the type. - */ -static bool __kprobes decode_regs(kprobe_opcode_t* pinsn, u32 regs) -{ - kprobe_opcode_t insn = *pinsn; - kprobe_opcode_t mask = 0xf; /* Start at least significant nibble */ - - for (; regs != 0; regs >>= 4, mask <<= 4) { - - kprobe_opcode_t new_bits = INSN_NEW_BITS; - - switch (regs & 0xf) { - - case REG_TYPE_NONE: - /* Nibble not a register, skip to next */ - continue; - - case REG_TYPE_ANY: - /* Any register is allowed */ - break; - - case REG_TYPE_SAMEAS16: - /* Replace register with same as at bit position 16 */ - new_bits = INSN_SAMEAS16_BITS; - break; - - case REG_TYPE_SP: - /* Only allow SP (R13) */ - if ((insn ^ 0xdddddddd) & mask) - goto reject; - break; - - case REG_TYPE_PC: - /* Only allow PC (R15) */ - if ((insn ^ 0xffffffff) & mask) - goto reject; - break; - - case REG_TYPE_NOSP: - /* Reject SP (R13) */ - if (((insn ^ 0xdddddddd) & mask) == 0) - goto reject; - break; - - case REG_TYPE_NOSPPC: - case REG_TYPE_NOSPPCX: - /* Reject SP and PC (R13 and R15) */ - if (((insn ^ 0xdddddddd) & 0xdddddddd & mask) == 0) - goto reject; - break; - - case REG_TYPE_NOPCWB: - if (!is_writeback(insn)) - break; /* No writeback, so any register is OK */ - /* fall through... */ - case REG_TYPE_NOPC: - case REG_TYPE_NOPCX: - /* Reject PC (R15) */ - if (((insn ^ 0xffffffff) & mask) == 0) - goto reject; - break; - } - - /* Replace value of nibble with new register number... */ - insn &= ~mask; - insn |= new_bits & mask; - } - - *pinsn = insn; - return true; - -reject: - return false; -} - -static const int decode_struct_sizes[NUM_DECODE_TYPES] = { - [DECODE_TYPE_TABLE] = sizeof(struct decode_table), - [DECODE_TYPE_CUSTOM] = sizeof(struct decode_custom), - [DECODE_TYPE_SIMULATE] = sizeof(struct decode_simulate), - [DECODE_TYPE_EMULATE] = sizeof(struct decode_emulate), - [DECODE_TYPE_OR] = sizeof(struct decode_or), - [DECODE_TYPE_REJECT] = sizeof(struct decode_reject) -}; - -/* - * kprobe_decode_insn operates on data tables in order to decode an ARM - * architecture instruction onto which a kprobe has been placed. - * - * These instruction decoding tables are a concatenation of entries each - * of which consist of one of the following structs: - * - * decode_table - * decode_custom - * decode_simulate - * decode_emulate - * decode_or - * decode_reject - * - * Each of these starts with a struct decode_header which has the following - * fields: - * - * type_regs - * mask - * value - * - * The least significant DECODE_TYPE_BITS of type_regs contains a value - * from enum decode_type, this indicates which of the decode_* structs - * the entry contains. The value DECODE_TYPE_END indicates the end of the - * table. - * - * When the table is parsed, each entry is checked in turn to see if it - * matches the instruction to be decoded using the test: - * - * (insn & mask) == value - * - * If no match is found before the end of the table is reached then decoding - * fails with INSN_REJECTED. - * - * When a match is found, decode_regs() is called to validate and modify each - * of the registers encoded in the instruction; the data it uses to do this - * is (type_regs >> DECODE_TYPE_BITS). A validation failure will cause decoding - * to fail with INSN_REJECTED. - * - * Once the instruction has passed the above tests, further processing - * depends on the type of the table entry's decode struct. - * - */ -int __kprobes -kprobe_decode_insn(kprobe_opcode_t insn, struct arch_specific_insn *asi, - const union decode_item *table, bool thumb) -{ - const struct decode_header *h = (struct decode_header *)table; - const struct decode_header *next; - bool matched = false; - - insn = prepare_emulated_insn(insn, asi, thumb); - - for (;; h = next) { - enum decode_type type = h->type_regs.bits & DECODE_TYPE_MASK; - u32 regs = h->type_regs.bits >> DECODE_TYPE_BITS; - - if (type == DECODE_TYPE_END) - return INSN_REJECTED; - - next = (struct decode_header *) - ((uintptr_t)h + decode_struct_sizes[type]); - - if (!matched && (insn & h->mask.bits) != h->value.bits) - continue; - - if (!decode_regs(&insn, regs)) - return INSN_REJECTED; - - switch (type) { - - case DECODE_TYPE_TABLE: { - struct decode_table *d = (struct decode_table *)h; - next = (struct decode_header *)d->table.table; - break; - } - - case DECODE_TYPE_CUSTOM: { - struct decode_custom *d = (struct decode_custom *)h; - return (*d->decoder.decoder)(insn, asi); - } - - case DECODE_TYPE_SIMULATE: { - struct decode_simulate *d = (struct decode_simulate *)h; - asi->insn_handler = d->handler.handler; - return INSN_GOOD_NO_SLOT; - } - - case DECODE_TYPE_EMULATE: { - struct decode_emulate *d = (struct decode_emulate *)h; - asi->insn_handler = d->handler.handler; - set_emulated_insn(insn, asi, thumb); - return INSN_GOOD; - } - - case DECODE_TYPE_OR: - matched = true; - break; - - case DECODE_TYPE_REJECT: - default: - return INSN_REJECTED; - } - } - } diff --git a/arch/arm/kernel/kprobes-test-arm.c b/arch/arm/kernel/kprobes-test-arm.c index 839312905067..87839de77e5f 100644 --- a/arch/arm/kernel/kprobes-test-arm.c +++ b/arch/arm/kernel/kprobes-test-arm.c @@ -10,6 +10,7 @@ #include <linux/kernel.h> #include <linux/module.h> +#include <asm/system_info.h> #include "kprobes-test.h" diff --git a/arch/arm/kernel/kprobes-test.c b/arch/arm/kernel/kprobes-test.c index 0cd63d080c7b..c2fd06b4c389 100644 --- a/arch/arm/kernel/kprobes-test.c +++ b/arch/arm/kernel/kprobes-test.c @@ -201,10 +201,14 @@ #include <linux/module.h> #include <linux/slab.h> #include <linux/kprobes.h> - +#include <linux/errno.h> +#include <linux/stddef.h> +#include <linux/bug.h> #include <asm/opcodes.h> #include "kprobes.h" +#include "probes-arm.h" +#include "probes-thumb.h" #include "kprobes-test.h" @@ -1608,7 +1612,7 @@ static int __init run_all_tests(void) goto out; pr_info("ARM instruction simulation\n"); - ret = run_test_cases(kprobe_arm_test_cases, kprobe_decode_arm_table); + ret = run_test_cases(kprobe_arm_test_cases, probes_decode_arm_table); if (ret) goto out; @@ -1631,13 +1635,13 @@ static int __init run_all_tests(void) pr_info("16-bit Thumb instruction simulation\n"); ret = run_test_cases(kprobe_thumb16_test_cases, - kprobe_decode_thumb16_table); + probes_decode_thumb16_table); if (ret) goto out; pr_info("32-bit Thumb instruction simulation\n"); ret = run_test_cases(kprobe_thumb32_test_cases, - kprobe_decode_thumb32_table); + probes_decode_thumb32_table); if (ret) goto out; #endif diff --git a/arch/arm/kernel/kprobes-thumb.c b/arch/arm/kernel/kprobes-thumb.c index 6123daf397a7..6619188619ae 100644 --- a/arch/arm/kernel/kprobes-thumb.c +++ b/arch/arm/kernel/kprobes-thumb.c @@ -8,41 +8,25 @@ * published by the Free Software Foundation. */ +#include <linux/types.h> #include <linux/kernel.h> +#include <linux/ptrace.h> #include <linux/kprobes.h> -#include <linux/module.h> #include "kprobes.h" +#include "probes-thumb.h" +/* These emulation encodings are functionally equivalent... */ +#define t32_emulate_rd8rn16rm0ra12_noflags \ + t32_emulate_rdlo12rdhi8rn16rm0_noflags -/* - * True if current instruction is in an IT block. - */ -#define in_it_block(cpsr) ((cpsr & 0x06000c00) != 0x00000000) - -/* - * Return the condition code to check for the currently executing instruction. - * This is in ITSTATE<7:4> which is in CPSR<15:12> but is only valid if - * in_it_block returns true. - */ -#define current_cond(cpsr) ((cpsr >> 12) & 0xf) - -/* - * Return the PC value for a probe in thumb code. - * This is the address of the probed instruction plus 4. - * We subtract one because the address will have bit zero set to indicate - * a pointer to thumb code. - */ -static inline unsigned long __kprobes thumb_probe_pc(struct kprobe *p) -{ - return (unsigned long)p->addr - 1 + 4; -} +/* t32 thumb actions */ static void __kprobes -t32_simulate_table_branch(struct kprobe *p, struct pt_regs *regs) +t32_simulate_table_branch(probes_opcode_t insn, + struct arch_probes_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; - unsigned long pc = thumb_probe_pc(p); + unsigned long pc = regs->ARM_pc; int rn = (insn >> 16) & 0xf; int rm = insn & 0xf; @@ -59,19 +43,19 @@ t32_simulate_table_branch(struct kprobe *p, struct pt_regs *regs) } static void __kprobes -t32_simulate_mrs(struct kprobe *p, struct pt_regs *regs) +t32_simulate_mrs(probes_opcode_t insn, + struct arch_probes_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; int rd = (insn >> 8) & 0xf; unsigned long mask = 0xf8ff03df; /* Mask out execution state */ regs->uregs[rd] = regs->ARM_cpsr & mask; } static void __kprobes -t32_simulate_cond_branch(struct kprobe *p, struct pt_regs *regs) +t32_simulate_cond_branch(probes_opcode_t insn, + struct arch_probes_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; - unsigned long pc = thumb_probe_pc(p); + unsigned long pc = regs->ARM_pc; long offset = insn & 0x7ff; /* imm11 */ offset += (insn & 0x003f0000) >> 5; /* imm6 */ @@ -82,20 +66,21 @@ t32_simulate_cond_branch(struct kprobe *p, struct pt_regs *regs) regs->ARM_pc = pc + (offset * 2); } -static enum kprobe_insn __kprobes -t32_decode_cond_branch(kprobe_opcode_t insn, struct arch_specific_insn *asi) +static enum probes_insn __kprobes +t32_decode_cond_branch(probes_opcode_t insn, struct arch_probes_insn *asi, + const struct decode_header *d) { int cc = (insn >> 22) & 0xf; - asi->insn_check_cc = kprobe_condition_checks[cc]; + asi->insn_check_cc = probes_condition_checks[cc]; asi->insn_handler = t32_simulate_cond_branch; return INSN_GOOD_NO_SLOT; } static void __kprobes -t32_simulate_branch(struct kprobe *p, struct pt_regs *regs) +t32_simulate_branch(probes_opcode_t insn, + struct arch_probes_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; - unsigned long pc = thumb_probe_pc(p); + unsigned long pc = regs->ARM_pc; long offset = insn & 0x7ff; /* imm11 */ offset += (insn & 0x03ff0000) >> 5; /* imm10 */ @@ -108,7 +93,7 @@ t32_simulate_branch(struct kprobe *p, struct pt_regs *regs) if (insn & (1 << 14)) { /* BL or BLX */ - regs->ARM_lr = (unsigned long)p->addr + 4; + regs->ARM_lr = regs->ARM_pc | 1; if (!(insn & (1 << 12))) { /* BLX so switch to ARM mode */ regs->ARM_cpsr &= ~PSR_T_BIT; @@ -120,10 +105,10 @@ t32_simulate_branch(struct kprobe *p, struct pt_regs *regs) } static void __kprobes -t32_simulate_ldr_literal(struct kprobe *p, struct pt_regs *regs) +t32_simulate_ldr_literal(probes_opcode_t insn, + struct arch_probes_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; - unsigned long addr = thumb_probe_pc(p) & ~3; + unsigned long addr = regs->ARM_pc & ~3; int rt = (insn >> 12) & 0xf; unsigned long rtv; @@ -157,10 +142,11 @@ t32_simulate_ldr_literal(struct kprobe *p, struct pt_regs *regs) regs->uregs[rt] = rtv; } -static enum kprobe_insn __kprobes -t32_decode_ldmstm(kprobe_opcode_t insn, struct arch_specific_insn *asi) +static enum probes_insn __kprobes +t32_decode_ldmstm(probes_opcode_t insn, struct arch_probes_insn *asi, + const struct decode_header *d) { - enum kprobe_insn ret = kprobe_decode_ldmstm(insn, asi); + enum probes_insn ret = kprobe_decode_ldmstm(insn, asi, d); /* Fixup modified instruction to have halfwords in correct order...*/ insn = asi->insn[0]; @@ -171,10 +157,10 @@ t32_decode_ldmstm(kprobe_opcode_t insn, struct arch_specific_insn *asi) } static void __kprobes -t32_emulate_ldrdstrd(struct kprobe *p, struct pt_regs *regs) +t32_emulate_ldrdstrd(probes_opcode_t insn, + struct arch_probes_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; - unsigned long pc = thumb_probe_pc(p) & ~3; + unsigned long pc = regs->ARM_pc & ~3; int rt1 = (insn >> 12) & 0xf; int rt2 = (insn >> 8) & 0xf; int rn = (insn >> 16) & 0xf; @@ -187,7 +173,7 @@ t32_emulate_ldrdstrd(struct kprobe *p, struct pt_regs *regs) __asm__ __volatile__ ( "blx %[fn]" : "=r" (rt1v), "=r" (rt2v), "=r" (rnv) - : "0" (rt1v), "1" (rt2v), "2" (rnv), [fn] "r" (p->ainsn.insn_fn) + : "0" (rt1v), "1" (rt2v), "2" (rnv), [fn] "r" (asi->insn_fn) : "lr", "memory", "cc" ); @@ -198,9 +184,9 @@ t32_emulate_ldrdstrd(struct kprobe *p, struct pt_regs *regs) } static void __kprobes -t32_emulate_ldrstr(struct kprobe *p, struct pt_regs *regs) +t32_emulate_ldrstr(probes_opcode_t insn, + struct arch_probes_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; int rt = (insn >> 12) & 0xf; int rn = (insn >> 16) & 0xf; int rm = insn & 0xf; @@ -212,7 +198,7 @@ t32_emulate_ldrstr(struct kprobe *p, struct pt_regs *regs) __asm__ __volatile__ ( "blx %[fn]" : "=r" (rtv), "=r" (rnv) - : "0" (rtv), "1" (rnv), "r" (rmv), [fn] "r" (p->ainsn.insn_fn) + : "0" (rtv), "1" (rnv), "r" (rmv), [fn] "r" (asi->insn_fn) : "lr", "memory", "cc" ); @@ -224,9 +210,9 @@ t32_emulate_ldrstr(struct kprobe *p, struct pt_regs *regs) } static void __kprobes -t32_emulate_rd8rn16rm0_rwflags(struct kprobe *p, struct pt_regs *regs) +t32_emulate_rd8rn16rm0_rwflags(probes_opcode_t insn, + struct arch_probes_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; int rd = (insn >> 8) & 0xf; int rn = (insn >> 16) & 0xf; int rm = insn & 0xf; @@ -242,7 +228,7 @@ t32_emulate_rd8rn16rm0_rwflags(struct kprobe *p, struct pt_regs *regs) "mrs %[cpsr], cpsr \n\t" : "=r" (rdv), [cpsr] "=r" (cpsr) : "0" (rdv), "r" (rnv), "r" (rmv), - "1" (cpsr), [fn] "r" (p->ainsn.insn_fn) + "1" (cpsr), [fn] "r" (asi->insn_fn) : "lr", "memory", "cc" ); @@ -251,10 +237,10 @@ t32_emulate_rd8rn16rm0_rwflags(struct kprobe *p, struct pt_regs *regs) } static void __kprobes -t32_emulate_rd8pc16_noflags(struct kprobe *p, struct pt_regs *regs) +t32_emulate_rd8pc16_noflags(probes_opcode_t insn, + struct arch_probes_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; - unsigned long pc = thumb_probe_pc(p); + unsigned long pc = regs->ARM_pc; int rd = (insn >> 8) & 0xf; register unsigned long rdv asm("r1") = regs->uregs[rd]; @@ -263,7 +249,7 @@ t32_emulate_rd8pc16_noflags(struct kprobe *p, struct pt_regs *regs) __asm__ __volatile__ ( "blx %[fn]" : "=r" (rdv) - : "0" (rdv), "r" (rnv), [fn] "r" (p->ainsn.insn_fn) + : "0" (rdv), "r" (rnv), [fn] "r" (asi->insn_fn) : "lr", "memory", "cc" ); @@ -271,9 +257,9 @@ t32_emulate_rd8pc16_noflags(struct kprobe *p, struct pt_regs *regs) } static void __kprobes -t32_emulate_rd8rn16_noflags(struct kprobe *p, struct pt_regs *regs) +t32_emulate_rd8rn16_noflags(probes_opcode_t insn, + struct arch_probes_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; int rd = (insn >> 8) & 0xf; int rn = (insn >> 16) & 0xf; @@ -283,7 +269,7 @@ t32_emulate_rd8rn16_noflags(struct kprobe *p, struct pt_regs *regs) __asm__ __volatile__ ( "blx %[fn]" : "=r" (rdv) - : "0" (rdv), "r" (rnv), [fn] "r" (p->ainsn.insn_fn) + : "0" (rdv), "r" (rnv), [fn] "r" (asi->insn_fn) : "lr", "memory", "cc" ); @@ -291,9 +277,10 @@ t32_emulate_rd8rn16_noflags(struct kprobe *p, struct pt_regs *regs) } static void __kprobes -t32_emulate_rdlo12rdhi8rn16rm0_noflags(struct kprobe *p, struct pt_regs *regs) +t32_emulate_rdlo12rdhi8rn16rm0_noflags(probes_opcode_t insn, + struct arch_probes_insn *asi, + struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; int rdlo = (insn >> 12) & 0xf; int rdhi = (insn >> 8) & 0xf; int rn = (insn >> 16) & 0xf; @@ -308,674 +295,43 @@ t32_emulate_rdlo12rdhi8rn16rm0_noflags(struct kprobe *p, struct pt_regs *regs) "blx %[fn]" : "=r" (rdlov), "=r" (rdhiv) : "0" (rdlov), "1" (rdhiv), "r" (rnv), "r" (rmv), - [fn] "r" (p->ainsn.insn_fn) + [fn] "r" (asi->insn_fn) : "lr", "memory", "cc" ); regs->uregs[rdlo] = rdlov; regs->uregs[rdhi] = rdhiv; } - -/* These emulation encodings are functionally equivalent... */ -#define t32_emulate_rd8rn16rm0ra12_noflags \ - t32_emulate_rdlo12rdhi8rn16rm0_noflags - -static const union decode_item t32_table_1110_100x_x0xx[] = { - /* Load/store multiple instructions */ - - /* Rn is PC 1110 100x x0xx 1111 xxxx xxxx xxxx xxxx */ - DECODE_REJECT (0xfe4f0000, 0xe80f0000), - - /* SRS 1110 1000 00x0 xxxx xxxx xxxx xxxx xxxx */ - /* RFE 1110 1000 00x1 xxxx xxxx xxxx xxxx xxxx */ - DECODE_REJECT (0xffc00000, 0xe8000000), - /* SRS 1110 1001 10x0 xxxx xxxx xxxx xxxx xxxx */ - /* RFE 1110 1001 10x1 xxxx xxxx xxxx xxxx xxxx */ - DECODE_REJECT (0xffc00000, 0xe9800000), - - /* STM Rn, {...pc} 1110 100x x0x0 xxxx 1xxx xxxx xxxx xxxx */ - DECODE_REJECT (0xfe508000, 0xe8008000), - /* LDM Rn, {...lr,pc} 1110 100x x0x1 xxxx 11xx xxxx xxxx xxxx */ - DECODE_REJECT (0xfe50c000, 0xe810c000), - /* LDM/STM Rn, {...sp} 1110 100x x0xx xxxx xx1x xxxx xxxx xxxx */ - DECODE_REJECT (0xfe402000, 0xe8002000), - - /* STMIA 1110 1000 10x0 xxxx xxxx xxxx xxxx xxxx */ - /* LDMIA 1110 1000 10x1 xxxx xxxx xxxx xxxx xxxx */ - /* STMDB 1110 1001 00x0 xxxx xxxx xxxx xxxx xxxx */ - /* LDMDB 1110 1001 00x1 xxxx xxxx xxxx xxxx xxxx */ - DECODE_CUSTOM (0xfe400000, 0xe8000000, t32_decode_ldmstm), - - DECODE_END -}; - -static const union decode_item t32_table_1110_100x_x1xx[] = { - /* Load/store dual, load/store exclusive, table branch */ - - /* STRD (immediate) 1110 1000 x110 xxxx xxxx xxxx xxxx xxxx */ - /* LDRD (immediate) 1110 1000 x111 xxxx xxxx xxxx xxxx xxxx */ - DECODE_OR (0xff600000, 0xe8600000), - /* STRD (immediate) 1110 1001 x1x0 xxxx xxxx xxxx xxxx xxxx */ - /* LDRD (immediate) 1110 1001 x1x1 xxxx xxxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0xff400000, 0xe9400000, t32_emulate_ldrdstrd, - REGS(NOPCWB, NOSPPC, NOSPPC, 0, 0)), - - /* TBB 1110 1000 1101 xxxx xxxx xxxx 0000 xxxx */ - /* TBH 1110 1000 1101 xxxx xxxx xxxx 0001 xxxx */ - DECODE_SIMULATEX(0xfff000e0, 0xe8d00000, t32_simulate_table_branch, - REGS(NOSP, 0, 0, 0, NOSPPC)), - - /* STREX 1110 1000 0100 xxxx xxxx xxxx xxxx xxxx */ - /* LDREX 1110 1000 0101 xxxx xxxx xxxx xxxx xxxx */ - /* STREXB 1110 1000 1100 xxxx xxxx xxxx 0100 xxxx */ - /* STREXH 1110 1000 1100 xxxx xxxx xxxx 0101 xxxx */ - /* STREXD 1110 1000 1100 xxxx xxxx xxxx 0111 xxxx */ - /* LDREXB 1110 1000 1101 xxxx xxxx xxxx 0100 xxxx */ - /* LDREXH 1110 1000 1101 xxxx xxxx xxxx 0101 xxxx */ - /* LDREXD 1110 1000 1101 xxxx xxxx xxxx 0111 xxxx */ - /* And unallocated instructions... */ - DECODE_END -}; - -static const union decode_item t32_table_1110_101x[] = { - /* Data-processing (shifted register) */ - - /* TST 1110 1010 0001 xxxx xxxx 1111 xxxx xxxx */ - /* TEQ 1110 1010 1001 xxxx xxxx 1111 xxxx xxxx */ - DECODE_EMULATEX (0xff700f00, 0xea100f00, t32_emulate_rd8rn16rm0_rwflags, - REGS(NOSPPC, 0, 0, 0, NOSPPC)), - - /* CMN 1110 1011 0001 xxxx xxxx 1111 xxxx xxxx */ - DECODE_OR (0xfff00f00, 0xeb100f00), - /* CMP 1110 1011 1011 xxxx xxxx 1111 xxxx xxxx */ - DECODE_EMULATEX (0xfff00f00, 0xebb00f00, t32_emulate_rd8rn16rm0_rwflags, - REGS(NOPC, 0, 0, 0, NOSPPC)), - - /* MOV 1110 1010 010x 1111 xxxx xxxx xxxx xxxx */ - /* MVN 1110 1010 011x 1111 xxxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0xffcf0000, 0xea4f0000, t32_emulate_rd8rn16rm0_rwflags, - REGS(0, 0, NOSPPC, 0, NOSPPC)), - - /* ??? 1110 1010 101x xxxx xxxx xxxx xxxx xxxx */ - /* ??? 1110 1010 111x xxxx xxxx xxxx xxxx xxxx */ - DECODE_REJECT (0xffa00000, 0xeaa00000), - /* ??? 1110 1011 001x xxxx xxxx xxxx xxxx xxxx */ - DECODE_REJECT (0xffe00000, 0xeb200000), - /* ??? 1110 1011 100x xxxx xxxx xxxx xxxx xxxx */ - DECODE_REJECT (0xffe00000, 0xeb800000), - /* ??? 1110 1011 111x xxxx xxxx xxxx xxxx xxxx */ - DECODE_REJECT (0xffe00000, 0xebe00000), - - /* ADD/SUB SP, SP, Rm, LSL #0..3 */ - /* 1110 1011 x0xx 1101 x000 1101 xx00 xxxx */ - DECODE_EMULATEX (0xff4f7f30, 0xeb0d0d00, t32_emulate_rd8rn16rm0_rwflags, - REGS(SP, 0, SP, 0, NOSPPC)), - - /* ADD/SUB SP, SP, Rm, shift */ - /* 1110 1011 x0xx 1101 xxxx 1101 xxxx xxxx */ - DECODE_REJECT (0xff4f0f00, 0xeb0d0d00), - - /* ADD/SUB Rd, SP, Rm, shift */ - /* 1110 1011 x0xx 1101 xxxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0xff4f0000, 0xeb0d0000, t32_emulate_rd8rn16rm0_rwflags, - REGS(SP, 0, NOPC, 0, NOSPPC)), - - /* AND 1110 1010 000x xxxx xxxx xxxx xxxx xxxx */ - /* BIC 1110 1010 001x xxxx xxxx xxxx xxxx xxxx */ - /* ORR 1110 1010 010x xxxx xxxx xxxx xxxx xxxx */ - /* ORN 1110 1010 011x xxxx xxxx xxxx xxxx xxxx */ - /* EOR 1110 1010 100x xxxx xxxx xxxx xxxx xxxx */ - /* PKH 1110 1010 110x xxxx xxxx xxxx xxxx xxxx */ - /* ADD 1110 1011 000x xxxx xxxx xxxx xxxx xxxx */ - /* ADC 1110 1011 010x xxxx xxxx xxxx xxxx xxxx */ - /* SBC 1110 1011 011x xxxx xxxx xxxx xxxx xxxx */ - /* SUB 1110 1011 101x xxxx xxxx xxxx xxxx xxxx */ - /* RSB 1110 1011 110x xxxx xxxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0xfe000000, 0xea000000, t32_emulate_rd8rn16rm0_rwflags, - REGS(NOSPPC, 0, NOSPPC, 0, NOSPPC)), - - DECODE_END -}; - -static const union decode_item t32_table_1111_0x0x___0[] = { - /* Data-processing (modified immediate) */ - - /* TST 1111 0x00 0001 xxxx 0xxx 1111 xxxx xxxx */ - /* TEQ 1111 0x00 1001 xxxx 0xxx 1111 xxxx xxxx */ - DECODE_EMULATEX (0xfb708f00, 0xf0100f00, t32_emulate_rd8rn16rm0_rwflags, - REGS(NOSPPC, 0, 0, 0, 0)), - - /* CMN 1111 0x01 0001 xxxx 0xxx 1111 xxxx xxxx */ - DECODE_OR (0xfbf08f00, 0xf1100f00), - /* CMP 1111 0x01 1011 xxxx 0xxx 1111 xxxx xxxx */ - DECODE_EMULATEX (0xfbf08f00, 0xf1b00f00, t32_emulate_rd8rn16rm0_rwflags, - REGS(NOPC, 0, 0, 0, 0)), - - /* MOV 1111 0x00 010x 1111 0xxx xxxx xxxx xxxx */ - /* MVN 1111 0x00 011x 1111 0xxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0xfbcf8000, 0xf04f0000, t32_emulate_rd8rn16rm0_rwflags, - REGS(0, 0, NOSPPC, 0, 0)), - - /* ??? 1111 0x00 101x xxxx 0xxx xxxx xxxx xxxx */ - DECODE_REJECT (0xfbe08000, 0xf0a00000), - /* ??? 1111 0x00 110x xxxx 0xxx xxxx xxxx xxxx */ - /* ??? 1111 0x00 111x xxxx 0xxx xxxx xxxx xxxx */ - DECODE_REJECT (0xfbc08000, 0xf0c00000), - /* ??? 1111 0x01 001x xxxx 0xxx xxxx xxxx xxxx */ - DECODE_REJECT (0xfbe08000, 0xf1200000), - /* ??? 1111 0x01 100x xxxx 0xxx xxxx xxxx xxxx */ - DECODE_REJECT (0xfbe08000, 0xf1800000), - /* ??? 1111 0x01 111x xxxx 0xxx xxxx xxxx xxxx */ - DECODE_REJECT (0xfbe08000, 0xf1e00000), - - /* ADD Rd, SP, #imm 1111 0x01 000x 1101 0xxx xxxx xxxx xxxx */ - /* SUB Rd, SP, #imm 1111 0x01 101x 1101 0xxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0xfb4f8000, 0xf10d0000, t32_emulate_rd8rn16rm0_rwflags, - REGS(SP, 0, NOPC, 0, 0)), - - /* AND 1111 0x00 000x xxxx 0xxx xxxx xxxx xxxx */ - /* BIC 1111 0x00 001x xxxx 0xxx xxxx xxxx xxxx */ - /* ORR 1111 0x00 010x xxxx 0xxx xxxx xxxx xxxx */ - /* ORN 1111 0x00 011x xxxx 0xxx xxxx xxxx xxxx */ - /* EOR 1111 0x00 100x xxxx 0xxx xxxx xxxx xxxx */ - /* ADD 1111 0x01 000x xxxx 0xxx xxxx xxxx xxxx */ - /* ADC 1111 0x01 010x xxxx 0xxx xxxx xxxx xxxx */ - /* SBC 1111 0x01 011x xxxx 0xxx xxxx xxxx xxxx */ - /* SUB 1111 0x01 101x xxxx 0xxx xxxx xxxx xxxx */ - /* RSB 1111 0x01 110x xxxx 0xxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0xfa008000, 0xf0000000, t32_emulate_rd8rn16rm0_rwflags, - REGS(NOSPPC, 0, NOSPPC, 0, 0)), - - DECODE_END -}; - -static const union decode_item t32_table_1111_0x1x___0[] = { - /* Data-processing (plain binary immediate) */ - - /* ADDW Rd, PC, #imm 1111 0x10 0000 1111 0xxx xxxx xxxx xxxx */ - DECODE_OR (0xfbff8000, 0xf20f0000), - /* SUBW Rd, PC, #imm 1111 0x10 1010 1111 0xxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0xfbff8000, 0xf2af0000, t32_emulate_rd8pc16_noflags, - REGS(PC, 0, NOSPPC, 0, 0)), - - /* ADDW SP, SP, #imm 1111 0x10 0000 1101 0xxx 1101 xxxx xxxx */ - DECODE_OR (0xfbff8f00, 0xf20d0d00), - /* SUBW SP, SP, #imm 1111 0x10 1010 1101 0xxx 1101 xxxx xxxx */ - DECODE_EMULATEX (0xfbff8f00, 0xf2ad0d00, t32_emulate_rd8rn16_noflags, - REGS(SP, 0, SP, 0, 0)), - - /* ADDW 1111 0x10 0000 xxxx 0xxx xxxx xxxx xxxx */ - DECODE_OR (0xfbf08000, 0xf2000000), - /* SUBW 1111 0x10 1010 xxxx 0xxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0xfbf08000, 0xf2a00000, t32_emulate_rd8rn16_noflags, - REGS(NOPCX, 0, NOSPPC, 0, 0)), - - /* MOVW 1111 0x10 0100 xxxx 0xxx xxxx xxxx xxxx */ - /* MOVT 1111 0x10 1100 xxxx 0xxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0xfb708000, 0xf2400000, t32_emulate_rd8rn16_noflags, - REGS(0, 0, NOSPPC, 0, 0)), - - /* SSAT16 1111 0x11 0010 xxxx 0000 xxxx 00xx xxxx */ - /* SSAT 1111 0x11 00x0 xxxx 0xxx xxxx xxxx xxxx */ - /* USAT16 1111 0x11 1010 xxxx 0000 xxxx 00xx xxxx */ - /* USAT 1111 0x11 10x0 xxxx 0xxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0xfb508000, 0xf3000000, t32_emulate_rd8rn16rm0_rwflags, - REGS(NOSPPC, 0, NOSPPC, 0, 0)), - - /* SFBX 1111 0x11 0100 xxxx 0xxx xxxx xxxx xxxx */ - /* UFBX 1111 0x11 1100 xxxx 0xxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0xfb708000, 0xf3400000, t32_emulate_rd8rn16_noflags, - REGS(NOSPPC, 0, NOSPPC, 0, 0)), - - /* BFC 1111 0x11 0110 1111 0xxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0xfbff8000, 0xf36f0000, t32_emulate_rd8rn16_noflags, - REGS(0, 0, NOSPPC, 0, 0)), - - /* BFI 1111 0x11 0110 xxxx 0xxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0xfbf08000, 0xf3600000, t32_emulate_rd8rn16_noflags, - REGS(NOSPPCX, 0, NOSPPC, 0, 0)), - - DECODE_END -}; - -static const union decode_item t32_table_1111_0xxx___1[] = { - /* Branches and miscellaneous control */ - - /* YIELD 1111 0011 1010 xxxx 10x0 x000 0000 0001 */ - DECODE_OR (0xfff0d7ff, 0xf3a08001), - /* SEV 1111 0011 1010 xxxx 10x0 x000 0000 0100 */ - DECODE_EMULATE (0xfff0d7ff, 0xf3a08004, kprobe_emulate_none), - /* NOP 1111 0011 1010 xxxx 10x0 x000 0000 0000 */ - /* WFE 1111 0011 1010 xxxx 10x0 x000 0000 0010 */ - /* WFI 1111 0011 1010 xxxx 10x0 x000 0000 0011 */ - DECODE_SIMULATE (0xfff0d7fc, 0xf3a08000, kprobe_simulate_nop), - - /* MRS Rd, CPSR 1111 0011 1110 xxxx 10x0 xxxx xxxx xxxx */ - DECODE_SIMULATEX(0xfff0d000, 0xf3e08000, t32_simulate_mrs, - REGS(0, 0, NOSPPC, 0, 0)), - - /* - * Unsupported instructions - * 1111 0x11 1xxx xxxx 10x0 xxxx xxxx xxxx - * - * MSR 1111 0011 100x xxxx 10x0 xxxx xxxx xxxx - * DBG hint 1111 0011 1010 xxxx 10x0 x000 1111 xxxx - * Unallocated hints 1111 0011 1010 xxxx 10x0 x000 xxxx xxxx - * CPS 1111 0011 1010 xxxx 10x0 xxxx xxxx xxxx - * CLREX/DSB/DMB/ISB 1111 0011 1011 xxxx 10x0 xxxx xxxx xxxx - * BXJ 1111 0011 1100 xxxx 10x0 xxxx xxxx xxxx - * SUBS PC,LR,#<imm8> 1111 0011 1101 xxxx 10x0 xxxx xxxx xxxx - * MRS Rd, SPSR 1111 0011 1111 xxxx 10x0 xxxx xxxx xxxx - * SMC 1111 0111 1111 xxxx 1000 xxxx xxxx xxxx - * UNDEFINED 1111 0111 1111 xxxx 1010 xxxx xxxx xxxx - * ??? 1111 0111 1xxx xxxx 1010 xxxx xxxx xxxx - */ - DECODE_REJECT (0xfb80d000, 0xf3808000), - - /* Bcc 1111 0xxx xxxx xxxx 10x0 xxxx xxxx xxxx */ - DECODE_CUSTOM (0xf800d000, 0xf0008000, t32_decode_cond_branch), - - /* BLX 1111 0xxx xxxx xxxx 11x0 xxxx xxxx xxx0 */ - DECODE_OR (0xf800d001, 0xf000c000), - /* B 1111 0xxx xxxx xxxx 10x1 xxxx xxxx xxxx */ - /* BL 1111 0xxx xxxx xxxx 11x1 xxxx xxxx xxxx */ - DECODE_SIMULATE (0xf8009000, 0xf0009000, t32_simulate_branch), - - DECODE_END -}; - -static const union decode_item t32_table_1111_100x_x0x1__1111[] = { - /* Memory hints */ - - /* PLD (literal) 1111 1000 x001 1111 1111 xxxx xxxx xxxx */ - /* PLI (literal) 1111 1001 x001 1111 1111 xxxx xxxx xxxx */ - DECODE_SIMULATE (0xfe7ff000, 0xf81ff000, kprobe_simulate_nop), - - /* PLD{W} (immediate) 1111 1000 10x1 xxxx 1111 xxxx xxxx xxxx */ - DECODE_OR (0xffd0f000, 0xf890f000), - /* PLD{W} (immediate) 1111 1000 00x1 xxxx 1111 1100 xxxx xxxx */ - DECODE_OR (0xffd0ff00, 0xf810fc00), - /* PLI (immediate) 1111 1001 1001 xxxx 1111 xxxx xxxx xxxx */ - DECODE_OR (0xfff0f000, 0xf990f000), - /* PLI (immediate) 1111 1001 0001 xxxx 1111 1100 xxxx xxxx */ - DECODE_SIMULATEX(0xfff0ff00, 0xf910fc00, kprobe_simulate_nop, - REGS(NOPCX, 0, 0, 0, 0)), - - /* PLD{W} (register) 1111 1000 00x1 xxxx 1111 0000 00xx xxxx */ - DECODE_OR (0xffd0ffc0, 0xf810f000), - /* PLI (register) 1111 1001 0001 xxxx 1111 0000 00xx xxxx */ - DECODE_SIMULATEX(0xfff0ffc0, 0xf910f000, kprobe_simulate_nop, - REGS(NOPCX, 0, 0, 0, NOSPPC)), - - /* Other unallocated instructions... */ - DECODE_END -}; - -static const union decode_item t32_table_1111_100x[] = { - /* Store/Load single data item */ - - /* ??? 1111 100x x11x xxxx xxxx xxxx xxxx xxxx */ - DECODE_REJECT (0xfe600000, 0xf8600000), - - /* ??? 1111 1001 0101 xxxx xxxx xxxx xxxx xxxx */ - DECODE_REJECT (0xfff00000, 0xf9500000), - - /* ??? 1111 100x 0xxx xxxx xxxx 10x0 xxxx xxxx */ - DECODE_REJECT (0xfe800d00, 0xf8000800), - - /* STRBT 1111 1000 0000 xxxx xxxx 1110 xxxx xxxx */ - /* STRHT 1111 1000 0010 xxxx xxxx 1110 xxxx xxxx */ - /* STRT 1111 1000 0100 xxxx xxxx 1110 xxxx xxxx */ - /* LDRBT 1111 1000 0001 xxxx xxxx 1110 xxxx xxxx */ - /* LDRSBT 1111 1001 0001 xxxx xxxx 1110 xxxx xxxx */ - /* LDRHT 1111 1000 0011 xxxx xxxx 1110 xxxx xxxx */ - /* LDRSHT 1111 1001 0011 xxxx xxxx 1110 xxxx xxxx */ - /* LDRT 1111 1000 0101 xxxx xxxx 1110 xxxx xxxx */ - DECODE_REJECT (0xfe800f00, 0xf8000e00), - - /* STR{,B,H} Rn,[PC...] 1111 1000 xxx0 1111 xxxx xxxx xxxx xxxx */ - DECODE_REJECT (0xff1f0000, 0xf80f0000), - - /* STR{,B,H} PC,[Rn...] 1111 1000 xxx0 xxxx 1111 xxxx xxxx xxxx */ - DECODE_REJECT (0xff10f000, 0xf800f000), - - /* LDR (literal) 1111 1000 x101 1111 xxxx xxxx xxxx xxxx */ - DECODE_SIMULATEX(0xff7f0000, 0xf85f0000, t32_simulate_ldr_literal, - REGS(PC, ANY, 0, 0, 0)), - - /* STR (immediate) 1111 1000 0100 xxxx xxxx 1xxx xxxx xxxx */ - /* LDR (immediate) 1111 1000 0101 xxxx xxxx 1xxx xxxx xxxx */ - DECODE_OR (0xffe00800, 0xf8400800), - /* STR (immediate) 1111 1000 1100 xxxx xxxx xxxx xxxx xxxx */ - /* LDR (immediate) 1111 1000 1101 xxxx xxxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0xffe00000, 0xf8c00000, t32_emulate_ldrstr, - REGS(NOPCX, ANY, 0, 0, 0)), - - /* STR (register) 1111 1000 0100 xxxx xxxx 0000 00xx xxxx */ - /* LDR (register) 1111 1000 0101 xxxx xxxx 0000 00xx xxxx */ - DECODE_EMULATEX (0xffe00fc0, 0xf8400000, t32_emulate_ldrstr, - REGS(NOPCX, ANY, 0, 0, NOSPPC)), - - /* LDRB (literal) 1111 1000 x001 1111 xxxx xxxx xxxx xxxx */ - /* LDRSB (literal) 1111 1001 x001 1111 xxxx xxxx xxxx xxxx */ - /* LDRH (literal) 1111 1000 x011 1111 xxxx xxxx xxxx xxxx */ - /* LDRSH (literal) 1111 1001 x011 1111 xxxx xxxx xxxx xxxx */ - DECODE_SIMULATEX(0xfe5f0000, 0xf81f0000, t32_simulate_ldr_literal, - REGS(PC, NOSPPCX, 0, 0, 0)), - - /* STRB (immediate) 1111 1000 0000 xxxx xxxx 1xxx xxxx xxxx */ - /* STRH (immediate) 1111 1000 0010 xxxx xxxx 1xxx xxxx xxxx */ - /* LDRB (immediate) 1111 1000 0001 xxxx xxxx 1xxx xxxx xxxx */ - /* LDRSB (immediate) 1111 1001 0001 xxxx xxxx 1xxx xxxx xxxx */ - /* LDRH (immediate) 1111 1000 0011 xxxx xxxx 1xxx xxxx xxxx */ - /* LDRSH (immediate) 1111 1001 0011 xxxx xxxx 1xxx xxxx xxxx */ - DECODE_OR (0xfec00800, 0xf8000800), - /* STRB (immediate) 1111 1000 1000 xxxx xxxx xxxx xxxx xxxx */ - /* STRH (immediate) 1111 1000 1010 xxxx xxxx xxxx xxxx xxxx */ - /* LDRB (immediate) 1111 1000 1001 xxxx xxxx xxxx xxxx xxxx */ - /* LDRSB (immediate) 1111 1001 1001 xxxx xxxx xxxx xxxx xxxx */ - /* LDRH (immediate) 1111 1000 1011 xxxx xxxx xxxx xxxx xxxx */ - /* LDRSH (immediate) 1111 1001 1011 xxxx xxxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0xfec00000, 0xf8800000, t32_emulate_ldrstr, - REGS(NOPCX, NOSPPCX, 0, 0, 0)), - - /* STRB (register) 1111 1000 0000 xxxx xxxx 0000 00xx xxxx */ - /* STRH (register) 1111 1000 0010 xxxx xxxx 0000 00xx xxxx */ - /* LDRB (register) 1111 1000 0001 xxxx xxxx 0000 00xx xxxx */ - /* LDRSB (register) 1111 1001 0001 xxxx xxxx 0000 00xx xxxx */ - /* LDRH (register) 1111 1000 0011 xxxx xxxx 0000 00xx xxxx */ - /* LDRSH (register) 1111 1001 0011 xxxx xxxx 0000 00xx xxxx */ - DECODE_EMULATEX (0xfe800fc0, 0xf8000000, t32_emulate_ldrstr, - REGS(NOPCX, NOSPPCX, 0, 0, NOSPPC)), - - /* Other unallocated instructions... */ - DECODE_END -}; - -static const union decode_item t32_table_1111_1010___1111[] = { - /* Data-processing (register) */ - - /* ??? 1111 1010 011x xxxx 1111 xxxx 1xxx xxxx */ - DECODE_REJECT (0xffe0f080, 0xfa60f080), - - /* SXTH 1111 1010 0000 1111 1111 xxxx 1xxx xxxx */ - /* UXTH 1111 1010 0001 1111 1111 xxxx 1xxx xxxx */ - /* SXTB16 1111 1010 0010 1111 1111 xxxx 1xxx xxxx */ - /* UXTB16 1111 1010 0011 1111 1111 xxxx 1xxx xxxx */ - /* SXTB 1111 1010 0100 1111 1111 xxxx 1xxx xxxx */ - /* UXTB 1111 1010 0101 1111 1111 xxxx 1xxx xxxx */ - DECODE_EMULATEX (0xff8ff080, 0xfa0ff080, t32_emulate_rd8rn16rm0_rwflags, - REGS(0, 0, NOSPPC, 0, NOSPPC)), - - - /* ??? 1111 1010 1xxx xxxx 1111 xxxx 0x11 xxxx */ - DECODE_REJECT (0xff80f0b0, 0xfa80f030), - /* ??? 1111 1010 1x11 xxxx 1111 xxxx 0xxx xxxx */ - DECODE_REJECT (0xffb0f080, 0xfab0f000), - - /* SADD16 1111 1010 1001 xxxx 1111 xxxx 0000 xxxx */ - /* SASX 1111 1010 1010 xxxx 1111 xxxx 0000 xxxx */ - /* SSAX 1111 1010 1110 xxxx 1111 xxxx 0000 xxxx */ - /* SSUB16 1111 1010 1101 xxxx 1111 xxxx 0000 xxxx */ - /* SADD8 1111 1010 1000 xxxx 1111 xxxx 0000 xxxx */ - /* SSUB8 1111 1010 1100 xxxx 1111 xxxx 0000 xxxx */ - - /* QADD16 1111 1010 1001 xxxx 1111 xxxx 0001 xxxx */ - /* QASX 1111 1010 1010 xxxx 1111 xxxx 0001 xxxx */ - /* QSAX 1111 1010 1110 xxxx 1111 xxxx 0001 xxxx */ - /* QSUB16 1111 1010 1101 xxxx 1111 xxxx 0001 xxxx */ - /* QADD8 1111 1010 1000 xxxx 1111 xxxx 0001 xxxx */ - /* QSUB8 1111 1010 1100 xxxx 1111 xxxx 0001 xxxx */ - - /* SHADD16 1111 1010 1001 xxxx 1111 xxxx 0010 xxxx */ - /* SHASX 1111 1010 1010 xxxx 1111 xxxx 0010 xxxx */ - /* SHSAX 1111 1010 1110 xxxx 1111 xxxx 0010 xxxx */ - /* SHSUB16 1111 1010 1101 xxxx 1111 xxxx 0010 xxxx */ - /* SHADD8 1111 1010 1000 xxxx 1111 xxxx 0010 xxxx */ - /* SHSUB8 1111 1010 1100 xxxx 1111 xxxx 0010 xxxx */ - - /* UADD16 1111 1010 1001 xxxx 1111 xxxx 0100 xxxx */ - /* UASX 1111 1010 1010 xxxx 1111 xxxx 0100 xxxx */ - /* USAX 1111 1010 1110 xxxx 1111 xxxx 0100 xxxx */ - /* USUB16 1111 1010 1101 xxxx 1111 xxxx 0100 xxxx */ - /* UADD8 1111 1010 1000 xxxx 1111 xxxx 0100 xxxx */ - /* USUB8 1111 1010 1100 xxxx 1111 xxxx 0100 xxxx */ - - /* UQADD16 1111 1010 1001 xxxx 1111 xxxx 0101 xxxx */ - /* UQASX 1111 1010 1010 xxxx 1111 xxxx 0101 xxxx */ - /* UQSAX 1111 1010 1110 xxxx 1111 xxxx 0101 xxxx */ - /* UQSUB16 1111 1010 1101 xxxx 1111 xxxx 0101 xxxx */ - /* UQADD8 1111 1010 1000 xxxx 1111 xxxx 0101 xxxx */ - /* UQSUB8 1111 1010 1100 xxxx 1111 xxxx 0101 xxxx */ - - /* UHADD16 1111 1010 1001 xxxx 1111 xxxx 0110 xxxx */ - /* UHASX 1111 1010 1010 xxxx 1111 xxxx 0110 xxxx */ - /* UHSAX 1111 1010 1110 xxxx 1111 xxxx 0110 xxxx */ - /* UHSUB16 1111 1010 1101 xxxx 1111 xxxx 0110 xxxx */ - /* UHADD8 1111 1010 1000 xxxx 1111 xxxx 0110 xxxx */ - /* UHSUB8 1111 1010 1100 xxxx 1111 xxxx 0110 xxxx */ - DECODE_OR (0xff80f080, 0xfa80f000), - - /* SXTAH 1111 1010 0000 xxxx 1111 xxxx 1xxx xxxx */ - /* UXTAH 1111 1010 0001 xxxx 1111 xxxx 1xxx xxxx */ - /* SXTAB16 1111 1010 0010 xxxx 1111 xxxx 1xxx xxxx */ - /* UXTAB16 1111 1010 0011 xxxx 1111 xxxx 1xxx xxxx */ - /* SXTAB 1111 1010 0100 xxxx 1111 xxxx 1xxx xxxx */ - /* UXTAB 1111 1010 0101 xxxx 1111 xxxx 1xxx xxxx */ - DECODE_OR (0xff80f080, 0xfa00f080), - - /* QADD 1111 1010 1000 xxxx 1111 xxxx 1000 xxxx */ - /* QDADD 1111 1010 1000 xxxx 1111 xxxx 1001 xxxx */ - /* QSUB 1111 1010 1000 xxxx 1111 xxxx 1010 xxxx */ - /* QDSUB 1111 1010 1000 xxxx 1111 xxxx 1011 xxxx */ - DECODE_OR (0xfff0f0c0, 0xfa80f080), - - /* SEL 1111 1010 1010 xxxx 1111 xxxx 1000 xxxx */ - DECODE_OR (0xfff0f0f0, 0xfaa0f080), - - /* LSL 1111 1010 000x xxxx 1111 xxxx 0000 xxxx */ - /* LSR 1111 1010 001x xxxx 1111 xxxx 0000 xxxx */ - /* ASR 1111 1010 010x xxxx 1111 xxxx 0000 xxxx */ - /* ROR 1111 1010 011x xxxx 1111 xxxx 0000 xxxx */ - DECODE_EMULATEX (0xff80f0f0, 0xfa00f000, t32_emulate_rd8rn16rm0_rwflags, - REGS(NOSPPC, 0, NOSPPC, 0, NOSPPC)), - - /* CLZ 1111 1010 1010 xxxx 1111 xxxx 1000 xxxx */ - DECODE_OR (0xfff0f0f0, 0xfab0f080), - - /* REV 1111 1010 1001 xxxx 1111 xxxx 1000 xxxx */ - /* REV16 1111 1010 1001 xxxx 1111 xxxx 1001 xxxx */ - /* RBIT 1111 1010 1001 xxxx 1111 xxxx 1010 xxxx */ - /* REVSH 1111 1010 1001 xxxx 1111 xxxx 1011 xxxx */ - DECODE_EMULATEX (0xfff0f0c0, 0xfa90f080, t32_emulate_rd8rn16_noflags, - REGS(NOSPPC, 0, NOSPPC, 0, SAMEAS16)), - - /* Other unallocated instructions... */ - DECODE_END -}; - -static const union decode_item t32_table_1111_1011_0[] = { - /* Multiply, multiply accumulate, and absolute difference */ - - /* ??? 1111 1011 0000 xxxx 1111 xxxx 0001 xxxx */ - DECODE_REJECT (0xfff0f0f0, 0xfb00f010), - /* ??? 1111 1011 0111 xxxx 1111 xxxx 0001 xxxx */ - DECODE_REJECT (0xfff0f0f0, 0xfb70f010), - - /* SMULxy 1111 1011 0001 xxxx 1111 xxxx 00xx xxxx */ - DECODE_OR (0xfff0f0c0, 0xfb10f000), - /* MUL 1111 1011 0000 xxxx 1111 xxxx 0000 xxxx */ - /* SMUAD{X} 1111 1011 0010 xxxx 1111 xxxx 000x xxxx */ - /* SMULWy 1111 1011 0011 xxxx 1111 xxxx 000x xxxx */ - /* SMUSD{X} 1111 1011 0100 xxxx 1111 xxxx 000x xxxx */ - /* SMMUL{R} 1111 1011 0101 xxxx 1111 xxxx 000x xxxx */ - /* USAD8 1111 1011 0111 xxxx 1111 xxxx 0000 xxxx */ - DECODE_EMULATEX (0xff80f0e0, 0xfb00f000, t32_emulate_rd8rn16rm0_rwflags, - REGS(NOSPPC, 0, NOSPPC, 0, NOSPPC)), - - /* ??? 1111 1011 0111 xxxx xxxx xxxx 0001 xxxx */ - DECODE_REJECT (0xfff000f0, 0xfb700010), - - /* SMLAxy 1111 1011 0001 xxxx xxxx xxxx 00xx xxxx */ - DECODE_OR (0xfff000c0, 0xfb100000), - /* MLA 1111 1011 0000 xxxx xxxx xxxx 0000 xxxx */ - /* MLS 1111 1011 0000 xxxx xxxx xxxx 0001 xxxx */ - /* SMLAD{X} 1111 1011 0010 xxxx xxxx xxxx 000x xxxx */ - /* SMLAWy 1111 1011 0011 xxxx xxxx xxxx 000x xxxx */ - /* SMLSD{X} 1111 1011 0100 xxxx xxxx xxxx 000x xxxx */ - /* SMMLA{R} 1111 1011 0101 xxxx xxxx xxxx 000x xxxx */ - /* SMMLS{R} 1111 1011 0110 xxxx xxxx xxxx 000x xxxx */ - /* USADA8 1111 1011 0111 xxxx xxxx xxxx 0000 xxxx */ - DECODE_EMULATEX (0xff8000c0, 0xfb000000, t32_emulate_rd8rn16rm0ra12_noflags, - REGS(NOSPPC, NOSPPCX, NOSPPC, 0, NOSPPC)), - - /* Other unallocated instructions... */ - DECODE_END -}; - -static const union decode_item t32_table_1111_1011_1[] = { - /* Long multiply, long multiply accumulate, and divide */ - - /* UMAAL 1111 1011 1110 xxxx xxxx xxxx 0110 xxxx */ - DECODE_OR (0xfff000f0, 0xfbe00060), - /* SMLALxy 1111 1011 1100 xxxx xxxx xxxx 10xx xxxx */ - DECODE_OR (0xfff000c0, 0xfbc00080), - /* SMLALD{X} 1111 1011 1100 xxxx xxxx xxxx 110x xxxx */ - /* SMLSLD{X} 1111 1011 1101 xxxx xxxx xxxx 110x xxxx */ - DECODE_OR (0xffe000e0, 0xfbc000c0), - /* SMULL 1111 1011 1000 xxxx xxxx xxxx 0000 xxxx */ - /* UMULL 1111 1011 1010 xxxx xxxx xxxx 0000 xxxx */ - /* SMLAL 1111 1011 1100 xxxx xxxx xxxx 0000 xxxx */ - /* UMLAL 1111 1011 1110 xxxx xxxx xxxx 0000 xxxx */ - DECODE_EMULATEX (0xff9000f0, 0xfb800000, t32_emulate_rdlo12rdhi8rn16rm0_noflags, - REGS(NOSPPC, NOSPPC, NOSPPC, 0, NOSPPC)), - - /* SDIV 1111 1011 1001 xxxx xxxx xxxx 1111 xxxx */ - /* UDIV 1111 1011 1011 xxxx xxxx xxxx 1111 xxxx */ - /* Other unallocated instructions... */ - DECODE_END -}; - -const union decode_item kprobe_decode_thumb32_table[] = { - - /* - * Load/store multiple instructions - * 1110 100x x0xx xxxx xxxx xxxx xxxx xxxx - */ - DECODE_TABLE (0xfe400000, 0xe8000000, t32_table_1110_100x_x0xx), - - /* - * Load/store dual, load/store exclusive, table branch - * 1110 100x x1xx xxxx xxxx xxxx xxxx xxxx - */ - DECODE_TABLE (0xfe400000, 0xe8400000, t32_table_1110_100x_x1xx), - - /* - * Data-processing (shifted register) - * 1110 101x xxxx xxxx xxxx xxxx xxxx xxxx - */ - DECODE_TABLE (0xfe000000, 0xea000000, t32_table_1110_101x), - - /* - * Coprocessor instructions - * 1110 11xx xxxx xxxx xxxx xxxx xxxx xxxx - */ - DECODE_REJECT (0xfc000000, 0xec000000), - - /* - * Data-processing (modified immediate) - * 1111 0x0x xxxx xxxx 0xxx xxxx xxxx xxxx - */ - DECODE_TABLE (0xfa008000, 0xf0000000, t32_table_1111_0x0x___0), - - /* - * Data-processing (plain binary immediate) - * 1111 0x1x xxxx xxxx 0xxx xxxx xxxx xxxx - */ - DECODE_TABLE (0xfa008000, 0xf2000000, t32_table_1111_0x1x___0), - - /* - * Branches and miscellaneous control - * 1111 0xxx xxxx xxxx 1xxx xxxx xxxx xxxx - */ - DECODE_TABLE (0xf8008000, 0xf0008000, t32_table_1111_0xxx___1), - - /* - * Advanced SIMD element or structure load/store instructions - * 1111 1001 xxx0 xxxx xxxx xxxx xxxx xxxx - */ - DECODE_REJECT (0xff100000, 0xf9000000), - - /* - * Memory hints - * 1111 100x x0x1 xxxx 1111 xxxx xxxx xxxx - */ - DECODE_TABLE (0xfe50f000, 0xf810f000, t32_table_1111_100x_x0x1__1111), - - /* - * Store single data item - * 1111 1000 xxx0 xxxx xxxx xxxx xxxx xxxx - * Load single data items - * 1111 100x xxx1 xxxx xxxx xxxx xxxx xxxx - */ - DECODE_TABLE (0xfe000000, 0xf8000000, t32_table_1111_100x), - - /* - * Data-processing (register) - * 1111 1010 xxxx xxxx 1111 xxxx xxxx xxxx - */ - DECODE_TABLE (0xff00f000, 0xfa00f000, t32_table_1111_1010___1111), - - /* - * Multiply, multiply accumulate, and absolute difference - * 1111 1011 0xxx xxxx xxxx xxxx xxxx xxxx - */ - DECODE_TABLE (0xff800000, 0xfb000000, t32_table_1111_1011_0), - - /* - * Long multiply, long multiply accumulate, and divide - * 1111 1011 1xxx xxxx xxxx xxxx xxxx xxxx - */ - DECODE_TABLE (0xff800000, 0xfb800000, t32_table_1111_1011_1), - - /* - * Coprocessor instructions - * 1111 11xx xxxx xxxx xxxx xxxx xxxx xxxx - */ - DECODE_END -}; -#ifdef CONFIG_ARM_KPROBES_TEST_MODULE -EXPORT_SYMBOL_GPL(kprobe_decode_thumb32_table); -#endif +/* t16 thumb actions */ static void __kprobes -t16_simulate_bxblx(struct kprobe *p, struct pt_regs *regs) +t16_simulate_bxblx(probes_opcode_t insn, + struct arch_probes_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; - unsigned long pc = thumb_probe_pc(p); + unsigned long pc = regs->ARM_pc + 2; int rm = (insn >> 3) & 0xf; unsigned long rmv = (rm == 15) ? pc : regs->uregs[rm]; if (insn & (1 << 7)) /* BLX ? */ - regs->ARM_lr = (unsigned long)p->addr + 2; + regs->ARM_lr = regs->ARM_pc | 1; bx_write_pc(rmv, regs); } static void __kprobes -t16_simulate_ldr_literal(struct kprobe *p, struct pt_regs *regs) +t16_simulate_ldr_literal(probes_opcode_t insn, + struct arch_probes_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; - unsigned long* base = (unsigned long *)(thumb_probe_pc(p) & ~3); + unsigned long *base = (unsigned long *)((regs->ARM_pc + 2) & ~3); long index = insn & 0xff; int rt = (insn >> 8) & 0x7; regs->uregs[rt] = base[index]; } static void __kprobes -t16_simulate_ldrstr_sp_relative(struct kprobe *p, struct pt_regs *regs) +t16_simulate_ldrstr_sp_relative(probes_opcode_t insn, + struct arch_probes_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; unsigned long* base = (unsigned long *)regs->ARM_sp; long index = insn & 0xff; int rt = (insn >> 8) & 0x7; @@ -986,20 +342,20 @@ t16_simulate_ldrstr_sp_relative(struct kprobe *p, struct pt_regs *regs) } static void __kprobes -t16_simulate_reladr(struct kprobe *p, struct pt_regs *regs) +t16_simulate_reladr(probes_opcode_t insn, + struct arch_probes_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; unsigned long base = (insn & 0x800) ? regs->ARM_sp - : (thumb_probe_pc(p) & ~3); + : ((regs->ARM_pc + 2) & ~3); long offset = insn & 0xff; int rt = (insn >> 8) & 0x7; regs->uregs[rt] = base + offset * 4; } static void __kprobes -t16_simulate_add_sp_imm(struct kprobe *p, struct pt_regs *regs) +t16_simulate_add_sp_imm(probes_opcode_t insn, + struct arch_probes_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; long imm = insn & 0x7f; if (insn & 0x80) /* SUB */ regs->ARM_sp -= imm * 4; @@ -1008,21 +364,22 @@ t16_simulate_add_sp_imm(struct kprobe *p, struct pt_regs *regs) } static void __kprobes -t16_simulate_cbz(struct kprobe *p, struct pt_regs *regs) +t16_simulate_cbz(probes_opcode_t insn, + struct arch_probes_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; int rn = insn & 0x7; - kprobe_opcode_t nonzero = regs->uregs[rn] ? insn : ~insn; + probes_opcode_t nonzero = regs->uregs[rn] ? insn : ~insn; if (nonzero & 0x800) { long i = insn & 0x200; long imm5 = insn & 0xf8; - unsigned long pc = thumb_probe_pc(p); + unsigned long pc = regs->ARM_pc + 2; regs->ARM_pc = pc + (i >> 3) + (imm5 >> 2); } } static void __kprobes -t16_simulate_it(struct kprobe *p, struct pt_regs *regs) +t16_simulate_it(probes_opcode_t insn, + struct arch_probes_insn *asi, struct pt_regs *regs) { /* * The 8 IT state bits are split into two parts in CPSR: @@ -1030,7 +387,6 @@ t16_simulate_it(struct kprobe *p, struct pt_regs *regs) * ITSTATE<7:2> are in CPSR<15:10> * The new IT state is in the lower byte of insn. */ - kprobe_opcode_t insn = p->opcode; unsigned long cpsr = regs->ARM_cpsr; cpsr &= ~PSR_IT_MASK; cpsr |= (insn & 0xfc) << 8; @@ -1039,50 +395,54 @@ t16_simulate_it(struct kprobe *p, struct pt_regs *regs) } static void __kprobes -t16_singlestep_it(struct kprobe *p, struct pt_regs *regs) +t16_singlestep_it(probes_opcode_t insn, + struct arch_probes_insn *asi, struct pt_regs *regs) { regs->ARM_pc += 2; - t16_simulate_it(p, regs); + t16_simulate_it(insn, asi, regs); } -static enum kprobe_insn __kprobes -t16_decode_it(kprobe_opcode_t insn, struct arch_specific_insn *asi) +static enum probes_insn __kprobes +t16_decode_it(probes_opcode_t insn, struct arch_probes_insn *asi, + const struct decode_header *d) { asi->insn_singlestep = t16_singlestep_it; return INSN_GOOD_NO_SLOT; } static void __kprobes -t16_simulate_cond_branch(struct kprobe *p, struct pt_regs *regs) +t16_simulate_cond_branch(probes_opcode_t insn, + struct arch_probes_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; - unsigned long pc = thumb_probe_pc(p); + unsigned long pc = regs->ARM_pc + 2; long offset = insn & 0x7f; offset -= insn & 0x80; /* Apply sign bit */ regs->ARM_pc = pc + (offset * 2); } -static enum kprobe_insn __kprobes -t16_decode_cond_branch(kprobe_opcode_t insn, struct arch_specific_insn *asi) +static enum probes_insn __kprobes +t16_decode_cond_branch(probes_opcode_t insn, struct arch_probes_insn *asi, + const struct decode_header *d) { int cc = (insn >> 8) & 0xf; - asi->insn_check_cc = kprobe_condition_checks[cc]; + asi->insn_check_cc = probes_condition_checks[cc]; asi->insn_handler = t16_simulate_cond_branch; return INSN_GOOD_NO_SLOT; } static void __kprobes -t16_simulate_branch(struct kprobe *p, struct pt_regs *regs) +t16_simulate_branch(probes_opcode_t insn, + struct arch_probes_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; - unsigned long pc = thumb_probe_pc(p); + unsigned long pc = regs->ARM_pc + 2; long offset = insn & 0x3ff; offset -= insn & 0x400; /* Apply sign bit */ regs->ARM_pc = pc + (offset * 2); } static unsigned long __kprobes -t16_emulate_loregs(struct kprobe *p, struct pt_regs *regs) +t16_emulate_loregs(probes_opcode_t insn, + struct arch_probes_insn *asi, struct pt_regs *regs) { unsigned long oldcpsr = regs->ARM_cpsr; unsigned long newcpsr; @@ -1095,7 +455,7 @@ t16_emulate_loregs(struct kprobe *p, struct pt_regs *regs) "mrs %[newcpsr], cpsr \n\t" : [newcpsr] "=r" (newcpsr) : [oldcpsr] "r" (oldcpsr), [regs] "r" (regs), - [fn] "r" (p->ainsn.insn_fn) + [fn] "r" (asi->insn_fn) : "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7", "lr", "memory", "cc" ); @@ -1104,24 +464,26 @@ t16_emulate_loregs(struct kprobe *p, struct pt_regs *regs) } static void __kprobes -t16_emulate_loregs_rwflags(struct kprobe *p, struct pt_regs *regs) +t16_emulate_loregs_rwflags(probes_opcode_t insn, + struct arch_probes_insn *asi, struct pt_regs *regs) { - regs->ARM_cpsr = t16_emulate_loregs(p, regs); + regs->ARM_cpsr = t16_emulate_loregs(insn, asi, regs); } static void __kprobes -t16_emulate_loregs_noitrwflags(struct kprobe *p, struct pt_regs *regs) +t16_emulate_loregs_noitrwflags(probes_opcode_t insn, + struct arch_probes_insn *asi, struct pt_regs *regs) { - unsigned long cpsr = t16_emulate_loregs(p, regs); + unsigned long cpsr = t16_emulate_loregs(insn, asi, regs); if (!in_it_block(cpsr)) regs->ARM_cpsr = cpsr; } static void __kprobes -t16_emulate_hiregs(struct kprobe *p, struct pt_regs *regs) +t16_emulate_hiregs(probes_opcode_t insn, + struct arch_probes_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; - unsigned long pc = thumb_probe_pc(p); + unsigned long pc = regs->ARM_pc + 2; int rdn = (insn & 0x7) | ((insn & 0x80) >> 4); int rm = (insn >> 3) & 0xf; @@ -1137,7 +499,7 @@ t16_emulate_hiregs(struct kprobe *p, struct pt_regs *regs) "blx %[fn] \n\t" "mrs %[cpsr], cpsr \n\t" : "=r" (rdnv), [cpsr] "=r" (cpsr) - : "0" (rdnv), "r" (rmv), "1" (cpsr), [fn] "r" (p->ainsn.insn_fn) + : "0" (rdnv), "r" (rmv), "1" (cpsr), [fn] "r" (asi->insn_fn) : "lr", "memory", "cc" ); @@ -1148,8 +510,9 @@ t16_emulate_hiregs(struct kprobe *p, struct pt_regs *regs) regs->ARM_cpsr = (regs->ARM_cpsr & ~APSR_MASK) | (cpsr & APSR_MASK); } -static enum kprobe_insn __kprobes -t16_decode_hiregs(kprobe_opcode_t insn, struct arch_specific_insn *asi) +static enum probes_insn __kprobes +t16_decode_hiregs(probes_opcode_t insn, struct arch_probes_insn *asi, + const struct decode_header *d) { insn &= ~0x00ff; insn |= 0x001; /* Set Rdn = R1 and Rm = R0 */ @@ -1159,7 +522,8 @@ t16_decode_hiregs(kprobe_opcode_t insn, struct arch_specific_insn *asi) } static void __kprobes -t16_emulate_push(struct kprobe *p, struct pt_regs *regs) +t16_emulate_push(probes_opcode_t insn, + struct arch_probes_insn *asi, struct pt_regs *regs) { __asm__ __volatile__ ( "ldr r9, [%[regs], #13*4] \n\t" @@ -1168,14 +532,15 @@ t16_emulate_push(struct kprobe *p, struct pt_regs *regs) "blx %[fn] \n\t" "str r9, [%[regs], #13*4] \n\t" : - : [regs] "r" (regs), [fn] "r" (p->ainsn.insn_fn) + : [regs] "r" (regs), [fn] "r" (asi->insn_fn) : "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "lr", "memory", "cc" ); } -static enum kprobe_insn __kprobes -t16_decode_push(kprobe_opcode_t insn, struct arch_specific_insn *asi) +static enum probes_insn __kprobes +t16_decode_push(probes_opcode_t insn, struct arch_probes_insn *asi, + const struct decode_header *d) { /* * To simulate a PUSH we use a Thumb-2 "STMDB R9!, {registers}" @@ -1189,7 +554,8 @@ t16_decode_push(kprobe_opcode_t insn, struct arch_specific_insn *asi) } static void __kprobes -t16_emulate_pop_nopc(struct kprobe *p, struct pt_regs *regs) +t16_emulate_pop_nopc(probes_opcode_t insn, + struct arch_probes_insn *asi, struct pt_regs *regs) { __asm__ __volatile__ ( "ldr r9, [%[regs], #13*4] \n\t" @@ -1198,14 +564,15 @@ t16_emulate_pop_nopc(struct kprobe *p, struct pt_regs *regs) "stmia %[regs], {r0-r7} \n\t" "str r9, [%[regs], #13*4] \n\t" : - : [regs] "r" (regs), [fn] "r" (p->ainsn.insn_fn) + : [regs] "r" (regs), [fn] "r" (asi->insn_fn) : "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7", "r9", "lr", "memory", "cc" ); } static void __kprobes -t16_emulate_pop_pc(struct kprobe *p, struct pt_regs *regs) +t16_emulate_pop_pc(probes_opcode_t insn, + struct arch_probes_insn *asi, struct pt_regs *regs) { register unsigned long pc asm("r8"); @@ -1216,7 +583,7 @@ t16_emulate_pop_pc(struct kprobe *p, struct pt_regs *regs) "stmia %[regs], {r0-r7} \n\t" "str r9, [%[regs], #13*4] \n\t" : "=r" (pc) - : [regs] "r" (regs), [fn] "r" (p->ainsn.insn_fn) + : [regs] "r" (regs), [fn] "r" (asi->insn_fn) : "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7", "r9", "lr", "memory", "cc" ); @@ -1224,8 +591,9 @@ t16_emulate_pop_pc(struct kprobe *p, struct pt_regs *regs) bx_write_pc(pc, regs); } -static enum kprobe_insn __kprobes -t16_decode_pop(kprobe_opcode_t insn, struct arch_specific_insn *asi) +static enum probes_insn __kprobes +t16_decode_pop(probes_opcode_t insn, struct arch_probes_insn *asi, + const struct decode_header *d) { /* * To simulate a POP we use a Thumb-2 "LDMDB R9!, {registers}" @@ -1239,231 +607,56 @@ t16_decode_pop(kprobe_opcode_t insn, struct arch_specific_insn *asi) return INSN_GOOD; } -static const union decode_item t16_table_1011[] = { - /* Miscellaneous 16-bit instructions */ - - /* ADD (SP plus immediate) 1011 0000 0xxx xxxx */ - /* SUB (SP minus immediate) 1011 0000 1xxx xxxx */ - DECODE_SIMULATE (0xff00, 0xb000, t16_simulate_add_sp_imm), - - /* CBZ 1011 00x1 xxxx xxxx */ - /* CBNZ 1011 10x1 xxxx xxxx */ - DECODE_SIMULATE (0xf500, 0xb100, t16_simulate_cbz), - - /* SXTH 1011 0010 00xx xxxx */ - /* SXTB 1011 0010 01xx xxxx */ - /* UXTH 1011 0010 10xx xxxx */ - /* UXTB 1011 0010 11xx xxxx */ - /* REV 1011 1010 00xx xxxx */ - /* REV16 1011 1010 01xx xxxx */ - /* ??? 1011 1010 10xx xxxx */ - /* REVSH 1011 1010 11xx xxxx */ - DECODE_REJECT (0xffc0, 0xba80), - DECODE_EMULATE (0xf500, 0xb000, t16_emulate_loregs_rwflags), - - /* PUSH 1011 010x xxxx xxxx */ - DECODE_CUSTOM (0xfe00, 0xb400, t16_decode_push), - /* POP 1011 110x xxxx xxxx */ - DECODE_CUSTOM (0xfe00, 0xbc00, t16_decode_pop), - - /* - * If-Then, and hints - * 1011 1111 xxxx xxxx - */ - - /* YIELD 1011 1111 0001 0000 */ - DECODE_OR (0xffff, 0xbf10), - /* SEV 1011 1111 0100 0000 */ - DECODE_EMULATE (0xffff, 0xbf40, kprobe_emulate_none), - /* NOP 1011 1111 0000 0000 */ - /* WFE 1011 1111 0010 0000 */ - /* WFI 1011 1111 0011 0000 */ - DECODE_SIMULATE (0xffcf, 0xbf00, kprobe_simulate_nop), - /* Unassigned hints 1011 1111 xxxx 0000 */ - DECODE_REJECT (0xff0f, 0xbf00), - /* IT 1011 1111 xxxx xxxx */ - DECODE_CUSTOM (0xff00, 0xbf00, t16_decode_it), - - /* SETEND 1011 0110 010x xxxx */ - /* CPS 1011 0110 011x xxxx */ - /* BKPT 1011 1110 xxxx xxxx */ - /* And unallocated instructions... */ - DECODE_END +const union decode_action kprobes_t16_actions[NUM_PROBES_T16_ACTIONS] = { + [PROBES_T16_ADD_SP] = {.handler = t16_simulate_add_sp_imm}, + [PROBES_T16_CBZ] = {.handler = t16_simulate_cbz}, + [PROBES_T16_SIGN_EXTEND] = {.handler = t16_emulate_loregs_rwflags}, + [PROBES_T16_PUSH] = {.decoder = t16_decode_push}, + [PROBES_T16_POP] = {.decoder = t16_decode_pop}, + [PROBES_T16_SEV] = {.handler = probes_emulate_none}, + [PROBES_T16_WFE] = {.handler = probes_simulate_nop}, + [PROBES_T16_IT] = {.decoder = t16_decode_it}, + [PROBES_T16_CMP] = {.handler = t16_emulate_loregs_rwflags}, + [PROBES_T16_ADDSUB] = {.handler = t16_emulate_loregs_noitrwflags}, + [PROBES_T16_LOGICAL] = {.handler = t16_emulate_loregs_noitrwflags}, + [PROBES_T16_LDR_LIT] = {.handler = t16_simulate_ldr_literal}, + [PROBES_T16_BLX] = {.handler = t16_simulate_bxblx}, + [PROBES_T16_HIREGOPS] = {.decoder = t16_decode_hiregs}, + [PROBES_T16_LDRHSTRH] = {.handler = t16_emulate_loregs_rwflags}, + [PROBES_T16_LDRSTR] = {.handler = t16_simulate_ldrstr_sp_relative}, + [PROBES_T16_ADR] = {.handler = t16_simulate_reladr}, + [PROBES_T16_LDMSTM] = {.handler = t16_emulate_loregs_rwflags}, + [PROBES_T16_BRANCH_COND] = {.decoder = t16_decode_cond_branch}, + [PROBES_T16_BRANCH] = {.handler = t16_simulate_branch}, }; -const union decode_item kprobe_decode_thumb16_table[] = { - - /* - * Shift (immediate), add, subtract, move, and compare - * 00xx xxxx xxxx xxxx - */ - - /* CMP (immediate) 0010 1xxx xxxx xxxx */ - DECODE_EMULATE (0xf800, 0x2800, t16_emulate_loregs_rwflags), - - /* ADD (register) 0001 100x xxxx xxxx */ - /* SUB (register) 0001 101x xxxx xxxx */ - /* LSL (immediate) 0000 0xxx xxxx xxxx */ - /* LSR (immediate) 0000 1xxx xxxx xxxx */ - /* ASR (immediate) 0001 0xxx xxxx xxxx */ - /* ADD (immediate, Thumb) 0001 110x xxxx xxxx */ - /* SUB (immediate, Thumb) 0001 111x xxxx xxxx */ - /* MOV (immediate) 0010 0xxx xxxx xxxx */ - /* ADD (immediate, Thumb) 0011 0xxx xxxx xxxx */ - /* SUB (immediate, Thumb) 0011 1xxx xxxx xxxx */ - DECODE_EMULATE (0xc000, 0x0000, t16_emulate_loregs_noitrwflags), - - /* - * 16-bit Thumb data-processing instructions - * 0100 00xx xxxx xxxx - */ - - /* TST (register) 0100 0010 00xx xxxx */ - DECODE_EMULATE (0xffc0, 0x4200, t16_emulate_loregs_rwflags), - /* CMP (register) 0100 0010 10xx xxxx */ - /* CMN (register) 0100 0010 11xx xxxx */ - DECODE_EMULATE (0xff80, 0x4280, t16_emulate_loregs_rwflags), - /* AND (register) 0100 0000 00xx xxxx */ - /* EOR (register) 0100 0000 01xx xxxx */ - /* LSL (register) 0100 0000 10xx xxxx */ - /* LSR (register) 0100 0000 11xx xxxx */ - /* ASR (register) 0100 0001 00xx xxxx */ - /* ADC (register) 0100 0001 01xx xxxx */ - /* SBC (register) 0100 0001 10xx xxxx */ - /* ROR (register) 0100 0001 11xx xxxx */ - /* RSB (immediate) 0100 0010 01xx xxxx */ - /* ORR (register) 0100 0011 00xx xxxx */ - /* MUL 0100 0011 00xx xxxx */ - /* BIC (register) 0100 0011 10xx xxxx */ - /* MVN (register) 0100 0011 10xx xxxx */ - DECODE_EMULATE (0xfc00, 0x4000, t16_emulate_loregs_noitrwflags), - - /* - * Special data instructions and branch and exchange - * 0100 01xx xxxx xxxx - */ - - /* BLX pc 0100 0111 1111 1xxx */ - DECODE_REJECT (0xfff8, 0x47f8), - - /* BX (register) 0100 0111 0xxx xxxx */ - /* BLX (register) 0100 0111 1xxx xxxx */ - DECODE_SIMULATE (0xff00, 0x4700, t16_simulate_bxblx), - - /* ADD pc, pc 0100 0100 1111 1111 */ - DECODE_REJECT (0xffff, 0x44ff), - - /* ADD (register) 0100 0100 xxxx xxxx */ - /* CMP (register) 0100 0101 xxxx xxxx */ - /* MOV (register) 0100 0110 xxxx xxxx */ - DECODE_CUSTOM (0xfc00, 0x4400, t16_decode_hiregs), - - /* - * Load from Literal Pool - * LDR (literal) 0100 1xxx xxxx xxxx - */ - DECODE_SIMULATE (0xf800, 0x4800, t16_simulate_ldr_literal), - - /* - * 16-bit Thumb Load/store instructions - * 0101 xxxx xxxx xxxx - * 011x xxxx xxxx xxxx - * 100x xxxx xxxx xxxx - */ - - /* STR (register) 0101 000x xxxx xxxx */ - /* STRH (register) 0101 001x xxxx xxxx */ - /* STRB (register) 0101 010x xxxx xxxx */ - /* LDRSB (register) 0101 011x xxxx xxxx */ - /* LDR (register) 0101 100x xxxx xxxx */ - /* LDRH (register) 0101 101x xxxx xxxx */ - /* LDRB (register) 0101 110x xxxx xxxx */ - /* LDRSH (register) 0101 111x xxxx xxxx */ - /* STR (immediate, Thumb) 0110 0xxx xxxx xxxx */ - /* LDR (immediate, Thumb) 0110 1xxx xxxx xxxx */ - /* STRB (immediate, Thumb) 0111 0xxx xxxx xxxx */ - /* LDRB (immediate, Thumb) 0111 1xxx xxxx xxxx */ - DECODE_EMULATE (0xc000, 0x4000, t16_emulate_loregs_rwflags), - /* STRH (immediate, Thumb) 1000 0xxx xxxx xxxx */ - /* LDRH (immediate, Thumb) 1000 1xxx xxxx xxxx */ - DECODE_EMULATE (0xf000, 0x8000, t16_emulate_loregs_rwflags), - /* STR (immediate, Thumb) 1001 0xxx xxxx xxxx */ - /* LDR (immediate, Thumb) 1001 1xxx xxxx xxxx */ - DECODE_SIMULATE (0xf000, 0x9000, t16_simulate_ldrstr_sp_relative), - - /* - * Generate PC-/SP-relative address - * ADR (literal) 1010 0xxx xxxx xxxx - * ADD (SP plus immediate) 1010 1xxx xxxx xxxx - */ - DECODE_SIMULATE (0xf000, 0xa000, t16_simulate_reladr), - - /* - * Miscellaneous 16-bit instructions - * 1011 xxxx xxxx xxxx - */ - DECODE_TABLE (0xf000, 0xb000, t16_table_1011), - - /* STM 1100 0xxx xxxx xxxx */ - /* LDM 1100 1xxx xxxx xxxx */ - DECODE_EMULATE (0xf000, 0xc000, t16_emulate_loregs_rwflags), - - /* - * Conditional branch, and Supervisor Call - */ - - /* Permanently UNDEFINED 1101 1110 xxxx xxxx */ - /* SVC 1101 1111 xxxx xxxx */ - DECODE_REJECT (0xfe00, 0xde00), - - /* Conditional branch 1101 xxxx xxxx xxxx */ - DECODE_CUSTOM (0xf000, 0xd000, t16_decode_cond_branch), - - /* - * Unconditional branch - * B 1110 0xxx xxxx xxxx - */ - DECODE_SIMULATE (0xf800, 0xe000, t16_simulate_branch), - - DECODE_END +const union decode_action kprobes_t32_actions[NUM_PROBES_T32_ACTIONS] = { + [PROBES_T32_LDMSTM] = {.decoder = t32_decode_ldmstm}, + [PROBES_T32_LDRDSTRD] = {.handler = t32_emulate_ldrdstrd}, + [PROBES_T32_TABLE_BRANCH] = {.handler = t32_simulate_table_branch}, + [PROBES_T32_TST] = {.handler = t32_emulate_rd8rn16rm0_rwflags}, + [PROBES_T32_MOV] = {.handler = t32_emulate_rd8rn16rm0_rwflags}, + [PROBES_T32_ADDSUB] = {.handler = t32_emulate_rd8rn16rm0_rwflags}, + [PROBES_T32_LOGICAL] = {.handler = t32_emulate_rd8rn16rm0_rwflags}, + [PROBES_T32_CMP] = {.handler = t32_emulate_rd8rn16rm0_rwflags}, + [PROBES_T32_ADDWSUBW_PC] = {.handler = t32_emulate_rd8pc16_noflags,}, + [PROBES_T32_ADDWSUBW] = {.handler = t32_emulate_rd8rn16_noflags}, + [PROBES_T32_MOVW] = {.handler = t32_emulate_rd8rn16_noflags}, + [PROBES_T32_SAT] = {.handler = t32_emulate_rd8rn16rm0_rwflags}, + [PROBES_T32_BITFIELD] = {.handler = t32_emulate_rd8rn16_noflags}, + [PROBES_T32_SEV] = {.handler = probes_emulate_none}, + [PROBES_T32_WFE] = {.handler = probes_simulate_nop}, + [PROBES_T32_MRS] = {.handler = t32_simulate_mrs}, + [PROBES_T32_BRANCH_COND] = {.decoder = t32_decode_cond_branch}, + [PROBES_T32_BRANCH] = {.handler = t32_simulate_branch}, + [PROBES_T32_PLDI] = {.handler = probes_simulate_nop}, + [PROBES_T32_LDR_LIT] = {.handler = t32_simulate_ldr_literal}, + [PROBES_T32_LDRSTR] = {.handler = t32_emulate_ldrstr}, + [PROBES_T32_SIGN_EXTEND] = {.handler = t32_emulate_rd8rn16rm0_rwflags}, + [PROBES_T32_MEDIA] = {.handler = t32_emulate_rd8rn16rm0_rwflags}, + [PROBES_T32_REVERSE] = {.handler = t32_emulate_rd8rn16_noflags}, + [PROBES_T32_MUL_ADD] = {.handler = t32_emulate_rd8rn16rm0_rwflags}, + [PROBES_T32_MUL_ADD2] = {.handler = t32_emulate_rd8rn16rm0ra12_noflags}, + [PROBES_T32_MUL_ADD_LONG] = { + .handler = t32_emulate_rdlo12rdhi8rn16rm0_noflags}, }; -#ifdef CONFIG_ARM_KPROBES_TEST_MODULE -EXPORT_SYMBOL_GPL(kprobe_decode_thumb16_table); -#endif - -static unsigned long __kprobes thumb_check_cc(unsigned long cpsr) -{ - if (unlikely(in_it_block(cpsr))) - return kprobe_condition_checks[current_cond(cpsr)](cpsr); - return true; -} - -static void __kprobes thumb16_singlestep(struct kprobe *p, struct pt_regs *regs) -{ - regs->ARM_pc += 2; - p->ainsn.insn_handler(p, regs); - regs->ARM_cpsr = it_advance(regs->ARM_cpsr); -} - -static void __kprobes thumb32_singlestep(struct kprobe *p, struct pt_regs *regs) -{ - regs->ARM_pc += 4; - p->ainsn.insn_handler(p, regs); - regs->ARM_cpsr = it_advance(regs->ARM_cpsr); -} - -enum kprobe_insn __kprobes -thumb16_kprobe_decode_insn(kprobe_opcode_t insn, struct arch_specific_insn *asi) -{ - asi->insn_singlestep = thumb16_singlestep; - asi->insn_check_cc = thumb_check_cc; - return kprobe_decode_insn(insn, asi, kprobe_decode_thumb16_table, true); -} - -enum kprobe_insn __kprobes -thumb32_kprobe_decode_insn(kprobe_opcode_t insn, struct arch_specific_insn *asi) -{ - asi->insn_singlestep = thumb32_singlestep; - asi->insn_check_cc = thumb_check_cc; - return kprobe_decode_insn(insn, asi, kprobe_decode_thumb32_table, true); -} diff --git a/arch/arm/kernel/kprobes.c b/arch/arm/kernel/kprobes.c index a7b621ece23d..8795f9f819d5 100644 --- a/arch/arm/kernel/kprobes.c +++ b/arch/arm/kernel/kprobes.c @@ -27,8 +27,12 @@ #include <linux/stringify.h> #include <asm/traps.h> #include <asm/cacheflush.h> +#include <linux/percpu.h> +#include <linux/bug.h> #include "kprobes.h" +#include "probes-arm.h" +#include "probes-thumb.h" #include "patch.h" #define MIN_STACK_SIZE(addr) \ @@ -54,6 +58,7 @@ int __kprobes arch_prepare_kprobe(struct kprobe *p) unsigned long addr = (unsigned long)p->addr; bool thumb; kprobe_decode_insn_t *decode_insn; + const union decode_action *actions; int is; if (in_exception_text(addr)) @@ -66,21 +71,25 @@ int __kprobes arch_prepare_kprobe(struct kprobe *p) if (is_wide_instruction(insn)) { insn <<= 16; insn |= ((u16 *)addr)[1]; - decode_insn = thumb32_kprobe_decode_insn; - } else - decode_insn = thumb16_kprobe_decode_insn; + decode_insn = thumb32_probes_decode_insn; + actions = kprobes_t32_actions; + } else { + decode_insn = thumb16_probes_decode_insn; + actions = kprobes_t16_actions; + } #else /* !CONFIG_THUMB2_KERNEL */ thumb = false; if (addr & 0x3) return -EINVAL; insn = *p->addr; - decode_insn = arm_kprobe_decode_insn; + decode_insn = arm_probes_decode_insn; + actions = kprobes_arm_actions; #endif p->opcode = insn; p->ainsn.insn = tmp_insn; - switch ((*decode_insn)(insn, &p->ainsn)) { + switch ((*decode_insn)(insn, &p->ainsn, true, actions)) { case INSN_REJECTED: /* not supported */ return -EINVAL; @@ -92,7 +101,7 @@ int __kprobes arch_prepare_kprobe(struct kprobe *p) p->ainsn.insn[is] = tmp_insn[is]; flush_insns(p->ainsn.insn, sizeof(p->ainsn.insn[0]) * MAX_INSN_SIZE); - p->ainsn.insn_fn = (kprobe_insn_fn_t *) + p->ainsn.insn_fn = (probes_insn_fn_t *) ((uintptr_t)p->ainsn.insn | thumb); break; @@ -197,7 +206,7 @@ singlestep_skip(struct kprobe *p, struct pt_regs *regs) static inline void __kprobes singlestep(struct kprobe *p, struct pt_regs *regs, struct kprobe_ctlblk *kcb) { - p->ainsn.insn_singlestep(p, regs); + p->ainsn.insn_singlestep(p->opcode, &p->ainsn, regs); } /* @@ -607,7 +616,7 @@ static struct undef_hook kprobes_arm_break_hook = { int __init arch_init_kprobes() { - arm_kprobe_decode_init(); + arm_probes_decode_init(); #ifdef CONFIG_THUMB2_KERNEL register_undef_hook(&kprobes_thumb16_break_hook); register_undef_hook(&kprobes_thumb32_break_hook); diff --git a/arch/arm/kernel/kprobes.h b/arch/arm/kernel/kprobes.h index 38945f78f9f1..9a2712ecefc3 100644 --- a/arch/arm/kernel/kprobes.h +++ b/arch/arm/kernel/kprobes.h @@ -19,6 +19,8 @@ #ifndef _ARM_KERNEL_KPROBES_H #define _ARM_KERNEL_KPROBES_H +#include "probes.h" + /* * These undefined instructions must be unique and * reserved solely for kprobes' use. @@ -27,402 +29,24 @@ #define KPROBE_THUMB16_BREAKPOINT_INSTRUCTION 0xde18 #define KPROBE_THUMB32_BREAKPOINT_INSTRUCTION 0xf7f0a018 +enum probes_insn __kprobes +kprobe_decode_ldmstm(kprobe_opcode_t insn, struct arch_probes_insn *asi, + const struct decode_header *h); -enum kprobe_insn { - INSN_REJECTED, - INSN_GOOD, - INSN_GOOD_NO_SLOT -}; - -typedef enum kprobe_insn (kprobe_decode_insn_t)(kprobe_opcode_t, - struct arch_specific_insn *); +typedef enum probes_insn (kprobe_decode_insn_t)(probes_opcode_t, + struct arch_probes_insn *, + bool, + const union decode_action *); #ifdef CONFIG_THUMB2_KERNEL -enum kprobe_insn thumb16_kprobe_decode_insn(kprobe_opcode_t, - struct arch_specific_insn *); -enum kprobe_insn thumb32_kprobe_decode_insn(kprobe_opcode_t, - struct arch_specific_insn *); +extern const union decode_action kprobes_t32_actions[]; +extern const union decode_action kprobes_t16_actions[]; #else /* !CONFIG_THUMB2_KERNEL */ -enum kprobe_insn arm_kprobe_decode_insn(kprobe_opcode_t, - struct arch_specific_insn *); -#endif - -void __init arm_kprobe_decode_init(void); - -extern kprobe_check_cc * const kprobe_condition_checks[16]; - - -#if __LINUX_ARM_ARCH__ >= 7 - -/* str_pc_offset is architecturally defined from ARMv7 onwards */ -#define str_pc_offset 8 -#define find_str_pc_offset() - -#else /* __LINUX_ARM_ARCH__ < 7 */ - -/* We need a run-time check to determine str_pc_offset */ -extern int str_pc_offset; -void __init find_str_pc_offset(void); +extern const union decode_action kprobes_arm_actions[]; #endif - -/* - * Update ITSTATE after normal execution of an IT block instruction. - * - * The 8 IT state bits are split into two parts in CPSR: - * ITSTATE<1:0> are in CPSR<26:25> - * ITSTATE<7:2> are in CPSR<15:10> - */ -static inline unsigned long it_advance(unsigned long cpsr) - { - if ((cpsr & 0x06000400) == 0) { - /* ITSTATE<2:0> == 0 means end of IT block, so clear IT state */ - cpsr &= ~PSR_IT_MASK; - } else { - /* We need to shift left ITSTATE<4:0> */ - const unsigned long mask = 0x06001c00; /* Mask ITSTATE<4:0> */ - unsigned long it = cpsr & mask; - it <<= 1; - it |= it >> (27 - 10); /* Carry ITSTATE<2> to correct place */ - it &= mask; - cpsr &= ~mask; - cpsr |= it; - } - return cpsr; -} - -static inline void __kprobes bx_write_pc(long pcv, struct pt_regs *regs) -{ - long cpsr = regs->ARM_cpsr; - if (pcv & 0x1) { - cpsr |= PSR_T_BIT; - pcv &= ~0x1; - } else { - cpsr &= ~PSR_T_BIT; - pcv &= ~0x2; /* Avoid UNPREDICTABLE address allignment */ - } - regs->ARM_cpsr = cpsr; - regs->ARM_pc = pcv; -} - - -#if __LINUX_ARM_ARCH__ >= 6 - -/* Kernels built for >= ARMv6 should never run on <= ARMv5 hardware, so... */ -#define load_write_pc_interworks true -#define test_load_write_pc_interworking() - -#else /* __LINUX_ARM_ARCH__ < 6 */ - -/* We need run-time testing to determine if load_write_pc() should interwork. */ -extern bool load_write_pc_interworks; -void __init test_load_write_pc_interworking(void); - -#endif - -static inline void __kprobes load_write_pc(long pcv, struct pt_regs *regs) -{ - if (load_write_pc_interworks) - bx_write_pc(pcv, regs); - else - regs->ARM_pc = pcv; -} - - -#if __LINUX_ARM_ARCH__ >= 7 - -#define alu_write_pc_interworks true -#define test_alu_write_pc_interworking() - -#elif __LINUX_ARM_ARCH__ <= 5 - -/* Kernels built for <= ARMv5 should never run on >= ARMv6 hardware, so... */ -#define alu_write_pc_interworks false -#define test_alu_write_pc_interworking() - -#else /* __LINUX_ARM_ARCH__ == 6 */ - -/* We could be an ARMv6 binary on ARMv7 hardware so we need a run-time check. */ -extern bool alu_write_pc_interworks; -void __init test_alu_write_pc_interworking(void); - -#endif /* __LINUX_ARM_ARCH__ == 6 */ - -static inline void __kprobes alu_write_pc(long pcv, struct pt_regs *regs) -{ - if (alu_write_pc_interworks) - bx_write_pc(pcv, regs); - else - regs->ARM_pc = pcv; -} - - -void __kprobes kprobe_simulate_nop(struct kprobe *p, struct pt_regs *regs); -void __kprobes kprobe_emulate_none(struct kprobe *p, struct pt_regs *regs); - -enum kprobe_insn __kprobes -kprobe_decode_ldmstm(kprobe_opcode_t insn, struct arch_specific_insn *asi); - -/* - * Test if load/store instructions writeback the address register. - * if P (bit 24) == 0 or W (bit 21) == 1 - */ -#define is_writeback(insn) ((insn ^ 0x01000000) & 0x01200000) - -/* - * The following definitions and macros are used to build instruction - * decoding tables for use by kprobe_decode_insn. - * - * These tables are a concatenation of entries each of which consist of one of - * the decode_* structs. All of the fields in every type of decode structure - * are of the union type decode_item, therefore the entire decode table can be - * viewed as an array of these and declared like: - * - * static const union decode_item table_name[] = {}; - * - * In order to construct each entry in the table, macros are used to - * initialise a number of sequential decode_item values in a layout which - * matches the relevant struct. E.g. DECODE_SIMULATE initialise a struct - * decode_simulate by initialising four decode_item objects like this... - * - * {.bits = _type}, - * {.bits = _mask}, - * {.bits = _value}, - * {.handler = _handler}, - * - * Initialising a specified member of the union means that the compiler - * will produce a warning if the argument is of an incorrect type. - * - * Below is a list of each of the macros used to initialise entries and a - * description of the action performed when that entry is matched to an - * instruction. A match is found when (instruction & mask) == value. - * - * DECODE_TABLE(mask, value, table) - * Instruction decoding jumps to parsing the new sub-table 'table'. - * - * DECODE_CUSTOM(mask, value, decoder) - * The custom function 'decoder' is called to the complete decoding - * of an instruction. - * - * DECODE_SIMULATE(mask, value, handler) - * Set the probes instruction handler to 'handler', this will be used - * to simulate the instruction when the probe is hit. Decoding returns - * with INSN_GOOD_NO_SLOT. - * - * DECODE_EMULATE(mask, value, handler) - * Set the probes instruction handler to 'handler', this will be used - * to emulate the instruction when the probe is hit. The modified - * instruction (see below) is placed in the probes instruction slot so it - * may be called by the emulation code. Decoding returns with INSN_GOOD. - * - * DECODE_REJECT(mask, value) - * Instruction decoding fails with INSN_REJECTED - * - * DECODE_OR(mask, value) - * This allows the mask/value test of multiple table entries to be - * logically ORed. Once an 'or' entry is matched the decoding action to - * be performed is that of the next entry which isn't an 'or'. E.g. - * - * DECODE_OR (mask1, value1) - * DECODE_OR (mask2, value2) - * DECODE_SIMULATE (mask3, value3, simulation_handler) - * - * This means that if any of the three mask/value pairs match the - * instruction being decoded, then 'simulation_handler' will be used - * for it. - * - * Both the SIMULATE and EMULATE macros have a second form which take an - * additional 'regs' argument. - * - * DECODE_SIMULATEX(mask, value, handler, regs) - * DECODE_EMULATEX (mask, value, handler, regs) - * - * These are used to specify what kind of CPU register is encoded in each of the - * least significant 5 nibbles of the instruction being decoded. The regs value - * is specified using the REGS macro, this takes any of the REG_TYPE_* values - * from enum decode_reg_type as arguments; only the '*' part of the name is - * given. E.g. - * - * REGS(0, ANY, NOPC, 0, ANY) - * - * This indicates an instruction is encoded like: - * - * bits 19..16 ignore - * bits 15..12 any register allowed here - * bits 11.. 8 any register except PC allowed here - * bits 7.. 4 ignore - * bits 3.. 0 any register allowed here - * - * This register specification is checked after a decode table entry is found to - * match an instruction (through the mask/value test). Any invalid register then - * found in the instruction will cause decoding to fail with INSN_REJECTED. In - * the above example this would happen if bits 11..8 of the instruction were - * 1111, indicating R15 or PC. - * - * As well as checking for legal combinations of registers, this data is also - * used to modify the registers encoded in the instructions so that an - * emulation routines can use it. (See decode_regs() and INSN_NEW_BITS.) - * - * Here is a real example which matches ARM instructions of the form - * "AND <Rd>,<Rn>,<Rm>,<shift> <Rs>" - * - * DECODE_EMULATEX (0x0e000090, 0x00000010, emulate_rd12rn16rm0rs8_rwflags, - * REGS(ANY, ANY, NOPC, 0, ANY)), - * ^ ^ ^ ^ - * Rn Rd Rs Rm - * - * Decoding the instruction "AND R4, R5, R6, ASL R15" will be rejected because - * Rs == R15 - * - * Decoding the instruction "AND R4, R5, R6, ASL R7" will be accepted and the - * instruction will be modified to "AND R0, R2, R3, ASL R1" and then placed into - * the kprobes instruction slot. This can then be called later by the handler - * function emulate_rd12rn16rm0rs8_rwflags in order to simulate the instruction. - */ - -enum decode_type { - DECODE_TYPE_END, - DECODE_TYPE_TABLE, - DECODE_TYPE_CUSTOM, - DECODE_TYPE_SIMULATE, - DECODE_TYPE_EMULATE, - DECODE_TYPE_OR, - DECODE_TYPE_REJECT, - NUM_DECODE_TYPES /* Must be last enum */ -}; - -#define DECODE_TYPE_BITS 4 -#define DECODE_TYPE_MASK ((1 << DECODE_TYPE_BITS) - 1) - -enum decode_reg_type { - REG_TYPE_NONE = 0, /* Not a register, ignore */ - REG_TYPE_ANY, /* Any register allowed */ - REG_TYPE_SAMEAS16, /* Register should be same as that at bits 19..16 */ - REG_TYPE_SP, /* Register must be SP */ - REG_TYPE_PC, /* Register must be PC */ - REG_TYPE_NOSP, /* Register must not be SP */ - REG_TYPE_NOSPPC, /* Register must not be SP or PC */ - REG_TYPE_NOPC, /* Register must not be PC */ - REG_TYPE_NOPCWB, /* No PC if load/store write-back flag also set */ - - /* The following types are used when the encoding for PC indicates - * another instruction form. This distiction only matters for test - * case coverage checks. - */ - REG_TYPE_NOPCX, /* Register must not be PC */ - REG_TYPE_NOSPPCX, /* Register must not be SP or PC */ - - /* Alias to allow '0' arg to be used in REGS macro. */ - REG_TYPE_0 = REG_TYPE_NONE -}; - -#define REGS(r16, r12, r8, r4, r0) \ - ((REG_TYPE_##r16) << 16) + \ - ((REG_TYPE_##r12) << 12) + \ - ((REG_TYPE_##r8) << 8) + \ - ((REG_TYPE_##r4) << 4) + \ - (REG_TYPE_##r0) - -union decode_item { - u32 bits; - const union decode_item *table; - kprobe_insn_handler_t *handler; - kprobe_decode_insn_t *decoder; -}; - - -#define DECODE_END \ - {.bits = DECODE_TYPE_END} - - -struct decode_header { - union decode_item type_regs; - union decode_item mask; - union decode_item value; -}; - -#define DECODE_HEADER(_type, _mask, _value, _regs) \ - {.bits = (_type) | ((_regs) << DECODE_TYPE_BITS)}, \ - {.bits = (_mask)}, \ - {.bits = (_value)} - - -struct decode_table { - struct decode_header header; - union decode_item table; -}; - -#define DECODE_TABLE(_mask, _value, _table) \ - DECODE_HEADER(DECODE_TYPE_TABLE, _mask, _value, 0), \ - {.table = (_table)} - - -struct decode_custom { - struct decode_header header; - union decode_item decoder; -}; - -#define DECODE_CUSTOM(_mask, _value, _decoder) \ - DECODE_HEADER(DECODE_TYPE_CUSTOM, _mask, _value, 0), \ - {.decoder = (_decoder)} - - -struct decode_simulate { - struct decode_header header; - union decode_item handler; -}; - -#define DECODE_SIMULATEX(_mask, _value, _handler, _regs) \ - DECODE_HEADER(DECODE_TYPE_SIMULATE, _mask, _value, _regs), \ - {.handler = (_handler)} - -#define DECODE_SIMULATE(_mask, _value, _handler) \ - DECODE_SIMULATEX(_mask, _value, _handler, 0) - - -struct decode_emulate { - struct decode_header header; - union decode_item handler; -}; - -#define DECODE_EMULATEX(_mask, _value, _handler, _regs) \ - DECODE_HEADER(DECODE_TYPE_EMULATE, _mask, _value, _regs), \ - {.handler = (_handler)} - -#define DECODE_EMULATE(_mask, _value, _handler) \ - DECODE_EMULATEX(_mask, _value, _handler, 0) - - -struct decode_or { - struct decode_header header; -}; - -#define DECODE_OR(_mask, _value) \ - DECODE_HEADER(DECODE_TYPE_OR, _mask, _value, 0) - - -struct decode_reject { - struct decode_header header; -}; - -#define DECODE_REJECT(_mask, _value) \ - DECODE_HEADER(DECODE_TYPE_REJECT, _mask, _value, 0) - - -#ifdef CONFIG_THUMB2_KERNEL -extern const union decode_item kprobe_decode_thumb16_table[]; -extern const union decode_item kprobe_decode_thumb32_table[]; -#else -extern const union decode_item kprobe_decode_arm_table[]; -#endif - - -int kprobe_decode_insn(kprobe_opcode_t insn, struct arch_specific_insn *asi, - const union decode_item *table, bool thumb16); - - #endif /* _ARM_KERNEL_KPROBES_H */ diff --git a/arch/arm/kernel/perf_event.c b/arch/arm/kernel/perf_event.c index 789d846a9184..a6bc431cde70 100644 --- a/arch/arm/kernel/perf_event.c +++ b/arch/arm/kernel/perf_event.c @@ -16,6 +16,8 @@ #include <linux/platform_device.h> #include <linux/pm_runtime.h> #include <linux/uaccess.h> +#include <linux/irq.h> +#include <linux/irqdesc.h> #include <asm/irq_regs.h> #include <asm/pmu.h> @@ -205,6 +207,8 @@ armpmu_del(struct perf_event *event, int flags) armpmu_stop(event, PERF_EF_UPDATE); hw_events->events[idx] = NULL; clear_bit(idx, hw_events->used_mask); + if (armpmu->clear_event_idx) + armpmu->clear_event_idx(hw_events, event); perf_event_update_userpage(event); } @@ -295,14 +299,27 @@ validate_group(struct perf_event *event) static irqreturn_t armpmu_dispatch_irq(int irq, void *dev) { - struct arm_pmu *armpmu = (struct arm_pmu *) dev; - struct platform_device *plat_device = armpmu->plat_device; - struct arm_pmu_platdata *plat = dev_get_platdata(&plat_device->dev); + struct arm_pmu *armpmu; + struct platform_device *plat_device; + struct arm_pmu_platdata *plat; + int ret; + u64 start_clock, finish_clock; + if (irq_is_percpu(irq)) + dev = *(void **)dev; + armpmu = dev; + plat_device = armpmu->plat_device; + plat = dev_get_platdata(&plat_device->dev); + + start_clock = sched_clock(); if (plat && plat->handle_irq) - return plat->handle_irq(irq, dev, armpmu->handle_irq); + ret = plat->handle_irq(irq, dev, armpmu->handle_irq); else - return armpmu->handle_irq(irq, dev); + ret = armpmu->handle_irq(irq, dev); + finish_clock = sched_clock(); + + perf_sample_event_took(finish_clock - start_clock); + return ret; } static void diff --git a/arch/arm/kernel/perf_event_cpu.c b/arch/arm/kernel/perf_event_cpu.c index 20d553c9f5e2..51798d7854ac 100644 --- a/arch/arm/kernel/perf_event_cpu.c +++ b/arch/arm/kernel/perf_event_cpu.c @@ -25,6 +25,8 @@ #include <linux/platform_device.h> #include <linux/slab.h> #include <linux/spinlock.h> +#include <linux/irq.h> +#include <linux/irqdesc.h> #include <asm/cputype.h> #include <asm/irq_regs.h> @@ -33,6 +35,7 @@ /* Set at runtime when we know what CPU type we are. */ static struct arm_pmu *cpu_pmu; +static DEFINE_PER_CPU(struct arm_pmu *, percpu_pmu); static DEFINE_PER_CPU(struct perf_event * [ARMPMU_MAX_HWEVENTS], hw_events); static DEFINE_PER_CPU(unsigned long [BITS_TO_LONGS(ARMPMU_MAX_HWEVENTS)], used_mask); static DEFINE_PER_CPU(struct pmu_hw_events, cpu_hw_events); @@ -71,6 +74,26 @@ static struct pmu_hw_events *cpu_pmu_get_cpu_events(void) return this_cpu_ptr(&cpu_hw_events); } +static void cpu_pmu_enable_percpu_irq(void *data) +{ + struct arm_pmu *cpu_pmu = data; + struct platform_device *pmu_device = cpu_pmu->plat_device; + int irq = platform_get_irq(pmu_device, 0); + + enable_percpu_irq(irq, IRQ_TYPE_NONE); + cpumask_set_cpu(smp_processor_id(), &cpu_pmu->active_irqs); +} + +static void cpu_pmu_disable_percpu_irq(void *data) +{ + struct arm_pmu *cpu_pmu = data; + struct platform_device *pmu_device = cpu_pmu->plat_device; + int irq = platform_get_irq(pmu_device, 0); + + cpumask_clear_cpu(smp_processor_id(), &cpu_pmu->active_irqs); + disable_percpu_irq(irq); +} + static void cpu_pmu_free_irq(struct arm_pmu *cpu_pmu) { int i, irq, irqs; @@ -78,12 +101,18 @@ static void cpu_pmu_free_irq(struct arm_pmu *cpu_pmu) irqs = min(pmu_device->num_resources, num_possible_cpus()); - for (i = 0; i < irqs; ++i) { - if (!cpumask_test_and_clear_cpu(i, &cpu_pmu->active_irqs)) - continue; - irq = platform_get_irq(pmu_device, i); - if (irq >= 0) - free_irq(irq, cpu_pmu); + irq = platform_get_irq(pmu_device, 0); + if (irq >= 0 && irq_is_percpu(irq)) { + on_each_cpu(cpu_pmu_disable_percpu_irq, cpu_pmu, 1); + free_percpu_irq(irq, &percpu_pmu); + } else { + for (i = 0; i < irqs; ++i) { + if (!cpumask_test_and_clear_cpu(i, &cpu_pmu->active_irqs)) + continue; + irq = platform_get_irq(pmu_device, i); + if (irq >= 0) + free_irq(irq, cpu_pmu); + } } } @@ -101,33 +130,44 @@ static int cpu_pmu_request_irq(struct arm_pmu *cpu_pmu, irq_handler_t handler) return -ENODEV; } - for (i = 0; i < irqs; ++i) { - err = 0; - irq = platform_get_irq(pmu_device, i); - if (irq < 0) - continue; - - /* - * If we have a single PMU interrupt that we can't shift, - * assume that we're running on a uniprocessor machine and - * continue. Otherwise, continue without this interrupt. - */ - if (irq_set_affinity(irq, cpumask_of(i)) && irqs > 1) { - pr_warning("unable to set irq affinity (irq=%d, cpu=%u)\n", - irq, i); - continue; - } - - err = request_irq(irq, handler, - IRQF_NOBALANCING | IRQF_NO_THREAD, "arm-pmu", - cpu_pmu); + irq = platform_get_irq(pmu_device, 0); + if (irq >= 0 && irq_is_percpu(irq)) { + err = request_percpu_irq(irq, handler, "arm-pmu", &percpu_pmu); if (err) { pr_err("unable to request IRQ%d for ARM PMU counters\n", irq); return err; } - - cpumask_set_cpu(i, &cpu_pmu->active_irqs); + on_each_cpu(cpu_pmu_enable_percpu_irq, cpu_pmu, 1); + } else { + for (i = 0; i < irqs; ++i) { + err = 0; + irq = platform_get_irq(pmu_device, i); + if (irq < 0) + continue; + + /* + * If we have a single PMU interrupt that we can't shift, + * assume that we're running on a uniprocessor machine and + * continue. Otherwise, continue without this interrupt. + */ + if (irq_set_affinity(irq, cpumask_of(i)) && irqs > 1) { + pr_warning("unable to set irq affinity (irq=%d, cpu=%u)\n", + irq, i); + continue; + } + + err = request_irq(irq, handler, + IRQF_NOBALANCING | IRQF_NO_THREAD, "arm-pmu", + cpu_pmu); + if (err) { + pr_err("unable to request IRQ%d for ARM PMU counters\n", + irq); + return err; + } + + cpumask_set_cpu(i, &cpu_pmu->active_irqs); + } } return 0; @@ -141,6 +181,7 @@ static void cpu_pmu_init(struct arm_pmu *cpu_pmu) events->events = per_cpu(hw_events, cpu); events->used_mask = per_cpu(used_mask, cpu); raw_spin_lock_init(&events->pmu_lock); + per_cpu(percpu_pmu, cpu) = cpu_pmu; } cpu_pmu->get_hw_events = cpu_pmu_get_cpu_events; @@ -181,6 +222,7 @@ static struct notifier_block cpu_pmu_hotplug_notifier = { */ static struct of_device_id cpu_pmu_of_device_ids[] = { {.compatible = "arm,cortex-a15-pmu", .data = armv7_a15_pmu_init}, + {.compatible = "arm,cortex-a12-pmu", .data = armv7_a12_pmu_init}, {.compatible = "arm,cortex-a9-pmu", .data = armv7_a9_pmu_init}, {.compatible = "arm,cortex-a8-pmu", .data = armv7_a8_pmu_init}, {.compatible = "arm,cortex-a7-pmu", .data = armv7_a7_pmu_init}, @@ -188,6 +230,7 @@ static struct of_device_id cpu_pmu_of_device_ids[] = { {.compatible = "arm,arm11mpcore-pmu", .data = armv6mpcore_pmu_init}, {.compatible = "arm,arm1176-pmu", .data = armv6pmu_init}, {.compatible = "arm,arm1136-pmu", .data = armv6pmu_init}, + {.compatible = "qcom,krait-pmu", .data = krait_pmu_init}, {}, }; @@ -225,15 +268,6 @@ static int probe_current_pmu(struct arm_pmu *pmu) case ARM_CPU_PART_CORTEX_A9: ret = armv7_a9_pmu_init(pmu); break; - case ARM_CPU_PART_CORTEX_A5: - ret = armv7_a5_pmu_init(pmu); - break; - case ARM_CPU_PART_CORTEX_A15: - ret = armv7_a15_pmu_init(pmu); - break; - case ARM_CPU_PART_CORTEX_A7: - ret = armv7_a7_pmu_init(pmu); - break; } /* Intel CPUs [xscale]. */ } else if (implementor == ARM_CPU_IMP_INTEL) { @@ -270,6 +304,9 @@ static int cpu_pmu_device_probe(struct platform_device *pdev) return -ENOMEM; } + cpu_pmu = pmu; + cpu_pmu->plat_device = pdev; + if (node && (of_id = of_match_node(cpu_pmu_of_device_ids, pdev->dev.of_node))) { init_fn = of_id->data; ret = init_fn(pmu); @@ -282,8 +319,6 @@ static int cpu_pmu_device_probe(struct platform_device *pdev) goto out_free; } - cpu_pmu = pmu; - cpu_pmu->plat_device = pdev; cpu_pmu_init(cpu_pmu); ret = armpmu_register(cpu_pmu, PERF_TYPE_RAW); diff --git a/arch/arm/kernel/perf_event_v7.c b/arch/arm/kernel/perf_event_v7.c index 039cffb053a7..f4ef3981ed02 100644 --- a/arch/arm/kernel/perf_event_v7.c +++ b/arch/arm/kernel/perf_event_v7.c @@ -18,6 +18,10 @@ #ifdef CONFIG_CPU_V7 +#include <asm/cp15.h> +#include <asm/vfp.h> +#include "../vfp/vfpinstr.h" + /* * Common ARMv7 event types * @@ -109,6 +113,33 @@ enum armv7_a15_perf_types { ARMV7_A15_PERFCTR_PC_WRITE_SPEC = 0x76, }; +/* ARMv7 Cortex-A12 specific event types */ +enum armv7_a12_perf_types { + ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_READ = 0x40, + ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_WRITE = 0x41, + + ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_READ = 0x50, + ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_WRITE = 0x51, + + ARMV7_A12_PERFCTR_PC_WRITE_SPEC = 0x76, + + ARMV7_A12_PERFCTR_PF_TLB_REFILL = 0xe7, +}; + +/* ARMv7 Krait specific event types */ +enum krait_perf_types { + KRAIT_PMRESR0_GROUP0 = 0xcc, + KRAIT_PMRESR1_GROUP0 = 0xd0, + KRAIT_PMRESR2_GROUP0 = 0xd4, + KRAIT_VPMRESR0_GROUP0 = 0xd8, + + KRAIT_PERFCTR_L1_ICACHE_ACCESS = 0x10011, + KRAIT_PERFCTR_L1_ICACHE_MISS = 0x10010, + + KRAIT_PERFCTR_L1_ITLB_ACCESS = 0x12222, + KRAIT_PERFCTR_L1_DTLB_ACCESS = 0x12210, +}; + /* * Cortex-A8 HW events mapping * @@ -732,6 +763,262 @@ static const unsigned armv7_a7_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] }; /* + * Cortex-A12 HW events mapping + */ +static const unsigned armv7_a12_perf_map[PERF_COUNT_HW_MAX] = { + [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, + [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED, + [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, + [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL, + [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_A12_PERFCTR_PC_WRITE_SPEC, + [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, + [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_BUS_CYCLES, + [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = HW_OP_UNSUPPORTED, + [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = HW_OP_UNSUPPORTED, +}; + +static const unsigned armv7_a12_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] + [PERF_COUNT_HW_CACHE_OP_MAX] + [PERF_COUNT_HW_CACHE_RESULT_MAX] = { + [C(L1D)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_READ, + [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_WRITE, + [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(L1I)] = { + /* + * Not all performance counters differentiate between read + * and write accesses/misses so we're not always strictly + * correct, but it's the best we can do. Writes and reads get + * combined in these cases. + */ + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_ICACHE_ACCESS, + [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(LL)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_READ, + [C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACHE_REFILL, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_WRITE, + [C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACHE_REFILL, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(DTLB)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = ARMV7_A12_PERFCTR_PF_TLB_REFILL, + }, + }, + [C(ITLB)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(BPU)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, + [C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, + [C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(NODE)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, +}; + +/* + * Krait HW events mapping + */ +static const unsigned krait_perf_map[PERF_COUNT_HW_MAX] = { + [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, + [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED, + [PERF_COUNT_HW_CACHE_REFERENCES] = HW_OP_UNSUPPORTED, + [PERF_COUNT_HW_CACHE_MISSES] = HW_OP_UNSUPPORTED, + [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE, + [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, + [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_CLOCK_CYCLES, +}; + +static const unsigned krait_perf_map_no_branch[PERF_COUNT_HW_MAX] = { + [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, + [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED, + [PERF_COUNT_HW_CACHE_REFERENCES] = HW_OP_UNSUPPORTED, + [PERF_COUNT_HW_CACHE_MISSES] = HW_OP_UNSUPPORTED, + [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = HW_OP_UNSUPPORTED, + [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, + [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_CLOCK_CYCLES, +}; + +static const unsigned krait_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] + [PERF_COUNT_HW_CACHE_OP_MAX] + [PERF_COUNT_HW_CACHE_RESULT_MAX] = { + [C(L1D)] = { + /* + * The performance counters don't differentiate between read + * and write accesses/misses so this isn't strictly correct, + * but it's the best we can do. Writes and reads get + * combined. + */ + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, + [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, + [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(L1I)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_ICACHE_ACCESS, + [C(RESULT_MISS)] = KRAIT_PERFCTR_L1_ICACHE_MISS, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(LL)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(DTLB)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_DTLB_ACCESS, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_DTLB_ACCESS, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(ITLB)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_ITLB_ACCESS, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_ITLB_ACCESS, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(BPU)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, + [C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, + [C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(NODE)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, +}; + +/* * Perf Events' indices */ #define ARMV7_IDX_CYCLE_COUNTER 0 @@ -1212,6 +1499,24 @@ static int armv7_a7_map_event(struct perf_event *event) &armv7_a7_perf_cache_map, 0xFF); } +static int armv7_a12_map_event(struct perf_event *event) +{ + return armpmu_map_event(event, &armv7_a12_perf_map, + &armv7_a12_perf_cache_map, 0xFF); +} + +static int krait_map_event(struct perf_event *event) +{ + return armpmu_map_event(event, &krait_perf_map, + &krait_perf_cache_map, 0xFFFFF); +} + +static int krait_map_event_no_branch(struct perf_event *event) +{ + return armpmu_map_event(event, &krait_perf_map_no_branch, + &krait_perf_cache_map, 0xFFFFF); +} + static void armv7pmu_init(struct arm_pmu *cpu_pmu) { cpu_pmu->handle_irq = armv7pmu_handle_irq; @@ -1283,6 +1588,408 @@ static int armv7_a7_pmu_init(struct arm_pmu *cpu_pmu) cpu_pmu->set_event_filter = armv7pmu_set_event_filter; return 0; } + +static int armv7_a12_pmu_init(struct arm_pmu *cpu_pmu) +{ + armv7pmu_init(cpu_pmu); + cpu_pmu->name = "ARMv7 Cortex-A12"; + cpu_pmu->map_event = armv7_a12_map_event; + cpu_pmu->num_events = armv7_read_num_pmnc_events(); + cpu_pmu->set_event_filter = armv7pmu_set_event_filter; + return 0; +} + +/* + * Krait Performance Monitor Region Event Selection Register (PMRESRn) + * + * 31 30 24 16 8 0 + * +--------------------------------+ + * PMRESR0 | EN | CC | CC | CC | CC | N = 1, R = 0 + * +--------------------------------+ + * PMRESR1 | EN | CC | CC | CC | CC | N = 1, R = 1 + * +--------------------------------+ + * PMRESR2 | EN | CC | CC | CC | CC | N = 1, R = 2 + * +--------------------------------+ + * VPMRESR0 | EN | CC | CC | CC | CC | N = 2, R = ? + * +--------------------------------+ + * EN | G=3 | G=2 | G=1 | G=0 + * + * Event Encoding: + * + * hwc->config_base = 0xNRCCG + * + * N = prefix, 1 for Krait CPU (PMRESRn), 2 for Venum VFP (VPMRESR) + * R = region register + * CC = class of events the group G is choosing from + * G = group or particular event + * + * Example: 0x12021 is a Krait CPU event in PMRESR2's group 1 with code 2 + * + * A region (R) corresponds to a piece of the CPU (execution unit, instruction + * unit, etc.) while the event code (CC) corresponds to a particular class of + * events (interrupts for example). An event code is broken down into + * groups (G) that can be mapped into the PMU (irq, fiqs, and irq+fiqs for + * example). + */ + +#define KRAIT_EVENT (1 << 16) +#define VENUM_EVENT (2 << 16) +#define KRAIT_EVENT_MASK (KRAIT_EVENT | VENUM_EVENT) +#define PMRESRn_EN BIT(31) + +static u32 krait_read_pmresrn(int n) +{ + u32 val; + + switch (n) { + case 0: + asm volatile("mrc p15, 1, %0, c9, c15, 0" : "=r" (val)); + break; + case 1: + asm volatile("mrc p15, 1, %0, c9, c15, 1" : "=r" (val)); + break; + case 2: + asm volatile("mrc p15, 1, %0, c9, c15, 2" : "=r" (val)); + break; + default: + BUG(); /* Should be validated in krait_pmu_get_event_idx() */ + } + + return val; +} + +static void krait_write_pmresrn(int n, u32 val) +{ + switch (n) { + case 0: + asm volatile("mcr p15, 1, %0, c9, c15, 0" : : "r" (val)); + break; + case 1: + asm volatile("mcr p15, 1, %0, c9, c15, 1" : : "r" (val)); + break; + case 2: + asm volatile("mcr p15, 1, %0, c9, c15, 2" : : "r" (val)); + break; + default: + BUG(); /* Should be validated in krait_pmu_get_event_idx() */ + } +} + +static u32 krait_read_vpmresr0(void) +{ + u32 val; + asm volatile("mrc p10, 7, %0, c11, c0, 0" : "=r" (val)); + return val; +} + +static void krait_write_vpmresr0(u32 val) +{ + asm volatile("mcr p10, 7, %0, c11, c0, 0" : : "r" (val)); +} + +static void krait_pre_vpmresr0(u32 *venum_orig_val, u32 *fp_orig_val) +{ + u32 venum_new_val; + u32 fp_new_val; + + BUG_ON(preemptible()); + /* CPACR Enable CP10 and CP11 access */ + *venum_orig_val = get_copro_access(); + venum_new_val = *venum_orig_val | CPACC_SVC(10) | CPACC_SVC(11); + set_copro_access(venum_new_val); + + /* Enable FPEXC */ + *fp_orig_val = fmrx(FPEXC); + fp_new_val = *fp_orig_val | FPEXC_EN; + fmxr(FPEXC, fp_new_val); +} + +static void krait_post_vpmresr0(u32 venum_orig_val, u32 fp_orig_val) +{ + BUG_ON(preemptible()); + /* Restore FPEXC */ + fmxr(FPEXC, fp_orig_val); + isb(); + /* Restore CPACR */ + set_copro_access(venum_orig_val); +} + +static u32 krait_get_pmresrn_event(unsigned int region) +{ + static const u32 pmresrn_table[] = { KRAIT_PMRESR0_GROUP0, + KRAIT_PMRESR1_GROUP0, + KRAIT_PMRESR2_GROUP0 }; + return pmresrn_table[region]; +} + +static void krait_evt_setup(int idx, u32 config_base) +{ + u32 val; + u32 mask; + u32 vval, fval; + unsigned int region; + unsigned int group; + unsigned int code; + unsigned int group_shift; + bool venum_event; + + venum_event = !!(config_base & VENUM_EVENT); + region = (config_base >> 12) & 0xf; + code = (config_base >> 4) & 0xff; + group = (config_base >> 0) & 0xf; + + group_shift = group * 8; + mask = 0xff << group_shift; + + /* Configure evtsel for the region and group */ + if (venum_event) + val = KRAIT_VPMRESR0_GROUP0; + else + val = krait_get_pmresrn_event(region); + val += group; + /* Mix in mode-exclusion bits */ + val |= config_base & (ARMV7_EXCLUDE_USER | ARMV7_EXCLUDE_PL1); + armv7_pmnc_write_evtsel(idx, val); + + asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0)); + + if (venum_event) { + krait_pre_vpmresr0(&vval, &fval); + val = krait_read_vpmresr0(); + val &= ~mask; + val |= code << group_shift; + val |= PMRESRn_EN; + krait_write_vpmresr0(val); + krait_post_vpmresr0(vval, fval); + } else { + val = krait_read_pmresrn(region); + val &= ~mask; + val |= code << group_shift; + val |= PMRESRn_EN; + krait_write_pmresrn(region, val); + } +} + +static u32 krait_clear_pmresrn_group(u32 val, int group) +{ + u32 mask; + int group_shift; + + group_shift = group * 8; + mask = 0xff << group_shift; + val &= ~mask; + + /* Don't clear enable bit if entire region isn't disabled */ + if (val & ~PMRESRn_EN) + return val |= PMRESRn_EN; + + return 0; +} + +static void krait_clearpmu(u32 config_base) +{ + u32 val; + u32 vval, fval; + unsigned int region; + unsigned int group; + bool venum_event; + + venum_event = !!(config_base & VENUM_EVENT); + region = (config_base >> 12) & 0xf; + group = (config_base >> 0) & 0xf; + + if (venum_event) { + krait_pre_vpmresr0(&vval, &fval); + val = krait_read_vpmresr0(); + val = krait_clear_pmresrn_group(val, group); + krait_write_vpmresr0(val); + krait_post_vpmresr0(vval, fval); + } else { + val = krait_read_pmresrn(region); + val = krait_clear_pmresrn_group(val, group); + krait_write_pmresrn(region, val); + } +} + +static void krait_pmu_disable_event(struct perf_event *event) +{ + unsigned long flags; + struct hw_perf_event *hwc = &event->hw; + int idx = hwc->idx; + struct pmu_hw_events *events = cpu_pmu->get_hw_events(); + + /* Disable counter and interrupt */ + raw_spin_lock_irqsave(&events->pmu_lock, flags); + + /* Disable counter */ + armv7_pmnc_disable_counter(idx); + + /* + * Clear pmresr code (if destined for PMNx counters) + */ + if (hwc->config_base & KRAIT_EVENT_MASK) + krait_clearpmu(hwc->config_base); + + /* Disable interrupt for this counter */ + armv7_pmnc_disable_intens(idx); + + raw_spin_unlock_irqrestore(&events->pmu_lock, flags); +} + +static void krait_pmu_enable_event(struct perf_event *event) +{ + unsigned long flags; + struct hw_perf_event *hwc = &event->hw; + int idx = hwc->idx; + struct pmu_hw_events *events = cpu_pmu->get_hw_events(); + + /* + * Enable counter and interrupt, and set the counter to count + * the event that we're interested in. + */ + raw_spin_lock_irqsave(&events->pmu_lock, flags); + + /* Disable counter */ + armv7_pmnc_disable_counter(idx); + + /* + * Set event (if destined for PMNx counters) + * We set the event for the cycle counter because we + * have the ability to perform event filtering. + */ + if (hwc->config_base & KRAIT_EVENT_MASK) + krait_evt_setup(idx, hwc->config_base); + else + armv7_pmnc_write_evtsel(idx, hwc->config_base); + + /* Enable interrupt for this counter */ + armv7_pmnc_enable_intens(idx); + + /* Enable counter */ + armv7_pmnc_enable_counter(idx); + + raw_spin_unlock_irqrestore(&events->pmu_lock, flags); +} + +static void krait_pmu_reset(void *info) +{ + u32 vval, fval; + + armv7pmu_reset(info); + + /* Clear all pmresrs */ + krait_write_pmresrn(0, 0); + krait_write_pmresrn(1, 0); + krait_write_pmresrn(2, 0); + + krait_pre_vpmresr0(&vval, &fval); + krait_write_vpmresr0(0); + krait_post_vpmresr0(vval, fval); +} + +static int krait_event_to_bit(struct perf_event *event, unsigned int region, + unsigned int group) +{ + int bit; + struct hw_perf_event *hwc = &event->hw; + struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); + + if (hwc->config_base & VENUM_EVENT) + bit = KRAIT_VPMRESR0_GROUP0; + else + bit = krait_get_pmresrn_event(region); + bit -= krait_get_pmresrn_event(0); + bit += group; + /* + * Lower bits are reserved for use by the counters (see + * armv7pmu_get_event_idx() for more info) + */ + bit += ARMV7_IDX_COUNTER_LAST(cpu_pmu) + 1; + + return bit; +} + +/* + * We check for column exclusion constraints here. + * Two events cant use the same group within a pmresr register. + */ +static int krait_pmu_get_event_idx(struct pmu_hw_events *cpuc, + struct perf_event *event) +{ + int idx; + int bit; + unsigned int prefix; + unsigned int region; + unsigned int code; + unsigned int group; + bool krait_event; + struct hw_perf_event *hwc = &event->hw; + + region = (hwc->config_base >> 12) & 0xf; + code = (hwc->config_base >> 4) & 0xff; + group = (hwc->config_base >> 0) & 0xf; + krait_event = !!(hwc->config_base & KRAIT_EVENT_MASK); + + if (krait_event) { + /* Ignore invalid events */ + if (group > 3 || region > 2) + return -EINVAL; + prefix = hwc->config_base & KRAIT_EVENT_MASK; + if (prefix != KRAIT_EVENT && prefix != VENUM_EVENT) + return -EINVAL; + if (prefix == VENUM_EVENT && (code & 0xe0)) + return -EINVAL; + + bit = krait_event_to_bit(event, region, group); + if (test_and_set_bit(bit, cpuc->used_mask)) + return -EAGAIN; + } + + idx = armv7pmu_get_event_idx(cpuc, event); + if (idx < 0 && krait_event) + clear_bit(bit, cpuc->used_mask); + + return idx; +} + +static void krait_pmu_clear_event_idx(struct pmu_hw_events *cpuc, + struct perf_event *event) +{ + int bit; + struct hw_perf_event *hwc = &event->hw; + unsigned int region; + unsigned int group; + bool krait_event; + + region = (hwc->config_base >> 12) & 0xf; + group = (hwc->config_base >> 0) & 0xf; + krait_event = !!(hwc->config_base & KRAIT_EVENT_MASK); + + if (krait_event) { + bit = krait_event_to_bit(event, region, group); + clear_bit(bit, cpuc->used_mask); + } +} + +static int krait_pmu_init(struct arm_pmu *cpu_pmu) +{ + armv7pmu_init(cpu_pmu); + cpu_pmu->name = "ARMv7 Krait"; + /* Some early versions of Krait don't support PC write events */ + if (of_property_read_bool(cpu_pmu->plat_device->dev.of_node, + "qcom,no-pc-write")) + cpu_pmu->map_event = krait_map_event_no_branch; + else + cpu_pmu->map_event = krait_map_event; + cpu_pmu->num_events = armv7_read_num_pmnc_events(); + cpu_pmu->set_event_filter = armv7pmu_set_event_filter; + cpu_pmu->reset = krait_pmu_reset; + cpu_pmu->enable = krait_pmu_enable_event; + cpu_pmu->disable = krait_pmu_disable_event; + cpu_pmu->get_event_idx = krait_pmu_get_event_idx; + cpu_pmu->clear_event_idx = krait_pmu_clear_event_idx; + return 0; +} #else static inline int armv7_a8_pmu_init(struct arm_pmu *cpu_pmu) { @@ -1308,4 +2015,14 @@ static inline int armv7_a7_pmu_init(struct arm_pmu *cpu_pmu) { return -ENODEV; } + +static inline int armv7_a12_pmu_init(struct arm_pmu *cpu_pmu) +{ + return -ENODEV; +} + +static inline int krait_pmu_init(struct arm_pmu *cpu_pmu) +{ + return -ENODEV; +} #endif /* CONFIG_CPU_V7 */ diff --git a/arch/arm/kernel/probes-arm.c b/arch/arm/kernel/probes-arm.c new file mode 100644 index 000000000000..51a13a027989 --- /dev/null +++ b/arch/arm/kernel/probes-arm.c @@ -0,0 +1,734 @@ +/* + * arch/arm/kernel/probes-arm.c + * + * Some code moved here from arch/arm/kernel/kprobes-arm.c + * + * Copyright (C) 2006, 2007 Motorola Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ + +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/stddef.h> +#include <linux/ptrace.h> + +#include "probes.h" +#include "probes-arm.h" + +#define sign_extend(x, signbit) ((x) | (0 - ((x) & (1 << (signbit))))) + +#define branch_displacement(insn) sign_extend(((insn) & 0xffffff) << 2, 25) + +/* + * To avoid the complications of mimicing single-stepping on a + * processor without a Next-PC or a single-step mode, and to + * avoid having to deal with the side-effects of boosting, we + * simulate or emulate (almost) all ARM instructions. + * + * "Simulation" is where the instruction's behavior is duplicated in + * C code. "Emulation" is where the original instruction is rewritten + * and executed, often by altering its registers. + * + * By having all behavior of the kprobe'd instruction completed before + * returning from the kprobe_handler(), all locks (scheduler and + * interrupt) can safely be released. There is no need for secondary + * breakpoints, no race with MP or preemptable kernels, nor having to + * clean up resources counts at a later time impacting overall system + * performance. By rewriting the instruction, only the minimum registers + * need to be loaded and saved back optimizing performance. + * + * Calling the insnslot_*_rwflags version of a function doesn't hurt + * anything even when the CPSR flags aren't updated by the + * instruction. It's just a little slower in return for saving + * a little space by not having a duplicate function that doesn't + * update the flags. (The same optimization can be said for + * instructions that do or don't perform register writeback) + * Also, instructions can either read the flags, only write the + * flags, or read and write the flags. To save combinations + * rather than for sheer performance, flag functions just assume + * read and write of flags. + */ + +void __kprobes simulate_bbl(probes_opcode_t insn, + struct arch_probes_insn *asi, struct pt_regs *regs) +{ + long iaddr = (long) regs->ARM_pc - 4; + int disp = branch_displacement(insn); + + if (insn & (1 << 24)) + regs->ARM_lr = iaddr + 4; + + regs->ARM_pc = iaddr + 8 + disp; +} + +void __kprobes simulate_blx1(probes_opcode_t insn, + struct arch_probes_insn *asi, struct pt_regs *regs) +{ + long iaddr = (long) regs->ARM_pc - 4; + int disp = branch_displacement(insn); + + regs->ARM_lr = iaddr + 4; + regs->ARM_pc = iaddr + 8 + disp + ((insn >> 23) & 0x2); + regs->ARM_cpsr |= PSR_T_BIT; +} + +void __kprobes simulate_blx2bx(probes_opcode_t insn, + struct arch_probes_insn *asi, struct pt_regs *regs) +{ + int rm = insn & 0xf; + long rmv = regs->uregs[rm]; + + if (insn & (1 << 5)) + regs->ARM_lr = (long) regs->ARM_pc; + + regs->ARM_pc = rmv & ~0x1; + regs->ARM_cpsr &= ~PSR_T_BIT; + if (rmv & 0x1) + regs->ARM_cpsr |= PSR_T_BIT; +} + +void __kprobes simulate_mrs(probes_opcode_t insn, + struct arch_probes_insn *asi, struct pt_regs *regs) +{ + int rd = (insn >> 12) & 0xf; + unsigned long mask = 0xf8ff03df; /* Mask out execution state */ + regs->uregs[rd] = regs->ARM_cpsr & mask; +} + +void __kprobes simulate_mov_ipsp(probes_opcode_t insn, + struct arch_probes_insn *asi, struct pt_regs *regs) +{ + regs->uregs[12] = regs->uregs[13]; +} + +/* + * For the instruction masking and comparisons in all the "space_*" + * functions below, Do _not_ rearrange the order of tests unless + * you're very, very sure of what you are doing. For the sake of + * efficiency, the masks for some tests sometimes assume other test + * have been done prior to them so the number of patterns to test + * for an instruction set can be as broad as possible to reduce the + * number of tests needed. + */ + +static const union decode_item arm_1111_table[] = { + /* Unconditional instructions */ + + /* memory hint 1111 0100 x001 xxxx xxxx xxxx xxxx xxxx */ + /* PLDI (immediate) 1111 0100 x101 xxxx xxxx xxxx xxxx xxxx */ + /* PLDW (immediate) 1111 0101 x001 xxxx xxxx xxxx xxxx xxxx */ + /* PLD (immediate) 1111 0101 x101 xxxx xxxx xxxx xxxx xxxx */ + DECODE_SIMULATE (0xfe300000, 0xf4100000, PROBES_PRELOAD_IMM), + + /* memory hint 1111 0110 x001 xxxx xxxx xxxx xxx0 xxxx */ + /* PLDI (register) 1111 0110 x101 xxxx xxxx xxxx xxx0 xxxx */ + /* PLDW (register) 1111 0111 x001 xxxx xxxx xxxx xxx0 xxxx */ + /* PLD (register) 1111 0111 x101 xxxx xxxx xxxx xxx0 xxxx */ + DECODE_SIMULATE (0xfe300010, 0xf6100000, PROBES_PRELOAD_REG), + + /* BLX (immediate) 1111 101x xxxx xxxx xxxx xxxx xxxx xxxx */ + DECODE_SIMULATE (0xfe000000, 0xfa000000, PROBES_BRANCH_IMM), + + /* CPS 1111 0001 0000 xxx0 xxxx xxxx xx0x xxxx */ + /* SETEND 1111 0001 0000 0001 xxxx xxxx 0000 xxxx */ + /* SRS 1111 100x x1x0 xxxx xxxx xxxx xxxx xxxx */ + /* RFE 1111 100x x0x1 xxxx xxxx xxxx xxxx xxxx */ + + /* Coprocessor instructions... */ + /* MCRR2 1111 1100 0100 xxxx xxxx xxxx xxxx xxxx */ + /* MRRC2 1111 1100 0101 xxxx xxxx xxxx xxxx xxxx */ + /* LDC2 1111 110x xxx1 xxxx xxxx xxxx xxxx xxxx */ + /* STC2 1111 110x xxx0 xxxx xxxx xxxx xxxx xxxx */ + /* CDP2 1111 1110 xxxx xxxx xxxx xxxx xxx0 xxxx */ + /* MCR2 1111 1110 xxx0 xxxx xxxx xxxx xxx1 xxxx */ + /* MRC2 1111 1110 xxx1 xxxx xxxx xxxx xxx1 xxxx */ + + /* Other unallocated instructions... */ + DECODE_END +}; + +static const union decode_item arm_cccc_0001_0xx0____0xxx_table[] = { + /* Miscellaneous instructions */ + + /* MRS cpsr cccc 0001 0000 xxxx xxxx xxxx 0000 xxxx */ + DECODE_SIMULATEX(0x0ff000f0, 0x01000000, PROBES_MRS, + REGS(0, NOPC, 0, 0, 0)), + + /* BX cccc 0001 0010 xxxx xxxx xxxx 0001 xxxx */ + DECODE_SIMULATE (0x0ff000f0, 0x01200010, PROBES_BRANCH_REG), + + /* BLX (register) cccc 0001 0010 xxxx xxxx xxxx 0011 xxxx */ + DECODE_SIMULATEX(0x0ff000f0, 0x01200030, PROBES_BRANCH_REG, + REGS(0, 0, 0, 0, NOPC)), + + /* CLZ cccc 0001 0110 xxxx xxxx xxxx 0001 xxxx */ + DECODE_EMULATEX (0x0ff000f0, 0x01600010, PROBES_CLZ, + REGS(0, NOPC, 0, 0, NOPC)), + + /* QADD cccc 0001 0000 xxxx xxxx xxxx 0101 xxxx */ + /* QSUB cccc 0001 0010 xxxx xxxx xxxx 0101 xxxx */ + /* QDADD cccc 0001 0100 xxxx xxxx xxxx 0101 xxxx */ + /* QDSUB cccc 0001 0110 xxxx xxxx xxxx 0101 xxxx */ + DECODE_EMULATEX (0x0f9000f0, 0x01000050, PROBES_SATURATING_ARITHMETIC, + REGS(NOPC, NOPC, 0, 0, NOPC)), + + /* BXJ cccc 0001 0010 xxxx xxxx xxxx 0010 xxxx */ + /* MSR cccc 0001 0x10 xxxx xxxx xxxx 0000 xxxx */ + /* MRS spsr cccc 0001 0100 xxxx xxxx xxxx 0000 xxxx */ + /* BKPT 1110 0001 0010 xxxx xxxx xxxx 0111 xxxx */ + /* SMC cccc 0001 0110 xxxx xxxx xxxx 0111 xxxx */ + /* And unallocated instructions... */ + DECODE_END +}; + +static const union decode_item arm_cccc_0001_0xx0____1xx0_table[] = { + /* Halfword multiply and multiply-accumulate */ + + /* SMLALxy cccc 0001 0100 xxxx xxxx xxxx 1xx0 xxxx */ + DECODE_EMULATEX (0x0ff00090, 0x01400080, PROBES_MUL1, + REGS(NOPC, NOPC, NOPC, 0, NOPC)), + + /* SMULWy cccc 0001 0010 xxxx xxxx xxxx 1x10 xxxx */ + DECODE_OR (0x0ff000b0, 0x012000a0), + /* SMULxy cccc 0001 0110 xxxx xxxx xxxx 1xx0 xxxx */ + DECODE_EMULATEX (0x0ff00090, 0x01600080, PROBES_MUL2, + REGS(NOPC, 0, NOPC, 0, NOPC)), + + /* SMLAxy cccc 0001 0000 xxxx xxxx xxxx 1xx0 xxxx */ + DECODE_OR (0x0ff00090, 0x01000080), + /* SMLAWy cccc 0001 0010 xxxx xxxx xxxx 1x00 xxxx */ + DECODE_EMULATEX (0x0ff000b0, 0x01200080, PROBES_MUL2, + REGS(NOPC, NOPC, NOPC, 0, NOPC)), + + DECODE_END +}; + +static const union decode_item arm_cccc_0000_____1001_table[] = { + /* Multiply and multiply-accumulate */ + + /* MUL cccc 0000 0000 xxxx xxxx xxxx 1001 xxxx */ + /* MULS cccc 0000 0001 xxxx xxxx xxxx 1001 xxxx */ + DECODE_EMULATEX (0x0fe000f0, 0x00000090, PROBES_MUL2, + REGS(NOPC, 0, NOPC, 0, NOPC)), + + /* MLA cccc 0000 0010 xxxx xxxx xxxx 1001 xxxx */ + /* MLAS cccc 0000 0011 xxxx xxxx xxxx 1001 xxxx */ + DECODE_OR (0x0fe000f0, 0x00200090), + /* MLS cccc 0000 0110 xxxx xxxx xxxx 1001 xxxx */ + DECODE_EMULATEX (0x0ff000f0, 0x00600090, PROBES_MUL2, + REGS(NOPC, NOPC, NOPC, 0, NOPC)), + + /* UMAAL cccc 0000 0100 xxxx xxxx xxxx 1001 xxxx */ + DECODE_OR (0x0ff000f0, 0x00400090), + /* UMULL cccc 0000 1000 xxxx xxxx xxxx 1001 xxxx */ + /* UMULLS cccc 0000 1001 xxxx xxxx xxxx 1001 xxxx */ + /* UMLAL cccc 0000 1010 xxxx xxxx xxxx 1001 xxxx */ + /* UMLALS cccc 0000 1011 xxxx xxxx xxxx 1001 xxxx */ + /* SMULL cccc 0000 1100 xxxx xxxx xxxx 1001 xxxx */ + /* SMULLS cccc 0000 1101 xxxx xxxx xxxx 1001 xxxx */ + /* SMLAL cccc 0000 1110 xxxx xxxx xxxx 1001 xxxx */ + /* SMLALS cccc 0000 1111 xxxx xxxx xxxx 1001 xxxx */ + DECODE_EMULATEX (0x0f8000f0, 0x00800090, PROBES_MUL1, + REGS(NOPC, NOPC, NOPC, 0, NOPC)), + + DECODE_END +}; + +static const union decode_item arm_cccc_0001_____1001_table[] = { + /* Synchronization primitives */ + +#if __LINUX_ARM_ARCH__ < 6 + /* Deprecated on ARMv6 and may be UNDEFINED on v7 */ + /* SMP/SWPB cccc 0001 0x00 xxxx xxxx xxxx 1001 xxxx */ + DECODE_EMULATEX (0x0fb000f0, 0x01000090, PROBES_SWP, + REGS(NOPC, NOPC, 0, 0, NOPC)), +#endif + /* LDREX/STREX{,D,B,H} cccc 0001 1xxx xxxx xxxx xxxx 1001 xxxx */ + /* And unallocated instructions... */ + DECODE_END +}; + +static const union decode_item arm_cccc_000x_____1xx1_table[] = { + /* Extra load/store instructions */ + + /* STRHT cccc 0000 xx10 xxxx xxxx xxxx 1011 xxxx */ + /* ??? cccc 0000 xx10 xxxx xxxx xxxx 11x1 xxxx */ + /* LDRHT cccc 0000 xx11 xxxx xxxx xxxx 1011 xxxx */ + /* LDRSBT cccc 0000 xx11 xxxx xxxx xxxx 1101 xxxx */ + /* LDRSHT cccc 0000 xx11 xxxx xxxx xxxx 1111 xxxx */ + DECODE_REJECT (0x0f200090, 0x00200090), + + /* LDRD/STRD lr,pc,{... cccc 000x x0x0 xxxx 111x xxxx 1101 xxxx */ + DECODE_REJECT (0x0e10e0d0, 0x0000e0d0), + + /* LDRD (register) cccc 000x x0x0 xxxx xxxx xxxx 1101 xxxx */ + /* STRD (register) cccc 000x x0x0 xxxx xxxx xxxx 1111 xxxx */ + DECODE_EMULATEX (0x0e5000d0, 0x000000d0, PROBES_LDRSTRD, + REGS(NOPCWB, NOPCX, 0, 0, NOPC)), + + /* LDRD (immediate) cccc 000x x1x0 xxxx xxxx xxxx 1101 xxxx */ + /* STRD (immediate) cccc 000x x1x0 xxxx xxxx xxxx 1111 xxxx */ + DECODE_EMULATEX (0x0e5000d0, 0x004000d0, PROBES_LDRSTRD, + REGS(NOPCWB, NOPCX, 0, 0, 0)), + + /* STRH (register) cccc 000x x0x0 xxxx xxxx xxxx 1011 xxxx */ + DECODE_EMULATEX (0x0e5000f0, 0x000000b0, PROBES_STORE_EXTRA, + REGS(NOPCWB, NOPC, 0, 0, NOPC)), + + /* LDRH (register) cccc 000x x0x1 xxxx xxxx xxxx 1011 xxxx */ + /* LDRSB (register) cccc 000x x0x1 xxxx xxxx xxxx 1101 xxxx */ + /* LDRSH (register) cccc 000x x0x1 xxxx xxxx xxxx 1111 xxxx */ + DECODE_EMULATEX (0x0e500090, 0x00100090, PROBES_LOAD_EXTRA, + REGS(NOPCWB, NOPC, 0, 0, NOPC)), + + /* STRH (immediate) cccc 000x x1x0 xxxx xxxx xxxx 1011 xxxx */ + DECODE_EMULATEX (0x0e5000f0, 0x004000b0, PROBES_STORE_EXTRA, + REGS(NOPCWB, NOPC, 0, 0, 0)), + + /* LDRH (immediate) cccc 000x x1x1 xxxx xxxx xxxx 1011 xxxx */ + /* LDRSB (immediate) cccc 000x x1x1 xxxx xxxx xxxx 1101 xxxx */ + /* LDRSH (immediate) cccc 000x x1x1 xxxx xxxx xxxx 1111 xxxx */ + DECODE_EMULATEX (0x0e500090, 0x00500090, PROBES_LOAD_EXTRA, + REGS(NOPCWB, NOPC, 0, 0, 0)), + + DECODE_END +}; + +static const union decode_item arm_cccc_000x_table[] = { + /* Data-processing (register) */ + + /* <op>S PC, ... cccc 000x xxx1 xxxx 1111 xxxx xxxx xxxx */ + DECODE_REJECT (0x0e10f000, 0x0010f000), + + /* MOV IP, SP 1110 0001 1010 0000 1100 0000 0000 1101 */ + DECODE_SIMULATE (0xffffffff, 0xe1a0c00d, PROBES_MOV_IP_SP), + + /* TST (register) cccc 0001 0001 xxxx xxxx xxxx xxx0 xxxx */ + /* TEQ (register) cccc 0001 0011 xxxx xxxx xxxx xxx0 xxxx */ + /* CMP (register) cccc 0001 0101 xxxx xxxx xxxx xxx0 xxxx */ + /* CMN (register) cccc 0001 0111 xxxx xxxx xxxx xxx0 xxxx */ + DECODE_EMULATEX (0x0f900010, 0x01100000, PROBES_DATA_PROCESSING_REG, + REGS(ANY, 0, 0, 0, ANY)), + + /* MOV (register) cccc 0001 101x xxxx xxxx xxxx xxx0 xxxx */ + /* MVN (register) cccc 0001 111x xxxx xxxx xxxx xxx0 xxxx */ + DECODE_EMULATEX (0x0fa00010, 0x01a00000, PROBES_DATA_PROCESSING_REG, + REGS(0, ANY, 0, 0, ANY)), + + /* AND (register) cccc 0000 000x xxxx xxxx xxxx xxx0 xxxx */ + /* EOR (register) cccc 0000 001x xxxx xxxx xxxx xxx0 xxxx */ + /* SUB (register) cccc 0000 010x xxxx xxxx xxxx xxx0 xxxx */ + /* RSB (register) cccc 0000 011x xxxx xxxx xxxx xxx0 xxxx */ + /* ADD (register) cccc 0000 100x xxxx xxxx xxxx xxx0 xxxx */ + /* ADC (register) cccc 0000 101x xxxx xxxx xxxx xxx0 xxxx */ + /* SBC (register) cccc 0000 110x xxxx xxxx xxxx xxx0 xxxx */ + /* RSC (register) cccc 0000 111x xxxx xxxx xxxx xxx0 xxxx */ + /* ORR (register) cccc 0001 100x xxxx xxxx xxxx xxx0 xxxx */ + /* BIC (register) cccc 0001 110x xxxx xxxx xxxx xxx0 xxxx */ + DECODE_EMULATEX (0x0e000010, 0x00000000, PROBES_DATA_PROCESSING_REG, + REGS(ANY, ANY, 0, 0, ANY)), + + /* TST (reg-shift reg) cccc 0001 0001 xxxx xxxx xxxx 0xx1 xxxx */ + /* TEQ (reg-shift reg) cccc 0001 0011 xxxx xxxx xxxx 0xx1 xxxx */ + /* CMP (reg-shift reg) cccc 0001 0101 xxxx xxxx xxxx 0xx1 xxxx */ + /* CMN (reg-shift reg) cccc 0001 0111 xxxx xxxx xxxx 0xx1 xxxx */ + DECODE_EMULATEX (0x0f900090, 0x01100010, PROBES_DATA_PROCESSING_REG, + REGS(ANY, 0, NOPC, 0, ANY)), + + /* MOV (reg-shift reg) cccc 0001 101x xxxx xxxx xxxx 0xx1 xxxx */ + /* MVN (reg-shift reg) cccc 0001 111x xxxx xxxx xxxx 0xx1 xxxx */ + DECODE_EMULATEX (0x0fa00090, 0x01a00010, PROBES_DATA_PROCESSING_REG, + REGS(0, ANY, NOPC, 0, ANY)), + + /* AND (reg-shift reg) cccc 0000 000x xxxx xxxx xxxx 0xx1 xxxx */ + /* EOR (reg-shift reg) cccc 0000 001x xxxx xxxx xxxx 0xx1 xxxx */ + /* SUB (reg-shift reg) cccc 0000 010x xxxx xxxx xxxx 0xx1 xxxx */ + /* RSB (reg-shift reg) cccc 0000 011x xxxx xxxx xxxx 0xx1 xxxx */ + /* ADD (reg-shift reg) cccc 0000 100x xxxx xxxx xxxx 0xx1 xxxx */ + /* ADC (reg-shift reg) cccc 0000 101x xxxx xxxx xxxx 0xx1 xxxx */ + /* SBC (reg-shift reg) cccc 0000 110x xxxx xxxx xxxx 0xx1 xxxx */ + /* RSC (reg-shift reg) cccc 0000 111x xxxx xxxx xxxx 0xx1 xxxx */ + /* ORR (reg-shift reg) cccc 0001 100x xxxx xxxx xxxx 0xx1 xxxx */ + /* BIC (reg-shift reg) cccc 0001 110x xxxx xxxx xxxx 0xx1 xxxx */ + DECODE_EMULATEX (0x0e000090, 0x00000010, PROBES_DATA_PROCESSING_REG, + REGS(ANY, ANY, NOPC, 0, ANY)), + + DECODE_END +}; + +static const union decode_item arm_cccc_001x_table[] = { + /* Data-processing (immediate) */ + + /* MOVW cccc 0011 0000 xxxx xxxx xxxx xxxx xxxx */ + /* MOVT cccc 0011 0100 xxxx xxxx xxxx xxxx xxxx */ + DECODE_EMULATEX (0x0fb00000, 0x03000000, PROBES_DATA_PROCESSING_IMM, + REGS(0, NOPC, 0, 0, 0)), + + /* YIELD cccc 0011 0010 0000 xxxx xxxx 0000 0001 */ + DECODE_OR (0x0fff00ff, 0x03200001), + /* SEV cccc 0011 0010 0000 xxxx xxxx 0000 0100 */ + DECODE_EMULATE (0x0fff00ff, 0x03200004, PROBES_EMULATE_NONE), + /* NOP cccc 0011 0010 0000 xxxx xxxx 0000 0000 */ + /* WFE cccc 0011 0010 0000 xxxx xxxx 0000 0010 */ + /* WFI cccc 0011 0010 0000 xxxx xxxx 0000 0011 */ + DECODE_SIMULATE (0x0fff00fc, 0x03200000, PROBES_SIMULATE_NOP), + /* DBG cccc 0011 0010 0000 xxxx xxxx ffff xxxx */ + /* unallocated hints cccc 0011 0010 0000 xxxx xxxx xxxx xxxx */ + /* MSR (immediate) cccc 0011 0x10 xxxx xxxx xxxx xxxx xxxx */ + DECODE_REJECT (0x0fb00000, 0x03200000), + + /* <op>S PC, ... cccc 001x xxx1 xxxx 1111 xxxx xxxx xxxx */ + DECODE_REJECT (0x0e10f000, 0x0210f000), + + /* TST (immediate) cccc 0011 0001 xxxx xxxx xxxx xxxx xxxx */ + /* TEQ (immediate) cccc 0011 0011 xxxx xxxx xxxx xxxx xxxx */ + /* CMP (immediate) cccc 0011 0101 xxxx xxxx xxxx xxxx xxxx */ + /* CMN (immediate) cccc 0011 0111 xxxx xxxx xxxx xxxx xxxx */ + DECODE_EMULATEX (0x0f900000, 0x03100000, PROBES_DATA_PROCESSING_IMM, + REGS(ANY, 0, 0, 0, 0)), + + /* MOV (immediate) cccc 0011 101x xxxx xxxx xxxx xxxx xxxx */ + /* MVN (immediate) cccc 0011 111x xxxx xxxx xxxx xxxx xxxx */ + DECODE_EMULATEX (0x0fa00000, 0x03a00000, PROBES_DATA_PROCESSING_IMM, + REGS(0, ANY, 0, 0, 0)), + + /* AND (immediate) cccc 0010 000x xxxx xxxx xxxx xxxx xxxx */ + /* EOR (immediate) cccc 0010 001x xxxx xxxx xxxx xxxx xxxx */ + /* SUB (immediate) cccc 0010 010x xxxx xxxx xxxx xxxx xxxx */ + /* RSB (immediate) cccc 0010 011x xxxx xxxx xxxx xxxx xxxx */ + /* ADD (immediate) cccc 0010 100x xxxx xxxx xxxx xxxx xxxx */ + /* ADC (immediate) cccc 0010 101x xxxx xxxx xxxx xxxx xxxx */ + /* SBC (immediate) cccc 0010 110x xxxx xxxx xxxx xxxx xxxx */ + /* RSC (immediate) cccc 0010 111x xxxx xxxx xxxx xxxx xxxx */ + /* ORR (immediate) cccc 0011 100x xxxx xxxx xxxx xxxx xxxx */ + /* BIC (immediate) cccc 0011 110x xxxx xxxx xxxx xxxx xxxx */ + DECODE_EMULATEX (0x0e000000, 0x02000000, PROBES_DATA_PROCESSING_IMM, + REGS(ANY, ANY, 0, 0, 0)), + + DECODE_END +}; + +static const union decode_item arm_cccc_0110_____xxx1_table[] = { + /* Media instructions */ + + /* SEL cccc 0110 1000 xxxx xxxx xxxx 1011 xxxx */ + DECODE_EMULATEX (0x0ff000f0, 0x068000b0, PROBES_SATURATE, + REGS(NOPC, NOPC, 0, 0, NOPC)), + + /* SSAT cccc 0110 101x xxxx xxxx xxxx xx01 xxxx */ + /* USAT cccc 0110 111x xxxx xxxx xxxx xx01 xxxx */ + DECODE_OR(0x0fa00030, 0x06a00010), + /* SSAT16 cccc 0110 1010 xxxx xxxx xxxx 0011 xxxx */ + /* USAT16 cccc 0110 1110 xxxx xxxx xxxx 0011 xxxx */ + DECODE_EMULATEX (0x0fb000f0, 0x06a00030, PROBES_SATURATE, + REGS(0, NOPC, 0, 0, NOPC)), + + /* REV cccc 0110 1011 xxxx xxxx xxxx 0011 xxxx */ + /* REV16 cccc 0110 1011 xxxx xxxx xxxx 1011 xxxx */ + /* RBIT cccc 0110 1111 xxxx xxxx xxxx 0011 xxxx */ + /* REVSH cccc 0110 1111 xxxx xxxx xxxx 1011 xxxx */ + DECODE_EMULATEX (0x0fb00070, 0x06b00030, PROBES_REV, + REGS(0, NOPC, 0, 0, NOPC)), + + /* ??? cccc 0110 0x00 xxxx xxxx xxxx xxx1 xxxx */ + DECODE_REJECT (0x0fb00010, 0x06000010), + /* ??? cccc 0110 0xxx xxxx xxxx xxxx 1011 xxxx */ + DECODE_REJECT (0x0f8000f0, 0x060000b0), + /* ??? cccc 0110 0xxx xxxx xxxx xxxx 1101 xxxx */ + DECODE_REJECT (0x0f8000f0, 0x060000d0), + /* SADD16 cccc 0110 0001 xxxx xxxx xxxx 0001 xxxx */ + /* SADDSUBX cccc 0110 0001 xxxx xxxx xxxx 0011 xxxx */ + /* SSUBADDX cccc 0110 0001 xxxx xxxx xxxx 0101 xxxx */ + /* SSUB16 cccc 0110 0001 xxxx xxxx xxxx 0111 xxxx */ + /* SADD8 cccc 0110 0001 xxxx xxxx xxxx 1001 xxxx */ + /* SSUB8 cccc 0110 0001 xxxx xxxx xxxx 1111 xxxx */ + /* QADD16 cccc 0110 0010 xxxx xxxx xxxx 0001 xxxx */ + /* QADDSUBX cccc 0110 0010 xxxx xxxx xxxx 0011 xxxx */ + /* QSUBADDX cccc 0110 0010 xxxx xxxx xxxx 0101 xxxx */ + /* QSUB16 cccc 0110 0010 xxxx xxxx xxxx 0111 xxxx */ + /* QADD8 cccc 0110 0010 xxxx xxxx xxxx 1001 xxxx */ + /* QSUB8 cccc 0110 0010 xxxx xxxx xxxx 1111 xxxx */ + /* SHADD16 cccc 0110 0011 xxxx xxxx xxxx 0001 xxxx */ + /* SHADDSUBX cccc 0110 0011 xxxx xxxx xxxx 0011 xxxx */ + /* SHSUBADDX cccc 0110 0011 xxxx xxxx xxxx 0101 xxxx */ + /* SHSUB16 cccc 0110 0011 xxxx xxxx xxxx 0111 xxxx */ + /* SHADD8 cccc 0110 0011 xxxx xxxx xxxx 1001 xxxx */ + /* SHSUB8 cccc 0110 0011 xxxx xxxx xxxx 1111 xxxx */ + /* UADD16 cccc 0110 0101 xxxx xxxx xxxx 0001 xxxx */ + /* UADDSUBX cccc 0110 0101 xxxx xxxx xxxx 0011 xxxx */ + /* USUBADDX cccc 0110 0101 xxxx xxxx xxxx 0101 xxxx */ + /* USUB16 cccc 0110 0101 xxxx xxxx xxxx 0111 xxxx */ + /* UADD8 cccc 0110 0101 xxxx xxxx xxxx 1001 xxxx */ + /* USUB8 cccc 0110 0101 xxxx xxxx xxxx 1111 xxxx */ + /* UQADD16 cccc 0110 0110 xxxx xxxx xxxx 0001 xxxx */ + /* UQADDSUBX cccc 0110 0110 xxxx xxxx xxxx 0011 xxxx */ + /* UQSUBADDX cccc 0110 0110 xxxx xxxx xxxx 0101 xxxx */ + /* UQSUB16 cccc 0110 0110 xxxx xxxx xxxx 0111 xxxx */ + /* UQADD8 cccc 0110 0110 xxxx xxxx xxxx 1001 xxxx */ + /* UQSUB8 cccc 0110 0110 xxxx xxxx xxxx 1111 xxxx */ + /* UHADD16 cccc 0110 0111 xxxx xxxx xxxx 0001 xxxx */ + /* UHADDSUBX cccc 0110 0111 xxxx xxxx xxxx 0011 xxxx */ + /* UHSUBADDX cccc 0110 0111 xxxx xxxx xxxx 0101 xxxx */ + /* UHSUB16 cccc 0110 0111 xxxx xxxx xxxx 0111 xxxx */ + /* UHADD8 cccc 0110 0111 xxxx xxxx xxxx 1001 xxxx */ + /* UHSUB8 cccc 0110 0111 xxxx xxxx xxxx 1111 xxxx */ + DECODE_EMULATEX (0x0f800010, 0x06000010, PROBES_MMI, + REGS(NOPC, NOPC, 0, 0, NOPC)), + + /* PKHBT cccc 0110 1000 xxxx xxxx xxxx x001 xxxx */ + /* PKHTB cccc 0110 1000 xxxx xxxx xxxx x101 xxxx */ + DECODE_EMULATEX (0x0ff00030, 0x06800010, PROBES_PACK, + REGS(NOPC, NOPC, 0, 0, NOPC)), + + /* ??? cccc 0110 1001 xxxx xxxx xxxx 0111 xxxx */ + /* ??? cccc 0110 1101 xxxx xxxx xxxx 0111 xxxx */ + DECODE_REJECT (0x0fb000f0, 0x06900070), + + /* SXTB16 cccc 0110 1000 1111 xxxx xxxx 0111 xxxx */ + /* SXTB cccc 0110 1010 1111 xxxx xxxx 0111 xxxx */ + /* SXTH cccc 0110 1011 1111 xxxx xxxx 0111 xxxx */ + /* UXTB16 cccc 0110 1100 1111 xxxx xxxx 0111 xxxx */ + /* UXTB cccc 0110 1110 1111 xxxx xxxx 0111 xxxx */ + /* UXTH cccc 0110 1111 1111 xxxx xxxx 0111 xxxx */ + DECODE_EMULATEX (0x0f8f00f0, 0x068f0070, PROBES_EXTEND, + REGS(0, NOPC, 0, 0, NOPC)), + + /* SXTAB16 cccc 0110 1000 xxxx xxxx xxxx 0111 xxxx */ + /* SXTAB cccc 0110 1010 xxxx xxxx xxxx 0111 xxxx */ + /* SXTAH cccc 0110 1011 xxxx xxxx xxxx 0111 xxxx */ + /* UXTAB16 cccc 0110 1100 xxxx xxxx xxxx 0111 xxxx */ + /* UXTAB cccc 0110 1110 xxxx xxxx xxxx 0111 xxxx */ + /* UXTAH cccc 0110 1111 xxxx xxxx xxxx 0111 xxxx */ + DECODE_EMULATEX (0x0f8000f0, 0x06800070, PROBES_EXTEND_ADD, + REGS(NOPCX, NOPC, 0, 0, NOPC)), + + DECODE_END +}; + +static const union decode_item arm_cccc_0111_____xxx1_table[] = { + /* Media instructions */ + + /* UNDEFINED cccc 0111 1111 xxxx xxxx xxxx 1111 xxxx */ + DECODE_REJECT (0x0ff000f0, 0x07f000f0), + + /* SMLALD cccc 0111 0100 xxxx xxxx xxxx 00x1 xxxx */ + /* SMLSLD cccc 0111 0100 xxxx xxxx xxxx 01x1 xxxx */ + DECODE_EMULATEX (0x0ff00090, 0x07400010, PROBES_MUL_ADD_LONG, + REGS(NOPC, NOPC, NOPC, 0, NOPC)), + + /* SMUAD cccc 0111 0000 xxxx 1111 xxxx 00x1 xxxx */ + /* SMUSD cccc 0111 0000 xxxx 1111 xxxx 01x1 xxxx */ + DECODE_OR (0x0ff0f090, 0x0700f010), + /* SMMUL cccc 0111 0101 xxxx 1111 xxxx 00x1 xxxx */ + DECODE_OR (0x0ff0f0d0, 0x0750f010), + /* USAD8 cccc 0111 1000 xxxx 1111 xxxx 0001 xxxx */ + DECODE_EMULATEX (0x0ff0f0f0, 0x0780f010, PROBES_MUL_ADD, + REGS(NOPC, 0, NOPC, 0, NOPC)), + + /* SMLAD cccc 0111 0000 xxxx xxxx xxxx 00x1 xxxx */ + /* SMLSD cccc 0111 0000 xxxx xxxx xxxx 01x1 xxxx */ + DECODE_OR (0x0ff00090, 0x07000010), + /* SMMLA cccc 0111 0101 xxxx xxxx xxxx 00x1 xxxx */ + DECODE_OR (0x0ff000d0, 0x07500010), + /* USADA8 cccc 0111 1000 xxxx xxxx xxxx 0001 xxxx */ + DECODE_EMULATEX (0x0ff000f0, 0x07800010, PROBES_MUL_ADD, + REGS(NOPC, NOPCX, NOPC, 0, NOPC)), + + /* SMMLS cccc 0111 0101 xxxx xxxx xxxx 11x1 xxxx */ + DECODE_EMULATEX (0x0ff000d0, 0x075000d0, PROBES_MUL_ADD, + REGS(NOPC, NOPC, NOPC, 0, NOPC)), + + /* SBFX cccc 0111 101x xxxx xxxx xxxx x101 xxxx */ + /* UBFX cccc 0111 111x xxxx xxxx xxxx x101 xxxx */ + DECODE_EMULATEX (0x0fa00070, 0x07a00050, PROBES_BITFIELD, + REGS(0, NOPC, 0, 0, NOPC)), + + /* BFC cccc 0111 110x xxxx xxxx xxxx x001 1111 */ + DECODE_EMULATEX (0x0fe0007f, 0x07c0001f, PROBES_BITFIELD, + REGS(0, NOPC, 0, 0, 0)), + + /* BFI cccc 0111 110x xxxx xxxx xxxx x001 xxxx */ + DECODE_EMULATEX (0x0fe00070, 0x07c00010, PROBES_BITFIELD, + REGS(0, NOPC, 0, 0, NOPCX)), + + DECODE_END +}; + +static const union decode_item arm_cccc_01xx_table[] = { + /* Load/store word and unsigned byte */ + + /* LDRB/STRB pc,[...] cccc 01xx x0xx xxxx xxxx xxxx xxxx xxxx */ + DECODE_REJECT (0x0c40f000, 0x0440f000), + + /* STRT cccc 01x0 x010 xxxx xxxx xxxx xxxx xxxx */ + /* LDRT cccc 01x0 x011 xxxx xxxx xxxx xxxx xxxx */ + /* STRBT cccc 01x0 x110 xxxx xxxx xxxx xxxx xxxx */ + /* LDRBT cccc 01x0 x111 xxxx xxxx xxxx xxxx xxxx */ + DECODE_REJECT (0x0d200000, 0x04200000), + + /* STR (immediate) cccc 010x x0x0 xxxx xxxx xxxx xxxx xxxx */ + /* STRB (immediate) cccc 010x x1x0 xxxx xxxx xxxx xxxx xxxx */ + DECODE_EMULATEX (0x0e100000, 0x04000000, PROBES_STORE, + REGS(NOPCWB, ANY, 0, 0, 0)), + + /* LDR (immediate) cccc 010x x0x1 xxxx xxxx xxxx xxxx xxxx */ + /* LDRB (immediate) cccc 010x x1x1 xxxx xxxx xxxx xxxx xxxx */ + DECODE_EMULATEX (0x0e100000, 0x04100000, PROBES_LOAD, + REGS(NOPCWB, ANY, 0, 0, 0)), + + /* STR (register) cccc 011x x0x0 xxxx xxxx xxxx xxxx xxxx */ + /* STRB (register) cccc 011x x1x0 xxxx xxxx xxxx xxxx xxxx */ + DECODE_EMULATEX (0x0e100000, 0x06000000, PROBES_STORE, + REGS(NOPCWB, ANY, 0, 0, NOPC)), + + /* LDR (register) cccc 011x x0x1 xxxx xxxx xxxx xxxx xxxx */ + /* LDRB (register) cccc 011x x1x1 xxxx xxxx xxxx xxxx xxxx */ + DECODE_EMULATEX (0x0e100000, 0x06100000, PROBES_LOAD, + REGS(NOPCWB, ANY, 0, 0, NOPC)), + + DECODE_END +}; + +static const union decode_item arm_cccc_100x_table[] = { + /* Block data transfer instructions */ + + /* LDM cccc 100x x0x1 xxxx xxxx xxxx xxxx xxxx */ + /* STM cccc 100x x0x0 xxxx xxxx xxxx xxxx xxxx */ + DECODE_CUSTOM (0x0e400000, 0x08000000, PROBES_LDMSTM), + + /* STM (user registers) cccc 100x x1x0 xxxx xxxx xxxx xxxx xxxx */ + /* LDM (user registers) cccc 100x x1x1 xxxx 0xxx xxxx xxxx xxxx */ + /* LDM (exception ret) cccc 100x x1x1 xxxx 1xxx xxxx xxxx xxxx */ + DECODE_END +}; + +const union decode_item probes_decode_arm_table[] = { + /* + * Unconditional instructions + * 1111 xxxx xxxx xxxx xxxx xxxx xxxx xxxx + */ + DECODE_TABLE (0xf0000000, 0xf0000000, arm_1111_table), + + /* + * Miscellaneous instructions + * cccc 0001 0xx0 xxxx xxxx xxxx 0xxx xxxx + */ + DECODE_TABLE (0x0f900080, 0x01000000, arm_cccc_0001_0xx0____0xxx_table), + + /* + * Halfword multiply and multiply-accumulate + * cccc 0001 0xx0 xxxx xxxx xxxx 1xx0 xxxx + */ + DECODE_TABLE (0x0f900090, 0x01000080, arm_cccc_0001_0xx0____1xx0_table), + + /* + * Multiply and multiply-accumulate + * cccc 0000 xxxx xxxx xxxx xxxx 1001 xxxx + */ + DECODE_TABLE (0x0f0000f0, 0x00000090, arm_cccc_0000_____1001_table), + + /* + * Synchronization primitives + * cccc 0001 xxxx xxxx xxxx xxxx 1001 xxxx + */ + DECODE_TABLE (0x0f0000f0, 0x01000090, arm_cccc_0001_____1001_table), + + /* + * Extra load/store instructions + * cccc 000x xxxx xxxx xxxx xxxx 1xx1 xxxx + */ + DECODE_TABLE (0x0e000090, 0x00000090, arm_cccc_000x_____1xx1_table), + + /* + * Data-processing (register) + * cccc 000x xxxx xxxx xxxx xxxx xxx0 xxxx + * Data-processing (register-shifted register) + * cccc 000x xxxx xxxx xxxx xxxx 0xx1 xxxx + */ + DECODE_TABLE (0x0e000000, 0x00000000, arm_cccc_000x_table), + + /* + * Data-processing (immediate) + * cccc 001x xxxx xxxx xxxx xxxx xxxx xxxx + */ + DECODE_TABLE (0x0e000000, 0x02000000, arm_cccc_001x_table), + + /* + * Media instructions + * cccc 011x xxxx xxxx xxxx xxxx xxx1 xxxx + */ + DECODE_TABLE (0x0f000010, 0x06000010, arm_cccc_0110_____xxx1_table), + DECODE_TABLE (0x0f000010, 0x07000010, arm_cccc_0111_____xxx1_table), + + /* + * Load/store word and unsigned byte + * cccc 01xx xxxx xxxx xxxx xxxx xxxx xxxx + */ + DECODE_TABLE (0x0c000000, 0x04000000, arm_cccc_01xx_table), + + /* + * Block data transfer instructions + * cccc 100x xxxx xxxx xxxx xxxx xxxx xxxx + */ + DECODE_TABLE (0x0e000000, 0x08000000, arm_cccc_100x_table), + + /* B cccc 1010 xxxx xxxx xxxx xxxx xxxx xxxx */ + /* BL cccc 1011 xxxx xxxx xxxx xxxx xxxx xxxx */ + DECODE_SIMULATE (0x0e000000, 0x0a000000, PROBES_BRANCH), + + /* + * Supervisor Call, and coprocessor instructions + */ + + /* MCRR cccc 1100 0100 xxxx xxxx xxxx xxxx xxxx */ + /* MRRC cccc 1100 0101 xxxx xxxx xxxx xxxx xxxx */ + /* LDC cccc 110x xxx1 xxxx xxxx xxxx xxxx xxxx */ + /* STC cccc 110x xxx0 xxxx xxxx xxxx xxxx xxxx */ + /* CDP cccc 1110 xxxx xxxx xxxx xxxx xxx0 xxxx */ + /* MCR cccc 1110 xxx0 xxxx xxxx xxxx xxx1 xxxx */ + /* MRC cccc 1110 xxx1 xxxx xxxx xxxx xxx1 xxxx */ + /* SVC cccc 1111 xxxx xxxx xxxx xxxx xxxx xxxx */ + DECODE_REJECT (0x0c000000, 0x0c000000), + + DECODE_END +}; +#ifdef CONFIG_ARM_KPROBES_TEST_MODULE +EXPORT_SYMBOL_GPL(probes_decode_arm_table); +#endif + +static void __kprobes arm_singlestep(probes_opcode_t insn, + struct arch_probes_insn *asi, struct pt_regs *regs) +{ + regs->ARM_pc += 4; + asi->insn_handler(insn, asi, regs); +} + +/* Return: + * INSN_REJECTED If instruction is one not allowed to kprobe, + * INSN_GOOD If instruction is supported and uses instruction slot, + * INSN_GOOD_NO_SLOT If instruction is supported but doesn't use its slot. + * + * For instructions we don't want to kprobe (INSN_REJECTED return result): + * These are generally ones that modify the processor state making + * them "hard" to simulate such as switches processor modes or + * make accesses in alternate modes. Any of these could be simulated + * if the work was put into it, but low return considering they + * should also be very rare. + */ +enum probes_insn __kprobes +arm_probes_decode_insn(probes_opcode_t insn, struct arch_probes_insn *asi, + bool emulate, const union decode_action *actions) +{ + asi->insn_singlestep = arm_singlestep; + asi->insn_check_cc = probes_condition_checks[insn>>28]; + return probes_decode_insn(insn, asi, probes_decode_arm_table, false, + emulate, actions); +} diff --git a/arch/arm/kernel/probes-arm.h b/arch/arm/kernel/probes-arm.h new file mode 100644 index 000000000000..ace6572f6e26 --- /dev/null +++ b/arch/arm/kernel/probes-arm.h @@ -0,0 +1,73 @@ +/* + * arch/arm/kernel/probes-arm.h + * + * Copyright 2013 Linaro Ltd. + * Written by: David A. Long + * + * The code contained herein is licensed under the GNU General Public + * License. You may obtain a copy of the GNU General Public License + * Version 2 or later at the following locations: + * + * http://www.opensource.org/licenses/gpl-license.html + * http://www.gnu.org/copyleft/gpl.html + */ + +#ifndef _ARM_KERNEL_PROBES_ARM_H +#define _ARM_KERNEL_PROBES_ARM_H + +enum probes_arm_action { + PROBES_EMULATE_NONE, + PROBES_SIMULATE_NOP, + PROBES_PRELOAD_IMM, + PROBES_PRELOAD_REG, + PROBES_BRANCH_IMM, + PROBES_BRANCH_REG, + PROBES_MRS, + PROBES_CLZ, + PROBES_SATURATING_ARITHMETIC, + PROBES_MUL1, + PROBES_MUL2, + PROBES_SWP, + PROBES_LDRSTRD, + PROBES_LOAD, + PROBES_STORE, + PROBES_LOAD_EXTRA, + PROBES_STORE_EXTRA, + PROBES_MOV_IP_SP, + PROBES_DATA_PROCESSING_REG, + PROBES_DATA_PROCESSING_IMM, + PROBES_MOV_HALFWORD, + PROBES_SEV, + PROBES_WFE, + PROBES_SATURATE, + PROBES_REV, + PROBES_MMI, + PROBES_PACK, + PROBES_EXTEND, + PROBES_EXTEND_ADD, + PROBES_MUL_ADD_LONG, + PROBES_MUL_ADD, + PROBES_BITFIELD, + PROBES_BRANCH, + PROBES_LDMSTM, + NUM_PROBES_ARM_ACTIONS +}; + +void __kprobes simulate_bbl(probes_opcode_t opcode, + struct arch_probes_insn *asi, struct pt_regs *regs); +void __kprobes simulate_blx1(probes_opcode_t opcode, + struct arch_probes_insn *asi, struct pt_regs *regs); +void __kprobes simulate_blx2bx(probes_opcode_t opcode, + struct arch_probes_insn *asi, struct pt_regs *regs); +void __kprobes simulate_mrs(probes_opcode_t opcode, + struct arch_probes_insn *asi, struct pt_regs *regs); +void __kprobes simulate_mov_ipsp(probes_opcode_t opcode, + struct arch_probes_insn *asi, struct pt_regs *regs); + +extern const union decode_item probes_decode_arm_table[]; + +enum probes_insn arm_probes_decode_insn(probes_opcode_t, + struct arch_probes_insn *, bool emulate, + const union decode_action *actions); + +#endif diff --git a/arch/arm/kernel/probes-thumb.c b/arch/arm/kernel/probes-thumb.c new file mode 100644 index 000000000000..4131351e812f --- /dev/null +++ b/arch/arm/kernel/probes-thumb.c @@ -0,0 +1,882 @@ +/* + * arch/arm/kernel/probes-thumb.c + * + * Copyright (C) 2011 Jon Medhurst <tixy@yxit.co.uk>. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/stddef.h> +#include <linux/kernel.h> +#include <linux/module.h> + +#include "probes.h" +#include "probes-thumb.h" + + +static const union decode_item t32_table_1110_100x_x0xx[] = { + /* Load/store multiple instructions */ + + /* Rn is PC 1110 100x x0xx 1111 xxxx xxxx xxxx xxxx */ + DECODE_REJECT (0xfe4f0000, 0xe80f0000), + + /* SRS 1110 1000 00x0 xxxx xxxx xxxx xxxx xxxx */ + /* RFE 1110 1000 00x1 xxxx xxxx xxxx xxxx xxxx */ + DECODE_REJECT (0xffc00000, 0xe8000000), + /* SRS 1110 1001 10x0 xxxx xxxx xxxx xxxx xxxx */ + /* RFE 1110 1001 10x1 xxxx xxxx xxxx xxxx xxxx */ + DECODE_REJECT (0xffc00000, 0xe9800000), + + /* STM Rn, {...pc} 1110 100x x0x0 xxxx 1xxx xxxx xxxx xxxx */ + DECODE_REJECT (0xfe508000, 0xe8008000), + /* LDM Rn, {...lr,pc} 1110 100x x0x1 xxxx 11xx xxxx xxxx xxxx */ + DECODE_REJECT (0xfe50c000, 0xe810c000), + /* LDM/STM Rn, {...sp} 1110 100x x0xx xxxx xx1x xxxx xxxx xxxx */ + DECODE_REJECT (0xfe402000, 0xe8002000), + + /* STMIA 1110 1000 10x0 xxxx xxxx xxxx xxxx xxxx */ + /* LDMIA 1110 1000 10x1 xxxx xxxx xxxx xxxx xxxx */ + /* STMDB 1110 1001 00x0 xxxx xxxx xxxx xxxx xxxx */ + /* LDMDB 1110 1001 00x1 xxxx xxxx xxxx xxxx xxxx */ + DECODE_CUSTOM (0xfe400000, 0xe8000000, PROBES_T32_LDMSTM), + + DECODE_END +}; + +static const union decode_item t32_table_1110_100x_x1xx[] = { + /* Load/store dual, load/store exclusive, table branch */ + + /* STRD (immediate) 1110 1000 x110 xxxx xxxx xxxx xxxx xxxx */ + /* LDRD (immediate) 1110 1000 x111 xxxx xxxx xxxx xxxx xxxx */ + DECODE_OR (0xff600000, 0xe8600000), + /* STRD (immediate) 1110 1001 x1x0 xxxx xxxx xxxx xxxx xxxx */ + /* LDRD (immediate) 1110 1001 x1x1 xxxx xxxx xxxx xxxx xxxx */ + DECODE_EMULATEX (0xff400000, 0xe9400000, PROBES_T32_LDRDSTRD, + REGS(NOPCWB, NOSPPC, NOSPPC, 0, 0)), + + /* TBB 1110 1000 1101 xxxx xxxx xxxx 0000 xxxx */ + /* TBH 1110 1000 1101 xxxx xxxx xxxx 0001 xxxx */ + DECODE_SIMULATEX(0xfff000e0, 0xe8d00000, PROBES_T32_TABLE_BRANCH, + REGS(NOSP, 0, 0, 0, NOSPPC)), + + /* STREX 1110 1000 0100 xxxx xxxx xxxx xxxx xxxx */ + /* LDREX 1110 1000 0101 xxxx xxxx xxxx xxxx xxxx */ + /* STREXB 1110 1000 1100 xxxx xxxx xxxx 0100 xxxx */ + /* STREXH 1110 1000 1100 xxxx xxxx xxxx 0101 xxxx */ + /* STREXD 1110 1000 1100 xxxx xxxx xxxx 0111 xxxx */ + /* LDREXB 1110 1000 1101 xxxx xxxx xxxx 0100 xxxx */ + /* LDREXH 1110 1000 1101 xxxx xxxx xxxx 0101 xxxx */ + /* LDREXD 1110 1000 1101 xxxx xxxx xxxx 0111 xxxx */ + /* And unallocated instructions... */ + DECODE_END +}; + +static const union decode_item t32_table_1110_101x[] = { + /* Data-processing (shifted register) */ + + /* TST 1110 1010 0001 xxxx xxxx 1111 xxxx xxxx */ + /* TEQ 1110 1010 1001 xxxx xxxx 1111 xxxx xxxx */ + DECODE_EMULATEX (0xff700f00, 0xea100f00, PROBES_T32_TST, + REGS(NOSPPC, 0, 0, 0, NOSPPC)), + + /* CMN 1110 1011 0001 xxxx xxxx 1111 xxxx xxxx */ + DECODE_OR (0xfff00f00, 0xeb100f00), + /* CMP 1110 1011 1011 xxxx xxxx 1111 xxxx xxxx */ + DECODE_EMULATEX (0xfff00f00, 0xebb00f00, PROBES_T32_TST, + REGS(NOPC, 0, 0, 0, NOSPPC)), + + /* MOV 1110 1010 010x 1111 xxxx xxxx xxxx xxxx */ + /* MVN 1110 1010 011x 1111 xxxx xxxx xxxx xxxx */ + DECODE_EMULATEX (0xffcf0000, 0xea4f0000, PROBES_T32_MOV, + REGS(0, 0, NOSPPC, 0, NOSPPC)), + + /* ??? 1110 1010 101x xxxx xxxx xxxx xxxx xxxx */ + /* ??? 1110 1010 111x xxxx xxxx xxxx xxxx xxxx */ + DECODE_REJECT (0xffa00000, 0xeaa00000), + /* ??? 1110 1011 001x xxxx xxxx xxxx xxxx xxxx */ + DECODE_REJECT (0xffe00000, 0xeb200000), + /* ??? 1110 1011 100x xxxx xxxx xxxx xxxx xxxx */ + DECODE_REJECT (0xffe00000, 0xeb800000), + /* ??? 1110 1011 111x xxxx xxxx xxxx xxxx xxxx */ + DECODE_REJECT (0xffe00000, 0xebe00000), + + /* ADD/SUB SP, SP, Rm, LSL #0..3 */ + /* 1110 1011 x0xx 1101 x000 1101 xx00 xxxx */ + DECODE_EMULATEX (0xff4f7f30, 0xeb0d0d00, PROBES_T32_ADDSUB, + REGS(SP, 0, SP, 0, NOSPPC)), + + /* ADD/SUB SP, SP, Rm, shift */ + /* 1110 1011 x0xx 1101 xxxx 1101 xxxx xxxx */ + DECODE_REJECT (0xff4f0f00, 0xeb0d0d00), + + /* ADD/SUB Rd, SP, Rm, shift */ + /* 1110 1011 x0xx 1101 xxxx xxxx xxxx xxxx */ + DECODE_EMULATEX (0xff4f0000, 0xeb0d0000, PROBES_T32_ADDSUB, + REGS(SP, 0, NOPC, 0, NOSPPC)), + + /* AND 1110 1010 000x xxxx xxxx xxxx xxxx xxxx */ + /* BIC 1110 1010 001x xxxx xxxx xxxx xxxx xxxx */ + /* ORR 1110 1010 010x xxxx xxxx xxxx xxxx xxxx */ + /* ORN 1110 1010 011x xxxx xxxx xxxx xxxx xxxx */ + /* EOR 1110 1010 100x xxxx xxxx xxxx xxxx xxxx */ + /* PKH 1110 1010 110x xxxx xxxx xxxx xxxx xxxx */ + /* ADD 1110 1011 000x xxxx xxxx xxxx xxxx xxxx */ + /* ADC 1110 1011 010x xxxx xxxx xxxx xxxx xxxx */ + /* SBC 1110 1011 011x xxxx xxxx xxxx xxxx xxxx */ + /* SUB 1110 1011 101x xxxx xxxx xxxx xxxx xxxx */ + /* RSB 1110 1011 110x xxxx xxxx xxxx xxxx xxxx */ + DECODE_EMULATEX (0xfe000000, 0xea000000, PROBES_T32_LOGICAL, + REGS(NOSPPC, 0, NOSPPC, 0, NOSPPC)), + + DECODE_END +}; + +static const union decode_item t32_table_1111_0x0x___0[] = { + /* Data-processing (modified immediate) */ + + /* TST 1111 0x00 0001 xxxx 0xxx 1111 xxxx xxxx */ + /* TEQ 1111 0x00 1001 xxxx 0xxx 1111 xxxx xxxx */ + DECODE_EMULATEX (0xfb708f00, 0xf0100f00, PROBES_T32_TST, + REGS(NOSPPC, 0, 0, 0, 0)), + + /* CMN 1111 0x01 0001 xxxx 0xxx 1111 xxxx xxxx */ + DECODE_OR (0xfbf08f00, 0xf1100f00), + /* CMP 1111 0x01 1011 xxxx 0xxx 1111 xxxx xxxx */ + DECODE_EMULATEX (0xfbf08f00, 0xf1b00f00, PROBES_T32_CMP, + REGS(NOPC, 0, 0, 0, 0)), + + /* MOV 1111 0x00 010x 1111 0xxx xxxx xxxx xxxx */ + /* MVN 1111 0x00 011x 1111 0xxx xxxx xxxx xxxx */ + DECODE_EMULATEX (0xfbcf8000, 0xf04f0000, PROBES_T32_MOV, + REGS(0, 0, NOSPPC, 0, 0)), + + /* ??? 1111 0x00 101x xxxx 0xxx xxxx xxxx xxxx */ + DECODE_REJECT (0xfbe08000, 0xf0a00000), + /* ??? 1111 0x00 110x xxxx 0xxx xxxx xxxx xxxx */ + /* ??? 1111 0x00 111x xxxx 0xxx xxxx xxxx xxxx */ + DECODE_REJECT (0xfbc08000, 0xf0c00000), + /* ??? 1111 0x01 001x xxxx 0xxx xxxx xxxx xxxx */ + DECODE_REJECT (0xfbe08000, 0xf1200000), + /* ??? 1111 0x01 100x xxxx 0xxx xxxx xxxx xxxx */ + DECODE_REJECT (0xfbe08000, 0xf1800000), + /* ??? 1111 0x01 111x xxxx 0xxx xxxx xxxx xxxx */ + DECODE_REJECT (0xfbe08000, 0xf1e00000), + + /* ADD Rd, SP, #imm 1111 0x01 000x 1101 0xxx xxxx xxxx xxxx */ + /* SUB Rd, SP, #imm 1111 0x01 101x 1101 0xxx xxxx xxxx xxxx */ + DECODE_EMULATEX (0xfb4f8000, 0xf10d0000, PROBES_T32_ADDSUB, + REGS(SP, 0, NOPC, 0, 0)), + + /* AND 1111 0x00 000x xxxx 0xxx xxxx xxxx xxxx */ + /* BIC 1111 0x00 001x xxxx 0xxx xxxx xxxx xxxx */ + /* ORR 1111 0x00 010x xxxx 0xxx xxxx xxxx xxxx */ + /* ORN 1111 0x00 011x xxxx 0xxx xxxx xxxx xxxx */ + /* EOR 1111 0x00 100x xxxx 0xxx xxxx xxxx xxxx */ + /* ADD 1111 0x01 000x xxxx 0xxx xxxx xxxx xxxx */ + /* ADC 1111 0x01 010x xxxx 0xxx xxxx xxxx xxxx */ + /* SBC 1111 0x01 011x xxxx 0xxx xxxx xxxx xxxx */ + /* SUB 1111 0x01 101x xxxx 0xxx xxxx xxxx xxxx */ + /* RSB 1111 0x01 110x xxxx 0xxx xxxx xxxx xxxx */ + DECODE_EMULATEX (0xfa008000, 0xf0000000, PROBES_T32_LOGICAL, + REGS(NOSPPC, 0, NOSPPC, 0, 0)), + + DECODE_END +}; + +static const union decode_item t32_table_1111_0x1x___0[] = { + /* Data-processing (plain binary immediate) */ + + /* ADDW Rd, PC, #imm 1111 0x10 0000 1111 0xxx xxxx xxxx xxxx */ + DECODE_OR (0xfbff8000, 0xf20f0000), + /* SUBW Rd, PC, #imm 1111 0x10 1010 1111 0xxx xxxx xxxx xxxx */ + DECODE_EMULATEX (0xfbff8000, 0xf2af0000, PROBES_T32_ADDWSUBW_PC, + REGS(PC, 0, NOSPPC, 0, 0)), + + /* ADDW SP, SP, #imm 1111 0x10 0000 1101 0xxx 1101 xxxx xxxx */ + DECODE_OR (0xfbff8f00, 0xf20d0d00), + /* SUBW SP, SP, #imm 1111 0x10 1010 1101 0xxx 1101 xxxx xxxx */ + DECODE_EMULATEX (0xfbff8f00, 0xf2ad0d00, PROBES_T32_ADDWSUBW, + REGS(SP, 0, SP, 0, 0)), + + /* ADDW 1111 0x10 0000 xxxx 0xxx xxxx xxxx xxxx */ + DECODE_OR (0xfbf08000, 0xf2000000), + /* SUBW 1111 0x10 1010 xxxx 0xxx xxxx xxxx xxxx */ + DECODE_EMULATEX (0xfbf08000, 0xf2a00000, PROBES_T32_ADDWSUBW, + REGS(NOPCX, 0, NOSPPC, 0, 0)), + + /* MOVW 1111 0x10 0100 xxxx 0xxx xxxx xxxx xxxx */ + /* MOVT 1111 0x10 1100 xxxx 0xxx xxxx xxxx xxxx */ + DECODE_EMULATEX (0xfb708000, 0xf2400000, PROBES_T32_MOVW, + REGS(0, 0, NOSPPC, 0, 0)), + + /* SSAT16 1111 0x11 0010 xxxx 0000 xxxx 00xx xxxx */ + /* SSAT 1111 0x11 00x0 xxxx 0xxx xxxx xxxx xxxx */ + /* USAT16 1111 0x11 1010 xxxx 0000 xxxx 00xx xxxx */ + /* USAT 1111 0x11 10x0 xxxx 0xxx xxxx xxxx xxxx */ + DECODE_EMULATEX (0xfb508000, 0xf3000000, PROBES_T32_SAT, + REGS(NOSPPC, 0, NOSPPC, 0, 0)), + + /* SFBX 1111 0x11 0100 xxxx 0xxx xxxx xxxx xxxx */ + /* UFBX 1111 0x11 1100 xxxx 0xxx xxxx xxxx xxxx */ + DECODE_EMULATEX (0xfb708000, 0xf3400000, PROBES_T32_BITFIELD, + REGS(NOSPPC, 0, NOSPPC, 0, 0)), + + /* BFC 1111 0x11 0110 1111 0xxx xxxx xxxx xxxx */ + DECODE_EMULATEX (0xfbff8000, 0xf36f0000, PROBES_T32_BITFIELD, + REGS(0, 0, NOSPPC, 0, 0)), + + /* BFI 1111 0x11 0110 xxxx 0xxx xxxx xxxx xxxx */ + DECODE_EMULATEX (0xfbf08000, 0xf3600000, PROBES_T32_BITFIELD, + REGS(NOSPPCX, 0, NOSPPC, 0, 0)), + + DECODE_END +}; + +static const union decode_item t32_table_1111_0xxx___1[] = { + /* Branches and miscellaneous control */ + + /* YIELD 1111 0011 1010 xxxx 10x0 x000 0000 0001 */ + DECODE_OR (0xfff0d7ff, 0xf3a08001), + /* SEV 1111 0011 1010 xxxx 10x0 x000 0000 0100 */ + DECODE_EMULATE (0xfff0d7ff, 0xf3a08004, PROBES_T32_SEV), + /* NOP 1111 0011 1010 xxxx 10x0 x000 0000 0000 */ + /* WFE 1111 0011 1010 xxxx 10x0 x000 0000 0010 */ + /* WFI 1111 0011 1010 xxxx 10x0 x000 0000 0011 */ + DECODE_SIMULATE (0xfff0d7fc, 0xf3a08000, PROBES_T32_WFE), + + /* MRS Rd, CPSR 1111 0011 1110 xxxx 10x0 xxxx xxxx xxxx */ + DECODE_SIMULATEX(0xfff0d000, 0xf3e08000, PROBES_T32_MRS, + REGS(0, 0, NOSPPC, 0, 0)), + + /* + * Unsupported instructions + * 1111 0x11 1xxx xxxx 10x0 xxxx xxxx xxxx + * + * MSR 1111 0011 100x xxxx 10x0 xxxx xxxx xxxx + * DBG hint 1111 0011 1010 xxxx 10x0 x000 1111 xxxx + * Unallocated hints 1111 0011 1010 xxxx 10x0 x000 xxxx xxxx + * CPS 1111 0011 1010 xxxx 10x0 xxxx xxxx xxxx + * CLREX/DSB/DMB/ISB 1111 0011 1011 xxxx 10x0 xxxx xxxx xxxx + * BXJ 1111 0011 1100 xxxx 10x0 xxxx xxxx xxxx + * SUBS PC,LR,#<imm8> 1111 0011 1101 xxxx 10x0 xxxx xxxx xxxx + * MRS Rd, SPSR 1111 0011 1111 xxxx 10x0 xxxx xxxx xxxx + * SMC 1111 0111 1111 xxxx 1000 xxxx xxxx xxxx + * UNDEFINED 1111 0111 1111 xxxx 1010 xxxx xxxx xxxx + * ??? 1111 0111 1xxx xxxx 1010 xxxx xxxx xxxx + */ + DECODE_REJECT (0xfb80d000, 0xf3808000), + + /* Bcc 1111 0xxx xxxx xxxx 10x0 xxxx xxxx xxxx */ + DECODE_CUSTOM (0xf800d000, 0xf0008000, PROBES_T32_BRANCH_COND), + + /* BLX 1111 0xxx xxxx xxxx 11x0 xxxx xxxx xxx0 */ + DECODE_OR (0xf800d001, 0xf000c000), + /* B 1111 0xxx xxxx xxxx 10x1 xxxx xxxx xxxx */ + /* BL 1111 0xxx xxxx xxxx 11x1 xxxx xxxx xxxx */ + DECODE_SIMULATE (0xf8009000, 0xf0009000, PROBES_T32_BRANCH), + + DECODE_END +}; + +static const union decode_item t32_table_1111_100x_x0x1__1111[] = { + /* Memory hints */ + + /* PLD (literal) 1111 1000 x001 1111 1111 xxxx xxxx xxxx */ + /* PLI (literal) 1111 1001 x001 1111 1111 xxxx xxxx xxxx */ + DECODE_SIMULATE (0xfe7ff000, 0xf81ff000, PROBES_T32_PLDI), + + /* PLD{W} (immediate) 1111 1000 10x1 xxxx 1111 xxxx xxxx xxxx */ + DECODE_OR (0xffd0f000, 0xf890f000), + /* PLD{W} (immediate) 1111 1000 00x1 xxxx 1111 1100 xxxx xxxx */ + DECODE_OR (0xffd0ff00, 0xf810fc00), + /* PLI (immediate) 1111 1001 1001 xxxx 1111 xxxx xxxx xxxx */ + DECODE_OR (0xfff0f000, 0xf990f000), + /* PLI (immediate) 1111 1001 0001 xxxx 1111 1100 xxxx xxxx */ + DECODE_SIMULATEX(0xfff0ff00, 0xf910fc00, PROBES_T32_PLDI, + REGS(NOPCX, 0, 0, 0, 0)), + + /* PLD{W} (register) 1111 1000 00x1 xxxx 1111 0000 00xx xxxx */ + DECODE_OR (0xffd0ffc0, 0xf810f000), + /* PLI (register) 1111 1001 0001 xxxx 1111 0000 00xx xxxx */ + DECODE_SIMULATEX(0xfff0ffc0, 0xf910f000, PROBES_T32_PLDI, + REGS(NOPCX, 0, 0, 0, NOSPPC)), + + /* Other unallocated instructions... */ + DECODE_END +}; + +static const union decode_item t32_table_1111_100x[] = { + /* Store/Load single data item */ + + /* ??? 1111 100x x11x xxxx xxxx xxxx xxxx xxxx */ + DECODE_REJECT (0xfe600000, 0xf8600000), + + /* ??? 1111 1001 0101 xxxx xxxx xxxx xxxx xxxx */ + DECODE_REJECT (0xfff00000, 0xf9500000), + + /* ??? 1111 100x 0xxx xxxx xxxx 10x0 xxxx xxxx */ + DECODE_REJECT (0xfe800d00, 0xf8000800), + + /* STRBT 1111 1000 0000 xxxx xxxx 1110 xxxx xxxx */ + /* STRHT 1111 1000 0010 xxxx xxxx 1110 xxxx xxxx */ + /* STRT 1111 1000 0100 xxxx xxxx 1110 xxxx xxxx */ + /* LDRBT 1111 1000 0001 xxxx xxxx 1110 xxxx xxxx */ + /* LDRSBT 1111 1001 0001 xxxx xxxx 1110 xxxx xxxx */ + /* LDRHT 1111 1000 0011 xxxx xxxx 1110 xxxx xxxx */ + /* LDRSHT 1111 1001 0011 xxxx xxxx 1110 xxxx xxxx */ + /* LDRT 1111 1000 0101 xxxx xxxx 1110 xxxx xxxx */ + DECODE_REJECT (0xfe800f00, 0xf8000e00), + + /* STR{,B,H} Rn,[PC...] 1111 1000 xxx0 1111 xxxx xxxx xxxx xxxx */ + DECODE_REJECT (0xff1f0000, 0xf80f0000), + + /* STR{,B,H} PC,[Rn...] 1111 1000 xxx0 xxxx 1111 xxxx xxxx xxxx */ + DECODE_REJECT (0xff10f000, 0xf800f000), + + /* LDR (literal) 1111 1000 x101 1111 xxxx xxxx xxxx xxxx */ + DECODE_SIMULATEX(0xff7f0000, 0xf85f0000, PROBES_T32_LDR_LIT, + REGS(PC, ANY, 0, 0, 0)), + + /* STR (immediate) 1111 1000 0100 xxxx xxxx 1xxx xxxx xxxx */ + /* LDR (immediate) 1111 1000 0101 xxxx xxxx 1xxx xxxx xxxx */ + DECODE_OR (0xffe00800, 0xf8400800), + /* STR (immediate) 1111 1000 1100 xxxx xxxx xxxx xxxx xxxx */ + /* LDR (immediate) 1111 1000 1101 xxxx xxxx xxxx xxxx xxxx */ + DECODE_EMULATEX (0xffe00000, 0xf8c00000, PROBES_T32_LDRSTR, + REGS(NOPCX, ANY, 0, 0, 0)), + + /* STR (register) 1111 1000 0100 xxxx xxxx 0000 00xx xxxx */ + /* LDR (register) 1111 1000 0101 xxxx xxxx 0000 00xx xxxx */ + DECODE_EMULATEX (0xffe00fc0, 0xf8400000, PROBES_T32_LDRSTR, + REGS(NOPCX, ANY, 0, 0, NOSPPC)), + + /* LDRB (literal) 1111 1000 x001 1111 xxxx xxxx xxxx xxxx */ + /* LDRSB (literal) 1111 1001 x001 1111 xxxx xxxx xxxx xxxx */ + /* LDRH (literal) 1111 1000 x011 1111 xxxx xxxx xxxx xxxx */ + /* LDRSH (literal) 1111 1001 x011 1111 xxxx xxxx xxxx xxxx */ + DECODE_SIMULATEX(0xfe5f0000, 0xf81f0000, PROBES_T32_LDR_LIT, + REGS(PC, NOSPPCX, 0, 0, 0)), + + /* STRB (immediate) 1111 1000 0000 xxxx xxxx 1xxx xxxx xxxx */ + /* STRH (immediate) 1111 1000 0010 xxxx xxxx 1xxx xxxx xxxx */ + /* LDRB (immediate) 1111 1000 0001 xxxx xxxx 1xxx xxxx xxxx */ + /* LDRSB (immediate) 1111 1001 0001 xxxx xxxx 1xxx xxxx xxxx */ + /* LDRH (immediate) 1111 1000 0011 xxxx xxxx 1xxx xxxx xxxx */ + /* LDRSH (immediate) 1111 1001 0011 xxxx xxxx 1xxx xxxx xxxx */ + DECODE_OR (0xfec00800, 0xf8000800), + /* STRB (immediate) 1111 1000 1000 xxxx xxxx xxxx xxxx xxxx */ + /* STRH (immediate) 1111 1000 1010 xxxx xxxx xxxx xxxx xxxx */ + /* LDRB (immediate) 1111 1000 1001 xxxx xxxx xxxx xxxx xxxx */ + /* LDRSB (immediate) 1111 1001 1001 xxxx xxxx xxxx xxxx xxxx */ + /* LDRH (immediate) 1111 1000 1011 xxxx xxxx xxxx xxxx xxxx */ + /* LDRSH (immediate) 1111 1001 1011 xxxx xxxx xxxx xxxx xxxx */ + DECODE_EMULATEX (0xfec00000, 0xf8800000, PROBES_T32_LDRSTR, + REGS(NOPCX, NOSPPCX, 0, 0, 0)), + + /* STRB (register) 1111 1000 0000 xxxx xxxx 0000 00xx xxxx */ + /* STRH (register) 1111 1000 0010 xxxx xxxx 0000 00xx xxxx */ + /* LDRB (register) 1111 1000 0001 xxxx xxxx 0000 00xx xxxx */ + /* LDRSB (register) 1111 1001 0001 xxxx xxxx 0000 00xx xxxx */ + /* LDRH (register) 1111 1000 0011 xxxx xxxx 0000 00xx xxxx */ + /* LDRSH (register) 1111 1001 0011 xxxx xxxx 0000 00xx xxxx */ + DECODE_EMULATEX (0xfe800fc0, 0xf8000000, PROBES_T32_LDRSTR, + REGS(NOPCX, NOSPPCX, 0, 0, NOSPPC)), + + /* Other unallocated instructions... */ + DECODE_END +}; + +static const union decode_item t32_table_1111_1010___1111[] = { + /* Data-processing (register) */ + + /* ??? 1111 1010 011x xxxx 1111 xxxx 1xxx xxxx */ + DECODE_REJECT (0xffe0f080, 0xfa60f080), + + /* SXTH 1111 1010 0000 1111 1111 xxxx 1xxx xxxx */ + /* UXTH 1111 1010 0001 1111 1111 xxxx 1xxx xxxx */ + /* SXTB16 1111 1010 0010 1111 1111 xxxx 1xxx xxxx */ + /* UXTB16 1111 1010 0011 1111 1111 xxxx 1xxx xxxx */ + /* SXTB 1111 1010 0100 1111 1111 xxxx 1xxx xxxx */ + /* UXTB 1111 1010 0101 1111 1111 xxxx 1xxx xxxx */ + DECODE_EMULATEX (0xff8ff080, 0xfa0ff080, PROBES_T32_SIGN_EXTEND, + REGS(0, 0, NOSPPC, 0, NOSPPC)), + + + /* ??? 1111 1010 1xxx xxxx 1111 xxxx 0x11 xxxx */ + DECODE_REJECT (0xff80f0b0, 0xfa80f030), + /* ??? 1111 1010 1x11 xxxx 1111 xxxx 0xxx xxxx */ + DECODE_REJECT (0xffb0f080, 0xfab0f000), + + /* SADD16 1111 1010 1001 xxxx 1111 xxxx 0000 xxxx */ + /* SASX 1111 1010 1010 xxxx 1111 xxxx 0000 xxxx */ + /* SSAX 1111 1010 1110 xxxx 1111 xxxx 0000 xxxx */ + /* SSUB16 1111 1010 1101 xxxx 1111 xxxx 0000 xxxx */ + /* SADD8 1111 1010 1000 xxxx 1111 xxxx 0000 xxxx */ + /* SSUB8 1111 1010 1100 xxxx 1111 xxxx 0000 xxxx */ + + /* QADD16 1111 1010 1001 xxxx 1111 xxxx 0001 xxxx */ + /* QASX 1111 1010 1010 xxxx 1111 xxxx 0001 xxxx */ + /* QSAX 1111 1010 1110 xxxx 1111 xxxx 0001 xxxx */ + /* QSUB16 1111 1010 1101 xxxx 1111 xxxx 0001 xxxx */ + /* QADD8 1111 1010 1000 xxxx 1111 xxxx 0001 xxxx */ + /* QSUB8 1111 1010 1100 xxxx 1111 xxxx 0001 xxxx */ + + /* SHADD16 1111 1010 1001 xxxx 1111 xxxx 0010 xxxx */ + /* SHASX 1111 1010 1010 xxxx 1111 xxxx 0010 xxxx */ + /* SHSAX 1111 1010 1110 xxxx 1111 xxxx 0010 xxxx */ + /* SHSUB16 1111 1010 1101 xxxx 1111 xxxx 0010 xxxx */ + /* SHADD8 1111 1010 1000 xxxx 1111 xxxx 0010 xxxx */ + /* SHSUB8 1111 1010 1100 xxxx 1111 xxxx 0010 xxxx */ + + /* UADD16 1111 1010 1001 xxxx 1111 xxxx 0100 xxxx */ + /* UASX 1111 1010 1010 xxxx 1111 xxxx 0100 xxxx */ + /* USAX 1111 1010 1110 xxxx 1111 xxxx 0100 xxxx */ + /* USUB16 1111 1010 1101 xxxx 1111 xxxx 0100 xxxx */ + /* UADD8 1111 1010 1000 xxxx 1111 xxxx 0100 xxxx */ + /* USUB8 1111 1010 1100 xxxx 1111 xxxx 0100 xxxx */ + + /* UQADD16 1111 1010 1001 xxxx 1111 xxxx 0101 xxxx */ + /* UQASX 1111 1010 1010 xxxx 1111 xxxx 0101 xxxx */ + /* UQSAX 1111 1010 1110 xxxx 1111 xxxx 0101 xxxx */ + /* UQSUB16 1111 1010 1101 xxxx 1111 xxxx 0101 xxxx */ + /* UQADD8 1111 1010 1000 xxxx 1111 xxxx 0101 xxxx */ + /* UQSUB8 1111 1010 1100 xxxx 1111 xxxx 0101 xxxx */ + + /* UHADD16 1111 1010 1001 xxxx 1111 xxxx 0110 xxxx */ + /* UHASX 1111 1010 1010 xxxx 1111 xxxx 0110 xxxx */ + /* UHSAX 1111 1010 1110 xxxx 1111 xxxx 0110 xxxx */ + /* UHSUB16 1111 1010 1101 xxxx 1111 xxxx 0110 xxxx */ + /* UHADD8 1111 1010 1000 xxxx 1111 xxxx 0110 xxxx */ + /* UHSUB8 1111 1010 1100 xxxx 1111 xxxx 0110 xxxx */ + DECODE_OR (0xff80f080, 0xfa80f000), + + /* SXTAH 1111 1010 0000 xxxx 1111 xxxx 1xxx xxxx */ + /* UXTAH 1111 1010 0001 xxxx 1111 xxxx 1xxx xxxx */ + /* SXTAB16 1111 1010 0010 xxxx 1111 xxxx 1xxx xxxx */ + /* UXTAB16 1111 1010 0011 xxxx 1111 xxxx 1xxx xxxx */ + /* SXTAB 1111 1010 0100 xxxx 1111 xxxx 1xxx xxxx */ + /* UXTAB 1111 1010 0101 xxxx 1111 xxxx 1xxx xxxx */ + DECODE_OR (0xff80f080, 0xfa00f080), + + /* QADD 1111 1010 1000 xxxx 1111 xxxx 1000 xxxx */ + /* QDADD 1111 1010 1000 xxxx 1111 xxxx 1001 xxxx */ + /* QSUB 1111 1010 1000 xxxx 1111 xxxx 1010 xxxx */ + /* QDSUB 1111 1010 1000 xxxx 1111 xxxx 1011 xxxx */ + DECODE_OR (0xfff0f0c0, 0xfa80f080), + + /* SEL 1111 1010 1010 xxxx 1111 xxxx 1000 xxxx */ + DECODE_OR (0xfff0f0f0, 0xfaa0f080), + + /* LSL 1111 1010 000x xxxx 1111 xxxx 0000 xxxx */ + /* LSR 1111 1010 001x xxxx 1111 xxxx 0000 xxxx */ + /* ASR 1111 1010 010x xxxx 1111 xxxx 0000 xxxx */ + /* ROR 1111 1010 011x xxxx 1111 xxxx 0000 xxxx */ + DECODE_EMULATEX (0xff80f0f0, 0xfa00f000, PROBES_T32_MEDIA, + REGS(NOSPPC, 0, NOSPPC, 0, NOSPPC)), + + /* CLZ 1111 1010 1010 xxxx 1111 xxxx 1000 xxxx */ + DECODE_OR (0xfff0f0f0, 0xfab0f080), + + /* REV 1111 1010 1001 xxxx 1111 xxxx 1000 xxxx */ + /* REV16 1111 1010 1001 xxxx 1111 xxxx 1001 xxxx */ + /* RBIT 1111 1010 1001 xxxx 1111 xxxx 1010 xxxx */ + /* REVSH 1111 1010 1001 xxxx 1111 xxxx 1011 xxxx */ + DECODE_EMULATEX (0xfff0f0c0, 0xfa90f080, PROBES_T32_REVERSE, + REGS(NOSPPC, 0, NOSPPC, 0, SAMEAS16)), + + /* Other unallocated instructions... */ + DECODE_END +}; + +static const union decode_item t32_table_1111_1011_0[] = { + /* Multiply, multiply accumulate, and absolute difference */ + + /* ??? 1111 1011 0000 xxxx 1111 xxxx 0001 xxxx */ + DECODE_REJECT (0xfff0f0f0, 0xfb00f010), + /* ??? 1111 1011 0111 xxxx 1111 xxxx 0001 xxxx */ + DECODE_REJECT (0xfff0f0f0, 0xfb70f010), + + /* SMULxy 1111 1011 0001 xxxx 1111 xxxx 00xx xxxx */ + DECODE_OR (0xfff0f0c0, 0xfb10f000), + /* MUL 1111 1011 0000 xxxx 1111 xxxx 0000 xxxx */ + /* SMUAD{X} 1111 1011 0010 xxxx 1111 xxxx 000x xxxx */ + /* SMULWy 1111 1011 0011 xxxx 1111 xxxx 000x xxxx */ + /* SMUSD{X} 1111 1011 0100 xxxx 1111 xxxx 000x xxxx */ + /* SMMUL{R} 1111 1011 0101 xxxx 1111 xxxx 000x xxxx */ + /* USAD8 1111 1011 0111 xxxx 1111 xxxx 0000 xxxx */ + DECODE_EMULATEX (0xff80f0e0, 0xfb00f000, PROBES_T32_MUL_ADD, + REGS(NOSPPC, 0, NOSPPC, 0, NOSPPC)), + + /* ??? 1111 1011 0111 xxxx xxxx xxxx 0001 xxxx */ + DECODE_REJECT (0xfff000f0, 0xfb700010), + + /* SMLAxy 1111 1011 0001 xxxx xxxx xxxx 00xx xxxx */ + DECODE_OR (0xfff000c0, 0xfb100000), + /* MLA 1111 1011 0000 xxxx xxxx xxxx 0000 xxxx */ + /* MLS 1111 1011 0000 xxxx xxxx xxxx 0001 xxxx */ + /* SMLAD{X} 1111 1011 0010 xxxx xxxx xxxx 000x xxxx */ + /* SMLAWy 1111 1011 0011 xxxx xxxx xxxx 000x xxxx */ + /* SMLSD{X} 1111 1011 0100 xxxx xxxx xxxx 000x xxxx */ + /* SMMLA{R} 1111 1011 0101 xxxx xxxx xxxx 000x xxxx */ + /* SMMLS{R} 1111 1011 0110 xxxx xxxx xxxx 000x xxxx */ + /* USADA8 1111 1011 0111 xxxx xxxx xxxx 0000 xxxx */ + DECODE_EMULATEX (0xff8000c0, 0xfb000000, PROBES_T32_MUL_ADD2, + REGS(NOSPPC, NOSPPCX, NOSPPC, 0, NOSPPC)), + + /* Other unallocated instructions... */ + DECODE_END +}; + +static const union decode_item t32_table_1111_1011_1[] = { + /* Long multiply, long multiply accumulate, and divide */ + + /* UMAAL 1111 1011 1110 xxxx xxxx xxxx 0110 xxxx */ + DECODE_OR (0xfff000f0, 0xfbe00060), + /* SMLALxy 1111 1011 1100 xxxx xxxx xxxx 10xx xxxx */ + DECODE_OR (0xfff000c0, 0xfbc00080), + /* SMLALD{X} 1111 1011 1100 xxxx xxxx xxxx 110x xxxx */ + /* SMLSLD{X} 1111 1011 1101 xxxx xxxx xxxx 110x xxxx */ + DECODE_OR (0xffe000e0, 0xfbc000c0), + /* SMULL 1111 1011 1000 xxxx xxxx xxxx 0000 xxxx */ + /* UMULL 1111 1011 1010 xxxx xxxx xxxx 0000 xxxx */ + /* SMLAL 1111 1011 1100 xxxx xxxx xxxx 0000 xxxx */ + /* UMLAL 1111 1011 1110 xxxx xxxx xxxx 0000 xxxx */ + DECODE_EMULATEX (0xff9000f0, 0xfb800000, PROBES_T32_MUL_ADD_LONG, + REGS(NOSPPC, NOSPPC, NOSPPC, 0, NOSPPC)), + + /* SDIV 1111 1011 1001 xxxx xxxx xxxx 1111 xxxx */ + /* UDIV 1111 1011 1011 xxxx xxxx xxxx 1111 xxxx */ + /* Other unallocated instructions... */ + DECODE_END +}; + +const union decode_item probes_decode_thumb32_table[] = { + + /* + * Load/store multiple instructions + * 1110 100x x0xx xxxx xxxx xxxx xxxx xxxx + */ + DECODE_TABLE (0xfe400000, 0xe8000000, t32_table_1110_100x_x0xx), + + /* + * Load/store dual, load/store exclusive, table branch + * 1110 100x x1xx xxxx xxxx xxxx xxxx xxxx + */ + DECODE_TABLE (0xfe400000, 0xe8400000, t32_table_1110_100x_x1xx), + + /* + * Data-processing (shifted register) + * 1110 101x xxxx xxxx xxxx xxxx xxxx xxxx + */ + DECODE_TABLE (0xfe000000, 0xea000000, t32_table_1110_101x), + + /* + * Coprocessor instructions + * 1110 11xx xxxx xxxx xxxx xxxx xxxx xxxx + */ + DECODE_REJECT (0xfc000000, 0xec000000), + + /* + * Data-processing (modified immediate) + * 1111 0x0x xxxx xxxx 0xxx xxxx xxxx xxxx + */ + DECODE_TABLE (0xfa008000, 0xf0000000, t32_table_1111_0x0x___0), + + /* + * Data-processing (plain binary immediate) + * 1111 0x1x xxxx xxxx 0xxx xxxx xxxx xxxx + */ + DECODE_TABLE (0xfa008000, 0xf2000000, t32_table_1111_0x1x___0), + + /* + * Branches and miscellaneous control + * 1111 0xxx xxxx xxxx 1xxx xxxx xxxx xxxx + */ + DECODE_TABLE (0xf8008000, 0xf0008000, t32_table_1111_0xxx___1), + + /* + * Advanced SIMD element or structure load/store instructions + * 1111 1001 xxx0 xxxx xxxx xxxx xxxx xxxx + */ + DECODE_REJECT (0xff100000, 0xf9000000), + + /* + * Memory hints + * 1111 100x x0x1 xxxx 1111 xxxx xxxx xxxx + */ + DECODE_TABLE (0xfe50f000, 0xf810f000, t32_table_1111_100x_x0x1__1111), + + /* + * Store single data item + * 1111 1000 xxx0 xxxx xxxx xxxx xxxx xxxx + * Load single data items + * 1111 100x xxx1 xxxx xxxx xxxx xxxx xxxx + */ + DECODE_TABLE (0xfe000000, 0xf8000000, t32_table_1111_100x), + + /* + * Data-processing (register) + * 1111 1010 xxxx xxxx 1111 xxxx xxxx xxxx + */ + DECODE_TABLE (0xff00f000, 0xfa00f000, t32_table_1111_1010___1111), + + /* + * Multiply, multiply accumulate, and absolute difference + * 1111 1011 0xxx xxxx xxxx xxxx xxxx xxxx + */ + DECODE_TABLE (0xff800000, 0xfb000000, t32_table_1111_1011_0), + + /* + * Long multiply, long multiply accumulate, and divide + * 1111 1011 1xxx xxxx xxxx xxxx xxxx xxxx + */ + DECODE_TABLE (0xff800000, 0xfb800000, t32_table_1111_1011_1), + + /* + * Coprocessor instructions + * 1111 11xx xxxx xxxx xxxx xxxx xxxx xxxx + */ + DECODE_END +}; +#ifdef CONFIG_ARM_KPROBES_TEST_MODULE +EXPORT_SYMBOL_GPL(probes_decode_thumb32_table); +#endif + +static const union decode_item t16_table_1011[] = { + /* Miscellaneous 16-bit instructions */ + + /* ADD (SP plus immediate) 1011 0000 0xxx xxxx */ + /* SUB (SP minus immediate) 1011 0000 1xxx xxxx */ + DECODE_SIMULATE (0xff00, 0xb000, PROBES_T16_ADD_SP), + + /* CBZ 1011 00x1 xxxx xxxx */ + /* CBNZ 1011 10x1 xxxx xxxx */ + DECODE_SIMULATE (0xf500, 0xb100, PROBES_T16_CBZ), + + /* SXTH 1011 0010 00xx xxxx */ + /* SXTB 1011 0010 01xx xxxx */ + /* UXTH 1011 0010 10xx xxxx */ + /* UXTB 1011 0010 11xx xxxx */ + /* REV 1011 1010 00xx xxxx */ + /* REV16 1011 1010 01xx xxxx */ + /* ??? 1011 1010 10xx xxxx */ + /* REVSH 1011 1010 11xx xxxx */ + DECODE_REJECT (0xffc0, 0xba80), + DECODE_EMULATE (0xf500, 0xb000, PROBES_T16_SIGN_EXTEND), + + /* PUSH 1011 010x xxxx xxxx */ + DECODE_CUSTOM (0xfe00, 0xb400, PROBES_T16_PUSH), + /* POP 1011 110x xxxx xxxx */ + DECODE_CUSTOM (0xfe00, 0xbc00, PROBES_T16_POP), + + /* + * If-Then, and hints + * 1011 1111 xxxx xxxx + */ + + /* YIELD 1011 1111 0001 0000 */ + DECODE_OR (0xffff, 0xbf10), + /* SEV 1011 1111 0100 0000 */ + DECODE_EMULATE (0xffff, 0xbf40, PROBES_T16_SEV), + /* NOP 1011 1111 0000 0000 */ + /* WFE 1011 1111 0010 0000 */ + /* WFI 1011 1111 0011 0000 */ + DECODE_SIMULATE (0xffcf, 0xbf00, PROBES_T16_WFE), + /* Unassigned hints 1011 1111 xxxx 0000 */ + DECODE_REJECT (0xff0f, 0xbf00), + /* IT 1011 1111 xxxx xxxx */ + DECODE_CUSTOM (0xff00, 0xbf00, PROBES_T16_IT), + + /* SETEND 1011 0110 010x xxxx */ + /* CPS 1011 0110 011x xxxx */ + /* BKPT 1011 1110 xxxx xxxx */ + /* And unallocated instructions... */ + DECODE_END +}; + +const union decode_item probes_decode_thumb16_table[] = { + + /* + * Shift (immediate), add, subtract, move, and compare + * 00xx xxxx xxxx xxxx + */ + + /* CMP (immediate) 0010 1xxx xxxx xxxx */ + DECODE_EMULATE (0xf800, 0x2800, PROBES_T16_CMP), + + /* ADD (register) 0001 100x xxxx xxxx */ + /* SUB (register) 0001 101x xxxx xxxx */ + /* LSL (immediate) 0000 0xxx xxxx xxxx */ + /* LSR (immediate) 0000 1xxx xxxx xxxx */ + /* ASR (immediate) 0001 0xxx xxxx xxxx */ + /* ADD (immediate, Thumb) 0001 110x xxxx xxxx */ + /* SUB (immediate, Thumb) 0001 111x xxxx xxxx */ + /* MOV (immediate) 0010 0xxx xxxx xxxx */ + /* ADD (immediate, Thumb) 0011 0xxx xxxx xxxx */ + /* SUB (immediate, Thumb) 0011 1xxx xxxx xxxx */ + DECODE_EMULATE (0xc000, 0x0000, PROBES_T16_ADDSUB), + + /* + * 16-bit Thumb data-processing instructions + * 0100 00xx xxxx xxxx + */ + + /* TST (register) 0100 0010 00xx xxxx */ + DECODE_EMULATE (0xffc0, 0x4200, PROBES_T16_CMP), + /* CMP (register) 0100 0010 10xx xxxx */ + /* CMN (register) 0100 0010 11xx xxxx */ + DECODE_EMULATE (0xff80, 0x4280, PROBES_T16_CMP), + /* AND (register) 0100 0000 00xx xxxx */ + /* EOR (register) 0100 0000 01xx xxxx */ + /* LSL (register) 0100 0000 10xx xxxx */ + /* LSR (register) 0100 0000 11xx xxxx */ + /* ASR (register) 0100 0001 00xx xxxx */ + /* ADC (register) 0100 0001 01xx xxxx */ + /* SBC (register) 0100 0001 10xx xxxx */ + /* ROR (register) 0100 0001 11xx xxxx */ + /* RSB (immediate) 0100 0010 01xx xxxx */ + /* ORR (register) 0100 0011 00xx xxxx */ + /* MUL 0100 0011 00xx xxxx */ + /* BIC (register) 0100 0011 10xx xxxx */ + /* MVN (register) 0100 0011 10xx xxxx */ + DECODE_EMULATE (0xfc00, 0x4000, PROBES_T16_LOGICAL), + + /* + * Special data instructions and branch and exchange + * 0100 01xx xxxx xxxx + */ + + /* BLX pc 0100 0111 1111 1xxx */ + DECODE_REJECT (0xfff8, 0x47f8), + + /* BX (register) 0100 0111 0xxx xxxx */ + /* BLX (register) 0100 0111 1xxx xxxx */ + DECODE_SIMULATE (0xff00, 0x4700, PROBES_T16_BLX), + + /* ADD pc, pc 0100 0100 1111 1111 */ + DECODE_REJECT (0xffff, 0x44ff), + + /* ADD (register) 0100 0100 xxxx xxxx */ + /* CMP (register) 0100 0101 xxxx xxxx */ + /* MOV (register) 0100 0110 xxxx xxxx */ + DECODE_CUSTOM (0xfc00, 0x4400, PROBES_T16_HIREGOPS), + + /* + * Load from Literal Pool + * LDR (literal) 0100 1xxx xxxx xxxx + */ + DECODE_SIMULATE (0xf800, 0x4800, PROBES_T16_LDR_LIT), + + /* + * 16-bit Thumb Load/store instructions + * 0101 xxxx xxxx xxxx + * 011x xxxx xxxx xxxx + * 100x xxxx xxxx xxxx + */ + + /* STR (register) 0101 000x xxxx xxxx */ + /* STRH (register) 0101 001x xxxx xxxx */ + /* STRB (register) 0101 010x xxxx xxxx */ + /* LDRSB (register) 0101 011x xxxx xxxx */ + /* LDR (register) 0101 100x xxxx xxxx */ + /* LDRH (register) 0101 101x xxxx xxxx */ + /* LDRB (register) 0101 110x xxxx xxxx */ + /* LDRSH (register) 0101 111x xxxx xxxx */ + /* STR (immediate, Thumb) 0110 0xxx xxxx xxxx */ + /* LDR (immediate, Thumb) 0110 1xxx xxxx xxxx */ + /* STRB (immediate, Thumb) 0111 0xxx xxxx xxxx */ + /* LDRB (immediate, Thumb) 0111 1xxx xxxx xxxx */ + DECODE_EMULATE (0xc000, 0x4000, PROBES_T16_LDRHSTRH), + /* STRH (immediate, Thumb) 1000 0xxx xxxx xxxx */ + /* LDRH (immediate, Thumb) 1000 1xxx xxxx xxxx */ + DECODE_EMULATE (0xf000, 0x8000, PROBES_T16_LDRHSTRH), + /* STR (immediate, Thumb) 1001 0xxx xxxx xxxx */ + /* LDR (immediate, Thumb) 1001 1xxx xxxx xxxx */ + DECODE_SIMULATE (0xf000, 0x9000, PROBES_T16_LDRSTR), + + /* + * Generate PC-/SP-relative address + * ADR (literal) 1010 0xxx xxxx xxxx + * ADD (SP plus immediate) 1010 1xxx xxxx xxxx + */ + DECODE_SIMULATE (0xf000, 0xa000, PROBES_T16_ADR), + + /* + * Miscellaneous 16-bit instructions + * 1011 xxxx xxxx xxxx + */ + DECODE_TABLE (0xf000, 0xb000, t16_table_1011), + + /* STM 1100 0xxx xxxx xxxx */ + /* LDM 1100 1xxx xxxx xxxx */ + DECODE_EMULATE (0xf000, 0xc000, PROBES_T16_LDMSTM), + + /* + * Conditional branch, and Supervisor Call + */ + + /* Permanently UNDEFINED 1101 1110 xxxx xxxx */ + /* SVC 1101 1111 xxxx xxxx */ + DECODE_REJECT (0xfe00, 0xde00), + + /* Conditional branch 1101 xxxx xxxx xxxx */ + DECODE_CUSTOM (0xf000, 0xd000, PROBES_T16_BRANCH_COND), + + /* + * Unconditional branch + * B 1110 0xxx xxxx xxxx + */ + DECODE_SIMULATE (0xf800, 0xe000, PROBES_T16_BRANCH), + + DECODE_END +}; +#ifdef CONFIG_ARM_KPROBES_TEST_MODULE +EXPORT_SYMBOL_GPL(probes_decode_thumb16_table); +#endif + +static unsigned long __kprobes thumb_check_cc(unsigned long cpsr) +{ + if (unlikely(in_it_block(cpsr))) + return probes_condition_checks[current_cond(cpsr)](cpsr); + return true; +} + +static void __kprobes thumb16_singlestep(probes_opcode_t opcode, + struct arch_probes_insn *asi, + struct pt_regs *regs) +{ + regs->ARM_pc += 2; + asi->insn_handler(opcode, asi, regs); + regs->ARM_cpsr = it_advance(regs->ARM_cpsr); +} + +static void __kprobes thumb32_singlestep(probes_opcode_t opcode, + struct arch_probes_insn *asi, + struct pt_regs *regs) +{ + regs->ARM_pc += 4; + asi->insn_handler(opcode, asi, regs); + regs->ARM_cpsr = it_advance(regs->ARM_cpsr); +} + +enum probes_insn __kprobes +thumb16_probes_decode_insn(probes_opcode_t insn, struct arch_probes_insn *asi, + bool emulate, const union decode_action *actions) +{ + asi->insn_singlestep = thumb16_singlestep; + asi->insn_check_cc = thumb_check_cc; + return probes_decode_insn(insn, asi, probes_decode_thumb16_table, true, + emulate, actions); +} + +enum probes_insn __kprobes +thumb32_probes_decode_insn(probes_opcode_t insn, struct arch_probes_insn *asi, + bool emulate, const union decode_action *actions) +{ + asi->insn_singlestep = thumb32_singlestep; + asi->insn_check_cc = thumb_check_cc; + return probes_decode_insn(insn, asi, probes_decode_thumb32_table, true, + emulate, actions); +} diff --git a/arch/arm/kernel/probes-thumb.h b/arch/arm/kernel/probes-thumb.h new file mode 100644 index 000000000000..7c6f6ebe514f --- /dev/null +++ b/arch/arm/kernel/probes-thumb.h @@ -0,0 +1,97 @@ +/* + * arch/arm/kernel/probes-thumb.h + * + * Copyright 2013 Linaro Ltd. + * Written by: David A. Long + * + * The code contained herein is licensed under the GNU General Public + * License. You may obtain a copy of the GNU General Public License + * Version 2 or later at the following locations: + * + * http://www.opensource.org/licenses/gpl-license.html + * http://www.gnu.org/copyleft/gpl.html + */ + +#ifndef _ARM_KERNEL_PROBES_THUMB_H +#define _ARM_KERNEL_PROBES_THUMB_H + +/* + * True if current instruction is in an IT block. + */ +#define in_it_block(cpsr) ((cpsr & 0x06000c00) != 0x00000000) + +/* + * Return the condition code to check for the currently executing instruction. + * This is in ITSTATE<7:4> which is in CPSR<15:12> but is only valid if + * in_it_block returns true. + */ +#define current_cond(cpsr) ((cpsr >> 12) & 0xf) + +enum probes_t32_action { + PROBES_T32_EMULATE_NONE, + PROBES_T32_SIMULATE_NOP, + PROBES_T32_LDMSTM, + PROBES_T32_LDRDSTRD, + PROBES_T32_TABLE_BRANCH, + PROBES_T32_TST, + PROBES_T32_CMP, + PROBES_T32_MOV, + PROBES_T32_ADDSUB, + PROBES_T32_LOGICAL, + PROBES_T32_ADDWSUBW_PC, + PROBES_T32_ADDWSUBW, + PROBES_T32_MOVW, + PROBES_T32_SAT, + PROBES_T32_BITFIELD, + PROBES_T32_SEV, + PROBES_T32_WFE, + PROBES_T32_MRS, + PROBES_T32_BRANCH_COND, + PROBES_T32_BRANCH, + PROBES_T32_PLDI, + PROBES_T32_LDR_LIT, + PROBES_T32_LDRSTR, + PROBES_T32_SIGN_EXTEND, + PROBES_T32_MEDIA, + PROBES_T32_REVERSE, + PROBES_T32_MUL_ADD, + PROBES_T32_MUL_ADD2, + PROBES_T32_MUL_ADD_LONG, + NUM_PROBES_T32_ACTIONS +}; + +enum probes_t16_action { + PROBES_T16_ADD_SP, + PROBES_T16_CBZ, + PROBES_T16_SIGN_EXTEND, + PROBES_T16_PUSH, + PROBES_T16_POP, + PROBES_T16_SEV, + PROBES_T16_WFE, + PROBES_T16_IT, + PROBES_T16_CMP, + PROBES_T16_ADDSUB, + PROBES_T16_LOGICAL, + PROBES_T16_BLX, + PROBES_T16_HIREGOPS, + PROBES_T16_LDR_LIT, + PROBES_T16_LDRHSTRH, + PROBES_T16_LDRSTR, + PROBES_T16_ADR, + PROBES_T16_LDMSTM, + PROBES_T16_BRANCH_COND, + PROBES_T16_BRANCH, + NUM_PROBES_T16_ACTIONS +}; + +extern const union decode_item probes_decode_thumb32_table[]; +extern const union decode_item probes_decode_thumb16_table[]; + +enum probes_insn __kprobes +thumb16_probes_decode_insn(probes_opcode_t insn, struct arch_probes_insn *asi, + bool emulate, const union decode_action *actions); +enum probes_insn __kprobes +thumb32_probes_decode_insn(probes_opcode_t insn, struct arch_probes_insn *asi, + bool emulate, const union decode_action *actions); + +#endif diff --git a/arch/arm/kernel/probes.c b/arch/arm/kernel/probes.c new file mode 100644 index 000000000000..b41873f33e69 --- /dev/null +++ b/arch/arm/kernel/probes.c @@ -0,0 +1,455 @@ +/* + * arch/arm/kernel/probes.c + * + * Copyright (C) 2011 Jon Medhurst <tixy@yxit.co.uk>. + * + * Some contents moved here from arch/arm/include/asm/kprobes-arm.c which is + * Copyright (C) 2006, 2007 Motorola Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/kernel.h> +#include <linux/types.h> +#include <asm/system_info.h> +#include <asm/ptrace.h> +#include <linux/bug.h> + +#include "probes.h" + + +#ifndef find_str_pc_offset + +/* + * For STR and STM instructions, an ARM core may choose to use either + * a +8 or a +12 displacement from the current instruction's address. + * Whichever value is chosen for a given core, it must be the same for + * both instructions and may not change. This function measures it. + */ + +int str_pc_offset; + +void __init find_str_pc_offset(void) +{ + int addr, scratch, ret; + + __asm__ ( + "sub %[ret], pc, #4 \n\t" + "str pc, %[addr] \n\t" + "ldr %[scr], %[addr] \n\t" + "sub %[ret], %[scr], %[ret] \n\t" + : [ret] "=r" (ret), [scr] "=r" (scratch), [addr] "+m" (addr)); + + str_pc_offset = ret; +} + +#endif /* !find_str_pc_offset */ + + +#ifndef test_load_write_pc_interworking + +bool load_write_pc_interworks; + +void __init test_load_write_pc_interworking(void) +{ + int arch = cpu_architecture(); + BUG_ON(arch == CPU_ARCH_UNKNOWN); + load_write_pc_interworks = arch >= CPU_ARCH_ARMv5T; +} + +#endif /* !test_load_write_pc_interworking */ + + +#ifndef test_alu_write_pc_interworking + +bool alu_write_pc_interworks; + +void __init test_alu_write_pc_interworking(void) +{ + int arch = cpu_architecture(); + BUG_ON(arch == CPU_ARCH_UNKNOWN); + alu_write_pc_interworks = arch >= CPU_ARCH_ARMv7; +} + +#endif /* !test_alu_write_pc_interworking */ + + +void __init arm_probes_decode_init(void) +{ + find_str_pc_offset(); + test_load_write_pc_interworking(); + test_alu_write_pc_interworking(); +} + + +static unsigned long __kprobes __check_eq(unsigned long cpsr) +{ + return cpsr & PSR_Z_BIT; +} + +static unsigned long __kprobes __check_ne(unsigned long cpsr) +{ + return (~cpsr) & PSR_Z_BIT; +} + +static unsigned long __kprobes __check_cs(unsigned long cpsr) +{ + return cpsr & PSR_C_BIT; +} + +static unsigned long __kprobes __check_cc(unsigned long cpsr) +{ + return (~cpsr) & PSR_C_BIT; +} + +static unsigned long __kprobes __check_mi(unsigned long cpsr) +{ + return cpsr & PSR_N_BIT; +} + +static unsigned long __kprobes __check_pl(unsigned long cpsr) +{ + return (~cpsr) & PSR_N_BIT; +} + +static unsigned long __kprobes __check_vs(unsigned long cpsr) +{ + return cpsr & PSR_V_BIT; +} + +static unsigned long __kprobes __check_vc(unsigned long cpsr) +{ + return (~cpsr) & PSR_V_BIT; +} + +static unsigned long __kprobes __check_hi(unsigned long cpsr) +{ + cpsr &= ~(cpsr >> 1); /* PSR_C_BIT &= ~PSR_Z_BIT */ + return cpsr & PSR_C_BIT; +} + +static unsigned long __kprobes __check_ls(unsigned long cpsr) +{ + cpsr &= ~(cpsr >> 1); /* PSR_C_BIT &= ~PSR_Z_BIT */ + return (~cpsr) & PSR_C_BIT; +} + +static unsigned long __kprobes __check_ge(unsigned long cpsr) +{ + cpsr ^= (cpsr << 3); /* PSR_N_BIT ^= PSR_V_BIT */ + return (~cpsr) & PSR_N_BIT; +} + +static unsigned long __kprobes __check_lt(unsigned long cpsr) +{ + cpsr ^= (cpsr << 3); /* PSR_N_BIT ^= PSR_V_BIT */ + return cpsr & PSR_N_BIT; +} + +static unsigned long __kprobes __check_gt(unsigned long cpsr) +{ + unsigned long temp = cpsr ^ (cpsr << 3); /* PSR_N_BIT ^= PSR_V_BIT */ + temp |= (cpsr << 1); /* PSR_N_BIT |= PSR_Z_BIT */ + return (~temp) & PSR_N_BIT; +} + +static unsigned long __kprobes __check_le(unsigned long cpsr) +{ + unsigned long temp = cpsr ^ (cpsr << 3); /* PSR_N_BIT ^= PSR_V_BIT */ + temp |= (cpsr << 1); /* PSR_N_BIT |= PSR_Z_BIT */ + return temp & PSR_N_BIT; +} + +static unsigned long __kprobes __check_al(unsigned long cpsr) +{ + return true; +} + +probes_check_cc * const probes_condition_checks[16] = { + &__check_eq, &__check_ne, &__check_cs, &__check_cc, + &__check_mi, &__check_pl, &__check_vs, &__check_vc, + &__check_hi, &__check_ls, &__check_ge, &__check_lt, + &__check_gt, &__check_le, &__check_al, &__check_al +}; + + +void __kprobes probes_simulate_nop(probes_opcode_t opcode, + struct arch_probes_insn *asi, + struct pt_regs *regs) +{ +} + +void __kprobes probes_emulate_none(probes_opcode_t opcode, + struct arch_probes_insn *asi, + struct pt_regs *regs) +{ + asi->insn_fn(); +} + +/* + * Prepare an instruction slot to receive an instruction for emulating. + * This is done by placing a subroutine return after the location where the + * instruction will be placed. We also modify ARM instructions to be + * unconditional as the condition code will already be checked before any + * emulation handler is called. + */ +static probes_opcode_t __kprobes +prepare_emulated_insn(probes_opcode_t insn, struct arch_probes_insn *asi, + bool thumb) +{ +#ifdef CONFIG_THUMB2_KERNEL + if (thumb) { + u16 *thumb_insn = (u16 *)asi->insn; + thumb_insn[1] = 0x4770; /* Thumb bx lr */ + thumb_insn[2] = 0x4770; /* Thumb bx lr */ + return insn; + } + asi->insn[1] = 0xe12fff1e; /* ARM bx lr */ +#else + asi->insn[1] = 0xe1a0f00e; /* mov pc, lr */ +#endif + /* Make an ARM instruction unconditional */ + if (insn < 0xe0000000) + insn = (insn | 0xe0000000) & ~0x10000000; + return insn; +} + +/* + * Write a (probably modified) instruction into the slot previously prepared by + * prepare_emulated_insn + */ +static void __kprobes +set_emulated_insn(probes_opcode_t insn, struct arch_probes_insn *asi, + bool thumb) +{ +#ifdef CONFIG_THUMB2_KERNEL + if (thumb) { + u16 *ip = (u16 *)asi->insn; + if (is_wide_instruction(insn)) + *ip++ = insn >> 16; + *ip++ = insn; + return; + } +#endif + asi->insn[0] = insn; +} + +/* + * When we modify the register numbers encoded in an instruction to be emulated, + * the new values come from this define. For ARM and 32-bit Thumb instructions + * this gives... + * + * bit position 16 12 8 4 0 + * ---------------+---+---+---+---+---+ + * register r2 r0 r1 -- r3 + */ +#define INSN_NEW_BITS 0x00020103 + +/* Each nibble has same value as that at INSN_NEW_BITS bit 16 */ +#define INSN_SAMEAS16_BITS 0x22222222 + +/* + * Validate and modify each of the registers encoded in an instruction. + * + * Each nibble in regs contains a value from enum decode_reg_type. For each + * non-zero value, the corresponding nibble in pinsn is validated and modified + * according to the type. + */ +static bool __kprobes decode_regs(probes_opcode_t *pinsn, u32 regs, bool modify) +{ + probes_opcode_t insn = *pinsn; + probes_opcode_t mask = 0xf; /* Start at least significant nibble */ + + for (; regs != 0; regs >>= 4, mask <<= 4) { + + probes_opcode_t new_bits = INSN_NEW_BITS; + + switch (regs & 0xf) { + + case REG_TYPE_NONE: + /* Nibble not a register, skip to next */ + continue; + + case REG_TYPE_ANY: + /* Any register is allowed */ + break; + + case REG_TYPE_SAMEAS16: + /* Replace register with same as at bit position 16 */ + new_bits = INSN_SAMEAS16_BITS; + break; + + case REG_TYPE_SP: + /* Only allow SP (R13) */ + if ((insn ^ 0xdddddddd) & mask) + goto reject; + break; + + case REG_TYPE_PC: + /* Only allow PC (R15) */ + if ((insn ^ 0xffffffff) & mask) + goto reject; + break; + + case REG_TYPE_NOSP: + /* Reject SP (R13) */ + if (((insn ^ 0xdddddddd) & mask) == 0) + goto reject; + break; + + case REG_TYPE_NOSPPC: + case REG_TYPE_NOSPPCX: + /* Reject SP and PC (R13 and R15) */ + if (((insn ^ 0xdddddddd) & 0xdddddddd & mask) == 0) + goto reject; + break; + + case REG_TYPE_NOPCWB: + if (!is_writeback(insn)) + break; /* No writeback, so any register is OK */ + /* fall through... */ + case REG_TYPE_NOPC: + case REG_TYPE_NOPCX: + /* Reject PC (R15) */ + if (((insn ^ 0xffffffff) & mask) == 0) + goto reject; + break; + } + + /* Replace value of nibble with new register number... */ + insn &= ~mask; + insn |= new_bits & mask; + } + + if (modify) + *pinsn = insn; + + return true; + +reject: + return false; +} + +static const int decode_struct_sizes[NUM_DECODE_TYPES] = { + [DECODE_TYPE_TABLE] = sizeof(struct decode_table), + [DECODE_TYPE_CUSTOM] = sizeof(struct decode_custom), + [DECODE_TYPE_SIMULATE] = sizeof(struct decode_simulate), + [DECODE_TYPE_EMULATE] = sizeof(struct decode_emulate), + [DECODE_TYPE_OR] = sizeof(struct decode_or), + [DECODE_TYPE_REJECT] = sizeof(struct decode_reject) +}; + +/* + * probes_decode_insn operates on data tables in order to decode an ARM + * architecture instruction onto which a kprobe has been placed. + * + * These instruction decoding tables are a concatenation of entries each + * of which consist of one of the following structs: + * + * decode_table + * decode_custom + * decode_simulate + * decode_emulate + * decode_or + * decode_reject + * + * Each of these starts with a struct decode_header which has the following + * fields: + * + * type_regs + * mask + * value + * + * The least significant DECODE_TYPE_BITS of type_regs contains a value + * from enum decode_type, this indicates which of the decode_* structs + * the entry contains. The value DECODE_TYPE_END indicates the end of the + * table. + * + * When the table is parsed, each entry is checked in turn to see if it + * matches the instruction to be decoded using the test: + * + * (insn & mask) == value + * + * If no match is found before the end of the table is reached then decoding + * fails with INSN_REJECTED. + * + * When a match is found, decode_regs() is called to validate and modify each + * of the registers encoded in the instruction; the data it uses to do this + * is (type_regs >> DECODE_TYPE_BITS). A validation failure will cause decoding + * to fail with INSN_REJECTED. + * + * Once the instruction has passed the above tests, further processing + * depends on the type of the table entry's decode struct. + * + */ +int __kprobes +probes_decode_insn(probes_opcode_t insn, struct arch_probes_insn *asi, + const union decode_item *table, bool thumb, + bool emulate, const union decode_action *actions) +{ + const struct decode_header *h = (struct decode_header *)table; + const struct decode_header *next; + bool matched = false; + + if (emulate) + insn = prepare_emulated_insn(insn, asi, thumb); + + for (;; h = next) { + enum decode_type type = h->type_regs.bits & DECODE_TYPE_MASK; + u32 regs = h->type_regs.bits >> DECODE_TYPE_BITS; + + if (type == DECODE_TYPE_END) + return INSN_REJECTED; + + next = (struct decode_header *) + ((uintptr_t)h + decode_struct_sizes[type]); + + if (!matched && (insn & h->mask.bits) != h->value.bits) + continue; + + if (!decode_regs(&insn, regs, emulate)) + return INSN_REJECTED; + + switch (type) { + + case DECODE_TYPE_TABLE: { + struct decode_table *d = (struct decode_table *)h; + next = (struct decode_header *)d->table.table; + break; + } + + case DECODE_TYPE_CUSTOM: { + struct decode_custom *d = (struct decode_custom *)h; + return actions[d->decoder.action].decoder(insn, asi, h); + } + + case DECODE_TYPE_SIMULATE: { + struct decode_simulate *d = (struct decode_simulate *)h; + asi->insn_handler = actions[d->handler.action].handler; + return INSN_GOOD_NO_SLOT; + } + + case DECODE_TYPE_EMULATE: { + struct decode_emulate *d = (struct decode_emulate *)h; + + if (!emulate) + return actions[d->handler.action].decoder(insn, + asi, h); + + asi->insn_handler = actions[d->handler.action].handler; + set_emulated_insn(insn, asi, thumb); + return INSN_GOOD; + } + + case DECODE_TYPE_OR: + matched = true; + break; + + case DECODE_TYPE_REJECT: + default: + return INSN_REJECTED; + } + } +} diff --git a/arch/arm/kernel/probes.h b/arch/arm/kernel/probes.h new file mode 100644 index 000000000000..dba9f2466a93 --- /dev/null +++ b/arch/arm/kernel/probes.h @@ -0,0 +1,407 @@ +/* + * arch/arm/kernel/probes.h + * + * Copyright (C) 2011 Jon Medhurst <tixy@yxit.co.uk>. + * + * Some contents moved here from arch/arm/include/asm/kprobes.h which is + * Copyright (C) 2006, 2007 Motorola Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ + +#ifndef _ARM_KERNEL_PROBES_H +#define _ARM_KERNEL_PROBES_H + +#include <linux/types.h> +#include <linux/stddef.h> +#include <asm/probes.h> + +void __init arm_probes_decode_init(void); + +extern probes_check_cc * const probes_condition_checks[16]; + +#if __LINUX_ARM_ARCH__ >= 7 + +/* str_pc_offset is architecturally defined from ARMv7 onwards */ +#define str_pc_offset 8 +#define find_str_pc_offset() + +#else /* __LINUX_ARM_ARCH__ < 7 */ + +/* We need a run-time check to determine str_pc_offset */ +extern int str_pc_offset; +void __init find_str_pc_offset(void); + +#endif + + +/* + * Update ITSTATE after normal execution of an IT block instruction. + * + * The 8 IT state bits are split into two parts in CPSR: + * ITSTATE<1:0> are in CPSR<26:25> + * ITSTATE<7:2> are in CPSR<15:10> + */ +static inline unsigned long it_advance(unsigned long cpsr) + { + if ((cpsr & 0x06000400) == 0) { + /* ITSTATE<2:0> == 0 means end of IT block, so clear IT state */ + cpsr &= ~PSR_IT_MASK; + } else { + /* We need to shift left ITSTATE<4:0> */ + const unsigned long mask = 0x06001c00; /* Mask ITSTATE<4:0> */ + unsigned long it = cpsr & mask; + it <<= 1; + it |= it >> (27 - 10); /* Carry ITSTATE<2> to correct place */ + it &= mask; + cpsr &= ~mask; + cpsr |= it; + } + return cpsr; +} + +static inline void __kprobes bx_write_pc(long pcv, struct pt_regs *regs) +{ + long cpsr = regs->ARM_cpsr; + if (pcv & 0x1) { + cpsr |= PSR_T_BIT; + pcv &= ~0x1; + } else { + cpsr &= ~PSR_T_BIT; + pcv &= ~0x2; /* Avoid UNPREDICTABLE address allignment */ + } + regs->ARM_cpsr = cpsr; + regs->ARM_pc = pcv; +} + + +#if __LINUX_ARM_ARCH__ >= 6 + +/* Kernels built for >= ARMv6 should never run on <= ARMv5 hardware, so... */ +#define load_write_pc_interworks true +#define test_load_write_pc_interworking() + +#else /* __LINUX_ARM_ARCH__ < 6 */ + +/* We need run-time testing to determine if load_write_pc() should interwork. */ +extern bool load_write_pc_interworks; +void __init test_load_write_pc_interworking(void); + +#endif + +static inline void __kprobes load_write_pc(long pcv, struct pt_regs *regs) +{ + if (load_write_pc_interworks) + bx_write_pc(pcv, regs); + else + regs->ARM_pc = pcv; +} + + +#if __LINUX_ARM_ARCH__ >= 7 + +#define alu_write_pc_interworks true +#define test_alu_write_pc_interworking() + +#elif __LINUX_ARM_ARCH__ <= 5 + +/* Kernels built for <= ARMv5 should never run on >= ARMv6 hardware, so... */ +#define alu_write_pc_interworks false +#define test_alu_write_pc_interworking() + +#else /* __LINUX_ARM_ARCH__ == 6 */ + +/* We could be an ARMv6 binary on ARMv7 hardware so we need a run-time check. */ +extern bool alu_write_pc_interworks; +void __init test_alu_write_pc_interworking(void); + +#endif /* __LINUX_ARM_ARCH__ == 6 */ + +static inline void __kprobes alu_write_pc(long pcv, struct pt_regs *regs) +{ + if (alu_write_pc_interworks) + bx_write_pc(pcv, regs); + else + regs->ARM_pc = pcv; +} + + +/* + * Test if load/store instructions writeback the address register. + * if P (bit 24) == 0 or W (bit 21) == 1 + */ +#define is_writeback(insn) ((insn ^ 0x01000000) & 0x01200000) + +/* + * The following definitions and macros are used to build instruction + * decoding tables for use by probes_decode_insn. + * + * These tables are a concatenation of entries each of which consist of one of + * the decode_* structs. All of the fields in every type of decode structure + * are of the union type decode_item, therefore the entire decode table can be + * viewed as an array of these and declared like: + * + * static const union decode_item table_name[] = {}; + * + * In order to construct each entry in the table, macros are used to + * initialise a number of sequential decode_item values in a layout which + * matches the relevant struct. E.g. DECODE_SIMULATE initialise a struct + * decode_simulate by initialising four decode_item objects like this... + * + * {.bits = _type}, + * {.bits = _mask}, + * {.bits = _value}, + * {.action = _handler}, + * + * Initialising a specified member of the union means that the compiler + * will produce a warning if the argument is of an incorrect type. + * + * Below is a list of each of the macros used to initialise entries and a + * description of the action performed when that entry is matched to an + * instruction. A match is found when (instruction & mask) == value. + * + * DECODE_TABLE(mask, value, table) + * Instruction decoding jumps to parsing the new sub-table 'table'. + * + * DECODE_CUSTOM(mask, value, decoder) + * The value of 'decoder' is used as an index into the array of + * action functions, and the retrieved decoder function is invoked + * to complete decoding of the instruction. + * + * DECODE_SIMULATE(mask, value, handler) + * The probes instruction handler is set to the value found by + * indexing into the action array using the value of 'handler'. This + * will be used to simulate the instruction when the probe is hit. + * Decoding returns with INSN_GOOD_NO_SLOT. + * + * DECODE_EMULATE(mask, value, handler) + * The probes instruction handler is set to the value found by + * indexing into the action array using the value of 'handler'. This + * will be used to emulate the instruction when the probe is hit. The + * modified instruction (see below) is placed in the probes instruction + * slot so it may be called by the emulation code. Decoding returns + * with INSN_GOOD. + * + * DECODE_REJECT(mask, value) + * Instruction decoding fails with INSN_REJECTED + * + * DECODE_OR(mask, value) + * This allows the mask/value test of multiple table entries to be + * logically ORed. Once an 'or' entry is matched the decoding action to + * be performed is that of the next entry which isn't an 'or'. E.g. + * + * DECODE_OR (mask1, value1) + * DECODE_OR (mask2, value2) + * DECODE_SIMULATE (mask3, value3, simulation_handler) + * + * This means that if any of the three mask/value pairs match the + * instruction being decoded, then 'simulation_handler' will be used + * for it. + * + * Both the SIMULATE and EMULATE macros have a second form which take an + * additional 'regs' argument. + * + * DECODE_SIMULATEX(mask, value, handler, regs) + * DECODE_EMULATEX (mask, value, handler, regs) + * + * These are used to specify what kind of CPU register is encoded in each of the + * least significant 5 nibbles of the instruction being decoded. The regs value + * is specified using the REGS macro, this takes any of the REG_TYPE_* values + * from enum decode_reg_type as arguments; only the '*' part of the name is + * given. E.g. + * + * REGS(0, ANY, NOPC, 0, ANY) + * + * This indicates an instruction is encoded like: + * + * bits 19..16 ignore + * bits 15..12 any register allowed here + * bits 11.. 8 any register except PC allowed here + * bits 7.. 4 ignore + * bits 3.. 0 any register allowed here + * + * This register specification is checked after a decode table entry is found to + * match an instruction (through the mask/value test). Any invalid register then + * found in the instruction will cause decoding to fail with INSN_REJECTED. In + * the above example this would happen if bits 11..8 of the instruction were + * 1111, indicating R15 or PC. + * + * As well as checking for legal combinations of registers, this data is also + * used to modify the registers encoded in the instructions so that an + * emulation routines can use it. (See decode_regs() and INSN_NEW_BITS.) + * + * Here is a real example which matches ARM instructions of the form + * "AND <Rd>,<Rn>,<Rm>,<shift> <Rs>" + * + * DECODE_EMULATEX (0x0e000090, 0x00000010, PROBES_DATA_PROCESSING_REG, + * REGS(ANY, ANY, NOPC, 0, ANY)), + * ^ ^ ^ ^ + * Rn Rd Rs Rm + * + * Decoding the instruction "AND R4, R5, R6, ASL R15" will be rejected because + * Rs == R15 + * + * Decoding the instruction "AND R4, R5, R6, ASL R7" will be accepted and the + * instruction will be modified to "AND R0, R2, R3, ASL R1" and then placed into + * the kprobes instruction slot. This can then be called later by the handler + * function emulate_rd12rn16rm0rs8_rwflags (a pointer to which is retrieved from + * the indicated slot in the action array), in order to simulate the instruction. + */ + +enum decode_type { + DECODE_TYPE_END, + DECODE_TYPE_TABLE, + DECODE_TYPE_CUSTOM, + DECODE_TYPE_SIMULATE, + DECODE_TYPE_EMULATE, + DECODE_TYPE_OR, + DECODE_TYPE_REJECT, + NUM_DECODE_TYPES /* Must be last enum */ +}; + +#define DECODE_TYPE_BITS 4 +#define DECODE_TYPE_MASK ((1 << DECODE_TYPE_BITS) - 1) + +enum decode_reg_type { + REG_TYPE_NONE = 0, /* Not a register, ignore */ + REG_TYPE_ANY, /* Any register allowed */ + REG_TYPE_SAMEAS16, /* Register should be same as that at bits 19..16 */ + REG_TYPE_SP, /* Register must be SP */ + REG_TYPE_PC, /* Register must be PC */ + REG_TYPE_NOSP, /* Register must not be SP */ + REG_TYPE_NOSPPC, /* Register must not be SP or PC */ + REG_TYPE_NOPC, /* Register must not be PC */ + REG_TYPE_NOPCWB, /* No PC if load/store write-back flag also set */ + + /* The following types are used when the encoding for PC indicates + * another instruction form. This distiction only matters for test + * case coverage checks. + */ + REG_TYPE_NOPCX, /* Register must not be PC */ + REG_TYPE_NOSPPCX, /* Register must not be SP or PC */ + + /* Alias to allow '0' arg to be used in REGS macro. */ + REG_TYPE_0 = REG_TYPE_NONE +}; + +#define REGS(r16, r12, r8, r4, r0) \ + (((REG_TYPE_##r16) << 16) + \ + ((REG_TYPE_##r12) << 12) + \ + ((REG_TYPE_##r8) << 8) + \ + ((REG_TYPE_##r4) << 4) + \ + (REG_TYPE_##r0)) + +union decode_item { + u32 bits; + const union decode_item *table; + int action; +}; + +struct decode_header; +typedef enum probes_insn (probes_custom_decode_t)(probes_opcode_t, + struct arch_probes_insn *, + const struct decode_header *); + +union decode_action { + probes_insn_handler_t *handler; + probes_custom_decode_t *decoder; +}; + +#define DECODE_END \ + {.bits = DECODE_TYPE_END} + + +struct decode_header { + union decode_item type_regs; + union decode_item mask; + union decode_item value; +}; + +#define DECODE_HEADER(_type, _mask, _value, _regs) \ + {.bits = (_type) | ((_regs) << DECODE_TYPE_BITS)}, \ + {.bits = (_mask)}, \ + {.bits = (_value)} + + +struct decode_table { + struct decode_header header; + union decode_item table; +}; + +#define DECODE_TABLE(_mask, _value, _table) \ + DECODE_HEADER(DECODE_TYPE_TABLE, _mask, _value, 0), \ + {.table = (_table)} + + +struct decode_custom { + struct decode_header header; + union decode_item decoder; +}; + +#define DECODE_CUSTOM(_mask, _value, _decoder) \ + DECODE_HEADER(DECODE_TYPE_CUSTOM, _mask, _value, 0), \ + {.action = (_decoder)} + + +struct decode_simulate { + struct decode_header header; + union decode_item handler; +}; + +#define DECODE_SIMULATEX(_mask, _value, _handler, _regs) \ + DECODE_HEADER(DECODE_TYPE_SIMULATE, _mask, _value, _regs), \ + {.action = (_handler)} + +#define DECODE_SIMULATE(_mask, _value, _handler) \ + DECODE_SIMULATEX(_mask, _value, _handler, 0) + + +struct decode_emulate { + struct decode_header header; + union decode_item handler; +}; + +#define DECODE_EMULATEX(_mask, _value, _handler, _regs) \ + DECODE_HEADER(DECODE_TYPE_EMULATE, _mask, _value, _regs), \ + {.action = (_handler)} + +#define DECODE_EMULATE(_mask, _value, _handler) \ + DECODE_EMULATEX(_mask, _value, _handler, 0) + + +struct decode_or { + struct decode_header header; +}; + +#define DECODE_OR(_mask, _value) \ + DECODE_HEADER(DECODE_TYPE_OR, _mask, _value, 0) + +enum probes_insn { + INSN_REJECTED, + INSN_GOOD, + INSN_GOOD_NO_SLOT +}; + +struct decode_reject { + struct decode_header header; +}; + +#define DECODE_REJECT(_mask, _value) \ + DECODE_HEADER(DECODE_TYPE_REJECT, _mask, _value, 0) + +probes_insn_handler_t probes_simulate_nop; +probes_insn_handler_t probes_emulate_none; + +int __kprobes +probes_decode_insn(probes_opcode_t insn, struct arch_probes_insn *asi, + const union decode_item *table, bool thumb, bool emulate, + const union decode_action *actions); + +#endif diff --git a/arch/arm/kernel/process.c b/arch/arm/kernel/process.c index adabeababeb0..806d287e3e53 100644 --- a/arch/arm/kernel/process.c +++ b/arch/arm/kernel/process.c @@ -47,14 +47,14 @@ unsigned long __stack_chk_guard __read_mostly; EXPORT_SYMBOL(__stack_chk_guard); #endif -static const char *processor_modes[] = { +static const char *processor_modes[] __maybe_unused = { "USER_26", "FIQ_26" , "IRQ_26" , "SVC_26" , "UK4_26" , "UK5_26" , "UK6_26" , "UK7_26" , "UK8_26" , "UK9_26" , "UK10_26", "UK11_26", "UK12_26", "UK13_26", "UK14_26", "UK15_26", "USER_32", "FIQ_32" , "IRQ_32" , "SVC_32" , "UK4_32" , "UK5_32" , "UK6_32" , "ABT_32" , "UK8_32" , "UK9_32" , "UK10_32", "UND_32" , "UK12_32", "UK13_32", "UK14_32", "SYS_32" }; -static const char *isa_modes[] = { +static const char *isa_modes[] __maybe_unused = { "ARM" , "Thumb" , "Jazelle", "ThumbEE" }; @@ -270,12 +270,17 @@ void __show_regs(struct pt_regs *regs) buf[3] = flags & PSR_V_BIT ? 'V' : 'v'; buf[4] = '\0'; +#ifndef CONFIG_CPU_V7M printk("Flags: %s IRQs o%s FIQs o%s Mode %s ISA %s Segment %s\n", buf, interrupts_enabled(regs) ? "n" : "ff", fast_interrupts_enabled(regs) ? "n" : "ff", processor_modes[processor_mode(regs)], isa_modes[isa_mode(regs)], get_fs() == get_ds() ? "kernel" : "user"); +#else + printk("xPSR: %08lx\n", regs->ARM_cpsr); +#endif + #ifdef CONFIG_CPU_CP15 { unsigned int ctrl; diff --git a/arch/arm/kernel/setup.c b/arch/arm/kernel/setup.c index 1e8b030dbefd..50e198c1e9c8 100644 --- a/arch/arm/kernel/setup.c +++ b/arch/arm/kernel/setup.c @@ -100,6 +100,9 @@ EXPORT_SYMBOL(system_serial_high); unsigned int elf_hwcap __read_mostly; EXPORT_SYMBOL(elf_hwcap); +unsigned int elf_hwcap2 __read_mostly; +EXPORT_SYMBOL(elf_hwcap2); + #ifdef MULTI_CPU struct processor processor __read_mostly; @@ -1005,6 +1008,15 @@ static const char *hwcap_str[] = { NULL }; +static const char *hwcap2_str[] = { + "aes", + "pmull", + "sha1", + "sha2", + "crc32", + NULL +}; + static int c_show(struct seq_file *m, void *v) { int i, j; @@ -1028,6 +1040,10 @@ static int c_show(struct seq_file *m, void *v) if (elf_hwcap & (1 << j)) seq_printf(m, "%s ", hwcap_str[j]); + for (j = 0; hwcap2_str[j]; j++) + if (elf_hwcap2 & (1 << j)) + seq_printf(m, "%s ", hwcap2_str[j]); + seq_printf(m, "\nCPU implementer\t: 0x%02x\n", cpuid >> 24); seq_printf(m, "CPU architecture: %s\n", proc_arch[cpu_architecture()]); diff --git a/arch/arm/kernel/signal.c b/arch/arm/kernel/signal.c index 04d63880037f..bd1983437205 100644 --- a/arch/arm/kernel/signal.c +++ b/arch/arm/kernel/signal.c @@ -13,6 +13,7 @@ #include <linux/personality.h> #include <linux/uaccess.h> #include <linux/tracehook.h> +#include <linux/uprobes.h> #include <asm/elf.h> #include <asm/cacheflush.h> @@ -590,6 +591,9 @@ do_work_pending(struct pt_regs *regs, unsigned int thread_flags, int syscall) return restart; } syscall = 0; + } else if (thread_flags & _TIF_UPROBE) { + clear_thread_flag(TIF_UPROBE); + uprobe_notify_resume(regs); } else { clear_thread_flag(TIF_NOTIFY_RESUME); tracehook_notify_resume(regs); diff --git a/arch/arm/kernel/unwind.c b/arch/arm/kernel/unwind.c index 00df012c4678..3c217694ebec 100644 --- a/arch/arm/kernel/unwind.c +++ b/arch/arm/kernel/unwind.c @@ -68,6 +68,12 @@ EXPORT_SYMBOL(__aeabi_unwind_cpp_pr2); struct unwind_ctrl_block { unsigned long vrs[16]; /* virtual register set */ const unsigned long *insn; /* pointer to the current instructions word */ + unsigned long sp_high; /* highest value of sp allowed */ + /* + * 1 : check for stack overflow for each register pop. + * 0 : save overhead if there is plenty of stack remaining. + */ + int check_each_pop; int entries; /* number of entries left to interpret */ int byte; /* current byte number in the instructions word */ }; @@ -235,12 +241,85 @@ static unsigned long unwind_get_byte(struct unwind_ctrl_block *ctrl) return ret; } +/* Before poping a register check whether it is feasible or not */ +static int unwind_pop_register(struct unwind_ctrl_block *ctrl, + unsigned long **vsp, unsigned int reg) +{ + if (unlikely(ctrl->check_each_pop)) + if (*vsp >= (unsigned long *)ctrl->sp_high) + return -URC_FAILURE; + + ctrl->vrs[reg] = *(*vsp)++; + return URC_OK; +} + +/* Helper functions to execute the instructions */ +static int unwind_exec_pop_subset_r4_to_r13(struct unwind_ctrl_block *ctrl, + unsigned long mask) +{ + unsigned long *vsp = (unsigned long *)ctrl->vrs[SP]; + int load_sp, reg = 4; + + load_sp = mask & (1 << (13 - 4)); + while (mask) { + if (mask & 1) + if (unwind_pop_register(ctrl, &vsp, reg)) + return -URC_FAILURE; + mask >>= 1; + reg++; + } + if (!load_sp) + ctrl->vrs[SP] = (unsigned long)vsp; + + return URC_OK; +} + +static int unwind_exec_pop_r4_to_rN(struct unwind_ctrl_block *ctrl, + unsigned long insn) +{ + unsigned long *vsp = (unsigned long *)ctrl->vrs[SP]; + int reg; + + /* pop R4-R[4+bbb] */ + for (reg = 4; reg <= 4 + (insn & 7); reg++) + if (unwind_pop_register(ctrl, &vsp, reg)) + return -URC_FAILURE; + + if (insn & 0x80) + if (unwind_pop_register(ctrl, &vsp, 14)) + return -URC_FAILURE; + + ctrl->vrs[SP] = (unsigned long)vsp; + + return URC_OK; +} + +static int unwind_exec_pop_subset_r0_to_r3(struct unwind_ctrl_block *ctrl, + unsigned long mask) +{ + unsigned long *vsp = (unsigned long *)ctrl->vrs[SP]; + int reg = 0; + + /* pop R0-R3 according to mask */ + while (mask) { + if (mask & 1) + if (unwind_pop_register(ctrl, &vsp, reg)) + return -URC_FAILURE; + mask >>= 1; + reg++; + } + ctrl->vrs[SP] = (unsigned long)vsp; + + return URC_OK; +} + /* * Execute the current unwind instruction. */ static int unwind_exec_insn(struct unwind_ctrl_block *ctrl) { unsigned long insn = unwind_get_byte(ctrl); + int ret = URC_OK; pr_debug("%s: insn = %08lx\n", __func__, insn); @@ -250,8 +329,6 @@ static int unwind_exec_insn(struct unwind_ctrl_block *ctrl) ctrl->vrs[SP] -= ((insn & 0x3f) << 2) + 4; else if ((insn & 0xf0) == 0x80) { unsigned long mask; - unsigned long *vsp = (unsigned long *)ctrl->vrs[SP]; - int load_sp, reg = 4; insn = (insn << 8) | unwind_get_byte(ctrl); mask = insn & 0x0fff; @@ -261,29 +338,16 @@ static int unwind_exec_insn(struct unwind_ctrl_block *ctrl) return -URC_FAILURE; } - /* pop R4-R15 according to mask */ - load_sp = mask & (1 << (13 - 4)); - while (mask) { - if (mask & 1) - ctrl->vrs[reg] = *vsp++; - mask >>= 1; - reg++; - } - if (!load_sp) - ctrl->vrs[SP] = (unsigned long)vsp; + ret = unwind_exec_pop_subset_r4_to_r13(ctrl, mask); + if (ret) + goto error; } else if ((insn & 0xf0) == 0x90 && (insn & 0x0d) != 0x0d) ctrl->vrs[SP] = ctrl->vrs[insn & 0x0f]; else if ((insn & 0xf0) == 0xa0) { - unsigned long *vsp = (unsigned long *)ctrl->vrs[SP]; - int reg; - - /* pop R4-R[4+bbb] */ - for (reg = 4; reg <= 4 + (insn & 7); reg++) - ctrl->vrs[reg] = *vsp++; - if (insn & 0x80) - ctrl->vrs[14] = *vsp++; - ctrl->vrs[SP] = (unsigned long)vsp; + ret = unwind_exec_pop_r4_to_rN(ctrl, insn); + if (ret) + goto error; } else if (insn == 0xb0) { if (ctrl->vrs[PC] == 0) ctrl->vrs[PC] = ctrl->vrs[LR]; @@ -291,8 +355,6 @@ static int unwind_exec_insn(struct unwind_ctrl_block *ctrl) ctrl->entries = 0; } else if (insn == 0xb1) { unsigned long mask = unwind_get_byte(ctrl); - unsigned long *vsp = (unsigned long *)ctrl->vrs[SP]; - int reg = 0; if (mask == 0 || mask & 0xf0) { pr_warning("unwind: Spare encoding %04lx\n", @@ -300,14 +362,9 @@ static int unwind_exec_insn(struct unwind_ctrl_block *ctrl) return -URC_FAILURE; } - /* pop R0-R3 according to mask */ - while (mask) { - if (mask & 1) - ctrl->vrs[reg] = *vsp++; - mask >>= 1; - reg++; - } - ctrl->vrs[SP] = (unsigned long)vsp; + ret = unwind_exec_pop_subset_r0_to_r3(ctrl, mask); + if (ret) + goto error; } else if (insn == 0xb2) { unsigned long uleb128 = unwind_get_byte(ctrl); @@ -320,7 +377,8 @@ static int unwind_exec_insn(struct unwind_ctrl_block *ctrl) pr_debug("%s: fp = %08lx sp = %08lx lr = %08lx pc = %08lx\n", __func__, ctrl->vrs[FP], ctrl->vrs[SP], ctrl->vrs[LR], ctrl->vrs[PC]); - return URC_OK; +error: + return ret; } /* @@ -329,13 +387,13 @@ static int unwind_exec_insn(struct unwind_ctrl_block *ctrl) */ int unwind_frame(struct stackframe *frame) { - unsigned long high, low; + unsigned long low; const struct unwind_idx *idx; struct unwind_ctrl_block ctrl; - /* only go to a higher address on the stack */ + /* store the highest address on the stack to avoid crossing it*/ low = frame->sp; - high = ALIGN(low, THREAD_SIZE); + ctrl.sp_high = ALIGN(low, THREAD_SIZE); pr_debug("%s(pc = %08lx lr = %08lx sp = %08lx)\n", __func__, frame->pc, frame->lr, frame->sp); @@ -382,11 +440,16 @@ int unwind_frame(struct stackframe *frame) return -URC_FAILURE; } + ctrl.check_each_pop = 0; + while (ctrl.entries > 0) { - int urc = unwind_exec_insn(&ctrl); + int urc; + if ((ctrl.sp_high - ctrl.vrs[SP]) < sizeof(ctrl.vrs)) + ctrl.check_each_pop = 1; + urc = unwind_exec_insn(&ctrl); if (urc < 0) return urc; - if (ctrl.vrs[SP] < low || ctrl.vrs[SP] >= high) + if (ctrl.vrs[SP] < low || ctrl.vrs[SP] >= ctrl.sp_high) return -URC_FAILURE; } diff --git a/arch/arm/kernel/uprobes-arm.c b/arch/arm/kernel/uprobes-arm.c new file mode 100644 index 000000000000..d3b655ff17da --- /dev/null +++ b/arch/arm/kernel/uprobes-arm.c @@ -0,0 +1,234 @@ +/* + * Copyright (C) 2012 Rabin Vincent <rabin at rab.in> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/kernel.h> +#include <linux/types.h> +#include <linux/stddef.h> +#include <linux/wait.h> +#include <linux/uprobes.h> +#include <linux/module.h> + +#include "probes.h" +#include "probes-arm.h" +#include "uprobes.h" + +static int uprobes_substitute_pc(unsigned long *pinsn, u32 oregs) +{ + probes_opcode_t insn = __mem_to_opcode_arm(*pinsn); + probes_opcode_t temp; + probes_opcode_t mask; + int freereg; + u32 free = 0xffff; + u32 regs; + + for (regs = oregs; regs; regs >>= 4, insn >>= 4) { + if ((regs & 0xf) == REG_TYPE_NONE) + continue; + + free &= ~(1 << (insn & 0xf)); + } + + /* No PC, no problem */ + if (free & (1 << 15)) + return 15; + + if (!free) + return -1; + + /* + * fls instead of ffs ensures that for "ldrd r0, r1, [pc]" we would + * pick LR instead of R1. + */ + freereg = free = fls(free) - 1; + + temp = __mem_to_opcode_arm(*pinsn); + insn = temp; + regs = oregs; + mask = 0xf; + + for (; regs; regs >>= 4, mask <<= 4, free <<= 4, temp >>= 4) { + if ((regs & 0xf) == REG_TYPE_NONE) + continue; + + if ((temp & 0xf) != 15) + continue; + + insn &= ~mask; + insn |= free & mask; + } + + *pinsn = __opcode_to_mem_arm(insn); + return freereg; +} + +static void uprobe_set_pc(struct arch_uprobe *auprobe, + struct arch_uprobe_task *autask, + struct pt_regs *regs) +{ + u32 pcreg = auprobe->pcreg; + + autask->backup = regs->uregs[pcreg]; + regs->uregs[pcreg] = regs->ARM_pc + 8; +} + +static void uprobe_unset_pc(struct arch_uprobe *auprobe, + struct arch_uprobe_task *autask, + struct pt_regs *regs) +{ + /* PC will be taken care of by common code */ + regs->uregs[auprobe->pcreg] = autask->backup; +} + +static void uprobe_aluwrite_pc(struct arch_uprobe *auprobe, + struct arch_uprobe_task *autask, + struct pt_regs *regs) +{ + u32 pcreg = auprobe->pcreg; + + alu_write_pc(regs->uregs[pcreg], regs); + regs->uregs[pcreg] = autask->backup; +} + +static void uprobe_write_pc(struct arch_uprobe *auprobe, + struct arch_uprobe_task *autask, + struct pt_regs *regs) +{ + u32 pcreg = auprobe->pcreg; + + load_write_pc(regs->uregs[pcreg], regs); + regs->uregs[pcreg] = autask->backup; +} + +enum probes_insn +decode_pc_ro(probes_opcode_t insn, struct arch_probes_insn *asi, + const struct decode_header *d) +{ + struct arch_uprobe *auprobe = container_of(asi, struct arch_uprobe, + asi); + struct decode_emulate *decode = (struct decode_emulate *) d; + u32 regs = decode->header.type_regs.bits >> DECODE_TYPE_BITS; + int reg; + + reg = uprobes_substitute_pc(&auprobe->ixol[0], regs); + if (reg == 15) + return INSN_GOOD; + + if (reg == -1) + return INSN_REJECTED; + + auprobe->pcreg = reg; + auprobe->prehandler = uprobe_set_pc; + auprobe->posthandler = uprobe_unset_pc; + + return INSN_GOOD; +} + +enum probes_insn +decode_wb_pc(probes_opcode_t insn, struct arch_probes_insn *asi, + const struct decode_header *d, bool alu) +{ + struct arch_uprobe *auprobe = container_of(asi, struct arch_uprobe, + asi); + enum probes_insn ret = decode_pc_ro(insn, asi, d); + + if (((insn >> 12) & 0xf) == 15) + auprobe->posthandler = alu ? uprobe_aluwrite_pc + : uprobe_write_pc; + + return ret; +} + +enum probes_insn +decode_rd12rn16rm0rs8_rwflags(probes_opcode_t insn, + struct arch_probes_insn *asi, + const struct decode_header *d) +{ + return decode_wb_pc(insn, asi, d, true); +} + +enum probes_insn +decode_ldr(probes_opcode_t insn, struct arch_probes_insn *asi, + const struct decode_header *d) +{ + return decode_wb_pc(insn, asi, d, false); +} + +enum probes_insn +uprobe_decode_ldmstm(probes_opcode_t insn, + struct arch_probes_insn *asi, + const struct decode_header *d) +{ + struct arch_uprobe *auprobe = container_of(asi, struct arch_uprobe, + asi); + unsigned reglist = insn & 0xffff; + int rn = (insn >> 16) & 0xf; + int lbit = insn & (1 << 20); + unsigned used = reglist | (1 << rn); + + if (rn == 15) + return INSN_REJECTED; + + if (!(used & (1 << 15))) + return INSN_GOOD; + + if (used & (1 << 14)) + return INSN_REJECTED; + + /* Use LR instead of PC */ + insn ^= 0xc000; + + auprobe->pcreg = 14; + auprobe->ixol[0] = __opcode_to_mem_arm(insn); + + auprobe->prehandler = uprobe_set_pc; + if (lbit) + auprobe->posthandler = uprobe_write_pc; + else + auprobe->posthandler = uprobe_unset_pc; + + return INSN_GOOD; +} + +const union decode_action uprobes_probes_actions[] = { + [PROBES_EMULATE_NONE] = {.handler = probes_simulate_nop}, + [PROBES_SIMULATE_NOP] = {.handler = probes_simulate_nop}, + [PROBES_PRELOAD_IMM] = {.handler = probes_simulate_nop}, + [PROBES_PRELOAD_REG] = {.handler = probes_simulate_nop}, + [PROBES_BRANCH_IMM] = {.handler = simulate_blx1}, + [PROBES_MRS] = {.handler = simulate_mrs}, + [PROBES_BRANCH_REG] = {.handler = simulate_blx2bx}, + [PROBES_CLZ] = {.handler = probes_simulate_nop}, + [PROBES_SATURATING_ARITHMETIC] = {.handler = probes_simulate_nop}, + [PROBES_MUL1] = {.handler = probes_simulate_nop}, + [PROBES_MUL2] = {.handler = probes_simulate_nop}, + [PROBES_SWP] = {.handler = probes_simulate_nop}, + [PROBES_LDRSTRD] = {.decoder = decode_pc_ro}, + [PROBES_LOAD_EXTRA] = {.decoder = decode_pc_ro}, + [PROBES_LOAD] = {.decoder = decode_ldr}, + [PROBES_STORE_EXTRA] = {.decoder = decode_pc_ro}, + [PROBES_STORE] = {.decoder = decode_pc_ro}, + [PROBES_MOV_IP_SP] = {.handler = simulate_mov_ipsp}, + [PROBES_DATA_PROCESSING_REG] = { + .decoder = decode_rd12rn16rm0rs8_rwflags}, + [PROBES_DATA_PROCESSING_IMM] = { + .decoder = decode_rd12rn16rm0rs8_rwflags}, + [PROBES_MOV_HALFWORD] = {.handler = probes_simulate_nop}, + [PROBES_SEV] = {.handler = probes_simulate_nop}, + [PROBES_WFE] = {.handler = probes_simulate_nop}, + [PROBES_SATURATE] = {.handler = probes_simulate_nop}, + [PROBES_REV] = {.handler = probes_simulate_nop}, + [PROBES_MMI] = {.handler = probes_simulate_nop}, + [PROBES_PACK] = {.handler = probes_simulate_nop}, + [PROBES_EXTEND] = {.handler = probes_simulate_nop}, + [PROBES_EXTEND_ADD] = {.handler = probes_simulate_nop}, + [PROBES_MUL_ADD_LONG] = {.handler = probes_simulate_nop}, + [PROBES_MUL_ADD] = {.handler = probes_simulate_nop}, + [PROBES_BITFIELD] = {.handler = probes_simulate_nop}, + [PROBES_BRANCH] = {.handler = simulate_bbl}, + [PROBES_LDMSTM] = {.decoder = uprobe_decode_ldmstm} +}; diff --git a/arch/arm/kernel/uprobes.c b/arch/arm/kernel/uprobes.c new file mode 100644 index 000000000000..f9bacee973bf --- /dev/null +++ b/arch/arm/kernel/uprobes.c @@ -0,0 +1,210 @@ +/* + * Copyright (C) 2012 Rabin Vincent <rabin at rab.in> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/kernel.h> +#include <linux/stddef.h> +#include <linux/errno.h> +#include <linux/highmem.h> +#include <linux/sched.h> +#include <linux/uprobes.h> +#include <linux/notifier.h> + +#include <asm/opcodes.h> +#include <asm/traps.h> + +#include "probes.h" +#include "probes-arm.h" +#include "uprobes.h" + +#define UPROBE_TRAP_NR UINT_MAX + +bool is_swbp_insn(uprobe_opcode_t *insn) +{ + return (__mem_to_opcode_arm(*insn) & 0x0fffffff) == + (UPROBE_SWBP_ARM_INSN & 0x0fffffff); +} + +int set_swbp(struct arch_uprobe *auprobe, struct mm_struct *mm, + unsigned long vaddr) +{ + return uprobe_write_opcode(mm, vaddr, + __opcode_to_mem_arm(auprobe->bpinsn)); +} + +bool arch_uprobe_ignore(struct arch_uprobe *auprobe, struct pt_regs *regs) +{ + if (!auprobe->asi.insn_check_cc(regs->ARM_cpsr)) { + regs->ARM_pc += 4; + return true; + } + + return false; +} + +bool arch_uprobe_skip_sstep(struct arch_uprobe *auprobe, struct pt_regs *regs) +{ + probes_opcode_t opcode; + + if (!auprobe->simulate) + return false; + + opcode = __mem_to_opcode_arm(*(unsigned int *) auprobe->insn); + + auprobe->asi.insn_singlestep(opcode, &auprobe->asi, regs); + + return true; +} + +unsigned long +arch_uretprobe_hijack_return_addr(unsigned long trampoline_vaddr, + struct pt_regs *regs) +{ + unsigned long orig_ret_vaddr; + + orig_ret_vaddr = regs->ARM_lr; + /* Replace the return addr with trampoline addr */ + regs->ARM_lr = trampoline_vaddr; + return orig_ret_vaddr; +} + +int arch_uprobe_analyze_insn(struct arch_uprobe *auprobe, struct mm_struct *mm, + unsigned long addr) +{ + unsigned int insn; + unsigned int bpinsn; + enum probes_insn ret; + + /* Thumb not yet support */ + if (addr & 0x3) + return -EINVAL; + + insn = __mem_to_opcode_arm(*(unsigned int *)auprobe->insn); + auprobe->ixol[0] = __opcode_to_mem_arm(insn); + auprobe->ixol[1] = __opcode_to_mem_arm(UPROBE_SS_ARM_INSN); + + ret = arm_probes_decode_insn(insn, &auprobe->asi, false, + uprobes_probes_actions); + switch (ret) { + case INSN_REJECTED: + return -EINVAL; + + case INSN_GOOD_NO_SLOT: + auprobe->simulate = true; + break; + + case INSN_GOOD: + default: + break; + } + + bpinsn = UPROBE_SWBP_ARM_INSN & 0x0fffffff; + if (insn >= 0xe0000000) + bpinsn |= 0xe0000000; /* Unconditional instruction */ + else + bpinsn |= insn & 0xf0000000; /* Copy condition from insn */ + + auprobe->bpinsn = bpinsn; + + return 0; +} + +int arch_uprobe_pre_xol(struct arch_uprobe *auprobe, struct pt_regs *regs) +{ + struct uprobe_task *utask = current->utask; + + if (auprobe->prehandler) + auprobe->prehandler(auprobe, &utask->autask, regs); + + utask->autask.saved_trap_no = current->thread.trap_no; + current->thread.trap_no = UPROBE_TRAP_NR; + regs->ARM_pc = utask->xol_vaddr; + + return 0; +} + +int arch_uprobe_post_xol(struct arch_uprobe *auprobe, struct pt_regs *regs) +{ + struct uprobe_task *utask = current->utask; + + WARN_ON_ONCE(current->thread.trap_no != UPROBE_TRAP_NR); + + current->thread.trap_no = utask->autask.saved_trap_no; + regs->ARM_pc = utask->vaddr + 4; + + if (auprobe->posthandler) + auprobe->posthandler(auprobe, &utask->autask, regs); + + return 0; +} + +bool arch_uprobe_xol_was_trapped(struct task_struct *t) +{ + if (t->thread.trap_no != UPROBE_TRAP_NR) + return true; + + return false; +} + +void arch_uprobe_abort_xol(struct arch_uprobe *auprobe, struct pt_regs *regs) +{ + struct uprobe_task *utask = current->utask; + + current->thread.trap_no = utask->autask.saved_trap_no; + instruction_pointer_set(regs, utask->vaddr); +} + +int arch_uprobe_exception_notify(struct notifier_block *self, + unsigned long val, void *data) +{ + return NOTIFY_DONE; +} + +static int uprobe_trap_handler(struct pt_regs *regs, unsigned int instr) +{ + unsigned long flags; + + local_irq_save(flags); + instr &= 0x0fffffff; + if (instr == (UPROBE_SWBP_ARM_INSN & 0x0fffffff)) + uprobe_pre_sstep_notifier(regs); + else if (instr == (UPROBE_SS_ARM_INSN & 0x0fffffff)) + uprobe_post_sstep_notifier(regs); + local_irq_restore(flags); + + return 0; +} + +unsigned long uprobe_get_swbp_addr(struct pt_regs *regs) +{ + return instruction_pointer(regs); +} + +static struct undef_hook uprobes_arm_break_hook = { + .instr_mask = 0x0fffffff, + .instr_val = (UPROBE_SWBP_ARM_INSN & 0x0fffffff), + .cpsr_mask = MODE_MASK, + .cpsr_val = USR_MODE, + .fn = uprobe_trap_handler, +}; + +static struct undef_hook uprobes_arm_ss_hook = { + .instr_mask = 0x0fffffff, + .instr_val = (UPROBE_SS_ARM_INSN & 0x0fffffff), + .cpsr_mask = MODE_MASK, + .cpsr_val = USR_MODE, + .fn = uprobe_trap_handler, +}; + +static int arch_uprobes_init(void) +{ + register_undef_hook(&uprobes_arm_break_hook); + register_undef_hook(&uprobes_arm_ss_hook); + + return 0; +} +device_initcall(arch_uprobes_init); diff --git a/arch/arm/kernel/uprobes.h b/arch/arm/kernel/uprobes.h new file mode 100644 index 000000000000..1d0c12dfbd03 --- /dev/null +++ b/arch/arm/kernel/uprobes.h @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2012 Rabin Vincent <rabin at rab.in> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#ifndef __ARM_KERNEL_UPROBES_H +#define __ARM_KERNEL_UPROBES_H + +enum probes_insn uprobe_decode_ldmstm(probes_opcode_t insn, + struct arch_probes_insn *asi, + const struct decode_header *d); + +enum probes_insn decode_ldr(probes_opcode_t insn, + struct arch_probes_insn *asi, + const struct decode_header *d); + +enum probes_insn +decode_rd12rn16rm0rs8_rwflags(probes_opcode_t insn, + struct arch_probes_insn *asi, + const struct decode_header *d); + +enum probes_insn +decode_wb_pc(probes_opcode_t insn, struct arch_probes_insn *asi, + const struct decode_header *d, bool alu); + +enum probes_insn +decode_pc_ro(probes_opcode_t insn, struct arch_probes_insn *asi, + const struct decode_header *d); + +extern const union decode_action uprobes_probes_actions[]; + +#endif diff --git a/arch/arm/lib/bitops.h b/arch/arm/lib/bitops.h index 52886b89706c..9f12ed1eea86 100644 --- a/arch/arm/lib/bitops.h +++ b/arch/arm/lib/bitops.h @@ -37,6 +37,11 @@ UNWIND( .fnstart ) add r1, r1, r0, lsl #2 @ Get word offset mov r3, r2, lsl r3 @ create mask smp_dmb +#if __LINUX_ARM_ARCH__ >= 7 && defined(CONFIG_SMP) + .arch_extension mp + ALT_SMP(W(pldw) [r1]) + ALT_UP(W(nop)) +#endif 1: ldrex r2, [r1] ands r0, r2, r3 @ save old value of bit \instr r2, r2, r3 @ toggle bit diff --git a/arch/arm/lib/copy_template.S b/arch/arm/lib/copy_template.S index 805e3f8fb007..3bc8eb811a73 100644 --- a/arch/arm/lib/copy_template.S +++ b/arch/arm/lib/copy_template.S @@ -197,24 +197,24 @@ 12: PLD( pld [r1, #124] ) 13: ldr4w r1, r4, r5, r6, r7, abort=19f - mov r3, lr, pull #\pull + mov r3, lr, lspull #\pull subs r2, r2, #32 ldr4w r1, r8, r9, ip, lr, abort=19f - orr r3, r3, r4, push #\push - mov r4, r4, pull #\pull - orr r4, r4, r5, push #\push - mov r5, r5, pull #\pull - orr r5, r5, r6, push #\push - mov r6, r6, pull #\pull - orr r6, r6, r7, push #\push - mov r7, r7, pull #\pull - orr r7, r7, r8, push #\push - mov r8, r8, pull #\pull - orr r8, r8, r9, push #\push - mov r9, r9, pull #\pull - orr r9, r9, ip, push #\push - mov ip, ip, pull #\pull - orr ip, ip, lr, push #\push + orr r3, r3, r4, lspush #\push + mov r4, r4, lspull #\pull + orr r4, r4, r5, lspush #\push + mov r5, r5, lspull #\pull + orr r5, r5, r6, lspush #\push + mov r6, r6, lspull #\pull + orr r6, r6, r7, lspush #\push + mov r7, r7, lspull #\pull + orr r7, r7, r8, lspush #\push + mov r8, r8, lspull #\pull + orr r8, r8, r9, lspush #\push + mov r9, r9, lspull #\pull + orr r9, r9, ip, lspush #\push + mov ip, ip, lspull #\pull + orr ip, ip, lr, lspush #\push str8w r0, r3, r4, r5, r6, r7, r8, r9, ip, , abort=19f bge 12b PLD( cmn r2, #96 ) @@ -225,10 +225,10 @@ 14: ands ip, r2, #28 beq 16f -15: mov r3, lr, pull #\pull +15: mov r3, lr, lspull #\pull ldr1w r1, lr, abort=21f subs ip, ip, #4 - orr r3, r3, lr, push #\push + orr r3, r3, lr, lspush #\push str1w r0, r3, abort=21f bgt 15b CALGN( cmp r2, #0 ) diff --git a/arch/arm/lib/csumpartialcopygeneric.S b/arch/arm/lib/csumpartialcopygeneric.S index d620a5f22a09..d6e742d24007 100644 --- a/arch/arm/lib/csumpartialcopygeneric.S +++ b/arch/arm/lib/csumpartialcopygeneric.S @@ -141,7 +141,7 @@ FN_ENTRY tst len, #2 mov r5, r4, get_byte_0 beq .Lexit - adcs sum, sum, r4, push #16 + adcs sum, sum, r4, lspush #16 strb r5, [dst], #1 mov r5, r4, get_byte_1 strb r5, [dst], #1 @@ -171,23 +171,23 @@ FN_ENTRY cmp ip, #2 beq .Lsrc2_aligned bhi .Lsrc3_aligned - mov r4, r5, pull #8 @ C = 0 + mov r4, r5, lspull #8 @ C = 0 bics ip, len, #15 beq 2f 1: load4l r5, r6, r7, r8 - orr r4, r4, r5, push #24 - mov r5, r5, pull #8 - orr r5, r5, r6, push #24 - mov r6, r6, pull #8 - orr r6, r6, r7, push #24 - mov r7, r7, pull #8 - orr r7, r7, r8, push #24 + orr r4, r4, r5, lspush #24 + mov r5, r5, lspull #8 + orr r5, r5, r6, lspush #24 + mov r6, r6, lspull #8 + orr r6, r6, r7, lspush #24 + mov r7, r7, lspull #8 + orr r7, r7, r8, lspush #24 stmia dst!, {r4, r5, r6, r7} adcs sum, sum, r4 adcs sum, sum, r5 adcs sum, sum, r6 adcs sum, sum, r7 - mov r4, r8, pull #8 + mov r4, r8, lspull #8 sub ip, ip, #16 teq ip, #0 bne 1b @@ -196,50 +196,50 @@ FN_ENTRY tst ip, #8 beq 3f load2l r5, r6 - orr r4, r4, r5, push #24 - mov r5, r5, pull #8 - orr r5, r5, r6, push #24 + orr r4, r4, r5, lspush #24 + mov r5, r5, lspull #8 + orr r5, r5, r6, lspush #24 stmia dst!, {r4, r5} adcs sum, sum, r4 adcs sum, sum, r5 - mov r4, r6, pull #8 + mov r4, r6, lspull #8 tst ip, #4 beq 4f 3: load1l r5 - orr r4, r4, r5, push #24 + orr r4, r4, r5, lspush #24 str r4, [dst], #4 adcs sum, sum, r4 - mov r4, r5, pull #8 + mov r4, r5, lspull #8 4: ands len, len, #3 beq .Ldone mov r5, r4, get_byte_0 tst len, #2 beq .Lexit - adcs sum, sum, r4, push #16 + adcs sum, sum, r4, lspush #16 strb r5, [dst], #1 mov r5, r4, get_byte_1 strb r5, [dst], #1 mov r5, r4, get_byte_2 b .Lexit -.Lsrc2_aligned: mov r4, r5, pull #16 +.Lsrc2_aligned: mov r4, r5, lspull #16 adds sum, sum, #0 bics ip, len, #15 beq 2f 1: load4l r5, r6, r7, r8 - orr r4, r4, r5, push #16 - mov r5, r5, pull #16 - orr r5, r5, r6, push #16 - mov r6, r6, pull #16 - orr r6, r6, r7, push #16 - mov r7, r7, pull #16 - orr r7, r7, r8, push #16 + orr r4, r4, r5, lspush #16 + mov r5, r5, lspull #16 + orr r5, r5, r6, lspush #16 + mov r6, r6, lspull #16 + orr r6, r6, r7, lspush #16 + mov r7, r7, lspull #16 + orr r7, r7, r8, lspush #16 stmia dst!, {r4, r5, r6, r7} adcs sum, sum, r4 adcs sum, sum, r5 adcs sum, sum, r6 adcs sum, sum, r7 - mov r4, r8, pull #16 + mov r4, r8, lspull #16 sub ip, ip, #16 teq ip, #0 bne 1b @@ -248,20 +248,20 @@ FN_ENTRY tst ip, #8 beq 3f load2l r5, r6 - orr r4, r4, r5, push #16 - mov r5, r5, pull #16 - orr r5, r5, r6, push #16 + orr r4, r4, r5, lspush #16 + mov r5, r5, lspull #16 + orr r5, r5, r6, lspush #16 stmia dst!, {r4, r5} adcs sum, sum, r4 adcs sum, sum, r5 - mov r4, r6, pull #16 + mov r4, r6, lspull #16 tst ip, #4 beq 4f 3: load1l r5 - orr r4, r4, r5, push #16 + orr r4, r4, r5, lspush #16 str r4, [dst], #4 adcs sum, sum, r4 - mov r4, r5, pull #16 + mov r4, r5, lspull #16 4: ands len, len, #3 beq .Ldone mov r5, r4, get_byte_0 @@ -276,24 +276,24 @@ FN_ENTRY load1b r5 b .Lexit -.Lsrc3_aligned: mov r4, r5, pull #24 +.Lsrc3_aligned: mov r4, r5, lspull #24 adds sum, sum, #0 bics ip, len, #15 beq 2f 1: load4l r5, r6, r7, r8 - orr r4, r4, r5, push #8 - mov r5, r5, pull #24 - orr r5, r5, r6, push #8 - mov r6, r6, pull #24 - orr r6, r6, r7, push #8 - mov r7, r7, pull #24 - orr r7, r7, r8, push #8 + orr r4, r4, r5, lspush #8 + mov r5, r5, lspull #24 + orr r5, r5, r6, lspush #8 + mov r6, r6, lspull #24 + orr r6, r6, r7, lspush #8 + mov r7, r7, lspull #24 + orr r7, r7, r8, lspush #8 stmia dst!, {r4, r5, r6, r7} adcs sum, sum, r4 adcs sum, sum, r5 adcs sum, sum, r6 adcs sum, sum, r7 - mov r4, r8, pull #24 + mov r4, r8, lspull #24 sub ip, ip, #16 teq ip, #0 bne 1b @@ -302,20 +302,20 @@ FN_ENTRY tst ip, #8 beq 3f load2l r5, r6 - orr r4, r4, r5, push #8 - mov r5, r5, pull #24 - orr r5, r5, r6, push #8 + orr r4, r4, r5, lspush #8 + mov r5, r5, lspull #24 + orr r5, r5, r6, lspush #8 stmia dst!, {r4, r5} adcs sum, sum, r4 adcs sum, sum, r5 - mov r4, r6, pull #24 + mov r4, r6, lspull #24 tst ip, #4 beq 4f 3: load1l r5 - orr r4, r4, r5, push #8 + orr r4, r4, r5, lspush #8 str r4, [dst], #4 adcs sum, sum, r4 - mov r4, r5, pull #24 + mov r4, r5, lspull #24 4: ands len, len, #3 beq .Ldone mov r5, r4, get_byte_0 @@ -326,7 +326,7 @@ FN_ENTRY load1l r4 mov r5, r4, get_byte_0 strb r5, [dst], #1 - adcs sum, sum, r4, push #24 + adcs sum, sum, r4, lspush #24 mov r5, r4, get_byte_1 b .Lexit FN_EXIT diff --git a/arch/arm/lib/io-readsl.S b/arch/arm/lib/io-readsl.S index 5fb97e7f9f4b..7a7430950c79 100644 --- a/arch/arm/lib/io-readsl.S +++ b/arch/arm/lib/io-readsl.S @@ -47,25 +47,25 @@ ENTRY(__raw_readsl) strb ip, [r1], #1 4: subs r2, r2, #1 - mov ip, r3, pull #24 + mov ip, r3, lspull #24 ldrne r3, [r0] - orrne ip, ip, r3, push #8 + orrne ip, ip, r3, lspush #8 strne ip, [r1], #4 bne 4b b 8f 5: subs r2, r2, #1 - mov ip, r3, pull #16 + mov ip, r3, lspull #16 ldrne r3, [r0] - orrne ip, ip, r3, push #16 + orrne ip, ip, r3, lspush #16 strne ip, [r1], #4 bne 5b b 7f 6: subs r2, r2, #1 - mov ip, r3, pull #8 + mov ip, r3, lspull #8 ldrne r3, [r0] - orrne ip, ip, r3, push #24 + orrne ip, ip, r3, lspush #24 strne ip, [r1], #4 bne 6b diff --git a/arch/arm/lib/io-writesl.S b/arch/arm/lib/io-writesl.S index 8d3b7813725c..d0d104a0dd11 100644 --- a/arch/arm/lib/io-writesl.S +++ b/arch/arm/lib/io-writesl.S @@ -41,26 +41,26 @@ ENTRY(__raw_writesl) blt 5f bgt 6f -4: mov ip, r3, pull #16 +4: mov ip, r3, lspull #16 ldr r3, [r1], #4 subs r2, r2, #1 - orr ip, ip, r3, push #16 + orr ip, ip, r3, lspush #16 str ip, [r0] bne 4b mov pc, lr -5: mov ip, r3, pull #8 +5: mov ip, r3, lspull #8 ldr r3, [r1], #4 subs r2, r2, #1 - orr ip, ip, r3, push #24 + orr ip, ip, r3, lspush #24 str ip, [r0] bne 5b mov pc, lr -6: mov ip, r3, pull #24 +6: mov ip, r3, lspull #24 ldr r3, [r1], #4 subs r2, r2, #1 - orr ip, ip, r3, push #8 + orr ip, ip, r3, lspush #8 str ip, [r0] bne 6b mov pc, lr diff --git a/arch/arm/lib/memmove.S b/arch/arm/lib/memmove.S index 938fc14f962d..d1fc0c0c342c 100644 --- a/arch/arm/lib/memmove.S +++ b/arch/arm/lib/memmove.S @@ -147,24 +147,24 @@ ENTRY(memmove) 12: PLD( pld [r1, #-128] ) 13: ldmdb r1!, {r7, r8, r9, ip} - mov lr, r3, push #\push + mov lr, r3, lspush #\push subs r2, r2, #32 ldmdb r1!, {r3, r4, r5, r6} - orr lr, lr, ip, pull #\pull - mov ip, ip, push #\push - orr ip, ip, r9, pull #\pull - mov r9, r9, push #\push - orr r9, r9, r8, pull #\pull - mov r8, r8, push #\push - orr r8, r8, r7, pull #\pull - mov r7, r7, push #\push - orr r7, r7, r6, pull #\pull - mov r6, r6, push #\push - orr r6, r6, r5, pull #\pull - mov r5, r5, push #\push - orr r5, r5, r4, pull #\pull - mov r4, r4, push #\push - orr r4, r4, r3, pull #\pull + orr lr, lr, ip, lspull #\pull + mov ip, ip, lspush #\push + orr ip, ip, r9, lspull #\pull + mov r9, r9, lspush #\push + orr r9, r9, r8, lspull #\pull + mov r8, r8, lspush #\push + orr r8, r8, r7, lspull #\pull + mov r7, r7, lspush #\push + orr r7, r7, r6, lspull #\pull + mov r6, r6, lspush #\push + orr r6, r6, r5, lspull #\pull + mov r5, r5, lspush #\push + orr r5, r5, r4, lspull #\pull + mov r4, r4, lspush #\push + orr r4, r4, r3, lspull #\pull stmdb r0!, {r4 - r9, ip, lr} bge 12b PLD( cmn r2, #96 ) @@ -175,10 +175,10 @@ ENTRY(memmove) 14: ands ip, r2, #28 beq 16f -15: mov lr, r3, push #\push +15: mov lr, r3, lspush #\push ldr r3, [r1, #-4]! subs ip, ip, #4 - orr lr, lr, r3, pull #\pull + orr lr, lr, r3, lspull #\pull str lr, [r0, #-4]! bgt 15b CALGN( cmp r2, #0 ) diff --git a/arch/arm/lib/uaccess.S b/arch/arm/lib/uaccess.S index 5c908b1cb8ed..e50520904b76 100644 --- a/arch/arm/lib/uaccess.S +++ b/arch/arm/lib/uaccess.S @@ -117,9 +117,9 @@ USER( TUSER( strgtb) r3, [r0], #1) @ May fault .Lc2u_1fupi: subs r2, r2, #4 addmi ip, r2, #4 bmi .Lc2u_1nowords - mov r3, r7, pull #8 + mov r3, r7, lspull #8 ldr r7, [r1], #4 - orr r3, r3, r7, push #24 + orr r3, r3, r7, lspush #24 USER( TUSER( str) r3, [r0], #4) @ May fault mov ip, r0, lsl #32 - PAGE_SHIFT rsb ip, ip, #0 @@ -131,30 +131,30 @@ USER( TUSER( str) r3, [r0], #4) @ May fault subs ip, ip, #16 blt .Lc2u_1rem8lp -.Lc2u_1cpy8lp: mov r3, r7, pull #8 +.Lc2u_1cpy8lp: mov r3, r7, lspull #8 ldmia r1!, {r4 - r7} subs ip, ip, #16 - orr r3, r3, r4, push #24 - mov r4, r4, pull #8 - orr r4, r4, r5, push #24 - mov r5, r5, pull #8 - orr r5, r5, r6, push #24 - mov r6, r6, pull #8 - orr r6, r6, r7, push #24 + orr r3, r3, r4, lspush #24 + mov r4, r4, lspull #8 + orr r4, r4, r5, lspush #24 + mov r5, r5, lspull #8 + orr r5, r5, r6, lspush #24 + mov r6, r6, lspull #8 + orr r6, r6, r7, lspush #24 stmia r0!, {r3 - r6} @ Shouldnt fault bpl .Lc2u_1cpy8lp .Lc2u_1rem8lp: tst ip, #8 - movne r3, r7, pull #8 + movne r3, r7, lspull #8 ldmneia r1!, {r4, r7} - orrne r3, r3, r4, push #24 - movne r4, r4, pull #8 - orrne r4, r4, r7, push #24 + orrne r3, r3, r4, lspush #24 + movne r4, r4, lspull #8 + orrne r4, r4, r7, lspush #24 stmneia r0!, {r3 - r4} @ Shouldnt fault tst ip, #4 - movne r3, r7, pull #8 + movne r3, r7, lspull #8 ldrne r7, [r1], #4 - orrne r3, r3, r7, push #24 + orrne r3, r3, r7, lspush #24 TUSER( strne) r3, [r0], #4 @ Shouldnt fault ands ip, ip, #3 beq .Lc2u_1fupi @@ -172,9 +172,9 @@ USER( TUSER( strgtb) r3, [r0], #1) @ May fault .Lc2u_2fupi: subs r2, r2, #4 addmi ip, r2, #4 bmi .Lc2u_2nowords - mov r3, r7, pull #16 + mov r3, r7, lspull #16 ldr r7, [r1], #4 - orr r3, r3, r7, push #16 + orr r3, r3, r7, lspush #16 USER( TUSER( str) r3, [r0], #4) @ May fault mov ip, r0, lsl #32 - PAGE_SHIFT rsb ip, ip, #0 @@ -186,30 +186,30 @@ USER( TUSER( str) r3, [r0], #4) @ May fault subs ip, ip, #16 blt .Lc2u_2rem8lp -.Lc2u_2cpy8lp: mov r3, r7, pull #16 +.Lc2u_2cpy8lp: mov r3, r7, lspull #16 ldmia r1!, {r4 - r7} subs ip, ip, #16 - orr r3, r3, r4, push #16 - mov r4, r4, pull #16 - orr r4, r4, r5, push #16 - mov r5, r5, pull #16 - orr r5, r5, r6, push #16 - mov r6, r6, pull #16 - orr r6, r6, r7, push #16 + orr r3, r3, r4, lspush #16 + mov r4, r4, lspull #16 + orr r4, r4, r5, lspush #16 + mov r5, r5, lspull #16 + orr r5, r5, r6, lspush #16 + mov r6, r6, lspull #16 + orr r6, r6, r7, lspush #16 stmia r0!, {r3 - r6} @ Shouldnt fault bpl .Lc2u_2cpy8lp .Lc2u_2rem8lp: tst ip, #8 - movne r3, r7, pull #16 + movne r3, r7, lspull #16 ldmneia r1!, {r4, r7} - orrne r3, r3, r4, push #16 - movne r4, r4, pull #16 - orrne r4, r4, r7, push #16 + orrne r3, r3, r4, lspush #16 + movne r4, r4, lspull #16 + orrne r4, r4, r7, lspush #16 stmneia r0!, {r3 - r4} @ Shouldnt fault tst ip, #4 - movne r3, r7, pull #16 + movne r3, r7, lspull #16 ldrne r7, [r1], #4 - orrne r3, r3, r7, push #16 + orrne r3, r3, r7, lspush #16 TUSER( strne) r3, [r0], #4 @ Shouldnt fault ands ip, ip, #3 beq .Lc2u_2fupi @@ -227,9 +227,9 @@ USER( TUSER( strgtb) r3, [r0], #1) @ May fault .Lc2u_3fupi: subs r2, r2, #4 addmi ip, r2, #4 bmi .Lc2u_3nowords - mov r3, r7, pull #24 + mov r3, r7, lspull #24 ldr r7, [r1], #4 - orr r3, r3, r7, push #8 + orr r3, r3, r7, lspush #8 USER( TUSER( str) r3, [r0], #4) @ May fault mov ip, r0, lsl #32 - PAGE_SHIFT rsb ip, ip, #0 @@ -241,30 +241,30 @@ USER( TUSER( str) r3, [r0], #4) @ May fault subs ip, ip, #16 blt .Lc2u_3rem8lp -.Lc2u_3cpy8lp: mov r3, r7, pull #24 +.Lc2u_3cpy8lp: mov r3, r7, lspull #24 ldmia r1!, {r4 - r7} subs ip, ip, #16 - orr r3, r3, r4, push #8 - mov r4, r4, pull #24 - orr r4, r4, r5, push #8 - mov r5, r5, pull #24 - orr r5, r5, r6, push #8 - mov r6, r6, pull #24 - orr r6, r6, r7, push #8 + orr r3, r3, r4, lspush #8 + mov r4, r4, lspull #24 + orr r4, r4, r5, lspush #8 + mov r5, r5, lspull #24 + orr r5, r5, r6, lspush #8 + mov r6, r6, lspull #24 + orr r6, r6, r7, lspush #8 stmia r0!, {r3 - r6} @ Shouldnt fault bpl .Lc2u_3cpy8lp .Lc2u_3rem8lp: tst ip, #8 - movne r3, r7, pull #24 + movne r3, r7, lspull #24 ldmneia r1!, {r4, r7} - orrne r3, r3, r4, push #8 - movne r4, r4, pull #24 - orrne r4, r4, r7, push #8 + orrne r3, r3, r4, lspush #8 + movne r4, r4, lspull #24 + orrne r4, r4, r7, lspush #8 stmneia r0!, {r3 - r4} @ Shouldnt fault tst ip, #4 - movne r3, r7, pull #24 + movne r3, r7, lspull #24 ldrne r7, [r1], #4 - orrne r3, r3, r7, push #8 + orrne r3, r3, r7, lspush #8 TUSER( strne) r3, [r0], #4 @ Shouldnt fault ands ip, ip, #3 beq .Lc2u_3fupi @@ -382,9 +382,9 @@ USER( TUSER( ldr) r7, [r1], #4) @ May fault .Lcfu_1fupi: subs r2, r2, #4 addmi ip, r2, #4 bmi .Lcfu_1nowords - mov r3, r7, pull #8 + mov r3, r7, lspull #8 USER( TUSER( ldr) r7, [r1], #4) @ May fault - orr r3, r3, r7, push #24 + orr r3, r3, r7, lspush #24 str r3, [r0], #4 mov ip, r1, lsl #32 - PAGE_SHIFT rsb ip, ip, #0 @@ -396,30 +396,30 @@ USER( TUSER( ldr) r7, [r1], #4) @ May fault subs ip, ip, #16 blt .Lcfu_1rem8lp -.Lcfu_1cpy8lp: mov r3, r7, pull #8 +.Lcfu_1cpy8lp: mov r3, r7, lspull #8 ldmia r1!, {r4 - r7} @ Shouldnt fault subs ip, ip, #16 - orr r3, r3, r4, push #24 - mov r4, r4, pull #8 - orr r4, r4, r5, push #24 - mov r5, r5, pull #8 - orr r5, r5, r6, push #24 - mov r6, r6, pull #8 - orr r6, r6, r7, push #24 + orr r3, r3, r4, lspush #24 + mov r4, r4, lspull #8 + orr r4, r4, r5, lspush #24 + mov r5, r5, lspull #8 + orr r5, r5, r6, lspush #24 + mov r6, r6, lspull #8 + orr r6, r6, r7, lspush #24 stmia r0!, {r3 - r6} bpl .Lcfu_1cpy8lp .Lcfu_1rem8lp: tst ip, #8 - movne r3, r7, pull #8 + movne r3, r7, lspull #8 ldmneia r1!, {r4, r7} @ Shouldnt fault - orrne r3, r3, r4, push #24 - movne r4, r4, pull #8 - orrne r4, r4, r7, push #24 + orrne r3, r3, r4, lspush #24 + movne r4, r4, lspull #8 + orrne r4, r4, r7, lspush #24 stmneia r0!, {r3 - r4} tst ip, #4 - movne r3, r7, pull #8 + movne r3, r7, lspull #8 USER( TUSER( ldrne) r7, [r1], #4) @ May fault - orrne r3, r3, r7, push #24 + orrne r3, r3, r7, lspush #24 strne r3, [r0], #4 ands ip, ip, #3 beq .Lcfu_1fupi @@ -437,9 +437,9 @@ USER( TUSER( ldrne) r7, [r1], #4) @ May fault .Lcfu_2fupi: subs r2, r2, #4 addmi ip, r2, #4 bmi .Lcfu_2nowords - mov r3, r7, pull #16 + mov r3, r7, lspull #16 USER( TUSER( ldr) r7, [r1], #4) @ May fault - orr r3, r3, r7, push #16 + orr r3, r3, r7, lspush #16 str r3, [r0], #4 mov ip, r1, lsl #32 - PAGE_SHIFT rsb ip, ip, #0 @@ -452,30 +452,30 @@ USER( TUSER( ldr) r7, [r1], #4) @ May fault blt .Lcfu_2rem8lp -.Lcfu_2cpy8lp: mov r3, r7, pull #16 +.Lcfu_2cpy8lp: mov r3, r7, lspull #16 ldmia r1!, {r4 - r7} @ Shouldnt fault subs ip, ip, #16 - orr r3, r3, r4, push #16 - mov r4, r4, pull #16 - orr r4, r4, r5, push #16 - mov r5, r5, pull #16 - orr r5, r5, r6, push #16 - mov r6, r6, pull #16 - orr r6, r6, r7, push #16 + orr r3, r3, r4, lspush #16 + mov r4, r4, lspull #16 + orr r4, r4, r5, lspush #16 + mov r5, r5, lspull #16 + orr r5, r5, r6, lspush #16 + mov r6, r6, lspull #16 + orr r6, r6, r7, lspush #16 stmia r0!, {r3 - r6} bpl .Lcfu_2cpy8lp .Lcfu_2rem8lp: tst ip, #8 - movne r3, r7, pull #16 + movne r3, r7, lspull #16 ldmneia r1!, {r4, r7} @ Shouldnt fault - orrne r3, r3, r4, push #16 - movne r4, r4, pull #16 - orrne r4, r4, r7, push #16 + orrne r3, r3, r4, lspush #16 + movne r4, r4, lspull #16 + orrne r4, r4, r7, lspush #16 stmneia r0!, {r3 - r4} tst ip, #4 - movne r3, r7, pull #16 + movne r3, r7, lspull #16 USER( TUSER( ldrne) r7, [r1], #4) @ May fault - orrne r3, r3, r7, push #16 + orrne r3, r3, r7, lspush #16 strne r3, [r0], #4 ands ip, ip, #3 beq .Lcfu_2fupi @@ -493,9 +493,9 @@ USER( TUSER( ldrgtb) r3, [r1], #0) @ May fault .Lcfu_3fupi: subs r2, r2, #4 addmi ip, r2, #4 bmi .Lcfu_3nowords - mov r3, r7, pull #24 + mov r3, r7, lspull #24 USER( TUSER( ldr) r7, [r1], #4) @ May fault - orr r3, r3, r7, push #8 + orr r3, r3, r7, lspush #8 str r3, [r0], #4 mov ip, r1, lsl #32 - PAGE_SHIFT rsb ip, ip, #0 @@ -507,30 +507,30 @@ USER( TUSER( ldr) r7, [r1], #4) @ May fault subs ip, ip, #16 blt .Lcfu_3rem8lp -.Lcfu_3cpy8lp: mov r3, r7, pull #24 +.Lcfu_3cpy8lp: mov r3, r7, lspull #24 ldmia r1!, {r4 - r7} @ Shouldnt fault - orr r3, r3, r4, push #8 - mov r4, r4, pull #24 - orr r4, r4, r5, push #8 - mov r5, r5, pull #24 - orr r5, r5, r6, push #8 - mov r6, r6, pull #24 - orr r6, r6, r7, push #8 + orr r3, r3, r4, lspush #8 + mov r4, r4, lspull #24 + orr r4, r4, r5, lspush #8 + mov r5, r5, lspull #24 + orr r5, r5, r6, lspush #8 + mov r6, r6, lspull #24 + orr r6, r6, r7, lspush #8 stmia r0!, {r3 - r6} subs ip, ip, #16 bpl .Lcfu_3cpy8lp .Lcfu_3rem8lp: tst ip, #8 - movne r3, r7, pull #24 + movne r3, r7, lspull #24 ldmneia r1!, {r4, r7} @ Shouldnt fault - orrne r3, r3, r4, push #8 - movne r4, r4, pull #24 - orrne r4, r4, r7, push #8 + orrne r3, r3, r4, lspush #8 + movne r4, r4, lspull #24 + orrne r4, r4, r7, lspush #8 stmneia r0!, {r3 - r4} tst ip, #4 - movne r3, r7, pull #24 + movne r3, r7, lspull #24 USER( TUSER( ldrne) r7, [r1], #4) @ May fault - orrne r3, r3, r7, push #8 + orrne r3, r3, r7, lspush #8 strne r3, [r0], #4 ands ip, ip, #3 beq .Lcfu_3fupi diff --git a/arch/arm/mach-cns3xxx/core.c b/arch/arm/mach-cns3xxx/core.c index e38b279f402c..384dc859e6c6 100644 --- a/arch/arm/mach-cns3xxx/core.c +++ b/arch/arm/mach-cns3xxx/core.c @@ -155,7 +155,7 @@ static irqreturn_t cns3xxx_timer_interrupt(int irq, void *dev_id) static struct irqaction cns3xxx_timer_irq = { .name = "timer", - .flags = IRQF_DISABLED | IRQF_TIMER | IRQF_IRQPOLL, + .flags = IRQF_TIMER | IRQF_IRQPOLL, .handler = cns3xxx_timer_interrupt, }; diff --git a/arch/arm/mach-ebsa110/core.c b/arch/arm/mach-ebsa110/core.c index 68ac934d4565..8254e716b095 100644 --- a/arch/arm/mach-ebsa110/core.c +++ b/arch/arm/mach-ebsa110/core.c @@ -206,7 +206,7 @@ ebsa110_timer_interrupt(int irq, void *dev_id) static struct irqaction ebsa110_timer_irq = { .name = "EBSA110 Timer Tick", - .flags = IRQF_DISABLED | IRQF_TIMER | IRQF_IRQPOLL, + .flags = IRQF_TIMER | IRQF_IRQPOLL, .handler = ebsa110_timer_interrupt, }; diff --git a/arch/arm/mach-footbridge/dc21285-timer.c b/arch/arm/mach-footbridge/dc21285-timer.c index 3971104d32d4..bf7aa7d298e7 100644 --- a/arch/arm/mach-footbridge/dc21285-timer.c +++ b/arch/arm/mach-footbridge/dc21285-timer.c @@ -105,7 +105,7 @@ static irqreturn_t timer1_interrupt(int irq, void *dev_id) static struct irqaction footbridge_timer_irq = { .name = "dc21285_timer1", .handler = timer1_interrupt, - .flags = IRQF_DISABLED | IRQF_TIMER | IRQF_IRQPOLL, + .flags = IRQF_TIMER | IRQF_IRQPOLL, .dev_id = &ckevt_dc21285, }; @@ -125,7 +125,7 @@ void __init footbridge_timer_init(void) clockevents_config_and_register(ce, rate, 0x4, 0xffffff); } -static u32 notrace footbridge_read_sched_clock(void) +static u64 notrace footbridge_read_sched_clock(void) { return ~*CSR_TIMER3_VALUE; } @@ -138,5 +138,5 @@ void __init footbridge_sched_clock(void) *CSR_TIMER3_CLR = 0; *CSR_TIMER3_CNTL = TIMER_CNTL_ENABLE | TIMER_CNTL_DIV16; - setup_sched_clock(footbridge_read_sched_clock, 24, rate); + sched_clock_register(footbridge_read_sched_clock, 24, rate); } diff --git a/arch/arm/mach-footbridge/dc21285.c b/arch/arm/mach-footbridge/dc21285.c index 7c2fdae9a38b..96a3d73ef4bf 100644 --- a/arch/arm/mach-footbridge/dc21285.c +++ b/arch/arm/mach-footbridge/dc21285.c @@ -334,15 +334,15 @@ void __init dc21285_preinit(void) /* * We don't care if these fail. */ - dc21285_request_irq(IRQ_PCI_SERR, dc21285_serr_irq, IRQF_DISABLED, + dc21285_request_irq(IRQ_PCI_SERR, dc21285_serr_irq, 0, "PCI system error", &serr_timer); - dc21285_request_irq(IRQ_PCI_PERR, dc21285_parity_irq, IRQF_DISABLED, + dc21285_request_irq(IRQ_PCI_PERR, dc21285_parity_irq, 0, "PCI parity error", &perr_timer); - dc21285_request_irq(IRQ_PCI_ABORT, dc21285_abort_irq, IRQF_DISABLED, + dc21285_request_irq(IRQ_PCI_ABORT, dc21285_abort_irq, 0, "PCI abort", NULL); - dc21285_request_irq(IRQ_DISCARD_TIMER, dc21285_discard_irq, IRQF_DISABLED, + dc21285_request_irq(IRQ_DISCARD_TIMER, dc21285_discard_irq, 0, "Discard timer", NULL); - dc21285_request_irq(IRQ_PCI_DPERR, dc21285_dparity_irq, IRQF_DISABLED, + dc21285_request_irq(IRQ_PCI_DPERR, dc21285_dparity_irq, 0, "PCI data parity", NULL); if (cfn_mode) { diff --git a/arch/arm/mach-footbridge/isa-timer.c b/arch/arm/mach-footbridge/isa-timer.c index d9301dd56354..b73f52e196b9 100644 --- a/arch/arm/mach-footbridge/isa-timer.c +++ b/arch/arm/mach-footbridge/isa-timer.c @@ -27,7 +27,7 @@ static irqreturn_t pit_timer_interrupt(int irq, void *dev_id) static struct irqaction pit_timer_irq = { .name = "pit", .handler = pit_timer_interrupt, - .flags = IRQF_DISABLED | IRQF_TIMER | IRQF_IRQPOLL, + .flags = IRQF_TIMER | IRQF_IRQPOLL, .dev_id = &i8253_clockevent, }; diff --git a/arch/arm/mach-gemini/idle.c b/arch/arm/mach-gemini/idle.c index 87dff4f5059e..ddf8ec9d203b 100644 --- a/arch/arm/mach-gemini/idle.c +++ b/arch/arm/mach-gemini/idle.c @@ -3,7 +3,7 @@ */ #include <linux/init.h> -#include <asm/system.h> +#include <asm/system_misc.h> #include <asm/proc-fns.h> static void gemini_idle(void) diff --git a/arch/arm/mach-integrator/integrator_ap.c b/arch/arm/mach-integrator/integrator_ap.c index 17c0fe627435..e4f27f0e56ac 100644 --- a/arch/arm/mach-integrator/integrator_ap.c +++ b/arch/arm/mach-integrator/integrator_ap.c @@ -358,7 +358,7 @@ static struct clock_event_device integrator_clockevent = { static struct irqaction integrator_timer_irq = { .name = "timer", - .flags = IRQF_DISABLED | IRQF_TIMER | IRQF_IRQPOLL, + .flags = IRQF_TIMER | IRQF_IRQPOLL, .handler = integrator_timer_interrupt, .dev_id = &integrator_clockevent, }; diff --git a/arch/arm/mach-ixp4xx/common.c b/arch/arm/mach-ixp4xx/common.c index 6d68aed6548a..a465f27bc263 100644 --- a/arch/arm/mach-ixp4xx/common.c +++ b/arch/arm/mach-ixp4xx/common.c @@ -312,7 +312,7 @@ static irqreturn_t ixp4xx_timer_interrupt(int irq, void *dev_id) static struct irqaction ixp4xx_timer_irq = { .name = "timer1", - .flags = IRQF_DISABLED | IRQF_TIMER | IRQF_IRQPOLL, + .flags = IRQF_TIMER | IRQF_IRQPOLL, .handler = ixp4xx_timer_interrupt, .dev_id = &clockevent_ixp4xx, }; diff --git a/arch/arm/mach-ixp4xx/dsmg600-setup.c b/arch/arm/mach-ixp4xx/dsmg600-setup.c index 736dc692d540..43ee06d3abe5 100644 --- a/arch/arm/mach-ixp4xx/dsmg600-setup.c +++ b/arch/arm/mach-ixp4xx/dsmg600-setup.c @@ -233,8 +233,7 @@ static int __init dsmg600_gpio_init(void) gpio_request(DSMG600_RB_GPIO, "reset button"); if (request_irq(gpio_to_irq(DSMG600_RB_GPIO), &dsmg600_reset_handler, - IRQF_DISABLED | IRQF_TRIGGER_LOW, - "DSM-G600 reset button", NULL) < 0) { + IRQF_TRIGGER_LOW, "DSM-G600 reset button", NULL) < 0) { printk(KERN_DEBUG "Reset Button IRQ %d not available\n", gpio_to_irq(DSMG600_RB_GPIO)); diff --git a/arch/arm/mach-ixp4xx/fsg-setup.c b/arch/arm/mach-ixp4xx/fsg-setup.c index 429966b756ed..5c4b0c4a1b37 100644 --- a/arch/arm/mach-ixp4xx/fsg-setup.c +++ b/arch/arm/mach-ixp4xx/fsg-setup.c @@ -208,16 +208,14 @@ static void __init fsg_init(void) platform_add_devices(fsg_devices, ARRAY_SIZE(fsg_devices)); if (request_irq(gpio_to_irq(FSG_RB_GPIO), &fsg_reset_handler, - IRQF_DISABLED | IRQF_TRIGGER_LOW, - "FSG reset button", NULL) < 0) { + IRQF_TRIGGER_LOW, "FSG reset button", NULL) < 0) { printk(KERN_DEBUG "Reset Button IRQ %d not available\n", gpio_to_irq(FSG_RB_GPIO)); } if (request_irq(gpio_to_irq(FSG_SB_GPIO), &fsg_power_handler, - IRQF_DISABLED | IRQF_TRIGGER_LOW, - "FSG power button", NULL) < 0) { + IRQF_TRIGGER_LOW, "FSG power button", NULL) < 0) { printk(KERN_DEBUG "Power Button IRQ %d not available\n", gpio_to_irq(FSG_SB_GPIO)); diff --git a/arch/arm/mach-ixp4xx/nas100d-setup.c b/arch/arm/mach-ixp4xx/nas100d-setup.c index 507cb5233537..4e0f762bc651 100644 --- a/arch/arm/mach-ixp4xx/nas100d-setup.c +++ b/arch/arm/mach-ixp4xx/nas100d-setup.c @@ -295,8 +295,7 @@ static void __init nas100d_init(void) pm_power_off = nas100d_power_off; if (request_irq(gpio_to_irq(NAS100D_RB_GPIO), &nas100d_reset_handler, - IRQF_DISABLED | IRQF_TRIGGER_LOW, - "NAS100D reset button", NULL) < 0) { + IRQF_TRIGGER_LOW, "NAS100D reset button", NULL) < 0) { printk(KERN_DEBUG "Reset Button IRQ %d not available\n", gpio_to_irq(NAS100D_RB_GPIO)); diff --git a/arch/arm/mach-ixp4xx/nslu2-setup.c b/arch/arm/mach-ixp4xx/nslu2-setup.c index ba5f1cda2a9d..88c025f52d8d 100644 --- a/arch/arm/mach-ixp4xx/nslu2-setup.c +++ b/arch/arm/mach-ixp4xx/nslu2-setup.c @@ -265,16 +265,14 @@ static void __init nslu2_init(void) pm_power_off = nslu2_power_off; if (request_irq(gpio_to_irq(NSLU2_RB_GPIO), &nslu2_reset_handler, - IRQF_DISABLED | IRQF_TRIGGER_LOW, - "NSLU2 reset button", NULL) < 0) { + IRQF_TRIGGER_LOW, "NSLU2 reset button", NULL) < 0) { printk(KERN_DEBUG "Reset Button IRQ %d not available\n", gpio_to_irq(NSLU2_RB_GPIO)); } if (request_irq(gpio_to_irq(NSLU2_PB_GPIO), &nslu2_power_handler, - IRQF_DISABLED | IRQF_TRIGGER_HIGH, - "NSLU2 power button", NULL) < 0) { + IRQF_TRIGGER_HIGH, "NSLU2 power button", NULL) < 0) { printk(KERN_DEBUG "Power Button IRQ %d not available\n", gpio_to_irq(NSLU2_PB_GPIO)); diff --git a/arch/arm/mach-ks8695/time.c b/arch/arm/mach-ks8695/time.c index 426c97662f5b..a197874bf382 100644 --- a/arch/arm/mach-ks8695/time.c +++ b/arch/arm/mach-ks8695/time.c @@ -122,7 +122,7 @@ static irqreturn_t ks8695_timer_interrupt(int irq, void *dev_id) static struct irqaction ks8695_timer_irq = { .name = "ks8695_tick", - .flags = IRQF_DISABLED | IRQF_TIMER, + .flags = IRQF_TIMER, .handler = ks8695_timer_interrupt, }; diff --git a/arch/arm/mach-lpc32xx/timer.c b/arch/arm/mach-lpc32xx/timer.c index 20eab63d10ba..4e5837299c04 100644 --- a/arch/arm/mach-lpc32xx/timer.c +++ b/arch/arm/mach-lpc32xx/timer.c @@ -90,7 +90,7 @@ static irqreturn_t lpc32xx_timer_interrupt(int irq, void *dev_id) static struct irqaction lpc32xx_timer_irq = { .name = "LPC32XX Timer Tick", - .flags = IRQF_DISABLED | IRQF_TIMER | IRQF_IRQPOLL, + .flags = IRQF_TIMER | IRQF_IRQPOLL, .handler = lpc32xx_timer_interrupt, }; diff --git a/arch/arm/mach-mmp/time.c b/arch/arm/mach-mmp/time.c index 024022d91fe3..bbcd2322fd27 100644 --- a/arch/arm/mach-mmp/time.c +++ b/arch/arm/mach-mmp/time.c @@ -186,7 +186,7 @@ static void __init timer_config(void) static struct irqaction timer_irq = { .name = "timer", - .flags = IRQF_DISABLED | IRQF_TIMER | IRQF_IRQPOLL, + .flags = IRQF_TIMER | IRQF_IRQPOLL, .handler = timer_interrupt, .dev_id = &ckevt, }; diff --git a/arch/arm/mach-netx/time.c b/arch/arm/mach-netx/time.c index 6df42e643031..3177c7a40930 100644 --- a/arch/arm/mach-netx/time.c +++ b/arch/arm/mach-netx/time.c @@ -99,7 +99,7 @@ netx_timer_interrupt(int irq, void *dev_id) static struct irqaction netx_timer_irq = { .name = "NetX Timer Tick", - .flags = IRQF_DISABLED | IRQF_TIMER | IRQF_IRQPOLL, + .flags = IRQF_TIMER | IRQF_IRQPOLL, .handler = netx_timer_interrupt, }; diff --git a/arch/arm/mach-omap1/dma.c b/arch/arm/mach-omap1/dma.c index 5bb8ce86d54b..4be601b638d7 100644 --- a/arch/arm/mach-omap1/dma.c +++ b/arch/arm/mach-omap1/dma.c @@ -32,55 +32,51 @@ #define OMAP1_DMA_BASE (0xfffed800) #define OMAP1_LOGICAL_DMA_CH_COUNT 17 -#define OMAP1_DMA_STRIDE 0x40 -static u32 errata; static u32 enable_1510_mode; -static u8 dma_stride; -static enum omap_reg_offsets dma_common_ch_start, dma_common_ch_end; - -static u16 reg_map[] = { - [GCR] = 0x400, - [GSCR] = 0x404, - [GRST1] = 0x408, - [HW_ID] = 0x442, - [PCH2_ID] = 0x444, - [PCH0_ID] = 0x446, - [PCH1_ID] = 0x448, - [PCHG_ID] = 0x44a, - [PCHD_ID] = 0x44c, - [CAPS_0] = 0x44e, - [CAPS_1] = 0x452, - [CAPS_2] = 0x456, - [CAPS_3] = 0x458, - [CAPS_4] = 0x45a, - [PCH2_SR] = 0x460, - [PCH0_SR] = 0x480, - [PCH1_SR] = 0x482, - [PCHD_SR] = 0x4c0, + +static const struct omap_dma_reg reg_map[] = { + [GCR] = { 0x0400, 0x00, OMAP_DMA_REG_16BIT }, + [GSCR] = { 0x0404, 0x00, OMAP_DMA_REG_16BIT }, + [GRST1] = { 0x0408, 0x00, OMAP_DMA_REG_16BIT }, + [HW_ID] = { 0x0442, 0x00, OMAP_DMA_REG_16BIT }, + [PCH2_ID] = { 0x0444, 0x00, OMAP_DMA_REG_16BIT }, + [PCH0_ID] = { 0x0446, 0x00, OMAP_DMA_REG_16BIT }, + [PCH1_ID] = { 0x0448, 0x00, OMAP_DMA_REG_16BIT }, + [PCHG_ID] = { 0x044a, 0x00, OMAP_DMA_REG_16BIT }, + [PCHD_ID] = { 0x044c, 0x00, OMAP_DMA_REG_16BIT }, + [CAPS_0] = { 0x044e, 0x00, OMAP_DMA_REG_2X16BIT }, + [CAPS_1] = { 0x0452, 0x00, OMAP_DMA_REG_2X16BIT }, + [CAPS_2] = { 0x0456, 0x00, OMAP_DMA_REG_16BIT }, + [CAPS_3] = { 0x0458, 0x00, OMAP_DMA_REG_16BIT }, + [CAPS_4] = { 0x045a, 0x00, OMAP_DMA_REG_16BIT }, + [PCH2_SR] = { 0x0460, 0x00, OMAP_DMA_REG_16BIT }, + [PCH0_SR] = { 0x0480, 0x00, OMAP_DMA_REG_16BIT }, + [PCH1_SR] = { 0x0482, 0x00, OMAP_DMA_REG_16BIT }, + [PCHD_SR] = { 0x04c0, 0x00, OMAP_DMA_REG_16BIT }, /* Common Registers */ - [CSDP] = 0x00, - [CCR] = 0x02, - [CICR] = 0x04, - [CSR] = 0x06, - [CEN] = 0x10, - [CFN] = 0x12, - [CSFI] = 0x14, - [CSEI] = 0x16, - [CPC] = 0x18, /* 15xx only */ - [CSAC] = 0x18, - [CDAC] = 0x1a, - [CDEI] = 0x1c, - [CDFI] = 0x1e, - [CLNK_CTRL] = 0x28, + [CSDP] = { 0x0000, 0x40, OMAP_DMA_REG_16BIT }, + [CCR] = { 0x0002, 0x40, OMAP_DMA_REG_16BIT }, + [CICR] = { 0x0004, 0x40, OMAP_DMA_REG_16BIT }, + [CSR] = { 0x0006, 0x40, OMAP_DMA_REG_16BIT }, + [CEN] = { 0x0010, 0x40, OMAP_DMA_REG_16BIT }, + [CFN] = { 0x0012, 0x40, OMAP_DMA_REG_16BIT }, + [CSFI] = { 0x0014, 0x40, OMAP_DMA_REG_16BIT }, + [CSEI] = { 0x0016, 0x40, OMAP_DMA_REG_16BIT }, + [CPC] = { 0x0018, 0x40, OMAP_DMA_REG_16BIT }, /* 15xx only */ + [CSAC] = { 0x0018, 0x40, OMAP_DMA_REG_16BIT }, + [CDAC] = { 0x001a, 0x40, OMAP_DMA_REG_16BIT }, + [CDEI] = { 0x001c, 0x40, OMAP_DMA_REG_16BIT }, + [CDFI] = { 0x001e, 0x40, OMAP_DMA_REG_16BIT }, + [CLNK_CTRL] = { 0x0028, 0x40, OMAP_DMA_REG_16BIT }, /* Channel specific register offsets */ - [CSSA] = 0x08, - [CDSA] = 0x0c, - [COLOR] = 0x20, - [CCR2] = 0x24, - [LCH_CTRL] = 0x2a, + [CSSA] = { 0x0008, 0x40, OMAP_DMA_REG_2X16BIT }, + [CDSA] = { 0x000c, 0x40, OMAP_DMA_REG_2X16BIT }, + [COLOR] = { 0x0020, 0x40, OMAP_DMA_REG_2X16BIT }, + [CCR2] = { 0x0024, 0x40, OMAP_DMA_REG_16BIT }, + [LCH_CTRL] = { 0x002a, 0x40, OMAP_DMA_REG_16BIT }, }; static struct resource res[] __initdata = { @@ -181,44 +177,36 @@ static struct resource res[] __initdata = { static void __iomem *dma_base; static inline void dma_write(u32 val, int reg, int lch) { - u8 stride; - u32 offset; + void __iomem *addr = dma_base; - stride = (reg >= dma_common_ch_start) ? dma_stride : 0; - offset = reg_map[reg] + (stride * lch); + addr += reg_map[reg].offset; + addr += reg_map[reg].stride * lch; - __raw_writew(val, dma_base + offset); - if ((reg > CLNK_CTRL && reg < CCEN) || - (reg > PCHD_ID && reg < CAPS_2)) { - u32 offset2 = reg_map[reg] + 2 + (stride * lch); - __raw_writew(val >> 16, dma_base + offset2); - } + __raw_writew(val, addr); + if (reg_map[reg].type == OMAP_DMA_REG_2X16BIT) + __raw_writew(val >> 16, addr + 2); } static inline u32 dma_read(int reg, int lch) { - u8 stride; - u32 offset, val; - - stride = (reg >= dma_common_ch_start) ? dma_stride : 0; - offset = reg_map[reg] + (stride * lch); - - val = __raw_readw(dma_base + offset); - if ((reg > CLNK_CTRL && reg < CCEN) || - (reg > PCHD_ID && reg < CAPS_2)) { - u16 upper; - u32 offset2 = reg_map[reg] + 2 + (stride * lch); - upper = __raw_readw(dma_base + offset2); - val |= (upper << 16); - } + void __iomem *addr = dma_base; + uint32_t val; + + addr += reg_map[reg].offset; + addr += reg_map[reg].stride * lch; + + val = __raw_readw(addr); + if (reg_map[reg].type == OMAP_DMA_REG_2X16BIT) + val |= __raw_readw(addr + 2) << 16; + return val; } static void omap1_clear_lch_regs(int lch) { - int i = dma_common_ch_start; + int i; - for (; i <= dma_common_ch_end; i += 1) + for (i = CPC; i <= COLOR; i += 1) dma_write(0, i, lch); } @@ -255,8 +243,9 @@ static void omap1_show_dma_caps(void) return; } -static u32 configure_dma_errata(void) +static unsigned configure_dma_errata(void) { + unsigned errata = 0; /* * Erratum 3.2/3.3: sometimes 0 is returned if CSAC/CDAC is @@ -272,11 +261,23 @@ static const struct platform_device_info omap_dma_dev_info = { .name = "omap-dma-engine", .id = -1, .dma_mask = DMA_BIT_MASK(32), + .res = res, + .num_res = 1, +}; + +static struct omap_system_dma_plat_info dma_plat_info __initdata = { + .reg_map = reg_map, + .channel_stride = 0x40, + .show_dma_caps = omap1_show_dma_caps, + .clear_lch_regs = omap1_clear_lch_regs, + .clear_dma = omap1_clear_dma, + .dma_write = dma_write, + .dma_read = dma_read, }; static int __init omap1_system_dma_init(void) { - struct omap_system_dma_plat_info *p; + struct omap_system_dma_plat_info p; struct omap_dma_dev_attr *d; struct platform_device *pdev, *dma_pdev; int ret; @@ -302,20 +303,12 @@ static int __init omap1_system_dma_init(void) goto exit_iounmap; } - p = kzalloc(sizeof(struct omap_system_dma_plat_info), GFP_KERNEL); - if (!p) { - dev_err(&pdev->dev, "%s: Unable to allocate 'p' for %s\n", - __func__, pdev->name); - ret = -ENOMEM; - goto exit_iounmap; - } - d = kzalloc(sizeof(struct omap_dma_dev_attr), GFP_KERNEL); if (!d) { dev_err(&pdev->dev, "%s: Unable to allocate 'd' for %s\n", __func__, pdev->name); ret = -ENOMEM; - goto exit_release_p; + goto exit_iounmap; } d->lch_count = OMAP1_LOGICAL_DMA_CH_COUNT; @@ -336,17 +329,6 @@ static int __init omap1_system_dma_init(void) d->dev_caps |= CLEAR_CSR_ON_READ; d->dev_caps |= IS_WORD_16; - - d->chan = kzalloc(sizeof(struct omap_dma_lch) * - (d->lch_count), GFP_KERNEL); - if (!d->chan) { - dev_err(&pdev->dev, - "%s: Memory allocation failed for d->chan!\n", - __func__); - ret = -ENOMEM; - goto exit_release_d; - } - if (cpu_is_omap15xx()) d->chan_count = 9; else if (cpu_is_omap16xx() || cpu_is_omap7xx()) { @@ -356,35 +338,24 @@ static int __init omap1_system_dma_init(void) d->chan_count = 9; } - p->dma_attr = d; - - p->show_dma_caps = omap1_show_dma_caps; - p->clear_lch_regs = omap1_clear_lch_regs; - p->clear_dma = omap1_clear_dma; - p->dma_write = dma_write; - p->dma_read = dma_read; - p->disable_irq_lch = NULL; - - p->errata = configure_dma_errata(); + p = dma_plat_info; + p.dma_attr = d; + p.errata = configure_dma_errata(); - ret = platform_device_add_data(pdev, p, sizeof(*p)); + ret = platform_device_add_data(pdev, &p, sizeof(p)); if (ret) { dev_err(&pdev->dev, "%s: Unable to add resources for %s%d\n", __func__, pdev->name, pdev->id); - goto exit_release_chan; + goto exit_release_d; } ret = platform_device_add(pdev); if (ret) { dev_err(&pdev->dev, "%s: Unable to add resources for %s%d\n", __func__, pdev->name, pdev->id); - goto exit_release_chan; + goto exit_release_d; } - dma_stride = OMAP1_DMA_STRIDE; - dma_common_ch_start = CPC; - dma_common_ch_end = COLOR; - dma_pdev = platform_device_register_full(&omap_dma_dev_info); if (IS_ERR(dma_pdev)) { ret = PTR_ERR(dma_pdev); @@ -395,12 +366,8 @@ static int __init omap1_system_dma_init(void) exit_release_pdev: platform_device_del(pdev); -exit_release_chan: - kfree(d->chan); exit_release_d: kfree(d); -exit_release_p: - kfree(p); exit_iounmap: iounmap(dma_base); exit_device_put: diff --git a/arch/arm/mach-omap2/am35xx-emac.c b/arch/arm/mach-omap2/am35xx-emac.c index 25b79a297365..6a6935caac1e 100644 --- a/arch/arm/mach-omap2/am35xx-emac.c +++ b/arch/arm/mach-omap2/am35xx-emac.c @@ -17,7 +17,6 @@ #include <linux/err.h> #include <linux/davinci_emac.h> -#include <asm/system.h> #include "omap_device.h" #include "am35xx.h" #include "control.h" diff --git a/arch/arm/mach-omap2/dma.c b/arch/arm/mach-omap2/dma.c index 49fd0d501c9b..5689c88d986d 100644 --- a/arch/arm/mach-omap2/dma.c +++ b/arch/arm/mach-omap2/dma.c @@ -35,97 +35,80 @@ #include "omap_hwmod.h" #include "omap_device.h" -#define OMAP2_DMA_STRIDE 0x60 - -static u32 errata; -static u8 dma_stride; - -static struct omap_dma_dev_attr *d; - -static enum omap_reg_offsets dma_common_ch_start, dma_common_ch_end; - -static u16 reg_map[] = { - [REVISION] = 0x00, - [GCR] = 0x78, - [IRQSTATUS_L0] = 0x08, - [IRQSTATUS_L1] = 0x0c, - [IRQSTATUS_L2] = 0x10, - [IRQSTATUS_L3] = 0x14, - [IRQENABLE_L0] = 0x18, - [IRQENABLE_L1] = 0x1c, - [IRQENABLE_L2] = 0x20, - [IRQENABLE_L3] = 0x24, - [SYSSTATUS] = 0x28, - [OCP_SYSCONFIG] = 0x2c, - [CAPS_0] = 0x64, - [CAPS_2] = 0x6c, - [CAPS_3] = 0x70, - [CAPS_4] = 0x74, +static enum omap_reg_offsets dma_common_ch_end; + +static const struct omap_dma_reg reg_map[] = { + [REVISION] = { 0x0000, 0x00, OMAP_DMA_REG_32BIT }, + [GCR] = { 0x0078, 0x00, OMAP_DMA_REG_32BIT }, + [IRQSTATUS_L0] = { 0x0008, 0x00, OMAP_DMA_REG_32BIT }, + [IRQSTATUS_L1] = { 0x000c, 0x00, OMAP_DMA_REG_32BIT }, + [IRQSTATUS_L2] = { 0x0010, 0x00, OMAP_DMA_REG_32BIT }, + [IRQSTATUS_L3] = { 0x0014, 0x00, OMAP_DMA_REG_32BIT }, + [IRQENABLE_L0] = { 0x0018, 0x00, OMAP_DMA_REG_32BIT }, + [IRQENABLE_L1] = { 0x001c, 0x00, OMAP_DMA_REG_32BIT }, + [IRQENABLE_L2] = { 0x0020, 0x00, OMAP_DMA_REG_32BIT }, + [IRQENABLE_L3] = { 0x0024, 0x00, OMAP_DMA_REG_32BIT }, + [SYSSTATUS] = { 0x0028, 0x00, OMAP_DMA_REG_32BIT }, + [OCP_SYSCONFIG] = { 0x002c, 0x00, OMAP_DMA_REG_32BIT }, + [CAPS_0] = { 0x0064, 0x00, OMAP_DMA_REG_32BIT }, + [CAPS_2] = { 0x006c, 0x00, OMAP_DMA_REG_32BIT }, + [CAPS_3] = { 0x0070, 0x00, OMAP_DMA_REG_32BIT }, + [CAPS_4] = { 0x0074, 0x00, OMAP_DMA_REG_32BIT }, /* Common register offsets */ - [CCR] = 0x80, - [CLNK_CTRL] = 0x84, - [CICR] = 0x88, - [CSR] = 0x8c, - [CSDP] = 0x90, - [CEN] = 0x94, - [CFN] = 0x98, - [CSEI] = 0xa4, - [CSFI] = 0xa8, - [CDEI] = 0xac, - [CDFI] = 0xb0, - [CSAC] = 0xb4, - [CDAC] = 0xb8, + [CCR] = { 0x0080, 0x60, OMAP_DMA_REG_32BIT }, + [CLNK_CTRL] = { 0x0084, 0x60, OMAP_DMA_REG_32BIT }, + [CICR] = { 0x0088, 0x60, OMAP_DMA_REG_32BIT }, + [CSR] = { 0x008c, 0x60, OMAP_DMA_REG_32BIT }, + [CSDP] = { 0x0090, 0x60, OMAP_DMA_REG_32BIT }, + [CEN] = { 0x0094, 0x60, OMAP_DMA_REG_32BIT }, + [CFN] = { 0x0098, 0x60, OMAP_DMA_REG_32BIT }, + [CSEI] = { 0x00a4, 0x60, OMAP_DMA_REG_32BIT }, + [CSFI] = { 0x00a8, 0x60, OMAP_DMA_REG_32BIT }, + [CDEI] = { 0x00ac, 0x60, OMAP_DMA_REG_32BIT }, + [CDFI] = { 0x00b0, 0x60, OMAP_DMA_REG_32BIT }, + [CSAC] = { 0x00b4, 0x60, OMAP_DMA_REG_32BIT }, + [CDAC] = { 0x00b8, 0x60, OMAP_DMA_REG_32BIT }, /* Channel specific register offsets */ - [CSSA] = 0x9c, - [CDSA] = 0xa0, - [CCEN] = 0xbc, - [CCFN] = 0xc0, - [COLOR] = 0xc4, + [CSSA] = { 0x009c, 0x60, OMAP_DMA_REG_32BIT }, + [CDSA] = { 0x00a0, 0x60, OMAP_DMA_REG_32BIT }, + [CCEN] = { 0x00bc, 0x60, OMAP_DMA_REG_32BIT }, + [CCFN] = { 0x00c0, 0x60, OMAP_DMA_REG_32BIT }, + [COLOR] = { 0x00c4, 0x60, OMAP_DMA_REG_32BIT }, /* OMAP4 specific registers */ - [CDP] = 0xd0, - [CNDP] = 0xd4, - [CCDN] = 0xd8, + [CDP] = { 0x00d0, 0x60, OMAP_DMA_REG_32BIT }, + [CNDP] = { 0x00d4, 0x60, OMAP_DMA_REG_32BIT }, + [CCDN] = { 0x00d8, 0x60, OMAP_DMA_REG_32BIT }, }; static void __iomem *dma_base; static inline void dma_write(u32 val, int reg, int lch) { - u8 stride; - u32 offset; + void __iomem *addr = dma_base; - stride = (reg >= dma_common_ch_start) ? dma_stride : 0; - offset = reg_map[reg] + (stride * lch); - __raw_writel(val, dma_base + offset); + addr += reg_map[reg].offset; + addr += reg_map[reg].stride * lch; + + __raw_writel(val, addr); } static inline u32 dma_read(int reg, int lch) { - u8 stride; - u32 offset, val; - - stride = (reg >= dma_common_ch_start) ? dma_stride : 0; - offset = reg_map[reg] + (stride * lch); - val = __raw_readl(dma_base + offset); - return val; -} + void __iomem *addr = dma_base; -static inline void omap2_disable_irq_lch(int lch) -{ - u32 val; + addr += reg_map[reg].offset; + addr += reg_map[reg].stride * lch; - val = dma_read(IRQENABLE_L0, lch); - val &= ~(1 << lch); - dma_write(val, IRQENABLE_L0, lch); + return __raw_readl(addr); } static void omap2_clear_dma(int lch) { - int i = dma_common_ch_start; + int i; - for (; i <= dma_common_ch_end; i += 1) + for (i = CSDP; i <= dma_common_ch_end; i += 1) dma_write(0, i, lch); } @@ -137,8 +120,9 @@ static void omap2_show_dma_caps(void) return; } -static u32 configure_dma_errata(void) +static unsigned configure_dma_errata(void) { + unsigned errata = 0; /* * Errata applicable for OMAP2430ES1.0 and all omap2420 @@ -220,48 +204,50 @@ static u32 configure_dma_errata(void) return errata; } +static struct omap_system_dma_plat_info dma_plat_info __initdata = { + .reg_map = reg_map, + .channel_stride = 0x60, + .show_dma_caps = omap2_show_dma_caps, + .clear_dma = omap2_clear_dma, + .dma_write = dma_write, + .dma_read = dma_read, +}; + +static struct platform_device_info omap_dma_dev_info = { + .name = "omap-dma-engine", + .id = -1, + .dma_mask = DMA_BIT_MASK(32), +}; + /* One time initializations */ static int __init omap2_system_dma_init_dev(struct omap_hwmod *oh, void *unused) { struct platform_device *pdev; - struct omap_system_dma_plat_info *p; + struct omap_system_dma_plat_info p; + struct omap_dma_dev_attr *d; struct resource *mem; char *name = "omap_dma_system"; - dma_stride = OMAP2_DMA_STRIDE; - dma_common_ch_start = CSDP; - - p = kzalloc(sizeof(struct omap_system_dma_plat_info), GFP_KERNEL); - if (!p) { - pr_err("%s: Unable to allocate pdata for %s:%s\n", - __func__, name, oh->name); - return -ENOMEM; - } - - p->dma_attr = (struct omap_dma_dev_attr *)oh->dev_attr; - p->disable_irq_lch = omap2_disable_irq_lch; - p->show_dma_caps = omap2_show_dma_caps; - p->clear_dma = omap2_clear_dma; - p->dma_write = dma_write; - p->dma_read = dma_read; - - p->clear_lch_regs = NULL; - - p->errata = configure_dma_errata(); + p = dma_plat_info; + p.dma_attr = (struct omap_dma_dev_attr *)oh->dev_attr; + p.errata = configure_dma_errata(); - pdev = omap_device_build(name, 0, oh, p, sizeof(*p)); - kfree(p); + pdev = omap_device_build(name, 0, oh, &p, sizeof(p)); if (IS_ERR(pdev)) { pr_err("%s: Can't build omap_device for %s:%s.\n", __func__, name, oh->name); return PTR_ERR(pdev); } + omap_dma_dev_info.res = pdev->resource; + omap_dma_dev_info.num_res = pdev->num_resources; + mem = platform_get_resource(pdev, IORESOURCE_MEM, 0); if (!mem) { dev_err(&pdev->dev, "%s: no mem resource\n", __func__); return -EINVAL; } + dma_base = ioremap(mem->start, resource_size(mem)); if (!dma_base) { dev_err(&pdev->dev, "%s: ioremap fail\n", __func__); @@ -269,13 +255,6 @@ static int __init omap2_system_dma_init_dev(struct omap_hwmod *oh, void *unused) } d = oh->dev_attr; - d->chan = kzalloc(sizeof(struct omap_dma_lch) * - (d->lch_count), GFP_KERNEL); - - if (!d->chan) { - dev_err(&pdev->dev, "%s: kzalloc fail\n", __func__); - return -ENOMEM; - } if (cpu_is_omap34xx() && (omap_type() != OMAP2_DEVICE_TYPE_GP)) d->dev_caps |= HS_CHANNELS_RESERVED; @@ -289,12 +268,6 @@ static int __init omap2_system_dma_init_dev(struct omap_hwmod *oh, void *unused) return 0; } -static const struct platform_device_info omap_dma_dev_info = { - .name = "omap-dma-engine", - .id = -1, - .dma_mask = DMA_BIT_MASK(32), -}; - static int __init omap2_system_dma_init(void) { struct platform_device *pdev; diff --git a/arch/arm/mach-rpc/dma.c b/arch/arm/mach-rpc/dma.c index 85883b2e0e49..6d3517dc4772 100644 --- a/arch/arm/mach-rpc/dma.c +++ b/arch/arm/mach-rpc/dma.c @@ -141,7 +141,7 @@ static int iomd_request_dma(unsigned int chan, dma_t *dma) struct iomd_dma *idma = container_of(dma, struct iomd_dma, dma); return request_irq(idma->irq, iomd_dma_handle, - IRQF_DISABLED, idma->dma.device_id, idma); + 0, idma->dma.device_id, idma); } static void iomd_free_dma(unsigned int chan, dma_t *dma) diff --git a/arch/arm/mach-rpc/time.c b/arch/arm/mach-rpc/time.c index 9a6def14df01..9a5158861ca9 100644 --- a/arch/arm/mach-rpc/time.c +++ b/arch/arm/mach-rpc/time.c @@ -75,7 +75,6 @@ ioc_timer_interrupt(int irq, void *dev_id) static struct irqaction ioc_timer_irq = { .name = "timer", - .flags = IRQF_DISABLED, .handler = ioc_timer_interrupt }; diff --git a/arch/arm/mach-sa1100/collie.c b/arch/arm/mach-sa1100/collie.c index 831a15824ec8..f9874ba60cc8 100644 --- a/arch/arm/mach-sa1100/collie.c +++ b/arch/arm/mach-sa1100/collie.c @@ -43,6 +43,7 @@ #include <asm/mach/arch.h> #include <asm/mach/flash.h> #include <asm/mach/map.h> +#include <asm/mach/irda.h> #include <asm/hardware/scoop.h> #include <asm/mach/sharpsl_param.h> @@ -96,6 +97,37 @@ static struct mcp_plat_data collie_mcp_data = { .codec_pdata = &collie_ucb1x00_data, }; +static int collie_ir_startup(struct device *dev) +{ + int rc = gpio_request(COLLIE_GPIO_IR_ON, "IrDA"); + if (rc) + return rc; + rc = gpio_direction_output(COLLIE_GPIO_IR_ON, 1); + + if (!rc) + return 0; + + gpio_free(COLLIE_GPIO_IR_ON); + return rc; +} + +static void collie_ir_shutdown(struct device *dev) +{ + gpio_free(COLLIE_GPIO_IR_ON); +} + +static int collie_ir_set_power(struct device *dev, unsigned int state) +{ + gpio_set_value(COLLIE_GPIO_IR_ON, !state); + return 0; +} + +static struct irda_platform_data collie_ir_data = { + .startup = collie_ir_startup, + .shutdown = collie_ir_shutdown, + .set_power = collie_ir_set_power, +}; + /* * Collie AC IN */ @@ -400,6 +432,7 @@ static void __init collie_init(void) sa11x0_register_mtd(&collie_flash_data, collie_flash_resources, ARRAY_SIZE(collie_flash_resources)); sa11x0_register_mcp(&collie_mcp_data); + sa11x0_register_irda(&collie_ir_data); sharpsl_save_param(); } diff --git a/arch/arm/mach-sa1100/h3100.c b/arch/arm/mach-sa1100/h3100.c index daa27c474c13..3c43219bc881 100644 --- a/arch/arm/mach-sa1100/h3100.c +++ b/arch/arm/mach-sa1100/h3100.c @@ -122,15 +122,8 @@ static struct irda_platform_data h3100_irda_data = { .shutdown = h3100_irda_shutdown, }; -static struct gpio_default_state h3100_default_gpio[] = { - { H3XXX_GPIO_COM_DCD, GPIO_MODE_IN, "COM DCD" }, - { H3XXX_GPIO_COM_CTS, GPIO_MODE_IN, "COM CTS" }, - { H3XXX_GPIO_COM_RTS, GPIO_MODE_OUT0, "COM RTS" }, -}; - static void __init h3100_mach_init(void) { - h3xxx_init_gpio(h3100_default_gpio, ARRAY_SIZE(h3100_default_gpio)); h3xxx_mach_init(); sa11x0_register_lcd(&h3100_lcd_info); diff --git a/arch/arm/mach-sa1100/h3600.c b/arch/arm/mach-sa1100/h3600.c index a663e7230141..5be54c214c7c 100644 --- a/arch/arm/mach-sa1100/h3600.c +++ b/arch/arm/mach-sa1100/h3600.c @@ -130,15 +130,8 @@ static struct irda_platform_data h3600_irda_data = { .shutdown = h3600_irda_shutdown, }; -static struct gpio_default_state h3600_default_gpio[] = { - { H3XXX_GPIO_COM_DCD, GPIO_MODE_IN, "COM DCD" }, - { H3XXX_GPIO_COM_CTS, GPIO_MODE_IN, "COM CTS" }, - { H3XXX_GPIO_COM_RTS, GPIO_MODE_OUT0, "COM RTS" }, -}; - static void __init h3600_mach_init(void) { - h3xxx_init_gpio(h3600_default_gpio, ARRAY_SIZE(h3600_default_gpio)); h3xxx_mach_init(); sa11x0_register_lcd(&h3600_lcd_info); diff --git a/arch/arm/mach-sa1100/h3xxx.c b/arch/arm/mach-sa1100/h3xxx.c index f17e7382242a..c79bf467fb7f 100644 --- a/arch/arm/mach-sa1100/h3xxx.c +++ b/arch/arm/mach-sa1100/h3xxx.c @@ -28,37 +28,6 @@ #include "generic.h" -void h3xxx_init_gpio(struct gpio_default_state *s, size_t n) -{ - while (n--) { - const char *name = s->name; - int err; - - if (!name) - name = "[init]"; - err = gpio_request(s->gpio, name); - if (err) { - printk(KERN_ERR "gpio%u: unable to request: %d\n", - s->gpio, err); - continue; - } - if (s->mode >= 0) { - err = gpio_direction_output(s->gpio, s->mode); - } else { - err = gpio_direction_input(s->gpio); - } - if (err) { - printk(KERN_ERR "gpio%u: unable to set direction: %d\n", - s->gpio, err); - continue; - } - if (!s->name) - gpio_free(s->gpio); - s++; - } -} - - /* * H3xxx flash support */ @@ -116,9 +85,34 @@ static struct resource h3xxx_flash_resource = /* * H3xxx uart support */ +static struct gpio h3xxx_uart_gpio[] = { + { H3XXX_GPIO_COM_DCD, GPIOF_IN, "COM DCD" }, + { H3XXX_GPIO_COM_CTS, GPIOF_IN, "COM CTS" }, + { H3XXX_GPIO_COM_RTS, GPIOF_OUT_INIT_LOW, "COM RTS" }, +}; + +static bool h3xxx_uart_request_gpios(void) +{ + static bool h3xxx_uart_gpio_ok; + int rc; + + if (h3xxx_uart_gpio_ok) + return true; + + rc = gpio_request_array(h3xxx_uart_gpio, ARRAY_SIZE(h3xxx_uart_gpio)); + if (rc) + pr_err("h3xxx_uart_request_gpios: error %d\n", rc); + else + h3xxx_uart_gpio_ok = true; + + return h3xxx_uart_gpio_ok; +} + static void h3xxx_uart_set_mctrl(struct uart_port *port, u_int mctrl) { if (port->mapbase == _Ser3UTCR0) { + if (!h3xxx_uart_request_gpios()) + return; gpio_set_value(H3XXX_GPIO_COM_RTS, !(mctrl & TIOCM_RTS)); } } @@ -128,6 +122,8 @@ static u_int h3xxx_uart_get_mctrl(struct uart_port *port) u_int ret = TIOCM_CD | TIOCM_CTS | TIOCM_DSR; if (port->mapbase == _Ser3UTCR0) { + if (!h3xxx_uart_request_gpios()) + return ret; /* * DCD and CTS bits are inverted in GPLR by RS232 transceiver */ diff --git a/arch/arm/mach-sa1100/include/mach/collie.h b/arch/arm/mach-sa1100/include/mach/collie.h index 50e1d850ee2e..b478ca180c19 100644 --- a/arch/arm/mach-sa1100/include/mach/collie.h +++ b/arch/arm/mach-sa1100/include/mach/collie.h @@ -80,7 +80,7 @@ extern void locomolcd_power(int on); #define COLLIE_TC35143_GPIO_VERSION0 UCB_IO_0 #define COLLIE_TC35143_GPIO_TBL_CHK UCB_IO_1 #define COLLIE_TC35143_GPIO_VPEN_ON UCB_IO_2 -#define COLLIE_TC35143_GPIO_IR_ON UCB_IO_3 +#define COLLIE_GPIO_IR_ON (COLLIE_TC35143_GPIO_BASE + 3) #define COLLIE_TC35143_GPIO_AMP_ON UCB_IO_4 #define COLLIE_TC35143_GPIO_VERSION1 UCB_IO_5 #define COLLIE_TC35143_GPIO_FS8KLPF UCB_IO_5 diff --git a/arch/arm/mach-sa1100/include/mach/h3xxx.h b/arch/arm/mach-sa1100/include/mach/h3xxx.h index c810620db53d..603d4343f7f6 100644 --- a/arch/arm/mach-sa1100/include/mach/h3xxx.h +++ b/arch/arm/mach-sa1100/include/mach/h3xxx.h @@ -79,17 +79,6 @@ #define H3600_EGPIO_LCD_5V_ON (H3XXX_EGPIO_BASE + 14) /* enable 5V to LCD. active high. */ #define H3600_EGPIO_LVDD_ON (H3XXX_EGPIO_BASE + 15) /* enable 9V and -6.5V to LCD. */ -struct gpio_default_state { - int gpio; - int mode; - const char *name; -}; - -#define GPIO_MODE_IN -1 -#define GPIO_MODE_OUT0 0 -#define GPIO_MODE_OUT1 1 - -void h3xxx_init_gpio(struct gpio_default_state *s, size_t n); void __init h3xxx_map_io(void); void __init h3xxx_mach_init(void); diff --git a/arch/arm/mach-sa1100/time.c b/arch/arm/mach-sa1100/time.c index 6fd4acb8f187..4852c08cb526 100644 --- a/arch/arm/mach-sa1100/time.c +++ b/arch/arm/mach-sa1100/time.c @@ -112,7 +112,7 @@ static struct clock_event_device ckevt_sa1100_osmr0 = { static struct irqaction sa1100_timer_irq = { .name = "ost0", - .flags = IRQF_DISABLED | IRQF_TIMER | IRQF_IRQPOLL, + .flags = IRQF_TIMER | IRQF_IRQPOLL, .handler = sa1100_ost0_interrupt, .dev_id = &ckevt_sa1100_osmr0, }; diff --git a/arch/arm/mach-spear/time.c b/arch/arm/mach-spear/time.c index d449673e40f7..218ba5b67d92 100644 --- a/arch/arm/mach-spear/time.c +++ b/arch/arm/mach-spear/time.c @@ -172,7 +172,7 @@ static irqreturn_t spear_timer_interrupt(int irq, void *dev_id) static struct irqaction spear_timer_irq = { .name = "timer", - .flags = IRQF_DISABLED | IRQF_TIMER, + .flags = IRQF_TIMER, .handler = spear_timer_interrupt }; diff --git a/arch/arm/mach-vexpress/Makefile b/arch/arm/mach-vexpress/Makefile index 0997e0b7494c..fc649bc09d0c 100644 --- a/arch/arm/mach-vexpress/Makefile +++ b/arch/arm/mach-vexpress/Makefile @@ -8,8 +8,11 @@ obj-y := v2m.o obj-$(CONFIG_ARCH_VEXPRESS_CA9X4) += ct-ca9x4.o obj-$(CONFIG_ARCH_VEXPRESS_DCSCB) += dcscb.o dcscb_setup.o CFLAGS_dcscb.o += -march=armv7-a +CFLAGS_REMOVE_dcscb.o = -pg obj-$(CONFIG_ARCH_VEXPRESS_SPC) += spc.o +CFLAGS_REMOVE_spc.o = -pg obj-$(CONFIG_ARCH_VEXPRESS_TC2_PM) += tc2_pm.o CFLAGS_tc2_pm.o += -march=armv7-a +CFLAGS_REMOVE_tc2_pm.o = -pg obj-$(CONFIG_SMP) += platsmp.o obj-$(CONFIG_HOTPLUG_CPU) += hotplug.o diff --git a/arch/arm/mach-w90x900/time.c b/arch/arm/mach-w90x900/time.c index 30fbca844575..9230d3725599 100644 --- a/arch/arm/mach-w90x900/time.c +++ b/arch/arm/mach-w90x900/time.c @@ -111,7 +111,7 @@ static irqreturn_t nuc900_timer0_interrupt(int irq, void *dev_id) static struct irqaction nuc900_timer0_irq = { .name = "nuc900-timer0", - .flags = IRQF_DISABLED | IRQF_TIMER | IRQF_IRQPOLL, + .flags = IRQF_TIMER | IRQF_IRQPOLL, .handler = nuc900_timer0_interrupt, }; diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig index 1f8fed94c2a4..ca8ecdee47d8 100644 --- a/arch/arm/mm/Kconfig +++ b/arch/arm/mm/Kconfig @@ -446,7 +446,6 @@ config CPU_32v5 config CPU_32v6 bool - select CPU_USE_DOMAINS if CPU_V6 && MMU select TLS_REG_EMUL if !CPU_32v6K && !MMU config CPU_32v6K @@ -671,7 +670,7 @@ config ARM_VIRT_EXT config SWP_EMULATE bool "Emulate SWP/SWPB instructions" - depends on !CPU_USE_DOMAINS && CPU_V7 + depends on CPU_V7 default y if SMP select HAVE_PROC_CPU if PROC_FS help diff --git a/arch/arm/mm/cache-feroceon-l2.c b/arch/arm/mm/cache-feroceon-l2.c index 48bc3c0a87ce..aae891820f8f 100644 --- a/arch/arm/mm/cache-feroceon-l2.c +++ b/arch/arm/mm/cache-feroceon-l2.c @@ -331,7 +331,9 @@ static void __init enable_l2(void) enable_icache(); if (d) enable_dcache(); - } + } else + pr_err(FW_BUG + "Feroceon L2: bootloader left the L2 cache on!\n"); } void __init feroceon_l2_init(int __l2_wt_override) diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index 4fe42ce720d2..f62aa0677e5c 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -284,9 +284,6 @@ static void __dma_free_buffer(struct page *page, size_t size) } #ifdef CONFIG_MMU -#ifdef CONFIG_HUGETLB_PAGE -#warning ARM Coherent DMA allocator does not (yet) support huge TLB -#endif static void *__alloc_from_contiguous(struct device *dev, size_t size, pgprot_t prot, struct page **ret_page, diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index a623cb3ad012..b68c6b22e1c8 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -516,6 +516,16 @@ static void __init build_mem_type_table(void) s2_device_pgprot = mem_types[MT_DEVICE].prot_pte_s2; /* + * We don't use domains on ARMv6 (since this causes problems with + * v6/v7 kernels), so we must use a separate memory type for user + * r/o, kernel r/w to map the vectors page. + */ +#ifndef CONFIG_ARM_LPAE + if (cpu_arch == CPU_ARCH_ARMv6) + vecs_pgprot |= L_PTE_MT_VECTORS; +#endif + + /* * ARMv6 and above have extended page tables. */ if (cpu_arch >= CPU_ARCH_ARMv6 && (cr & CR_XP)) { diff --git a/arch/arm/mm/proc-macros.S b/arch/arm/mm/proc-macros.S index e3c48a3fe063..ee1d80593958 100644 --- a/arch/arm/mm/proc-macros.S +++ b/arch/arm/mm/proc-macros.S @@ -112,13 +112,9 @@ * 100x 1 0 1 r/o no acc * 10x0 1 0 1 r/o no acc * 1011 0 0 1 r/w no acc - * 110x 0 1 0 r/w r/o - * 11x0 0 1 0 r/w r/o - * 1111 0 1 1 r/w r/w - * - * If !CONFIG_CPU_USE_DOMAINS, the following permissions are changed: * 110x 1 1 1 r/o r/o * 11x0 1 1 1 r/o r/o + * 1111 0 1 1 r/w r/w */ .macro armv6_mt_table pfx \pfx\()_mt_table: @@ -137,7 +133,7 @@ .long PTE_EXT_TEX(2) @ L_PTE_MT_DEV_NONSHARED .long 0x00 @ unused .long 0x00 @ unused - .long 0x00 @ unused + .long PTE_CACHEABLE | PTE_BUFFERABLE | PTE_EXT_APX @ L_PTE_MT_VECTORS .endm .macro armv6_set_pte_ext pfx @@ -158,24 +154,21 @@ tst r1, #L_PTE_USER orrne r3, r3, #PTE_EXT_AP1 -#ifdef CONFIG_CPU_USE_DOMAINS - @ allow kernel read/write access to read-only user pages tstne r3, #PTE_EXT_APX - bicne r3, r3, #PTE_EXT_APX | PTE_EXT_AP0 -#endif + + @ user read-only -> kernel read-only + bicne r3, r3, #PTE_EXT_AP0 tst r1, #L_PTE_XN orrne r3, r3, #PTE_EXT_XN - orr r3, r3, r2 + eor r3, r3, r2 tst r1, #L_PTE_YOUNG tstne r1, #L_PTE_PRESENT moveq r3, #0 -#ifndef CONFIG_CPU_USE_DOMAINS tstne r1, #L_PTE_NONE movne r3, #0 -#endif str r3, [r0] mcr p15, 0, r0, c7, c10, 1 @ flush_pte diff --git a/arch/arm/mm/proc-v7-2level.S b/arch/arm/mm/proc-v7-2level.S index bdd3be4be77a..1f52915f2b28 100644 --- a/arch/arm/mm/proc-v7-2level.S +++ b/arch/arm/mm/proc-v7-2level.S @@ -90,21 +90,14 @@ ENTRY(cpu_v7_set_pte_ext) tst r1, #L_PTE_USER orrne r3, r3, #PTE_EXT_AP1 -#ifdef CONFIG_CPU_USE_DOMAINS - @ allow kernel read/write access to read-only user pages - tstne r3, #PTE_EXT_APX - bicne r3, r3, #PTE_EXT_APX | PTE_EXT_AP0 -#endif tst r1, #L_PTE_XN orrne r3, r3, #PTE_EXT_XN tst r1, #L_PTE_YOUNG tstne r1, #L_PTE_VALID -#ifndef CONFIG_CPU_USE_DOMAINS eorne r1, r1, #L_PTE_NONE tstne r1, #L_PTE_NONE -#endif moveq r3, #0 ARM( str r3, [r0, #2048]! ) diff --git a/arch/arm/mm/proc-v7.S b/arch/arm/mm/proc-v7.S index 74f6033e76dd..195731d3813b 100644 --- a/arch/arm/mm/proc-v7.S +++ b/arch/arm/mm/proc-v7.S @@ -192,6 +192,7 @@ __v7_cr7mp_setup: mov r10, #(1 << 0) @ Cache/TLB ops broadcasting b 1f __v7_ca7mp_setup: +__v7_ca12mp_setup: __v7_ca15mp_setup: mov r10, #0 1: @@ -484,6 +485,16 @@ __v7_ca7mp_proc_info: .size __v7_ca7mp_proc_info, . - __v7_ca7mp_proc_info /* + * ARM Ltd. Cortex A12 processor. + */ + .type __v7_ca12mp_proc_info, #object +__v7_ca12mp_proc_info: + .long 0x410fc0d0 + .long 0xff0ffff0 + __v7_proc __v7_ca12mp_setup + .size __v7_ca12mp_proc_info, . - __v7_ca12mp_proc_info + + /* * ARM Ltd. Cortex A15 processor. */ .type __v7_ca15mp_proc_info, #object diff --git a/arch/arm/plat-iop/time.c b/arch/arm/plat-iop/time.c index d70b73364a3f..6ad65d8ae237 100644 --- a/arch/arm/plat-iop/time.c +++ b/arch/arm/plat-iop/time.c @@ -127,7 +127,7 @@ iop_timer_interrupt(int irq, void *dev_id) static struct irqaction iop_timer_irq = { .name = "IOP Timer Tick", .handler = iop_timer_interrupt, - .flags = IRQF_DISABLED | IRQF_TIMER | IRQF_IRQPOLL, + .flags = IRQF_TIMER | IRQF_IRQPOLL, .dev_id = &iop_clockevent, }; diff --git a/arch/arm/plat-omap/dma.c b/arch/arm/plat-omap/dma.c index 01619c2910e3..5f5b975887fc 100644 --- a/arch/arm/plat-omap/dma.c +++ b/arch/arm/plat-omap/dma.c @@ -2000,6 +2000,12 @@ void omap_dma_global_context_restore(void) omap_clear_dma(ch); } +struct omap_system_dma_plat_info *omap_get_plat_info(void) +{ + return p; +} +EXPORT_SYMBOL_GPL(omap_get_plat_info); + static int omap_system_dma_probe(struct platform_device *pdev) { int ch, ret = 0; @@ -2024,9 +2030,16 @@ static int omap_system_dma_probe(struct platform_device *pdev) dma_lch_count = d->lch_count; dma_chan_count = dma_lch_count; - dma_chan = d->chan; enable_1510_mode = d->dev_caps & ENABLE_1510_MODE; + dma_chan = devm_kcalloc(&pdev->dev, dma_lch_count, + sizeof(struct omap_dma_lch), GFP_KERNEL); + if (!dma_chan) { + dev_err(&pdev->dev, "%s: kzalloc fail\n", __func__); + return -ENOMEM; + } + + if (dma_omap2plus()) { dma_linked_lch = kzalloc(sizeof(struct dma_link_info) * dma_lch_count, GFP_KERNEL); @@ -2111,7 +2124,6 @@ exit_dma_irq_fail: } exit_dma_lch_fail: - kfree(dma_chan); return ret; } @@ -2131,7 +2143,6 @@ static int omap_system_dma_remove(struct platform_device *pdev) free_irq(dma_irq, (void *)(irq_rel + 1)); } } - kfree(dma_chan); return 0; } diff --git a/drivers/amba/bus.c b/drivers/amba/bus.c index 9e6029105607..3cf61a127ee5 100644 --- a/drivers/amba/bus.c +++ b/drivers/amba/bus.c @@ -83,7 +83,7 @@ static struct device_attribute amba_dev_attrs[] = { __ATTR_NULL, }; -#ifdef CONFIG_PM_RUNTIME +#ifdef CONFIG_PM /* * Hooks to provide runtime PM of the pclk (bus clock). It is safe to * enable/disable the bus clock at runtime PM suspend/resume as this @@ -123,7 +123,7 @@ static const struct dev_pm_ops amba_pm = { .thaw = pm_generic_thaw, .poweroff = pm_generic_poweroff, .restore = pm_generic_restore, - SET_RUNTIME_PM_OPS( + SET_PM_RUNTIME_PM_OPS( amba_pm_runtime_suspend, amba_pm_runtime_resume, NULL diff --git a/drivers/clocksource/timer-u300.c b/drivers/clocksource/timer-u300.c index e63d469661fd..5dcf756970e7 100644 --- a/drivers/clocksource/timer-u300.c +++ b/drivers/clocksource/timer-u300.c @@ -333,7 +333,7 @@ static irqreturn_t u300_timer_interrupt(int irq, void *dev_id) static struct irqaction u300_timer_irq = { .name = "U300 Timer Tick", - .flags = IRQF_DISABLED | IRQF_TIMER | IRQF_IRQPOLL, + .flags = IRQF_TIMER | IRQF_IRQPOLL, .handler = u300_timer_interrupt, }; diff --git a/drivers/dma/omap-dma.c b/drivers/dma/omap-dma.c index 362e7c49f2e1..64ceca2920b8 100644 --- a/drivers/dma/omap-dma.c +++ b/drivers/dma/omap-dma.c @@ -5,6 +5,7 @@ * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. */ +#include <linux/delay.h> #include <linux/dmaengine.h> #include <linux/dma-mapping.h> #include <linux/err.h> @@ -26,11 +27,21 @@ struct omap_dmadev { spinlock_t lock; struct tasklet_struct task; struct list_head pending; + void __iomem *base; + const struct omap_dma_reg *reg_map; + struct omap_system_dma_plat_info *plat; + bool legacy; + spinlock_t irq_lock; + uint32_t irq_enable_mask; + struct omap_chan *lch_map[32]; }; struct omap_chan { struct virt_dma_chan vc; struct list_head node; + void __iomem *channel_base; + const struct omap_dma_reg *reg_map; + uint32_t ccr; struct dma_slave_config cfg; unsigned dma_sig; @@ -54,19 +65,93 @@ struct omap_desc { dma_addr_t dev_addr; int16_t fi; /* for OMAP_DMA_SYNC_PACKET */ - uint8_t es; /* OMAP_DMA_DATA_TYPE_xxx */ - uint8_t sync_mode; /* OMAP_DMA_SYNC_xxx */ - uint8_t sync_type; /* OMAP_DMA_xxx_SYNC* */ - uint8_t periph_port; /* Peripheral port */ + uint8_t es; /* CSDP_DATA_TYPE_xxx */ + uint32_t ccr; /* CCR value */ + uint16_t clnk_ctrl; /* CLNK_CTRL value */ + uint16_t cicr; /* CICR value */ + uint32_t csdp; /* CSDP value */ unsigned sglen; struct omap_sg sg[0]; }; +enum { + CCR_FS = BIT(5), + CCR_READ_PRIORITY = BIT(6), + CCR_ENABLE = BIT(7), + CCR_AUTO_INIT = BIT(8), /* OMAP1 only */ + CCR_REPEAT = BIT(9), /* OMAP1 only */ + CCR_OMAP31_DISABLE = BIT(10), /* OMAP1 only */ + CCR_SUSPEND_SENSITIVE = BIT(8), /* OMAP2+ only */ + CCR_RD_ACTIVE = BIT(9), /* OMAP2+ only */ + CCR_WR_ACTIVE = BIT(10), /* OMAP2+ only */ + CCR_SRC_AMODE_CONSTANT = 0 << 12, + CCR_SRC_AMODE_POSTINC = 1 << 12, + CCR_SRC_AMODE_SGLIDX = 2 << 12, + CCR_SRC_AMODE_DBLIDX = 3 << 12, + CCR_DST_AMODE_CONSTANT = 0 << 14, + CCR_DST_AMODE_POSTINC = 1 << 14, + CCR_DST_AMODE_SGLIDX = 2 << 14, + CCR_DST_AMODE_DBLIDX = 3 << 14, + CCR_CONSTANT_FILL = BIT(16), + CCR_TRANSPARENT_COPY = BIT(17), + CCR_BS = BIT(18), + CCR_SUPERVISOR = BIT(22), + CCR_PREFETCH = BIT(23), + CCR_TRIGGER_SRC = BIT(24), + CCR_BUFFERING_DISABLE = BIT(25), + CCR_WRITE_PRIORITY = BIT(26), + CCR_SYNC_ELEMENT = 0, + CCR_SYNC_FRAME = CCR_FS, + CCR_SYNC_BLOCK = CCR_BS, + CCR_SYNC_PACKET = CCR_BS | CCR_FS, + + CSDP_DATA_TYPE_8 = 0, + CSDP_DATA_TYPE_16 = 1, + CSDP_DATA_TYPE_32 = 2, + CSDP_SRC_PORT_EMIFF = 0 << 2, /* OMAP1 only */ + CSDP_SRC_PORT_EMIFS = 1 << 2, /* OMAP1 only */ + CSDP_SRC_PORT_OCP_T1 = 2 << 2, /* OMAP1 only */ + CSDP_SRC_PORT_TIPB = 3 << 2, /* OMAP1 only */ + CSDP_SRC_PORT_OCP_T2 = 4 << 2, /* OMAP1 only */ + CSDP_SRC_PORT_MPUI = 5 << 2, /* OMAP1 only */ + CSDP_SRC_PACKED = BIT(6), + CSDP_SRC_BURST_1 = 0 << 7, + CSDP_SRC_BURST_16 = 1 << 7, + CSDP_SRC_BURST_32 = 2 << 7, + CSDP_SRC_BURST_64 = 3 << 7, + CSDP_DST_PORT_EMIFF = 0 << 9, /* OMAP1 only */ + CSDP_DST_PORT_EMIFS = 1 << 9, /* OMAP1 only */ + CSDP_DST_PORT_OCP_T1 = 2 << 9, /* OMAP1 only */ + CSDP_DST_PORT_TIPB = 3 << 9, /* OMAP1 only */ + CSDP_DST_PORT_OCP_T2 = 4 << 9, /* OMAP1 only */ + CSDP_DST_PORT_MPUI = 5 << 9, /* OMAP1 only */ + CSDP_DST_PACKED = BIT(13), + CSDP_DST_BURST_1 = 0 << 14, + CSDP_DST_BURST_16 = 1 << 14, + CSDP_DST_BURST_32 = 2 << 14, + CSDP_DST_BURST_64 = 3 << 14, + + CICR_TOUT_IE = BIT(0), /* OMAP1 only */ + CICR_DROP_IE = BIT(1), + CICR_HALF_IE = BIT(2), + CICR_FRAME_IE = BIT(3), + CICR_LAST_IE = BIT(4), + CICR_BLOCK_IE = BIT(5), + CICR_PKT_IE = BIT(7), /* OMAP2+ only */ + CICR_TRANS_ERR_IE = BIT(8), /* OMAP2+ only */ + CICR_SUPERVISOR_ERR_IE = BIT(10), /* OMAP2+ only */ + CICR_MISALIGNED_ERR_IE = BIT(11), /* OMAP2+ only */ + CICR_DRAIN_IE = BIT(12), /* OMAP2+ only */ + CICR_SUPER_BLOCK_IE = BIT(14), /* OMAP2+ only */ + + CLNK_CTRL_ENABLE_LNK = BIT(15), +}; + static const unsigned es_bytes[] = { - [OMAP_DMA_DATA_TYPE_S8] = 1, - [OMAP_DMA_DATA_TYPE_S16] = 2, - [OMAP_DMA_DATA_TYPE_S32] = 4, + [CSDP_DATA_TYPE_8] = 1, + [CSDP_DATA_TYPE_16] = 2, + [CSDP_DATA_TYPE_32] = 4, }; static struct of_dma_filter_info omap_dma_info = { @@ -93,28 +178,214 @@ static void omap_dma_desc_free(struct virt_dma_desc *vd) kfree(container_of(vd, struct omap_desc, vd)); } +static void omap_dma_write(uint32_t val, unsigned type, void __iomem *addr) +{ + switch (type) { + case OMAP_DMA_REG_16BIT: + writew_relaxed(val, addr); + break; + case OMAP_DMA_REG_2X16BIT: + writew_relaxed(val, addr); + writew_relaxed(val >> 16, addr + 2); + break; + case OMAP_DMA_REG_32BIT: + writel_relaxed(val, addr); + break; + default: + WARN_ON(1); + } +} + +static unsigned omap_dma_read(unsigned type, void __iomem *addr) +{ + unsigned val; + + switch (type) { + case OMAP_DMA_REG_16BIT: + val = readw_relaxed(addr); + break; + case OMAP_DMA_REG_2X16BIT: + val = readw_relaxed(addr); + val |= readw_relaxed(addr + 2) << 16; + break; + case OMAP_DMA_REG_32BIT: + val = readl_relaxed(addr); + break; + default: + WARN_ON(1); + val = 0; + } + + return val; +} + +static void omap_dma_glbl_write(struct omap_dmadev *od, unsigned reg, unsigned val) +{ + const struct omap_dma_reg *r = od->reg_map + reg; + + WARN_ON(r->stride); + + omap_dma_write(val, r->type, od->base + r->offset); +} + +static unsigned omap_dma_glbl_read(struct omap_dmadev *od, unsigned reg) +{ + const struct omap_dma_reg *r = od->reg_map + reg; + + WARN_ON(r->stride); + + return omap_dma_read(r->type, od->base + r->offset); +} + +static void omap_dma_chan_write(struct omap_chan *c, unsigned reg, unsigned val) +{ + const struct omap_dma_reg *r = c->reg_map + reg; + + omap_dma_write(val, r->type, c->channel_base + r->offset); +} + +static unsigned omap_dma_chan_read(struct omap_chan *c, unsigned reg) +{ + const struct omap_dma_reg *r = c->reg_map + reg; + + return omap_dma_read(r->type, c->channel_base + r->offset); +} + +static void omap_dma_clear_csr(struct omap_chan *c) +{ + if (dma_omap1()) + omap_dma_chan_read(c, CSR); + else + omap_dma_chan_write(c, CSR, ~0); +} + +static unsigned omap_dma_get_csr(struct omap_chan *c) +{ + unsigned val = omap_dma_chan_read(c, CSR); + + if (!dma_omap1()) + omap_dma_chan_write(c, CSR, val); + + return val; +} + +static void omap_dma_assign(struct omap_dmadev *od, struct omap_chan *c, + unsigned lch) +{ + c->channel_base = od->base + od->plat->channel_stride * lch; + + od->lch_map[lch] = c; +} + +static void omap_dma_start(struct omap_chan *c, struct omap_desc *d) +{ + struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device); + + if (__dma_omap15xx(od->plat->dma_attr)) + omap_dma_chan_write(c, CPC, 0); + else + omap_dma_chan_write(c, CDAC, 0); + + omap_dma_clear_csr(c); + + /* Enable interrupts */ + omap_dma_chan_write(c, CICR, d->cicr); + + /* Enable channel */ + omap_dma_chan_write(c, CCR, d->ccr | CCR_ENABLE); +} + +static void omap_dma_stop(struct omap_chan *c) +{ + struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device); + uint32_t val; + + /* disable irq */ + omap_dma_chan_write(c, CICR, 0); + + omap_dma_clear_csr(c); + + val = omap_dma_chan_read(c, CCR); + if (od->plat->errata & DMA_ERRATA_i541 && val & CCR_TRIGGER_SRC) { + uint32_t sysconfig; + unsigned i; + + sysconfig = omap_dma_glbl_read(od, OCP_SYSCONFIG); + val = sysconfig & ~DMA_SYSCONFIG_MIDLEMODE_MASK; + val |= DMA_SYSCONFIG_MIDLEMODE(DMA_IDLEMODE_NO_IDLE); + omap_dma_glbl_write(od, OCP_SYSCONFIG, val); + + val = omap_dma_chan_read(c, CCR); + val &= ~CCR_ENABLE; + omap_dma_chan_write(c, CCR, val); + + /* Wait for sDMA FIFO to drain */ + for (i = 0; ; i++) { + val = omap_dma_chan_read(c, CCR); + if (!(val & (CCR_RD_ACTIVE | CCR_WR_ACTIVE))) + break; + + if (i > 100) + break; + + udelay(5); + } + + if (val & (CCR_RD_ACTIVE | CCR_WR_ACTIVE)) + dev_err(c->vc.chan.device->dev, + "DMA drain did not complete on lch %d\n", + c->dma_ch); + + omap_dma_glbl_write(od, OCP_SYSCONFIG, sysconfig); + } else { + val &= ~CCR_ENABLE; + omap_dma_chan_write(c, CCR, val); + } + + mb(); + + if (!__dma_omap15xx(od->plat->dma_attr) && c->cyclic) { + val = omap_dma_chan_read(c, CLNK_CTRL); + + if (dma_omap1()) + val |= 1 << 14; /* set the STOP_LNK bit */ + else + val &= ~CLNK_CTRL_ENABLE_LNK; + + omap_dma_chan_write(c, CLNK_CTRL, val); + } +} + static void omap_dma_start_sg(struct omap_chan *c, struct omap_desc *d, unsigned idx) { struct omap_sg *sg = d->sg + idx; + unsigned cxsa, cxei, cxfi; - if (d->dir == DMA_DEV_TO_MEM) - omap_set_dma_dest_params(c->dma_ch, OMAP_DMA_PORT_EMIFF, - OMAP_DMA_AMODE_POST_INC, sg->addr, 0, 0); - else - omap_set_dma_src_params(c->dma_ch, OMAP_DMA_PORT_EMIFF, - OMAP_DMA_AMODE_POST_INC, sg->addr, 0, 0); + if (d->dir == DMA_DEV_TO_MEM) { + cxsa = CDSA; + cxei = CDEI; + cxfi = CDFI; + } else { + cxsa = CSSA; + cxei = CSEI; + cxfi = CSFI; + } - omap_set_dma_transfer_params(c->dma_ch, d->es, sg->en, sg->fn, - d->sync_mode, c->dma_sig, d->sync_type); + omap_dma_chan_write(c, cxsa, sg->addr); + omap_dma_chan_write(c, cxei, 0); + omap_dma_chan_write(c, cxfi, 0); + omap_dma_chan_write(c, CEN, sg->en); + omap_dma_chan_write(c, CFN, sg->fn); - omap_start_dma(c->dma_ch); + omap_dma_start(c, d); } static void omap_dma_start_desc(struct omap_chan *c) { struct virt_dma_desc *vd = vchan_next_desc(&c->vc); struct omap_desc *d; + unsigned cxsa, cxei, cxfi; if (!vd) { c->desc = NULL; @@ -126,12 +397,32 @@ static void omap_dma_start_desc(struct omap_chan *c) c->desc = d = to_omap_dma_desc(&vd->tx); c->sgidx = 0; - if (d->dir == DMA_DEV_TO_MEM) - omap_set_dma_src_params(c->dma_ch, d->periph_port, - OMAP_DMA_AMODE_CONSTANT, d->dev_addr, 0, d->fi); - else - omap_set_dma_dest_params(c->dma_ch, d->periph_port, - OMAP_DMA_AMODE_CONSTANT, d->dev_addr, 0, d->fi); + /* + * This provides the necessary barrier to ensure data held in + * DMA coherent memory is visible to the DMA engine prior to + * the transfer starting. + */ + mb(); + + omap_dma_chan_write(c, CCR, d->ccr); + if (dma_omap1()) + omap_dma_chan_write(c, CCR2, d->ccr >> 16); + + if (d->dir == DMA_DEV_TO_MEM) { + cxsa = CSSA; + cxei = CSEI; + cxfi = CSFI; + } else { + cxsa = CDSA; + cxei = CDEI; + cxfi = CDFI; + } + + omap_dma_chan_write(c, cxsa, d->dev_addr); + omap_dma_chan_write(c, cxei, 0); + omap_dma_chan_write(c, cxfi, d->fi); + omap_dma_chan_write(c, CSDP, d->csdp); + omap_dma_chan_write(c, CLNK_CTRL, d->clnk_ctrl); omap_dma_start_sg(c, d, 0); } @@ -186,24 +477,118 @@ static void omap_dma_sched(unsigned long data) } } +static irqreturn_t omap_dma_irq(int irq, void *devid) +{ + struct omap_dmadev *od = devid; + unsigned status, channel; + + spin_lock(&od->irq_lock); + + status = omap_dma_glbl_read(od, IRQSTATUS_L1); + status &= od->irq_enable_mask; + if (status == 0) { + spin_unlock(&od->irq_lock); + return IRQ_NONE; + } + + while ((channel = ffs(status)) != 0) { + unsigned mask, csr; + struct omap_chan *c; + + channel -= 1; + mask = BIT(channel); + status &= ~mask; + + c = od->lch_map[channel]; + if (c == NULL) { + /* This should never happen */ + dev_err(od->ddev.dev, "invalid channel %u\n", channel); + continue; + } + + csr = omap_dma_get_csr(c); + omap_dma_glbl_write(od, IRQSTATUS_L1, mask); + + omap_dma_callback(channel, csr, c); + } + + spin_unlock(&od->irq_lock); + + return IRQ_HANDLED; +} + static int omap_dma_alloc_chan_resources(struct dma_chan *chan) { + struct omap_dmadev *od = to_omap_dma_dev(chan->device); struct omap_chan *c = to_omap_dma_chan(chan); + int ret; + + if (od->legacy) { + ret = omap_request_dma(c->dma_sig, "DMA engine", + omap_dma_callback, c, &c->dma_ch); + } else { + ret = omap_request_dma(c->dma_sig, "DMA engine", NULL, NULL, + &c->dma_ch); + } + + dev_dbg(od->ddev.dev, "allocating channel %u for %u\n", + c->dma_ch, c->dma_sig); - dev_dbg(c->vc.chan.device->dev, "allocating channel for %u\n", c->dma_sig); + if (ret >= 0) { + omap_dma_assign(od, c, c->dma_ch); - return omap_request_dma(c->dma_sig, "DMA engine", - omap_dma_callback, c, &c->dma_ch); + if (!od->legacy) { + unsigned val; + + spin_lock_irq(&od->irq_lock); + val = BIT(c->dma_ch); + omap_dma_glbl_write(od, IRQSTATUS_L1, val); + od->irq_enable_mask |= val; + omap_dma_glbl_write(od, IRQENABLE_L1, od->irq_enable_mask); + + val = omap_dma_glbl_read(od, IRQENABLE_L0); + val &= ~BIT(c->dma_ch); + omap_dma_glbl_write(od, IRQENABLE_L0, val); + spin_unlock_irq(&od->irq_lock); + } + } + + if (dma_omap1()) { + if (__dma_omap16xx(od->plat->dma_attr)) { + c->ccr = CCR_OMAP31_DISABLE; + /* Duplicate what plat-omap/dma.c does */ + c->ccr |= c->dma_ch + 1; + } else { + c->ccr = c->dma_sig & 0x1f; + } + } else { + c->ccr = c->dma_sig & 0x1f; + c->ccr |= (c->dma_sig & ~0x1f) << 14; + } + if (od->plat->errata & DMA_ERRATA_IFRAME_BUFFERING) + c->ccr |= CCR_BUFFERING_DISABLE; + + return ret; } static void omap_dma_free_chan_resources(struct dma_chan *chan) { + struct omap_dmadev *od = to_omap_dma_dev(chan->device); struct omap_chan *c = to_omap_dma_chan(chan); + if (!od->legacy) { + spin_lock_irq(&od->irq_lock); + od->irq_enable_mask &= ~BIT(c->dma_ch); + omap_dma_glbl_write(od, IRQENABLE_L1, od->irq_enable_mask); + spin_unlock_irq(&od->irq_lock); + } + + c->channel_base = NULL; + od->lch_map[c->dma_ch] = NULL; vchan_free_chan_resources(&c->vc); omap_free_dma(c->dma_ch); - dev_dbg(c->vc.chan.device->dev, "freeing channel for %u\n", c->dma_sig); + dev_dbg(od->ddev.dev, "freeing channel for %u\n", c->dma_sig); } static size_t omap_dma_sg_size(struct omap_sg *sg) @@ -239,6 +624,74 @@ static size_t omap_dma_desc_size_pos(struct omap_desc *d, dma_addr_t addr) return size; } +/* + * OMAP 3.2/3.3 erratum: sometimes 0 is returned if CSAC/CDAC is + * read before the DMA controller finished disabling the channel. + */ +static uint32_t omap_dma_chan_read_3_3(struct omap_chan *c, unsigned reg) +{ + struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device); + uint32_t val; + + val = omap_dma_chan_read(c, reg); + if (val == 0 && od->plat->errata & DMA_ERRATA_3_3) + val = omap_dma_chan_read(c, reg); + + return val; +} + +static dma_addr_t omap_dma_get_src_pos(struct omap_chan *c) +{ + struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device); + dma_addr_t addr, cdac; + + if (__dma_omap15xx(od->plat->dma_attr)) { + addr = omap_dma_chan_read(c, CPC); + } else { + addr = omap_dma_chan_read_3_3(c, CSAC); + cdac = omap_dma_chan_read_3_3(c, CDAC); + + /* + * CDAC == 0 indicates that the DMA transfer on the channel has + * not been started (no data has been transferred so far). + * Return the programmed source start address in this case. + */ + if (cdac == 0) + addr = omap_dma_chan_read(c, CSSA); + } + + if (dma_omap1()) + addr |= omap_dma_chan_read(c, CSSA) & 0xffff0000; + + return addr; +} + +static dma_addr_t omap_dma_get_dst_pos(struct omap_chan *c) +{ + struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device); + dma_addr_t addr; + + if (__dma_omap15xx(od->plat->dma_attr)) { + addr = omap_dma_chan_read(c, CPC); + } else { + addr = omap_dma_chan_read_3_3(c, CDAC); + + /* + * CDAC == 0 indicates that the DMA transfer on the channel + * has not been started (no data has been transferred so + * far). Return the programmed destination start address in + * this case. + */ + if (addr == 0) + addr = omap_dma_chan_read(c, CDSA); + } + + if (dma_omap1()) + addr |= omap_dma_chan_read(c, CDSA) & 0xffff0000; + + return addr; +} + static enum dma_status omap_dma_tx_status(struct dma_chan *chan, dma_cookie_t cookie, struct dma_tx_state *txstate) { @@ -260,9 +713,9 @@ static enum dma_status omap_dma_tx_status(struct dma_chan *chan, dma_addr_t pos; if (d->dir == DMA_MEM_TO_DEV) - pos = omap_get_dma_src_pos(c->dma_ch); + pos = omap_dma_get_src_pos(c); else if (d->dir == DMA_DEV_TO_MEM) - pos = omap_get_dma_dst_pos(c->dma_ch); + pos = omap_dma_get_dst_pos(c); else pos = 0; @@ -304,24 +757,23 @@ static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg( struct dma_chan *chan, struct scatterlist *sgl, unsigned sglen, enum dma_transfer_direction dir, unsigned long tx_flags, void *context) { + struct omap_dmadev *od = to_omap_dma_dev(chan->device); struct omap_chan *c = to_omap_dma_chan(chan); enum dma_slave_buswidth dev_width; struct scatterlist *sgent; struct omap_desc *d; dma_addr_t dev_addr; - unsigned i, j = 0, es, en, frame_bytes, sync_type; + unsigned i, j = 0, es, en, frame_bytes; u32 burst; if (dir == DMA_DEV_TO_MEM) { dev_addr = c->cfg.src_addr; dev_width = c->cfg.src_addr_width; burst = c->cfg.src_maxburst; - sync_type = OMAP_DMA_SRC_SYNC; } else if (dir == DMA_MEM_TO_DEV) { dev_addr = c->cfg.dst_addr; dev_width = c->cfg.dst_addr_width; burst = c->cfg.dst_maxburst; - sync_type = OMAP_DMA_DST_SYNC; } else { dev_err(chan->device->dev, "%s: bad direction?\n", __func__); return NULL; @@ -330,13 +782,13 @@ static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg( /* Bus width translates to the element size (ES) */ switch (dev_width) { case DMA_SLAVE_BUSWIDTH_1_BYTE: - es = OMAP_DMA_DATA_TYPE_S8; + es = CSDP_DATA_TYPE_8; break; case DMA_SLAVE_BUSWIDTH_2_BYTES: - es = OMAP_DMA_DATA_TYPE_S16; + es = CSDP_DATA_TYPE_16; break; case DMA_SLAVE_BUSWIDTH_4_BYTES: - es = OMAP_DMA_DATA_TYPE_S32; + es = CSDP_DATA_TYPE_32; break; default: /* not reached */ return NULL; @@ -350,9 +802,31 @@ static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg( d->dir = dir; d->dev_addr = dev_addr; d->es = es; - d->sync_mode = OMAP_DMA_SYNC_FRAME; - d->sync_type = sync_type; - d->periph_port = OMAP_DMA_PORT_TIPB; + + d->ccr = c->ccr | CCR_SYNC_FRAME; + if (dir == DMA_DEV_TO_MEM) + d->ccr |= CCR_DST_AMODE_POSTINC | CCR_SRC_AMODE_CONSTANT; + else + d->ccr |= CCR_DST_AMODE_CONSTANT | CCR_SRC_AMODE_POSTINC; + + d->cicr = CICR_DROP_IE | CICR_BLOCK_IE; + d->csdp = es; + + if (dma_omap1()) { + d->cicr |= CICR_TOUT_IE; + + if (dir == DMA_DEV_TO_MEM) + d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_TIPB; + else + d->csdp |= CSDP_DST_PORT_TIPB | CSDP_SRC_PORT_EMIFF; + } else { + if (dir == DMA_DEV_TO_MEM) + d->ccr |= CCR_TRIGGER_SRC; + + d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE; + } + if (od->plat->errata & DMA_ERRATA_PARALLEL_CHANNELS) + d->clnk_ctrl = c->dma_ch; /* * Build our scatterlist entries: each contains the address, @@ -382,23 +856,22 @@ static struct dma_async_tx_descriptor *omap_dma_prep_dma_cyclic( size_t period_len, enum dma_transfer_direction dir, unsigned long flags, void *context) { + struct omap_dmadev *od = to_omap_dma_dev(chan->device); struct omap_chan *c = to_omap_dma_chan(chan); enum dma_slave_buswidth dev_width; struct omap_desc *d; dma_addr_t dev_addr; - unsigned es, sync_type; + unsigned es; u32 burst; if (dir == DMA_DEV_TO_MEM) { dev_addr = c->cfg.src_addr; dev_width = c->cfg.src_addr_width; burst = c->cfg.src_maxburst; - sync_type = OMAP_DMA_SRC_SYNC; } else if (dir == DMA_MEM_TO_DEV) { dev_addr = c->cfg.dst_addr; dev_width = c->cfg.dst_addr_width; burst = c->cfg.dst_maxburst; - sync_type = OMAP_DMA_DST_SYNC; } else { dev_err(chan->device->dev, "%s: bad direction?\n", __func__); return NULL; @@ -407,13 +880,13 @@ static struct dma_async_tx_descriptor *omap_dma_prep_dma_cyclic( /* Bus width translates to the element size (ES) */ switch (dev_width) { case DMA_SLAVE_BUSWIDTH_1_BYTE: - es = OMAP_DMA_DATA_TYPE_S8; + es = CSDP_DATA_TYPE_8; break; case DMA_SLAVE_BUSWIDTH_2_BYTES: - es = OMAP_DMA_DATA_TYPE_S16; + es = CSDP_DATA_TYPE_16; break; case DMA_SLAVE_BUSWIDTH_4_BYTES: - es = OMAP_DMA_DATA_TYPE_S32; + es = CSDP_DATA_TYPE_32; break; default: /* not reached */ return NULL; @@ -428,32 +901,51 @@ static struct dma_async_tx_descriptor *omap_dma_prep_dma_cyclic( d->dev_addr = dev_addr; d->fi = burst; d->es = es; - if (burst) - d->sync_mode = OMAP_DMA_SYNC_PACKET; - else - d->sync_mode = OMAP_DMA_SYNC_ELEMENT; - d->sync_type = sync_type; - d->periph_port = OMAP_DMA_PORT_MPUI; d->sg[0].addr = buf_addr; d->sg[0].en = period_len / es_bytes[es]; d->sg[0].fn = buf_len / period_len; d->sglen = 1; - if (!c->cyclic) { - c->cyclic = true; - omap_dma_link_lch(c->dma_ch, c->dma_ch); + d->ccr = c->ccr; + if (dir == DMA_DEV_TO_MEM) + d->ccr |= CCR_DST_AMODE_POSTINC | CCR_SRC_AMODE_CONSTANT; + else + d->ccr |= CCR_DST_AMODE_CONSTANT | CCR_SRC_AMODE_POSTINC; - if (flags & DMA_PREP_INTERRUPT) - omap_enable_dma_irq(c->dma_ch, OMAP_DMA_FRAME_IRQ); + d->cicr = CICR_DROP_IE; + if (flags & DMA_PREP_INTERRUPT) + d->cicr |= CICR_FRAME_IE; - omap_disable_dma_irq(c->dma_ch, OMAP_DMA_BLOCK_IRQ); - } + d->csdp = es; + + if (dma_omap1()) { + d->cicr |= CICR_TOUT_IE; + + if (dir == DMA_DEV_TO_MEM) + d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_MPUI; + else + d->csdp |= CSDP_DST_PORT_MPUI | CSDP_SRC_PORT_EMIFF; + } else { + if (burst) + d->ccr |= CCR_SYNC_PACKET; + else + d->ccr |= CCR_SYNC_ELEMENT; + + if (dir == DMA_DEV_TO_MEM) + d->ccr |= CCR_TRIGGER_SRC; + + d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE; - if (dma_omap2plus()) { - omap_set_dma_src_burst_mode(c->dma_ch, OMAP_DMA_DATA_BURST_16); - omap_set_dma_dest_burst_mode(c->dma_ch, OMAP_DMA_DATA_BURST_16); + d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64; } + if (__dma_omap15xx(od->plat->dma_attr)) + d->ccr |= CCR_AUTO_INIT | CCR_REPEAT; + else + d->clnk_ctrl = c->dma_ch | CLNK_CTRL_ENABLE_LNK; + + c->cyclic = true; + return vchan_tx_prep(&c->vc, &d->vd, flags); } @@ -483,20 +975,19 @@ static int omap_dma_terminate_all(struct omap_chan *c) /* * Stop DMA activity: we assume the callback will not be called - * after omap_stop_dma() returns (even if it does, it will see + * after omap_dma_stop() returns (even if it does, it will see * c->desc is NULL and exit.) */ if (c->desc) { c->desc = NULL; /* Avoid stopping the dma twice */ if (!c->paused) - omap_stop_dma(c->dma_ch); + omap_dma_stop(c); } if (c->cyclic) { c->cyclic = false; c->paused = false; - omap_dma_unlink_lch(c->dma_ch, c->dma_ch); } vchan_get_all_descriptors(&c->vc, &head); @@ -513,7 +1004,7 @@ static int omap_dma_pause(struct omap_chan *c) return -EINVAL; if (!c->paused) { - omap_stop_dma(c->dma_ch); + omap_dma_stop(c); c->paused = true; } @@ -527,7 +1018,7 @@ static int omap_dma_resume(struct omap_chan *c) return -EINVAL; if (c->paused) { - omap_start_dma(c->dma_ch); + omap_dma_start(c, c->desc); c->paused = false; } @@ -573,6 +1064,7 @@ static int omap_dma_chan_init(struct omap_dmadev *od, int dma_sig) if (!c) return -ENOMEM; + c->reg_map = od->reg_map; c->dma_sig = dma_sig; c->vc.desc_free = omap_dma_desc_free; vchan_init(&c->vc, &od->ddev); @@ -594,18 +1086,29 @@ static void omap_dma_free(struct omap_dmadev *od) tasklet_kill(&c->vc.task); kfree(c); } - kfree(od); } static int omap_dma_probe(struct platform_device *pdev) { struct omap_dmadev *od; - int rc, i; + struct resource *res; + int rc, i, irq; - od = kzalloc(sizeof(*od), GFP_KERNEL); + od = devm_kzalloc(&pdev->dev, sizeof(*od), GFP_KERNEL); if (!od) return -ENOMEM; + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + od->base = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(od->base)) + return PTR_ERR(od->base); + + od->plat = omap_get_plat_info(); + if (!od->plat) + return -EPROBE_DEFER; + + od->reg_map = od->plat->reg_map; + dma_cap_set(DMA_SLAVE, od->ddev.cap_mask); dma_cap_set(DMA_CYCLIC, od->ddev.cap_mask); od->ddev.device_alloc_chan_resources = omap_dma_alloc_chan_resources; @@ -619,6 +1122,7 @@ static int omap_dma_probe(struct platform_device *pdev) INIT_LIST_HEAD(&od->ddev.channels); INIT_LIST_HEAD(&od->pending); spin_lock_init(&od->lock); + spin_lock_init(&od->irq_lock); tasklet_init(&od->task, omap_dma_sched, (unsigned long)od); @@ -630,6 +1134,21 @@ static int omap_dma_probe(struct platform_device *pdev) } } + irq = platform_get_irq(pdev, 1); + if (irq <= 0) { + dev_info(&pdev->dev, "failed to get L1 IRQ: %d\n", irq); + od->legacy = true; + } else { + /* Disable all interrupts */ + od->irq_enable_mask = 0; + omap_dma_glbl_write(od, IRQENABLE_L1, 0); + + rc = devm_request_irq(&pdev->dev, irq, omap_dma_irq, + IRQF_SHARED, "omap-dma-engine", od); + if (rc) + return rc; + } + rc = dma_async_device_register(&od->ddev); if (rc) { pr_warn("OMAP-DMA: failed to register slave DMA engine device: %d\n", @@ -666,6 +1185,12 @@ static int omap_dma_remove(struct platform_device *pdev) of_dma_controller_free(pdev->dev.of_node); dma_async_device_unregister(&od->ddev); + + if (!od->legacy) { + /* Disable all interrupts */ + omap_dma_glbl_write(od, IRQENABLE_L0, 0); + } + omap_dma_free(od); return 0; diff --git a/drivers/mmc/host/mmci.h b/drivers/mmc/host/mmci.h index 168bc72f7a94..84c0e59b792a 100644 --- a/drivers/mmc/host/mmci.h +++ b/drivers/mmc/host/mmci.h @@ -38,10 +38,11 @@ #define MCI_CPSM_INTERRUPT (1 << 8) #define MCI_CPSM_PENDING (1 << 9) #define MCI_CPSM_ENABLE (1 << 10) -#define MCI_SDIO_SUSP (1 << 11) -#define MCI_ENCMD_COMPL (1 << 12) -#define MCI_NIEN (1 << 13) -#define MCI_CE_ATACMD (1 << 14) +/* Argument flag extenstions in the ST Micro versions */ +#define MCI_ST_SDIO_SUSP (1 << 11) +#define MCI_ST_ENCMD_COMPL (1 << 12) +#define MCI_ST_NIEN (1 << 13) +#define MCI_ST_CE_ATACMD (1 << 14) #define MMCIRESPCMD 0x010 #define MMCIRESPONSE0 0x014 diff --git a/drivers/usb/gadget/lpc32xx_udc.c b/drivers/usb/gadget/lpc32xx_udc.c index a139894c600f..e471580a2a3b 100644 --- a/drivers/usb/gadget/lpc32xx_udc.c +++ b/drivers/usb/gadget/lpc32xx_udc.c @@ -55,7 +55,6 @@ #include <mach/hardware.h> #include <linux/io.h> #include <asm/irq.h> -#include <asm/system.h> #include <mach/platform.h> #include <mach/irqs.h> diff --git a/include/linux/memblock.h b/include/linux/memblock.h index 1ef66360f0b0..8a20a51ed42d 100644 --- a/include/linux/memblock.h +++ b/include/linux/memblock.h @@ -252,6 +252,8 @@ static inline void memblock_dump_all(void) void memblock_set_current_limit(phys_addr_t limit); +phys_addr_t memblock_get_current_limit(void); + /* * pfn conversion functions * diff --git a/include/linux/omap-dma.h b/include/linux/omap-dma.h index 7af25a9c9c51..41a13e70f41f 100644 --- a/include/linux/omap-dma.h +++ b/include/linux/omap-dma.h @@ -268,14 +268,27 @@ struct omap_dma_dev_attr { u32 dev_caps; u16 lch_count; u16 chan_count; - struct omap_dma_lch *chan; +}; + +enum { + OMAP_DMA_REG_NONE, + OMAP_DMA_REG_16BIT, + OMAP_DMA_REG_2X16BIT, + OMAP_DMA_REG_32BIT, +}; + +struct omap_dma_reg { + u16 offset; + u8 stride; + u8 type; }; /* System DMA platform data structure */ struct omap_system_dma_plat_info { + const struct omap_dma_reg *reg_map; + unsigned channel_stride; struct omap_dma_dev_attr *dma_attr; u32 errata; - void (*disable_irq_lch)(int lch); void (*show_dma_caps)(void); void (*clear_lch_regs)(int lch); void (*clear_dma)(int lch); @@ -289,8 +302,12 @@ struct omap_system_dma_plat_info { #define dma_omap2plus() 0 #endif #define dma_omap1() (!dma_omap2plus()) -#define dma_omap15xx() ((dma_omap1() && (d->dev_caps & ENABLE_1510_MODE))) -#define dma_omap16xx() ((dma_omap1() && (d->dev_caps & ENABLE_16XX_MODE))) +#define __dma_omap15xx(d) (dma_omap1() && (d)->dev_caps & ENABLE_1510_MODE) +#define __dma_omap16xx(d) (dma_omap1() && (d)->dev_caps & ENABLE_16XX_MODE) +#define dma_omap15xx() __dma_omap15xx(d) +#define dma_omap16xx() __dma_omap16xx(d) + +extern struct omap_system_dma_plat_info *omap_get_plat_info(void); extern void omap_set_dma_priority(int lch, int dst_port, int priority); extern int omap_request_dma(int dev_id, const char *dev_name, diff --git a/include/linux/uprobes.h b/include/linux/uprobes.h index e32251e00e62..edff2b97b864 100644 --- a/include/linux/uprobes.h +++ b/include/linux/uprobes.h @@ -126,6 +126,7 @@ extern bool arch_uprobe_xol_was_trapped(struct task_struct *tsk); extern int arch_uprobe_exception_notify(struct notifier_block *self, unsigned long val, void *data); extern void arch_uprobe_abort_xol(struct arch_uprobe *aup, struct pt_regs *regs); extern unsigned long arch_uretprobe_hijack_return_addr(unsigned long trampoline_vaddr, struct pt_regs *regs); +extern bool __weak arch_uprobe_ignore(struct arch_uprobe *aup, struct pt_regs *regs); #else /* !CONFIG_UPROBES */ struct uprobes_state { }; diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c index 307d87c0991a..04709b66369d 100644 --- a/kernel/events/uprobes.c +++ b/kernel/events/uprobes.c @@ -1804,6 +1804,11 @@ static bool handle_trampoline(struct pt_regs *regs) return true; } +bool __weak arch_uprobe_ignore(struct arch_uprobe *aup, struct pt_regs *regs) +{ + return false; +} + /* * Run handler and ask thread to singlestep. * Ensure all non-fatal signals cannot interrupt thread while it singlesteps. @@ -1858,7 +1863,11 @@ static void handle_swbp(struct pt_regs *regs) if (!get_utask()) goto out; + if (arch_uprobe_ignore(&uprobe->arch, regs)) + goto out; + handler_chain(uprobe, regs); + if (can_skip_sstep(uprobe, regs)) goto out; diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig index 015f85aaca08..8639819f6cef 100644 --- a/kernel/trace/Kconfig +++ b/kernel/trace/Kconfig @@ -424,6 +424,7 @@ config UPROBE_EVENT bool "Enable uprobes-based dynamic events" depends on ARCH_SUPPORTS_UPROBES depends on MMU + depends on PERF_EVENTS select UPROBES select PROBE_EVENTS select TRACING diff --git a/mm/memblock.c b/mm/memblock.c index 39a31e7f0045..7fe5354e7552 100644 --- a/mm/memblock.c +++ b/mm/memblock.c @@ -1407,6 +1407,11 @@ void __init_memblock memblock_set_current_limit(phys_addr_t limit) memblock.current_limit = limit; } +phys_addr_t __init_memblock memblock_get_current_limit(void) +{ + return memblock.current_limit; +} + static void __init_memblock memblock_dump(struct memblock_type *type, char *name) { unsigned long long base, size; |