diff options
Diffstat (limited to 'arch/riscv')
95 files changed, 2766 insertions, 815 deletions
diff --git a/arch/riscv/Kbuild b/arch/riscv/Kbuild index d1d0aa70fdf1..4614c01ba5b3 100644 --- a/arch/riscv/Kbuild +++ b/arch/riscv/Kbuild @@ -1,3 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only obj-y += kernel/ mm/ net/ +obj-$(CONFIG_BUILTIN_DTB) += boot/dts/ diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index a31e1a41913a..7766e1289468 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -12,62 +12,81 @@ config 32BIT config RISCV def_bool y - select OF - select OF_EARLY_FLATTREE - select OF_IRQ + select ARCH_CLOCKSOURCE_INIT + select ARCH_SUPPORTS_ATOMIC_RMW select ARCH_HAS_BINFMT_FLAT + select ARCH_HAS_DEBUG_VM_PGTABLE + select ARCH_HAS_DEBUG_VIRTUAL if MMU + select ARCH_HAS_DEBUG_WX + select ARCH_HAS_GCOV_PROFILE_ALL + select ARCH_HAS_GIGANTIC_PAGE + select ARCH_HAS_KCOV + select ARCH_HAS_MMIOWB + select ARCH_HAS_PTE_SPECIAL + select ARCH_HAS_SET_DIRECT_MAP + select ARCH_HAS_SET_MEMORY + select ARCH_HAS_STRICT_KERNEL_RWX if MMU + select ARCH_OPTIONAL_KERNEL_RWX if ARCH_HAS_STRICT_KERNEL_RWX + select ARCH_OPTIONAL_KERNEL_RWX_DEFAULT + select ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT if MMU select ARCH_WANT_FRAME_POINTERS + select ARCH_WANT_HUGE_PMD_SHARE if 64BIT select CLONE_BACKWARDS + select CLINT_TIMER if !MMU select COMMON_CLK + select EDAC_SUPPORT + select GENERIC_ARCH_TOPOLOGY if SMP + select GENERIC_ATOMIC64 if !64BIT select GENERIC_CLOCKEVENTS + select GENERIC_GETTIMEOFDAY if HAVE_GENERIC_VDSO + select GENERIC_IOREMAP + select GENERIC_IRQ_MULTI_HANDLER select GENERIC_IRQ_SHOW select GENERIC_PCI_IOMAP + select GENERIC_PTDUMP if MMU select GENERIC_SCHED_CLOCK + select GENERIC_SMP_IDLE_THREAD select GENERIC_STRNCPY_FROM_USER if MMU select GENERIC_STRNLEN_USER if MMU - select GENERIC_SMP_IDLE_THREAD - select GENERIC_ATOMIC64 if !64BIT - select GENERIC_IOREMAP - select GENERIC_PTDUMP if MMU + select GENERIC_TIME_VSYSCALL if MMU && 64BIT + select HANDLE_DOMAIN_IRQ select HAVE_ARCH_AUDITSYSCALL + select HAVE_ARCH_JUMP_LABEL + select HAVE_ARCH_JUMP_LABEL_RELATIVE + select HAVE_ARCH_KASAN if MMU && 64BIT + select HAVE_ARCH_KGDB + select HAVE_ARCH_KGDB_QXFER_PKT + select HAVE_ARCH_MMAP_RND_BITS if MMU select HAVE_ARCH_SECCOMP_FILTER + select HAVE_ARCH_TRACEHOOK select HAVE_ASM_MODVERSIONS - select HAVE_MEMBLOCK_NODE_MAP + select HAVE_CONTEXT_TRACKING + select HAVE_DEBUG_KMEMLEAK select HAVE_DMA_CONTIGUOUS if MMU + select HAVE_EBPF_JIT if MMU select HAVE_FUTEX_CMPXCHG if FUTEX + select HAVE_GCC_PLUGINS + select HAVE_GENERIC_VDSO if MMU && 64BIT + select HAVE_PCI select HAVE_PERF_EVENTS select HAVE_PERF_REGS select HAVE_PERF_USER_STACK_DUMP + select HAVE_STACKPROTECTOR select HAVE_SYSCALL_TRACEPOINTS select IRQ_DOMAIN - select SPARSE_IRQ - select SYSCTL_EXCEPTION_TRACE - select HAVE_ARCH_TRACEHOOK - select HAVE_PCI select MODULES_USE_ELF_RELA if MODULES select MODULE_SECTIONS if MODULES - select THREAD_INFO_IN_TASK + select OF + select OF_EARLY_FLATTREE + select OF_IRQ select PCI_DOMAINS_GENERIC if PCI select PCI_MSI if PCI - select RISCV_TIMER - select GENERIC_IRQ_MULTI_HANDLER - select GENERIC_ARCH_TOPOLOGY if SMP - select ARCH_HAS_PTE_SPECIAL - select ARCH_HAS_MMIOWB - select ARCH_HAS_DEBUG_VIRTUAL if MMU - select HAVE_EBPF_JIT if MMU - select EDAC_SUPPORT - select ARCH_HAS_GIGANTIC_PAGE - select ARCH_HAS_SET_DIRECT_MAP - select ARCH_HAS_SET_MEMORY - select ARCH_HAS_STRICT_KERNEL_RWX if MMU - select ARCH_WANT_HUGE_PMD_SHARE if 64BIT + select RISCV_INTC + select RISCV_TIMER if RISCV_SBI select SPARSEMEM_STATIC if 32BIT - select ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT if MMU - select HAVE_ARCH_MMAP_RND_BITS if MMU - select ARCH_HAS_GCOV_PROFILE_ALL - select HAVE_COPY_THREAD_TLS - select HAVE_ARCH_KASAN if MMU && 64BIT + select SPARSE_IRQ + select SYSCTL_EXCEPTION_TRACE + select THREAD_INFO_IN_TASK config ARCH_MMAP_RND_BITS_MIN default 18 if 64BIT @@ -170,6 +189,9 @@ config PGTABLE_LEVELS default 3 if 64BIT default 2 +config LOCKDEP_SUPPORT + def_bool y + source "arch/riscv/Kconfig.socs" menu "Platform type" @@ -194,11 +216,11 @@ config ARCH_RV64I bool "RV64I" select 64BIT select ARCH_SUPPORTS_INT128 if CC_HAS_INT128 && GCC_VERSION >= 50000 - select HAVE_FUNCTION_TRACER - select HAVE_FUNCTION_GRAPH_TRACER - select HAVE_FTRACE_MCOUNT_RECORD select HAVE_DYNAMIC_FTRACE if MMU select HAVE_DYNAMIC_FTRACE_WITH_REGS if HAVE_DYNAMIC_FTRACE + select HAVE_FTRACE_MCOUNT_RECORD + select HAVE_FUNCTION_GRAPH_TRACER + select HAVE_FUNCTION_TRACER select SWIOTLB if MMU endchoice @@ -381,6 +403,11 @@ endchoice endmenu +config BUILTIN_DTB + def_bool n + depends on RISCV_M_MODE + depends on OF + menu "Power management options" source "kernel/power/Kconfig" diff --git a/arch/riscv/Kconfig.socs b/arch/riscv/Kconfig.socs index d646332e44f1..8a55f6156661 100644 --- a/arch/riscv/Kconfig.socs +++ b/arch/riscv/Kconfig.socs @@ -12,6 +12,7 @@ config SOC_SIFIVE config SOC_VIRT bool "QEMU Virt Machine" + select CLINT_TIMER if RISCV_M_MODE select POWER_RESET select POWER_RESET_SYSCON select POWER_RESET_SYSCON_POWEROFF @@ -24,11 +25,27 @@ config SOC_VIRT config SOC_KENDRYTE bool "Kendryte K210 SoC" depends on !MMU - select BUILTIN_DTB + select CLINT_TIMER if RISCV_M_MODE select SERIAL_SIFIVE if TTY select SERIAL_SIFIVE_CONSOLE if TTY select SIFIVE_PLIC help This enables support for Kendryte K210 SoC platform hardware. +config SOC_KENDRYTE_K210_DTB + def_bool y + depends on SOC_KENDRYTE_K210_DTB_BUILTIN + +config SOC_KENDRYTE_K210_DTB_BUILTIN + bool "Builtin device tree for the Kendryte K210" + depends on SOC_KENDRYTE + default y + select OF + select BUILTIN_DTB + select SOC_KENDRYTE_K210_DTB + help + Builds a device tree for the Kendryte K210 into the Linux image. + This option should be selected if no bootloader is being used. + If unsure, say Y. + endmenu diff --git a/arch/riscv/boot/Makefile b/arch/riscv/boot/Makefile index 3530c59b3ea7..c59fca695f9d 100644 --- a/arch/riscv/boot/Makefile +++ b/arch/riscv/boot/Makefile @@ -14,6 +14,8 @@ # Based on the ia64 and arm64 boot/Makefile. # +KCOV_INSTRUMENT := n + OBJCOPYFLAGS_Image :=-O binary -R .note -R .note.gnu.build-id -R .comment -S targets := Image loader diff --git a/arch/riscv/boot/dts/Makefile b/arch/riscv/boot/dts/Makefile index 557f0b519c8e..ca1f8cbd78c0 100644 --- a/arch/riscv/boot/dts/Makefile +++ b/arch/riscv/boot/dts/Makefile @@ -1,3 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 subdir-y += sifive subdir-y += kendryte + +obj-$(CONFIG_BUILTIN_DTB) := $(addsuffix /, $(subdir-y)) diff --git a/arch/riscv/boot/dts/kendryte/Makefile b/arch/riscv/boot/dts/kendryte/Makefile index 815444e69e89..1a88e616f18e 100644 --- a/arch/riscv/boot/dts/kendryte/Makefile +++ b/arch/riscv/boot/dts/kendryte/Makefile @@ -1,2 +1,4 @@ # SPDX-License-Identifier: GPL-2.0 -dtb-$(CONFIG_SOC_KENDRYTE) += k210.dtb +dtb-$(CONFIG_SOC_KENDRYTE_K210_DTB) += k210.dtb + +obj-$(CONFIG_SOC_KENDRYTE_K210_DTB_BUILTIN) += $(addsuffix .o, $(dtb-y)) diff --git a/arch/riscv/boot/dts/kendryte/k210.dtsi b/arch/riscv/boot/dts/kendryte/k210.dtsi index c1df56ccb8d5..d2d0ff645632 100644 --- a/arch/riscv/boot/dts/kendryte/k210.dtsi +++ b/arch/riscv/boot/dts/kendryte/k210.dtsi @@ -95,10 +95,12 @@ #clock-cells = <1>; }; - clint0: interrupt-controller@2000000 { + clint0: clint@2000000 { + #interrupt-cells = <1>; compatible = "riscv,clint0"; reg = <0x2000000 0xC000>; - interrupts-extended = <&cpu0_intc 3>, <&cpu1_intc 3>; + interrupts-extended = <&cpu0_intc 3 &cpu0_intc 7 + &cpu1_intc 3 &cpu1_intc 7>; clocks = <&sysctl K210_CLK_ACLK>; }; diff --git a/arch/riscv/configs/defconfig b/arch/riscv/configs/defconfig index 4da4886246a4..d58c93efb603 100644 --- a/arch/riscv/configs/defconfig +++ b/arch/riscv/configs/defconfig @@ -17,6 +17,7 @@ CONFIG_BPF_SYSCALL=y CONFIG_SOC_SIFIVE=y CONFIG_SOC_VIRT=y CONFIG_SMP=y +CONFIG_JUMP_LABEL=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y CONFIG_NET=y diff --git a/arch/riscv/configs/nommu_k210_defconfig b/arch/riscv/configs/nommu_k210_defconfig index 632aa2f95e57..cd1df62b13c7 100644 --- a/arch/riscv/configs/nommu_k210_defconfig +++ b/arch/riscv/configs/nommu_k210_defconfig @@ -2,14 +2,12 @@ CONFIG_LOG_BUF_SHIFT=15 CONFIG_PRINTK_SAFE_LOG_BUF_SHIFT=12 CONFIG_BLK_DEV_INITRD=y -CONFIG_INITRAMFS_SOURCE="" CONFIG_INITRAMFS_FORCE=y # CONFIG_RD_BZIP2 is not set # CONFIG_RD_LZMA is not set # CONFIG_RD_XZ is not set # CONFIG_RD_LZO is not set # CONFIG_RD_LZ4 is not set -# CONFIG_BOOT_CONFIG is not set CONFIG_CC_OPTIMIZE_FOR_SIZE=y # CONFIG_SYSFS_SYSCALL is not set # CONFIG_FHANDLE is not set @@ -35,8 +33,7 @@ CONFIG_SMP=y CONFIG_NR_CPUS=2 CONFIG_CMDLINE="earlycon console=ttySIF0" CONFIG_CMDLINE_FORCE=y -CONFIG_USE_BUILTIN_DTB=y -CONFIG_BUILTIN_DTB_SOURCE="kendryte/k210" +CONFIG_JUMP_LABEL=y # CONFIG_BLOCK is not set CONFIG_BINFMT_FLAT=y # CONFIG_COREDUMP is not set @@ -49,8 +46,8 @@ CONFIG_DEVTMPFS_MOUNT=y # CONFIG_SERIO is not set # CONFIG_LEGACY_PTYS is not set # CONFIG_LDISC_AUTOLOAD is not set -# CONFIG_DEVMEM is not set # CONFIG_HW_RANDOM is not set +# CONFIG_DEVMEM is not set # CONFIG_HWMON is not set # CONFIG_VGA_CONSOLE is not set # CONFIG_HID is not set @@ -62,6 +59,7 @@ CONFIG_DEVTMPFS_MOUNT=y CONFIG_LSM="[]" CONFIG_PRINTK_TIME=y # CONFIG_DEBUG_MISC is not set +CONFIG_PANIC_ON_OOPS=y # CONFIG_SCHED_DEBUG is not set # CONFIG_RCU_TRACE is not set # CONFIG_FTRACE is not set diff --git a/arch/riscv/configs/nommu_virt_defconfig b/arch/riscv/configs/nommu_virt_defconfig index cf74e179bf90..e046a0babde4 100644 --- a/arch/riscv/configs/nommu_virt_defconfig +++ b/arch/riscv/configs/nommu_virt_defconfig @@ -26,10 +26,12 @@ CONFIG_EXPERT=y CONFIG_SLOB=y # CONFIG_SLAB_MERGE_DEFAULT is not set # CONFIG_MMU is not set +CONFIG_SOC_VIRT=y CONFIG_MAXPHYSMEM_2GB=y CONFIG_SMP=y CONFIG_CMDLINE="root=/dev/vda rw earlycon=uart8250,mmio,0x10000000,115200n8 console=ttyS0" CONFIG_CMDLINE_FORCE=y +CONFIG_JUMP_LABEL=y # CONFIG_BLK_DEV_BSG is not set CONFIG_PARTITION_ADVANCED=y # CONFIG_MSDOS_PARTITION is not set @@ -48,7 +50,6 @@ CONFIG_VIRTIO_BLK=y # CONFIG_SERIO is not set # CONFIG_LEGACY_PTYS is not set # CONFIG_LDISC_AUTOLOAD is not set -# CONFIG_DEVMEM is not set CONFIG_SERIAL_8250=y # CONFIG_SERIAL_8250_DEPRECATED_OPTIONS is not set CONFIG_SERIAL_8250_CONSOLE=y @@ -56,16 +57,13 @@ CONFIG_SERIAL_8250_NR_UARTS=1 CONFIG_SERIAL_8250_RUNTIME_UARTS=1 CONFIG_SERIAL_OF_PLATFORM=y # CONFIG_HW_RANDOM is not set +# CONFIG_DEVMEM is not set # CONFIG_HWMON is not set -# CONFIG_LCD_CLASS_DEVICE is not set -# CONFIG_BACKLIGHT_CLASS_DEVICE is not set # CONFIG_VGA_CONSOLE is not set # CONFIG_HID is not set # CONFIG_USB_SUPPORT is not set CONFIG_VIRTIO_MMIO=y CONFIG_VIRTIO_MMIO_CMDLINE_DEVICES=y -CONFIG_SIFIVE_PLIC=y -# CONFIG_VALIDATE_FS_PARSER is not set CONFIG_EXT2_FS=y # CONFIG_DNOTIFY is not set # CONFIG_INOTIFY_USER is not set diff --git a/arch/riscv/configs/rv32_defconfig b/arch/riscv/configs/rv32_defconfig index 05bbf5240569..2c2cda6cc1c5 100644 --- a/arch/riscv/configs/rv32_defconfig +++ b/arch/riscv/configs/rv32_defconfig @@ -14,9 +14,11 @@ CONFIG_CHECKPOINT_RESTORE=y CONFIG_BLK_DEV_INITRD=y CONFIG_EXPERT=y CONFIG_BPF_SYSCALL=y +CONFIG_SOC_SIFIVE=y CONFIG_SOC_VIRT=y CONFIG_ARCH_RV32I=y CONFIG_SMP=y +CONFIG_JUMP_LABEL=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y CONFIG_NET=y @@ -61,6 +63,8 @@ CONFIG_HVC_RISCV_SBI=y CONFIG_VIRTIO_CONSOLE=y CONFIG_HW_RANDOM=y CONFIG_HW_RANDOM_VIRTIO=y +CONFIG_SPI=y +CONFIG_SPI_SIFIVE=y # CONFIG_PTP_1588_CLOCK is not set CONFIG_POWER_RESET=y CONFIG_DRM=y @@ -76,6 +80,8 @@ CONFIG_USB_OHCI_HCD=y CONFIG_USB_OHCI_HCD_PLATFORM=y CONFIG_USB_STORAGE=y CONFIG_USB_UAS=y +CONFIG_MMC=y +CONFIG_MMC_SPI=y CONFIG_RTC_CLASS=y CONFIG_VIRTIO_PCI=y CONFIG_VIRTIO_BALLOON=y diff --git a/arch/riscv/include/asm/atomic.h b/arch/riscv/include/asm/atomic.h index 96f95c9ebd97..400a8c8b6de7 100644 --- a/arch/riscv/include/asm/atomic.h +++ b/arch/riscv/include/asm/atomic.h @@ -19,8 +19,6 @@ #include <asm/cmpxchg.h> #include <asm/barrier.h> -#define ATOMIC_INIT(i) { (i) } - #define __atomic_acquire_fence() \ __asm__ __volatile__(RISCV_ACQUIRE_BARRIER "" ::: "memory") diff --git a/arch/riscv/include/asm/barrier.h b/arch/riscv/include/asm/barrier.h index 3f1737f301cc..d0e24aaa2aa0 100644 --- a/arch/riscv/include/asm/barrier.h +++ b/arch/riscv/include/asm/barrier.h @@ -58,8 +58,16 @@ do { \ * The AQ/RL pair provides a RCpc critical section, but there's not really any * way we can take advantage of that here because the ordering is only enforced * on that one lock. Thus, we're just doing a full fence. + * + * Since we allow writeX to be called from preemptive regions we need at least + * an "o" in the predecessor set to ensure device writes are visible before the + * task is marked as available for scheduling on a new hart. While I don't see + * any concrete reason we need a full IO fence, it seems safer to just upgrade + * this in order to avoid any IO crossing a scheduling boundary. In both + * instances the scheduler pairs this with an mb(), so nothing is necessary on + * the new hart. */ -#define smp_mb__after_spinlock() RISCV_FENCE(rw,rw) +#define smp_mb__after_spinlock() RISCV_FENCE(iorw,iorw) #include <asm-generic/barrier.h> diff --git a/arch/riscv/include/asm/cacheflush.h b/arch/riscv/include/asm/cacheflush.h index c8677c75f82c..23ff70350992 100644 --- a/arch/riscv/include/asm/cacheflush.h +++ b/arch/riscv/include/asm/cacheflush.h @@ -8,65 +8,6 @@ #include <linux/mm.h> -#define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 0 - -/* - * The cache doesn't need to be flushed when TLB entries change when - * the cache is mapped to physical memory, not virtual memory - */ -static inline void flush_cache_all(void) -{ -} - -static inline void flush_cache_mm(struct mm_struct *mm) -{ -} - -static inline void flush_cache_dup_mm(struct mm_struct *mm) -{ -} - -static inline void flush_cache_range(struct vm_area_struct *vma, - unsigned long start, - unsigned long end) -{ -} - -static inline void flush_cache_page(struct vm_area_struct *vma, - unsigned long vmaddr, - unsigned long pfn) -{ -} - -static inline void flush_dcache_mmap_lock(struct address_space *mapping) -{ -} - -static inline void flush_dcache_mmap_unlock(struct address_space *mapping) -{ -} - -static inline void flush_icache_page(struct vm_area_struct *vma, - struct page *page) -{ -} - -static inline void flush_cache_vmap(unsigned long start, unsigned long end) -{ -} - -static inline void flush_cache_vunmap(unsigned long start, unsigned long end) -{ -} - -#define copy_to_user_page(vma, page, vaddr, dst, src, len) \ - do { \ - memcpy(dst, src, len); \ - flush_icache_user_range(vma, page, vaddr, len); \ - } while (0) -#define copy_from_user_page(vma, page, vaddr, dst, src, len) \ - memcpy(dst, src, len) - static inline void local_flush_icache_all(void) { asm volatile ("fence.i" ::: "memory"); @@ -79,13 +20,15 @@ static inline void flush_dcache_page(struct page *page) if (test_bit(PG_dcache_clean, &page->flags)) clear_bit(PG_dcache_clean, &page->flags); } +#define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1 /* * RISC-V doesn't have an instruction to flush parts of the instruction cache, * so instead we just flush the whole thing. */ #define flush_icache_range(start, end) flush_icache_all() -#define flush_icache_user_range(vma, pg, addr, len) flush_icache_mm(vma->vm_mm, 0) +#define flush_icache_user_page(vma, pg, addr, len) \ + flush_icache_mm(vma->vm_mm, 0) #ifndef CONFIG_SMP @@ -105,4 +48,6 @@ void flush_icache_mm(struct mm_struct *mm, bool local); #define SYS_RISCV_FLUSH_ICACHE_LOCAL 1UL #define SYS_RISCV_FLUSH_ICACHE_ALL (SYS_RISCV_FLUSH_ICACHE_LOCAL) +#include <asm-generic/cacheflush.h> + #endif /* _ASM_RISCV_CACHEFLUSH_H */ diff --git a/arch/riscv/include/asm/cacheinfo.h b/arch/riscv/include/asm/cacheinfo.h new file mode 100644 index 000000000000..5d9662e9aba8 --- /dev/null +++ b/arch/riscv/include/asm/cacheinfo.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _ASM_RISCV_CACHEINFO_H +#define _ASM_RISCV_CACHEINFO_H + +#include <linux/cacheinfo.h> + +struct riscv_cacheinfo_ops { + const struct attribute_group * (*get_priv_group)(struct cacheinfo + *this_leaf); +}; + +void riscv_set_cacheinfo_ops(struct riscv_cacheinfo_ops *ops); + +#endif /* _ASM_RISCV_CACHEINFO_H */ diff --git a/arch/riscv/include/asm/clint.h b/arch/riscv/include/asm/clint.h index a279b17a6aad..0789fd37b40a 100644 --- a/arch/riscv/include/asm/clint.h +++ b/arch/riscv/include/asm/clint.h @@ -1,39 +1,26 @@ -/* SPDX-License-Identifier: GPL-2.0 */ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020 Google, Inc + */ + #ifndef _ASM_RISCV_CLINT_H -#define _ASM_RISCV_CLINT_H 1 +#define _ASM_RISCV_CLINT_H -#include <linux/io.h> -#include <linux/smp.h> +#include <linux/types.h> +#include <asm/mmio.h> #ifdef CONFIG_RISCV_M_MODE -extern u32 __iomem *clint_ipi_base; - -void clint_init_boot_cpu(void); - -static inline void clint_send_ipi_single(unsigned long hartid) -{ - writel(1, clint_ipi_base + hartid); -} - -static inline void clint_send_ipi_mask(const struct cpumask *mask) -{ - int cpu; - - for_each_cpu(cpu, mask) - clint_send_ipi_single(cpuid_to_hartid_map(cpu)); -} - -static inline void clint_clear_ipi(unsigned long hartid) -{ - writel(0, clint_ipi_base + hartid); -} -#else /* CONFIG_RISCV_M_MODE */ -#define clint_init_boot_cpu() do { } while (0) - -/* stubs to for code is only reachable under IS_ENABLED(CONFIG_RISCV_M_MODE): */ -void clint_send_ipi_single(unsigned long hartid); -void clint_send_ipi_mask(const struct cpumask *hartid_mask); -void clint_clear_ipi(unsigned long hartid); -#endif /* CONFIG_RISCV_M_MODE */ - -#endif /* _ASM_RISCV_CLINT_H */ +/* + * This lives in the CLINT driver, but is accessed directly by timex.h to avoid + * any overhead when accessing the MMIO timer. + * + * The ISA defines mtime as a 64-bit memory-mapped register that increments at + * a constant frequency, but it doesn't define some other constraints we depend + * on (most notably ordering constraints, but also some simpler stuff like the + * memory layout). Thus, this is called "clint_time_val" instead of something + * like "riscv_mtime", to signify that these non-ISA assumptions must hold. + */ +extern u64 __iomem *clint_time_val; +#endif + +#endif diff --git a/arch/riscv/include/asm/clocksource.h b/arch/riscv/include/asm/clocksource.h new file mode 100644 index 000000000000..482185566b0c --- /dev/null +++ b/arch/riscv/include/asm/clocksource.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_CLOCKSOURCE_H +#define _ASM_CLOCKSOURCE_H + +#include <asm/vdso/clocksource.h> + +#endif diff --git a/arch/riscv/include/asm/cmpxchg.h b/arch/riscv/include/asm/cmpxchg.h index d969bab4a26b..262e5bbb2776 100644 --- a/arch/riscv/include/asm/cmpxchg.h +++ b/arch/riscv/include/asm/cmpxchg.h @@ -179,7 +179,7 @@ " bnez %1, 0b\n" \ "1:\n" \ : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \ - : "rJ" (__old), "rJ" (__new) \ + : "rJ" ((long)__old), "rJ" (__new) \ : "memory"); \ break; \ case 8: \ @@ -224,7 +224,7 @@ RISCV_ACQUIRE_BARRIER \ "1:\n" \ : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \ - : "rJ" (__old), "rJ" (__new) \ + : "rJ" ((long)__old), "rJ" (__new) \ : "memory"); \ break; \ case 8: \ @@ -270,7 +270,7 @@ " bnez %1, 0b\n" \ "1:\n" \ : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \ - : "rJ" (__old), "rJ" (__new) \ + : "rJ" ((long)__old), "rJ" (__new) \ : "memory"); \ break; \ case 8: \ @@ -316,7 +316,7 @@ " fence rw, rw\n" \ "1:\n" \ : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \ - : "rJ" (__old), "rJ" (__new) \ + : "rJ" ((long)__old), "rJ" (__new) \ : "memory"); \ break; \ case 8: \ diff --git a/arch/riscv/include/asm/fixmap.h b/arch/riscv/include/asm/fixmap.h index 2368d49eb4ef..1ff075a8dfc7 100644 --- a/arch/riscv/include/asm/fixmap.h +++ b/arch/riscv/include/asm/fixmap.h @@ -8,8 +8,8 @@ #include <linux/kernel.h> #include <linux/sizes.h> +#include <linux/pgtable.h> #include <asm/page.h> -#include <asm/pgtable.h> #ifdef CONFIG_MMU /* diff --git a/arch/riscv/include/asm/ftrace.h b/arch/riscv/include/asm/ftrace.h index ace8a6e2d11d..845002cc2e57 100644 --- a/arch/riscv/include/asm/ftrace.h +++ b/arch/riscv/include/asm/ftrace.h @@ -66,6 +66,13 @@ do { \ * Let auipc+jalr be the basic *mcount unit*, so we make it 8 bytes here. */ #define MCOUNT_INSN_SIZE 8 + +#ifndef __ASSEMBLY__ +struct dyn_ftrace; +int ftrace_init_nop(struct module *mod, struct dyn_ftrace *rec); +#define ftrace_init_nop ftrace_init_nop +#endif + #endif #endif /* _ASM_RISCV_FTRACE_H */ diff --git a/arch/riscv/include/asm/gdb_xml.h b/arch/riscv/include/asm/gdb_xml.h new file mode 100644 index 000000000000..09342111f227 --- /dev/null +++ b/arch/riscv/include/asm/gdb_xml.h @@ -0,0 +1,116 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef __ASM_GDB_XML_H_ +#define __ASM_GDB_XML_H_ + +const char riscv_gdb_stub_feature[64] = + "PacketSize=800;qXfer:features:read+;"; + +static const char gdb_xfer_read_target[31] = "qXfer:features:read:target.xml:"; + +#ifdef CONFIG_64BIT +static const char gdb_xfer_read_cpuxml[39] = + "qXfer:features:read:riscv-64bit-cpu.xml"; + +static const char riscv_gdb_stub_target_desc[256] = +"l<?xml version=\"1.0\"?>" +"<!DOCTYPE target SYSTEM \"gdb-target.dtd\">" +"<target>" +"<xi:include href=\"riscv-64bit-cpu.xml\"/>" +"</target>"; + +static const char riscv_gdb_stub_cpuxml[2048] = +"l<?xml version=\"1.0\"?>" +"<!DOCTYPE feature SYSTEM \"gdb-target.dtd\">" +"<feature name=\"org.gnu.gdb.riscv.cpu\">" +"<reg name=\""DBG_REG_ZERO"\" bitsize=\"64\" type=\"int\" regnum=\"0\"/>" +"<reg name=\""DBG_REG_RA"\" bitsize=\"64\" type=\"code_ptr\"/>" +"<reg name=\""DBG_REG_SP"\" bitsize=\"64\" type=\"data_ptr\"/>" +"<reg name=\""DBG_REG_GP"\" bitsize=\"64\" type=\"data_ptr\"/>" +"<reg name=\""DBG_REG_TP"\" bitsize=\"64\" type=\"data_ptr\"/>" +"<reg name=\""DBG_REG_T0"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_T1"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_T2"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_FP"\" bitsize=\"64\" type=\"data_ptr\"/>" +"<reg name=\""DBG_REG_S1"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_A0"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_A1"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_A2"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_A3"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_A4"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_A5"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_A6"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_A7"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_S2"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_S3"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_S4"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_S5"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_S6"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_S7"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_S8"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_S9"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_S10"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_S11"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_T3"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_T4"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_T5"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_T6"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_EPC"\" bitsize=\"64\" type=\"code_ptr\"/>" +"<reg name=\""DBG_REG_STATUS"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_BADADDR"\" bitsize=\"64\" type=\"int\"/>" +"<reg name=\""DBG_REG_CAUSE"\" bitsize=\"64\" type=\"int\"/>" +"</feature>"; +#else +static const char gdb_xfer_read_cpuxml[39] = + "qXfer:features:read:riscv-32bit-cpu.xml"; + +static const char riscv_gdb_stub_target_desc[256] = +"l<?xml version=\"1.0\"?>" +"<!DOCTYPE target SYSTEM \"gdb-target.dtd\">" +"<target>" +"<xi:include href=\"riscv-32bit-cpu.xml\"/>" +"</target>"; + +static const char riscv_gdb_stub_cpuxml[2048] = +"l<?xml version=\"1.0\"?>" +"<!DOCTYPE feature SYSTEM \"gdb-target.dtd\">" +"<feature name=\"org.gnu.gdb.riscv.cpu\">" +"<reg name=\""DBG_REG_ZERO"\" bitsize=\"32\" type=\"int\" regnum=\"0\"/>" +"<reg name=\""DBG_REG_RA"\" bitsize=\"32\" type=\"code_ptr\"/>" +"<reg name=\""DBG_REG_SP"\" bitsize=\"32\" type=\"data_ptr\"/>" +"<reg name=\""DBG_REG_GP"\" bitsize=\"32\" type=\"data_ptr\"/>" +"<reg name=\""DBG_REG_TP"\" bitsize=\"32\" type=\"data_ptr\"/>" +"<reg name=\""DBG_REG_T0"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_T1"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_T2"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_FP"\" bitsize=\"32\" type=\"data_ptr\"/>" +"<reg name=\""DBG_REG_S1"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_A0"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_A1"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_A2"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_A3"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_A4"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_A5"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_A6"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_A7"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_S2"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_S3"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_S4"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_S5"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_S6"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_S7"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_S8"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_S9"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_S10"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_S11"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_T3"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_T4"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_T5"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_T6"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_EPC"\" bitsize=\"32\" type=\"code_ptr\"/>" +"<reg name=\""DBG_REG_STATUS"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_BADADDR"\" bitsize=\"32\" type=\"int\"/>" +"<reg name=\""DBG_REG_CAUSE"\" bitsize=\"32\" type=\"int\"/>" +"</feature>"; +#endif +#endif diff --git a/arch/riscv/include/asm/hugetlb.h b/arch/riscv/include/asm/hugetlb.h index 728a5db66597..a5c2ca1d1cd8 100644 --- a/arch/riscv/include/asm/hugetlb.h +++ b/arch/riscv/include/asm/hugetlb.h @@ -5,14 +5,4 @@ #include <asm-generic/hugetlb.h> #include <asm/page.h> -static inline int is_hugepage_only_range(struct mm_struct *mm, - unsigned long addr, - unsigned long len) { - return 0; -} - -static inline void arch_clear_hugepage_flags(struct page *page) -{ -} - #endif /* _ASM_RISCV_HUGETLB_H */ diff --git a/arch/riscv/include/asm/io.h b/arch/riscv/include/asm/io.h index 0f477206a4ed..3835c3295dc5 100644 --- a/arch/riscv/include/asm/io.h +++ b/arch/riscv/include/asm/io.h @@ -12,8 +12,8 @@ #define _ASM_RISCV_IO_H #include <linux/types.h> +#include <linux/pgtable.h> #include <asm/mmiowb.h> -#include <asm/pgtable.h> /* * MMIO access functions are separated out to break dependency cycles diff --git a/arch/riscv/include/asm/irq.h b/arch/riscv/include/asm/irq.h index 6e1b0e0325eb..9807ad164015 100644 --- a/arch/riscv/include/asm/irq.h +++ b/arch/riscv/include/asm/irq.h @@ -10,11 +10,6 @@ #include <linux/interrupt.h> #include <linux/linkage.h> -#define NR_IRQS 0 - -void riscv_timer_interrupt(void); -void riscv_software_interrupt(void); - #include <asm-generic/irq.h> #endif /* _ASM_RISCV_IRQ_H */ diff --git a/arch/riscv/include/asm/irq_work.h b/arch/riscv/include/asm/irq_work.h new file mode 100644 index 000000000000..d6c277992f76 --- /dev/null +++ b/arch/riscv/include/asm/irq_work.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_RISCV_IRQ_WORK_H +#define _ASM_RISCV_IRQ_WORK_H + +static inline bool arch_irq_work_has_interrupt(void) +{ + return true; +} +extern void arch_irq_work_raise(void); +#endif /* _ASM_RISCV_IRQ_WORK_H */ diff --git a/arch/riscv/include/asm/jump_label.h b/arch/riscv/include/asm/jump_label.h new file mode 100644 index 000000000000..38af2ec7b9bf --- /dev/null +++ b/arch/riscv/include/asm/jump_label.h @@ -0,0 +1,60 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020 Emil Renner Berthing + * + * Based on arch/arm64/include/asm/jump_label.h + */ +#ifndef __ASM_JUMP_LABEL_H +#define __ASM_JUMP_LABEL_H + +#ifndef __ASSEMBLY__ + +#include <linux/types.h> +#include <asm/asm.h> + +#define JUMP_LABEL_NOP_SIZE 4 + +static __always_inline bool arch_static_branch(struct static_key *key, + bool branch) +{ + asm_volatile_goto( + " .option push \n\t" + " .option norelax \n\t" + " .option norvc \n\t" + "1: nop \n\t" + " .option pop \n\t" + " .pushsection __jump_table, \"aw\" \n\t" + " .align " RISCV_LGPTR " \n\t" + " .long 1b - ., %l[label] - . \n\t" + " " RISCV_PTR " %0 - . \n\t" + " .popsection \n\t" + : : "i"(&((char *)key)[branch]) : : label); + + return false; +label: + return true; +} + +static __always_inline bool arch_static_branch_jump(struct static_key *key, + bool branch) +{ + asm_volatile_goto( + " .option push \n\t" + " .option norelax \n\t" + " .option norvc \n\t" + "1: jal zero, %l[label] \n\t" + " .option pop \n\t" + " .pushsection __jump_table, \"aw\" \n\t" + " .align " RISCV_LGPTR " \n\t" + " .long 1b - ., %l[label] - . \n\t" + " " RISCV_PTR " %0 - . \n\t" + " .popsection \n\t" + : : "i"(&((char *)key)[branch]) : : label); + + return false; +label: + return true; +} + +#endif /* __ASSEMBLY__ */ +#endif /* __ASM_JUMP_LABEL_H */ diff --git a/arch/riscv/include/asm/kasan.h b/arch/riscv/include/asm/kasan.h index b47045cb85ce..b04028c6218c 100644 --- a/arch/riscv/include/asm/kasan.h +++ b/arch/riscv/include/asm/kasan.h @@ -8,8 +8,6 @@ #ifdef CONFIG_KASAN -#include <asm/pgtable.h> - #define KASAN_SHADOW_SCALE_SHIFT 3 #define KASAN_SHADOW_SIZE (UL(1) << (38 - KASAN_SHADOW_SCALE_SHIFT)) diff --git a/arch/riscv/include/asm/kdebug.h b/arch/riscv/include/asm/kdebug.h new file mode 100644 index 000000000000..85ac00411f6e --- /dev/null +++ b/arch/riscv/include/asm/kdebug.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef _ASM_ARC_KDEBUG_H +#define _ASM_ARC_KDEBUG_H + +enum die_val { + DIE_UNUSED, + DIE_TRAP, + DIE_OOPS +}; + +#endif diff --git a/arch/riscv/include/asm/kgdb.h b/arch/riscv/include/asm/kgdb.h new file mode 100644 index 000000000000..46677daf708b --- /dev/null +++ b/arch/riscv/include/asm/kgdb.h @@ -0,0 +1,113 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef __ASM_KGDB_H_ +#define __ASM_KGDB_H_ + +#ifdef __KERNEL__ + +#define GDB_SIZEOF_REG sizeof(unsigned long) + +#define DBG_MAX_REG_NUM (36) +#define NUMREGBYTES ((DBG_MAX_REG_NUM) * GDB_SIZEOF_REG) +#define CACHE_FLUSH_IS_SAFE 1 +#define BUFMAX 2048 +#ifdef CONFIG_RISCV_ISA_C +#define BREAK_INSTR_SIZE 2 +#else +#define BREAK_INSTR_SIZE 4 +#endif + +#ifndef __ASSEMBLY__ + +extern unsigned long kgdb_compiled_break; + +static inline void arch_kgdb_breakpoint(void) +{ + asm(".global kgdb_compiled_break\n" + ".option norvc\n" + "kgdb_compiled_break: ebreak\n" + ".option rvc\n"); +} + +#endif /* !__ASSEMBLY__ */ + +#define DBG_REG_ZERO "zero" +#define DBG_REG_RA "ra" +#define DBG_REG_SP "sp" +#define DBG_REG_GP "gp" +#define DBG_REG_TP "tp" +#define DBG_REG_T0 "t0" +#define DBG_REG_T1 "t1" +#define DBG_REG_T2 "t2" +#define DBG_REG_FP "fp" +#define DBG_REG_S1 "s1" +#define DBG_REG_A0 "a0" +#define DBG_REG_A1 "a1" +#define DBG_REG_A2 "a2" +#define DBG_REG_A3 "a3" +#define DBG_REG_A4 "a4" +#define DBG_REG_A5 "a5" +#define DBG_REG_A6 "a6" +#define DBG_REG_A7 "a7" +#define DBG_REG_S2 "s2" +#define DBG_REG_S3 "s3" +#define DBG_REG_S4 "s4" +#define DBG_REG_S5 "s5" +#define DBG_REG_S6 "s6" +#define DBG_REG_S7 "s7" +#define DBG_REG_S8 "s8" +#define DBG_REG_S9 "s9" +#define DBG_REG_S10 "s10" +#define DBG_REG_S11 "s11" +#define DBG_REG_T3 "t3" +#define DBG_REG_T4 "t4" +#define DBG_REG_T5 "t5" +#define DBG_REG_T6 "t6" +#define DBG_REG_EPC "pc" +#define DBG_REG_STATUS "sstatus" +#define DBG_REG_BADADDR "stval" +#define DBG_REG_CAUSE "scause" + +#define DBG_REG_ZERO_OFF 0 +#define DBG_REG_RA_OFF 1 +#define DBG_REG_SP_OFF 2 +#define DBG_REG_GP_OFF 3 +#define DBG_REG_TP_OFF 4 +#define DBG_REG_T0_OFF 5 +#define DBG_REG_T1_OFF 6 +#define DBG_REG_T2_OFF 7 +#define DBG_REG_FP_OFF 8 +#define DBG_REG_S1_OFF 9 +#define DBG_REG_A0_OFF 10 +#define DBG_REG_A1_OFF 11 +#define DBG_REG_A2_OFF 12 +#define DBG_REG_A3_OFF 13 +#define DBG_REG_A4_OFF 14 +#define DBG_REG_A5_OFF 15 +#define DBG_REG_A6_OFF 16 +#define DBG_REG_A7_OFF 17 +#define DBG_REG_S2_OFF 18 +#define DBG_REG_S3_OFF 19 +#define DBG_REG_S4_OFF 20 +#define DBG_REG_S5_OFF 21 +#define DBG_REG_S6_OFF 22 +#define DBG_REG_S7_OFF 23 +#define DBG_REG_S8_OFF 24 +#define DBG_REG_S9_OFF 25 +#define DBG_REG_S10_OFF 26 +#define DBG_REG_S11_OFF 27 +#define DBG_REG_T3_OFF 28 +#define DBG_REG_T4_OFF 29 +#define DBG_REG_T5_OFF 30 +#define DBG_REG_T6_OFF 31 +#define DBG_REG_EPC_OFF 32 +#define DBG_REG_STATUS_OFF 33 +#define DBG_REG_BADADDR_OFF 34 +#define DBG_REG_CAUSE_OFF 35 + +extern const char riscv_gdb_stub_feature[64]; + +#define kgdb_arch_gdb_stub_feature riscv_gdb_stub_feature + +#endif +#endif diff --git a/arch/riscv/include/asm/mmio.h b/arch/riscv/include/asm/mmio.h index 56053c9838b2..aff6c33ab0c0 100644 --- a/arch/riscv/include/asm/mmio.h +++ b/arch/riscv/include/asm/mmio.h @@ -14,12 +14,6 @@ #include <linux/types.h> #include <asm/mmiowb.h> -#ifndef CONFIG_MMU -#define pgprot_noncached(x) (x) -#define pgprot_writecombine(x) (x) -#define pgprot_device(x) (x) -#endif /* CONFIG_MMU */ - /* Generic IO read/write. These perform native-endian accesses. */ #define __raw_writeb __raw_writeb static inline void __raw_writeb(u8 val, volatile void __iomem *addr) diff --git a/arch/riscv/include/asm/parse_asm.h b/arch/riscv/include/asm/parse_asm.h new file mode 100644 index 000000000000..f36368de839f --- /dev/null +++ b/arch/riscv/include/asm/parse_asm.h @@ -0,0 +1,219 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020 SiFive + */ + +#include <linux/bits.h> + +/* The bit field of immediate value in I-type instruction */ +#define I_IMM_SIGN_OPOFF 31 +#define I_IMM_11_0_OPOFF 20 +#define I_IMM_SIGN_OFF 12 +#define I_IMM_11_0_OFF 0 +#define I_IMM_11_0_MASK GENMASK(11, 0) + +/* The bit field of immediate value in J-type instruction */ +#define J_IMM_SIGN_OPOFF 31 +#define J_IMM_10_1_OPOFF 21 +#define J_IMM_11_OPOFF 20 +#define J_IMM_19_12_OPOFF 12 +#define J_IMM_SIGN_OFF 20 +#define J_IMM_10_1_OFF 1 +#define J_IMM_11_OFF 11 +#define J_IMM_19_12_OFF 12 +#define J_IMM_10_1_MASK GENMASK(9, 0) +#define J_IMM_11_MASK GENMASK(0, 0) +#define J_IMM_19_12_MASK GENMASK(7, 0) + +/* The bit field of immediate value in B-type instruction */ +#define B_IMM_SIGN_OPOFF 31 +#define B_IMM_10_5_OPOFF 25 +#define B_IMM_4_1_OPOFF 8 +#define B_IMM_11_OPOFF 7 +#define B_IMM_SIGN_OFF 12 +#define B_IMM_10_5_OFF 5 +#define B_IMM_4_1_OFF 1 +#define B_IMM_11_OFF 11 +#define B_IMM_10_5_MASK GENMASK(5, 0) +#define B_IMM_4_1_MASK GENMASK(3, 0) +#define B_IMM_11_MASK GENMASK(0, 0) + +/* The register offset in RVG instruction */ +#define RVG_RS1_OPOFF 15 +#define RVG_RS2_OPOFF 20 +#define RVG_RD_OPOFF 7 + +/* The bit field of immediate value in RVC J instruction */ +#define RVC_J_IMM_SIGN_OPOFF 12 +#define RVC_J_IMM_4_OPOFF 11 +#define RVC_J_IMM_9_8_OPOFF 9 +#define RVC_J_IMM_10_OPOFF 8 +#define RVC_J_IMM_6_OPOFF 7 +#define RVC_J_IMM_7_OPOFF 6 +#define RVC_J_IMM_3_1_OPOFF 3 +#define RVC_J_IMM_5_OPOFF 2 +#define RVC_J_IMM_SIGN_OFF 11 +#define RVC_J_IMM_4_OFF 4 +#define RVC_J_IMM_9_8_OFF 8 +#define RVC_J_IMM_10_OFF 10 +#define RVC_J_IMM_6_OFF 6 +#define RVC_J_IMM_7_OFF 7 +#define RVC_J_IMM_3_1_OFF 1 +#define RVC_J_IMM_5_OFF 5 +#define RVC_J_IMM_4_MASK GENMASK(0, 0) +#define RVC_J_IMM_9_8_MASK GENMASK(1, 0) +#define RVC_J_IMM_10_MASK GENMASK(0, 0) +#define RVC_J_IMM_6_MASK GENMASK(0, 0) +#define RVC_J_IMM_7_MASK GENMASK(0, 0) +#define RVC_J_IMM_3_1_MASK GENMASK(2, 0) +#define RVC_J_IMM_5_MASK GENMASK(0, 0) + +/* The bit field of immediate value in RVC B instruction */ +#define RVC_B_IMM_SIGN_OPOFF 12 +#define RVC_B_IMM_4_3_OPOFF 10 +#define RVC_B_IMM_7_6_OPOFF 5 +#define RVC_B_IMM_2_1_OPOFF 3 +#define RVC_B_IMM_5_OPOFF 2 +#define RVC_B_IMM_SIGN_OFF 8 +#define RVC_B_IMM_4_3_OFF 3 +#define RVC_B_IMM_7_6_OFF 6 +#define RVC_B_IMM_2_1_OFF 1 +#define RVC_B_IMM_5_OFF 5 +#define RVC_B_IMM_4_3_MASK GENMASK(1, 0) +#define RVC_B_IMM_7_6_MASK GENMASK(1, 0) +#define RVC_B_IMM_2_1_MASK GENMASK(1, 0) +#define RVC_B_IMM_5_MASK GENMASK(0, 0) + +/* The register offset in RVC op=C0 instruction */ +#define RVC_C0_RS1_OPOFF 7 +#define RVC_C0_RS2_OPOFF 2 +#define RVC_C0_RD_OPOFF 2 + +/* The register offset in RVC op=C1 instruction */ +#define RVC_C1_RS1_OPOFF 7 +#define RVC_C1_RS2_OPOFF 2 +#define RVC_C1_RD_OPOFF 7 + +/* The register offset in RVC op=C2 instruction */ +#define RVC_C2_RS1_OPOFF 7 +#define RVC_C2_RS2_OPOFF 2 +#define RVC_C2_RD_OPOFF 7 + +/* parts of opcode for RVG*/ +#define OPCODE_BRANCH 0x63 +#define OPCODE_JALR 0x67 +#define OPCODE_JAL 0x6f +#define OPCODE_SYSTEM 0x73 + +/* parts of opcode for RVC*/ +#define OPCODE_C_0 0x0 +#define OPCODE_C_1 0x1 +#define OPCODE_C_2 0x2 + +/* parts of funct3 code for I, M, A extension*/ +#define FUNCT3_JALR 0x0 +#define FUNCT3_BEQ 0x0 +#define FUNCT3_BNE 0x1000 +#define FUNCT3_BLT 0x4000 +#define FUNCT3_BGE 0x5000 +#define FUNCT3_BLTU 0x6000 +#define FUNCT3_BGEU 0x7000 + +/* parts of funct3 code for C extension*/ +#define FUNCT3_C_BEQZ 0xc000 +#define FUNCT3_C_BNEZ 0xe000 +#define FUNCT3_C_J 0xa000 +#define FUNCT3_C_JAL 0x2000 +#define FUNCT4_C_JR 0x8000 +#define FUNCT4_C_JALR 0xf000 + +#define FUNCT12_SRET 0x10200000 + +#define MATCH_JALR (FUNCT3_JALR | OPCODE_JALR) +#define MATCH_JAL (OPCODE_JAL) +#define MATCH_BEQ (FUNCT3_BEQ | OPCODE_BRANCH) +#define MATCH_BNE (FUNCT3_BNE | OPCODE_BRANCH) +#define MATCH_BLT (FUNCT3_BLT | OPCODE_BRANCH) +#define MATCH_BGE (FUNCT3_BGE | OPCODE_BRANCH) +#define MATCH_BLTU (FUNCT3_BLTU | OPCODE_BRANCH) +#define MATCH_BGEU (FUNCT3_BGEU | OPCODE_BRANCH) +#define MATCH_SRET (FUNCT12_SRET | OPCODE_SYSTEM) +#define MATCH_C_BEQZ (FUNCT3_C_BEQZ | OPCODE_C_1) +#define MATCH_C_BNEZ (FUNCT3_C_BNEZ | OPCODE_C_1) +#define MATCH_C_J (FUNCT3_C_J | OPCODE_C_1) +#define MATCH_C_JAL (FUNCT3_C_JAL | OPCODE_C_1) +#define MATCH_C_JR (FUNCT4_C_JR | OPCODE_C_2) +#define MATCH_C_JALR (FUNCT4_C_JALR | OPCODE_C_2) + +#define MASK_JALR 0x707f +#define MASK_JAL 0x7f +#define MASK_C_JALR 0xf07f +#define MASK_C_JR 0xf07f +#define MASK_C_JAL 0xe003 +#define MASK_C_J 0xe003 +#define MASK_BEQ 0x707f +#define MASK_BNE 0x707f +#define MASK_BLT 0x707f +#define MASK_BGE 0x707f +#define MASK_BLTU 0x707f +#define MASK_BGEU 0x707f +#define MASK_C_BEQZ 0xe003 +#define MASK_C_BNEZ 0xe003 +#define MASK_SRET 0xffffffff + +#define __INSN_LENGTH_MASK _UL(0x3) +#define __INSN_LENGTH_GE_32 _UL(0x3) +#define __INSN_OPCODE_MASK _UL(0x7F) +#define __INSN_BRANCH_OPCODE _UL(OPCODE_BRANCH) + +/* Define a series of is_XXX_insn functions to check if the value INSN + * is an instance of instruction XXX. + */ +#define DECLARE_INSN(INSN_NAME, INSN_MATCH, INSN_MASK) \ +static inline bool is_ ## INSN_NAME ## _insn(long insn) \ +{ \ + return (insn & (INSN_MASK)) == (INSN_MATCH); \ +} + +#define RV_IMM_SIGN(x) (-(((x) >> 31) & 1)) +#define RVC_IMM_SIGN(x) (-(((x) >> 12) & 1)) +#define RV_X(X, s, mask) (((X) >> (s)) & (mask)) +#define RVC_X(X, s, mask) RV_X(X, s, mask) + +#define EXTRACT_JTYPE_IMM(x) \ + ({typeof(x) x_ = (x); \ + (RV_X(x_, J_IMM_10_1_OPOFF, J_IMM_10_1_MASK) << J_IMM_10_1_OFF) | \ + (RV_X(x_, J_IMM_11_OPOFF, J_IMM_11_MASK) << J_IMM_11_OFF) | \ + (RV_X(x_, J_IMM_19_12_OPOFF, J_IMM_19_12_MASK) << J_IMM_19_12_OFF) | \ + (RV_IMM_SIGN(x_) << J_IMM_SIGN_OFF); }) + +#define EXTRACT_ITYPE_IMM(x) \ + ({typeof(x) x_ = (x); \ + (RV_X(x_, I_IMM_11_0_OPOFF, I_IMM_11_0_MASK)) | \ + (RV_IMM_SIGN(x_) << I_IMM_SIGN_OFF); }) + +#define EXTRACT_BTYPE_IMM(x) \ + ({typeof(x) x_ = (x); \ + (RV_X(x_, B_IMM_4_1_OPOFF, B_IMM_4_1_MASK) << B_IMM_4_1_OFF) | \ + (RV_X(x_, B_IMM_10_5_OPOFF, B_IMM_10_5_MASK) << B_IMM_10_5_OFF) | \ + (RV_X(x_, B_IMM_11_OPOFF, B_IMM_11_MASK) << B_IMM_11_OFF) | \ + (RV_IMM_SIGN(x_) << B_IMM_SIGN_OFF); }) + +#define EXTRACT_RVC_J_IMM(x) \ + ({typeof(x) x_ = (x); \ + (RVC_X(x_, RVC_J_IMM_3_1_OPOFF, RVC_J_IMM_3_1_MASK) << RVC_J_IMM_3_1_OFF) | \ + (RVC_X(x_, RVC_J_IMM_4_OPOFF, RVC_J_IMM_4_MASK) << RVC_J_IMM_4_OFF) | \ + (RVC_X(x_, RVC_J_IMM_5_OPOFF, RVC_J_IMM_5_MASK) << RVC_J_IMM_5_OFF) | \ + (RVC_X(x_, RVC_J_IMM_6_OPOFF, RVC_J_IMM_6_MASK) << RVC_J_IMM_6_OFF) | \ + (RVC_X(x_, RVC_J_IMM_7_OPOFF, RVC_J_IMM_7_MASK) << RVC_J_IMM_7_OFF) | \ + (RVC_X(x_, RVC_J_IMM_9_8_OPOFF, RVC_J_IMM_9_8_MASK) << RVC_J_IMM_9_8_OFF) | \ + (RVC_X(x_, RVC_J_IMM_10_OPOFF, RVC_J_IMM_10_MASK) << RVC_J_IMM_10_OFF) | \ + (RVC_IMM_SIGN(x_) << RVC_J_IMM_SIGN_OFF); }) + +#define EXTRACT_RVC_B_IMM(x) \ + ({typeof(x) x_ = (x); \ + (RVC_X(x_, RVC_B_IMM_2_1_OPOFF, RVC_B_IMM_2_1_MASK) << RVC_B_IMM_2_1_OFF) | \ + (RVC_X(x_, RVC_B_IMM_4_3_OPOFF, RVC_B_IMM_4_3_MASK) << RVC_B_IMM_4_3_OFF) | \ + (RVC_X(x_, RVC_B_IMM_5_OPOFF, RVC_B_IMM_5_MASK) << RVC_B_IMM_5_OFF) | \ + (RVC_X(x_, RVC_B_IMM_7_6_OPOFF, RVC_B_IMM_7_6_MASK) << RVC_B_IMM_7_6_OFF) | \ + (RVC_IMM_SIGN(x_) << RVC_B_IMM_SIGN_OFF); }) diff --git a/arch/riscv/include/asm/patch.h b/arch/riscv/include/asm/patch.h index b5918a6e0615..9a7d7346001e 100644 --- a/arch/riscv/include/asm/patch.h +++ b/arch/riscv/include/asm/patch.h @@ -6,7 +6,7 @@ #ifndef _ASM_RISCV_PATCH_H #define _ASM_RISCV_PATCH_H -int riscv_patch_text_nosync(void *addr, const void *insns, size_t len); -int riscv_patch_text(void *addr, u32 insn); +int patch_text_nosync(void *addr, const void *insns, size_t len); +int patch_text(void *addr, u32 insn); #endif /* _ASM_RISCV_PATCH_H */ diff --git a/arch/riscv/include/asm/pgalloc.h b/arch/riscv/include/asm/pgalloc.h index 3f601ee8233f..23b1544e0ca5 100644 --- a/arch/riscv/include/asm/pgalloc.h +++ b/arch/riscv/include/asm/pgalloc.h @@ -11,7 +11,7 @@ #include <asm/tlb.h> #ifdef CONFIG_MMU -#include <asm-generic/pgalloc.h> /* for pte_{alloc,free}_one */ +#include <asm-generic/pgalloc.h> static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd, pte_t *pte) @@ -55,24 +55,8 @@ static inline pgd_t *pgd_alloc(struct mm_struct *mm) return pgd; } -static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) -{ - free_page((unsigned long)pgd); -} - #ifndef __PAGETABLE_PMD_FOLDED -static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr) -{ - return (pmd_t *)__get_free_page( - GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_ZERO); -} - -static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) -{ - free_page((unsigned long)pmd); -} - #define __pmd_free_tlb(tlb, pmd, addr) pmd_free((tlb)->mm, pmd) #endif /* __PAGETABLE_PMD_FOLDED */ diff --git a/arch/riscv/include/asm/pgtable-64.h b/arch/riscv/include/asm/pgtable-64.h index b15f70a1fdfa..f3b0da64c6c8 100644 --- a/arch/riscv/include/asm/pgtable-64.h +++ b/arch/riscv/include/asm/pgtable-64.h @@ -70,13 +70,6 @@ static inline struct page *pud_page(pud_t pud) return pfn_to_page(pud_val(pud) >> _PAGE_PFN_SHIFT); } -#define pmd_index(addr) (((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1)) - -static inline pmd_t *pmd_offset(pud_t *pud, unsigned long addr) -{ - return (pmd_t *)pud_page_vaddr(*pud) + pmd_index(addr); -} - static inline pmd_t pfn_pmd(unsigned long pfn, pgprot_t prot) { return __pmd((pfn << _PAGE_PFN_SHIFT) | pgprot_val(prot)); diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index 35b60035b6b0..eaea1f717010 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -173,16 +173,6 @@ static inline unsigned long _pgd_pfn(pgd_t pgd) return pgd_val(pgd) >> _PAGE_PFN_SHIFT; } -#define pgd_index(addr) (((addr) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1)) - -/* Locate an entry in the page global directory */ -static inline pgd_t *pgd_offset(const struct mm_struct *mm, unsigned long addr) -{ - return mm->pgd + pgd_index(addr); -} -/* Locate an entry in the kernel page global directory */ -#define pgd_offset_k(addr) pgd_offset(&init_mm, (addr)) - static inline struct page *pmd_page(pmd_t pmd) { return pfn_to_page(pmd_val(pmd) >> _PAGE_PFN_SHIFT); @@ -209,16 +199,6 @@ static inline pte_t pfn_pte(unsigned long pfn, pgprot_t prot) #define mk_pte(page, prot) pfn_pte(page_to_pfn(page), prot) -#define pte_index(addr) (((addr) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) - -static inline pte_t *pte_offset_kernel(pmd_t *pmd, unsigned long addr) -{ - return (pte_t *)pmd_page_vaddr(*pmd) + pte_index(addr); -} - -#define pte_offset_map(dir, addr) pte_offset_kernel((dir), (addr)) -#define pte_unmap(pte) ((void)(pte)) - static inline int pte_present(pte_t pte) { return (pte_val(pte) & (_PAGE_PRESENT | _PAGE_PROT_NONE)); @@ -473,9 +453,9 @@ static inline int ptep_clear_flush_young(struct vm_area_struct *vma, #define PAGE_SHARED __pgprot(0) #define PAGE_KERNEL __pgprot(0) #define swapper_pg_dir NULL +#define TASK_SIZE 0xffffffffUL #define VMALLOC_START 0 - -#define TASK_SIZE 0xffffffffUL +#define VMALLOC_END TASK_SIZE static inline void __kernel_map_pages(struct page *page, int numpages, int enable) {} @@ -496,8 +476,6 @@ void paging_init(void); extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]; #define ZERO_PAGE(vaddr) (virt_to_page(empty_zero_page)) -#include <asm-generic/pgtable.h> - #endif /* !__ASSEMBLY__ */ #endif /* _ASM_RISCV_PGTABLE_H */ diff --git a/arch/riscv/include/asm/processor.h b/arch/riscv/include/asm/processor.h index 3ddb798264f1..bdddcd5c1b71 100644 --- a/arch/riscv/include/asm/processor.h +++ b/arch/riscv/include/asm/processor.h @@ -8,6 +8,8 @@ #include <linux/const.h> +#include <vdso/processor.h> + #include <asm/ptrace.h> /* @@ -58,16 +60,6 @@ static inline void release_thread(struct task_struct *dead_task) extern unsigned long get_wchan(struct task_struct *p); -static inline void cpu_relax(void) -{ -#ifdef __riscv_muldiv - int dummy; - /* In lieu of a halt instruction, induce a long-latency stall. */ - __asm__ __volatile__ ("div %0, %0, zero" : "=r" (dummy)); -#endif - barrier(); -} - static inline void wait_for_interrupt(void) { __asm__ __volatile__ ("wfi"); @@ -75,6 +67,7 @@ static inline void wait_for_interrupt(void) struct device_node; int riscv_of_processor_hartid(struct device_node *node); +int riscv_of_parent_hartid(struct device_node *node); extern void riscv_fill_hwcap(void); diff --git a/arch/riscv/include/asm/ptdump.h b/arch/riscv/include/asm/ptdump.h index e29af7191909..3c9ea6dd5af7 100644 --- a/arch/riscv/include/asm/ptdump.h +++ b/arch/riscv/include/asm/ptdump.h @@ -8,4 +8,15 @@ void ptdump_check_wx(void); +#ifdef CONFIG_DEBUG_WX +static inline void debug_checkwx(void) +{ + ptdump_check_wx(); +} +#else +static inline void debug_checkwx(void) +{ +} +#endif + #endif /* _ASM_RISCV_PTDUMP_H */ diff --git a/arch/riscv/include/asm/smp.h b/arch/riscv/include/asm/smp.h index f4c7cfda6b7f..df1f7c4cd433 100644 --- a/arch/riscv/include/asm/smp.h +++ b/arch/riscv/include/asm/smp.h @@ -15,6 +15,11 @@ struct seq_file; extern unsigned long boot_cpu_hartid; +struct riscv_ipi_ops { + void (*ipi_inject)(const struct cpumask *target); + void (*ipi_clear)(void); +}; + #ifdef CONFIG_SMP /* * Mapping between linux logical cpu index and hartid. @@ -28,6 +33,9 @@ void show_ipi_stats(struct seq_file *p, int prec); /* SMP initialization hook for setup_arch */ void __init setup_smp(void); +/* Called from C code, this handles an IPI. */ +void handle_IPI(struct pt_regs *regs); + /* Hook for the generic smp_call_function_many() routine. */ void arch_send_call_function_ipi_mask(struct cpumask *mask); @@ -37,6 +45,15 @@ void arch_send_call_function_single_ipi(int cpu); int riscv_hartid_to_cpuid(int hartid); void riscv_cpuid_to_hartid_mask(const struct cpumask *in, struct cpumask *out); +/* Set custom IPI operations */ +void riscv_set_ipi_ops(struct riscv_ipi_ops *ops); + +/* Clear IPI for current CPU */ +void riscv_clear_ipi(void); + +/* Secondary hart entry */ +asmlinkage void smp_callin(void); + /* * Obtains the hart ID of the currently executing task. This relies on * THREAD_INFO_IN_TASK, but we define that unconditionally. @@ -75,6 +92,14 @@ static inline void riscv_cpuid_to_hartid_mask(const struct cpumask *in, cpumask_set_cpu(boot_cpu_hartid, out); } +static inline void riscv_set_ipi_ops(struct riscv_ipi_ops *ops) +{ +} + +static inline void riscv_clear_ipi(void) +{ +} + #endif /* CONFIG_SMP */ #if defined(CONFIG_HOTPLUG_CPU) && (CONFIG_SMP) diff --git a/arch/riscv/include/asm/soc.h b/arch/riscv/include/asm/soc.h index 7cec1968c8b4..136a442ef876 100644 --- a/arch/riscv/include/asm/soc.h +++ b/arch/riscv/include/asm/soc.h @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0-or-later */ /* * Copyright (C) 2020 Western Digital Corporation or its affiliates. + * Copyright (C) 2020 Google, Inc */ #ifndef _ASM_RISCV_SOC_H @@ -20,4 +21,42 @@ void soc_early_init(void); extern unsigned long __soc_early_init_table_start; extern unsigned long __soc_early_init_table_end; +/* + * Allows Linux to provide a device tree, which is necessary for SOCs that + * don't provide a useful one on their own. + */ +struct soc_builtin_dtb { + unsigned long vendor_id; + unsigned long arch_id; + unsigned long imp_id; + void *(*dtb_func)(void); +}; + +/* + * The argument name must specify a valid DTS file name without the dts + * extension. + */ +#define SOC_BUILTIN_DTB_DECLARE(name, vendor, arch, impl) \ + extern void *__dtb_##name##_begin; \ + \ + static __init __used \ + void *__soc_builtin_dtb_f__##name(void) \ + { \ + return (void *)&__dtb_##name##_begin; \ + } \ + \ + static const struct soc_builtin_dtb __soc_builtin_dtb__##name \ + __used __section(__soc_builtin_dtb_table) = \ + { \ + .vendor_id = vendor, \ + .arch_id = arch, \ + .imp_id = impl, \ + .dtb_func = __soc_builtin_dtb_f__##name, \ + } + +extern unsigned long __soc_builtin_dtb_table_start; +extern unsigned long __soc_builtin_dtb_table_end; + +void *soc_lookup_builtin_dtb(void); + #endif diff --git a/arch/riscv/include/asm/stackprotector.h b/arch/riscv/include/asm/stackprotector.h new file mode 100644 index 000000000000..5962f8891f06 --- /dev/null +++ b/arch/riscv/include/asm/stackprotector.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _ASM_RISCV_STACKPROTECTOR_H +#define _ASM_RISCV_STACKPROTECTOR_H + +#include <linux/random.h> +#include <linux/version.h> + +extern unsigned long __stack_chk_guard; + +/* + * Initialize the stackprotector canary value. + * + * NOTE: this must only be called from functions that never return, + * and it must always be inlined. + */ +static __always_inline void boot_init_stack_canary(void) +{ + unsigned long canary; + + /* Try to get a semi random initial value. */ + get_random_bytes(&canary, sizeof(canary)); + canary ^= LINUX_VERSION_CODE; + canary &= CANARY_MASK; + + current->stack_canary = canary; + __stack_chk_guard = current->stack_canary; +} +#endif /* _ASM_RISCV_STACKPROTECTOR_H */ diff --git a/arch/riscv/include/asm/thread_info.h b/arch/riscv/include/asm/thread_info.h index 1dd12a0cbb2b..464a2bbc97ea 100644 --- a/arch/riscv/include/asm/thread_info.h +++ b/arch/riscv/include/asm/thread_info.h @@ -12,7 +12,11 @@ #include <linux/const.h> /* thread information allocation */ +#ifdef CONFIG_64BIT +#define THREAD_SIZE_ORDER (2) +#else #define THREAD_SIZE_ORDER (1) +#endif #define THREAD_SIZE (PAGE_SIZE << THREAD_SIZE_ORDER) #ifndef __ASSEMBLY__ diff --git a/arch/riscv/include/asm/timex.h b/arch/riscv/include/asm/timex.h index bad2a7c2cda5..ab104905d4db 100644 --- a/arch/riscv/include/asm/timex.h +++ b/arch/riscv/include/asm/timex.h @@ -7,41 +7,65 @@ #define _ASM_RISCV_TIMEX_H #include <asm/csr.h> -#include <asm/mmio.h> typedef unsigned long cycles_t; -extern u64 __iomem *riscv_time_val; -extern u64 __iomem *riscv_time_cmp; +#ifdef CONFIG_RISCV_M_MODE -#ifdef CONFIG_64BIT -#define mmio_get_cycles() readq_relaxed(riscv_time_val) -#else -#define mmio_get_cycles() readl_relaxed(riscv_time_val) -#define mmio_get_cycles_hi() readl_relaxed(((u32 *)riscv_time_val) + 1) -#endif +#include <asm/clint.h> +#ifdef CONFIG_64BIT static inline cycles_t get_cycles(void) { - if (IS_ENABLED(CONFIG_RISCV_SBI)) - return csr_read(CSR_TIME); - return mmio_get_cycles(); + return readq_relaxed(clint_time_val); +} +#else /* !CONFIG_64BIT */ +static inline u32 get_cycles(void) +{ + return readl_relaxed(((u32 *)clint_time_val)); } #define get_cycles get_cycles -#ifdef CONFIG_64BIT -static inline u64 get_cycles64(void) +static inline u32 get_cycles_hi(void) { + return readl_relaxed(((u32 *)clint_time_val) + 1); +} +#define get_cycles_hi get_cycles_hi +#endif /* CONFIG_64BIT */ + +/* + * Much like MIPS, we may not have a viable counter to use at an early point + * in the boot process. Unfortunately we don't have a fallback, so instead + * we just return 0. + */ +static inline unsigned long random_get_entropy(void) +{ + if (unlikely(clint_time_val == NULL)) + return 0; return get_cycles(); } -#else /* CONFIG_64BIT */ +#define random_get_entropy() random_get_entropy() + +#else /* CONFIG_RISCV_M_MODE */ + +static inline cycles_t get_cycles(void) +{ + return csr_read(CSR_TIME); +} +#define get_cycles get_cycles + static inline u32 get_cycles_hi(void) { - if (IS_ENABLED(CONFIG_RISCV_SBI)) - return csr_read(CSR_TIMEH); - return mmio_get_cycles_hi(); + return csr_read(CSR_TIMEH); } +#define get_cycles_hi get_cycles_hi +#ifdef CONFIG_64BIT +static inline u64 get_cycles64(void) +{ + return get_cycles(); +} +#else /* CONFIG_64BIT */ static inline u64 get_cycles64(void) { u32 hi, lo; @@ -55,6 +79,8 @@ static inline u64 get_cycles64(void) } #endif /* CONFIG_64BIT */ +#endif /* !CONFIG_RISCV_M_MODE */ + #define ARCH_HAS_READ_CURRENT_TIMER static inline int read_current_timer(unsigned long *timer_val) { diff --git a/arch/riscv/include/asm/uaccess.h b/arch/riscv/include/asm/uaccess.h index 8ce9d607b53d..f56c66b3f5fe 100644 --- a/arch/riscv/include/asm/uaccess.h +++ b/arch/riscv/include/asm/uaccess.h @@ -8,6 +8,8 @@ #ifndef _ASM_RISCV_UACCESS_H #define _ASM_RISCV_UACCESS_H +#include <asm/pgtable.h> /* for TASK_SIZE */ + /* * User space memory access functions */ @@ -62,11 +64,9 @@ static inline void set_fs(mm_segment_t fs) current_thread_info()->addr_limit = fs; } -#define segment_eq(a, b) ((a).seg == (b).seg) - +#define uaccess_kernel() (get_fs().seg == KERNEL_DS.seg) #define user_addr_max() (get_fs().seg) - /** * access_ok: - Checks if a user space pointer is valid * @addr: User space pointer to start of block to check diff --git a/arch/riscv/include/asm/vdso.h b/arch/riscv/include/asm/vdso.h index 7a7fce63c474..8454f746bbfd 100644 --- a/arch/riscv/include/asm/vdso.h +++ b/arch/riscv/include/asm/vdso.h @@ -10,8 +10,10 @@ #include <linux/types.h> +#ifndef GENERIC_TIME_VSYSCALL struct vdso_data { }; +#endif /* * The VDSO symbols are mapped into Linux so we can just use regular symbol diff --git a/arch/riscv/include/asm/vdso/clocksource.h b/arch/riscv/include/asm/vdso/clocksource.h new file mode 100644 index 000000000000..df6ea65c1dec --- /dev/null +++ b/arch/riscv/include/asm/vdso/clocksource.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_VDSOCLOCKSOURCE_H +#define __ASM_VDSOCLOCKSOURCE_H + +#define VDSO_ARCH_CLOCKMODES \ + VDSO_CLOCKMODE_ARCHTIMER + +#endif diff --git a/arch/riscv/include/asm/vdso/gettimeofday.h b/arch/riscv/include/asm/vdso/gettimeofday.h new file mode 100644 index 000000000000..f839f16e0d2a --- /dev/null +++ b/arch/riscv/include/asm/vdso/gettimeofday.h @@ -0,0 +1,81 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_VDSO_GETTIMEOFDAY_H +#define __ASM_VDSO_GETTIMEOFDAY_H + +#ifndef __ASSEMBLY__ + +#include <asm/barrier.h> +#include <asm/unistd.h> +#include <asm/csr.h> +#include <uapi/linux/time.h> + +#define VDSO_HAS_CLOCK_GETRES 1 + +static __always_inline +int gettimeofday_fallback(struct __kernel_old_timeval *_tv, + struct timezone *_tz) +{ + register struct __kernel_old_timeval *tv asm("a0") = _tv; + register struct timezone *tz asm("a1") = _tz; + register long ret asm("a0"); + register long nr asm("a7") = __NR_gettimeofday; + + asm volatile ("ecall\n" + : "=r" (ret) + : "r"(tv), "r"(tz), "r"(nr) + : "memory"); + + return ret; +} + +static __always_inline +long clock_gettime_fallback(clockid_t _clkid, struct __kernel_timespec *_ts) +{ + register clockid_t clkid asm("a0") = _clkid; + register struct __kernel_timespec *ts asm("a1") = _ts; + register long ret asm("a0"); + register long nr asm("a7") = __NR_clock_gettime; + + asm volatile ("ecall\n" + : "=r" (ret) + : "r"(clkid), "r"(ts), "r"(nr) + : "memory"); + + return ret; +} + +static __always_inline +int clock_getres_fallback(clockid_t _clkid, struct __kernel_timespec *_ts) +{ + register clockid_t clkid asm("a0") = _clkid; + register struct __kernel_timespec *ts asm("a1") = _ts; + register long ret asm("a0"); + register long nr asm("a7") = __NR_clock_getres; + + asm volatile ("ecall\n" + : "=r" (ret) + : "r"(clkid), "r"(ts), "r"(nr) + : "memory"); + + return ret; +} + +static __always_inline u64 __arch_get_hw_counter(s32 clock_mode, + const struct vdso_data *vd) +{ + /* + * The purpose of csr_read(CSR_TIME) is to trap the system into + * M-mode to obtain the value of CSR_TIME. Hence, unlike other + * architecture, no fence instructions surround the csr_read() + */ + return csr_read(CSR_TIME); +} + +static __always_inline const struct vdso_data *__arch_get_vdso_data(void) +{ + return _vdso_data; +} + +#endif /* !__ASSEMBLY__ */ + +#endif /* __ASM_VDSO_GETTIMEOFDAY_H */ diff --git a/arch/riscv/include/asm/vdso/processor.h b/arch/riscv/include/asm/vdso/processor.h new file mode 100644 index 000000000000..82a5693b1861 --- /dev/null +++ b/arch/riscv/include/asm/vdso/processor.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __ASM_VDSO_PROCESSOR_H +#define __ASM_VDSO_PROCESSOR_H + +#ifndef __ASSEMBLY__ + +static inline void cpu_relax(void) +{ +#ifdef __riscv_muldiv + int dummy; + /* In lieu of a halt instruction, induce a long-latency stall. */ + __asm__ __volatile__ ("div %0, %0, zero" : "=r" (dummy)); +#endif + barrier(); +} + +#endif /* __ASSEMBLY__ */ + +#endif /* __ASM_VDSO_PROCESSOR_H */ diff --git a/arch/riscv/include/asm/vdso/vsyscall.h b/arch/riscv/include/asm/vdso/vsyscall.h new file mode 100644 index 000000000000..82fd5d83bd60 --- /dev/null +++ b/arch/riscv/include/asm/vdso/vsyscall.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_VDSO_VSYSCALL_H +#define __ASM_VDSO_VSYSCALL_H + +#ifndef __ASSEMBLY__ + +#include <linux/timekeeper_internal.h> +#include <vdso/datapage.h> + +extern struct vdso_data *vdso_data; + +/* + * Update the vDSO data page to keep in sync with kernel timekeeping. + */ +static __always_inline struct vdso_data *__riscv_get_k_vdso_data(void) +{ + return vdso_data; +} + +#define __arch_get_k_vdso_data __riscv_get_k_vdso_data + +/* The asm-generic header needs to be included after the definitions above */ +#include <asm-generic/vdso/vsyscall.h> + +#endif /* !__ASSEMBLY__ */ + +#endif /* __ASM_VDSO_VSYSCALL_H */ diff --git a/arch/riscv/include/uapi/asm/hwcap.h b/arch/riscv/include/uapi/asm/hwcap.h index dee98ee28318..46dc3f5ee99f 100644 --- a/arch/riscv/include/uapi/asm/hwcap.h +++ b/arch/riscv/include/uapi/asm/hwcap.h @@ -11,7 +11,7 @@ /* * Linux saves the floating-point registers according to the ISA Linux is * executing on, as opposed to the ISA the user program is compiled for. This - * is necessary for a handful of esoteric use cases: for example, userpsace + * is necessary for a handful of esoteric use cases: for example, userspace * threading libraries must be able to examine the actual machine state in * order to fully reconstruct the state of a thread. */ diff --git a/arch/riscv/include/uapi/asm/unistd.h b/arch/riscv/include/uapi/asm/unistd.h index 13ce76cc5aff..4b989ae15d59 100644 --- a/arch/riscv/include/uapi/asm/unistd.h +++ b/arch/riscv/include/uapi/asm/unistd.h @@ -12,7 +12,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <https://www.gnu.org/licenses/>. */ #ifdef __LP64__ diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile index d8bbd3207100..dc93710f0b2f 100644 --- a/arch/riscv/kernel/Makefile +++ b/arch/riscv/kernel/Makefile @@ -31,7 +31,7 @@ obj-y += cacheinfo.o obj-y += patch.o obj-$(CONFIG_MMU) += vdso.o vdso/ -obj-$(CONFIG_RISCV_M_MODE) += clint.o traps_misaligned.o +obj-$(CONFIG_RISCV_M_MODE) += traps_misaligned.o obj-$(CONFIG_FPU) += fpu.o obj-$(CONFIG_SMP) += smpboot.o obj-$(CONFIG_SMP) += smp.o @@ -51,5 +51,8 @@ ifeq ($(CONFIG_RISCV_SBI), y) obj-$(CONFIG_SMP) += cpu_ops_sbi.o endif obj-$(CONFIG_HOTPLUG_CPU) += cpu-hotplug.o +obj-$(CONFIG_KGDB) += kgdb.o + +obj-$(CONFIG_JUMP_LABEL) += jump_label.o clean: diff --git a/arch/riscv/kernel/asm-offsets.c b/arch/riscv/kernel/asm-offsets.c index 07cb9c10de4e..db203442c08f 100644 --- a/arch/riscv/kernel/asm-offsets.c +++ b/arch/riscv/kernel/asm-offsets.c @@ -27,9 +27,6 @@ void asm_offsets(void) OFFSET(TASK_THREAD_S9, task_struct, thread.s[9]); OFFSET(TASK_THREAD_S10, task_struct, thread.s[10]); OFFSET(TASK_THREAD_S11, task_struct, thread.s[11]); - OFFSET(TASK_THREAD_SP, task_struct, thread.sp); - OFFSET(TASK_STACK, task_struct, stack); - OFFSET(TASK_TI, task_struct, thread_info); OFFSET(TASK_TI_FLAGS, task_struct, thread_info.flags); OFFSET(TASK_TI_PREEMPT_COUNT, task_struct, thread_info.preempt_count); OFFSET(TASK_TI_KERNEL_SP, task_struct, thread_info.kernel_sp); diff --git a/arch/riscv/kernel/cacheinfo.c b/arch/riscv/kernel/cacheinfo.c index 4c90c07d8c39..bd0f122965c3 100644 --- a/arch/riscv/kernel/cacheinfo.c +++ b/arch/riscv/kernel/cacheinfo.c @@ -7,6 +7,23 @@ #include <linux/cpu.h> #include <linux/of.h> #include <linux/of_device.h> +#include <asm/cacheinfo.h> + +static struct riscv_cacheinfo_ops *rv_cache_ops; + +void riscv_set_cacheinfo_ops(struct riscv_cacheinfo_ops *ops) +{ + rv_cache_ops = ops; +} +EXPORT_SYMBOL_GPL(riscv_set_cacheinfo_ops); + +const struct attribute_group * +cache_get_priv_group(struct cacheinfo *this_leaf) +{ + if (rv_cache_ops && rv_cache_ops->get_priv_group) + return rv_cache_ops->get_priv_group(this_leaf); + return NULL; +} static void ci_leaf_init(struct cacheinfo *this_leaf, struct device_node *node, diff --git a/arch/riscv/kernel/clint.c b/arch/riscv/kernel/clint.c deleted file mode 100644 index 3647980d14c3..000000000000 --- a/arch/riscv/kernel/clint.c +++ /dev/null @@ -1,44 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Copyright (c) 2019 Christoph Hellwig. - */ - -#include <linux/io.h> -#include <linux/of_address.h> -#include <linux/types.h> -#include <asm/clint.h> -#include <asm/csr.h> -#include <asm/timex.h> -#include <asm/smp.h> - -/* - * This is the layout used by the SiFive clint, which is also shared by the qemu - * virt platform, and the Kendryte KD210 at least. - */ -#define CLINT_IPI_OFF 0 -#define CLINT_TIME_CMP_OFF 0x4000 -#define CLINT_TIME_VAL_OFF 0xbff8 - -u32 __iomem *clint_ipi_base; - -void clint_init_boot_cpu(void) -{ - struct device_node *np; - void __iomem *base; - - np = of_find_compatible_node(NULL, NULL, "riscv,clint0"); - if (!np) { - panic("clint not found"); - return; - } - - base = of_iomap(np, 0); - if (!base) - panic("could not map CLINT"); - - clint_ipi_base = base + CLINT_IPI_OFF; - riscv_time_cmp = base + CLINT_TIME_CMP_OFF; - riscv_time_val = base + CLINT_TIME_VAL_OFF; - - clint_clear_ipi(boot_cpu_hartid); -} diff --git a/arch/riscv/kernel/cpu.c b/arch/riscv/kernel/cpu.c index 40a3c442ac5f..6d59e6906fdd 100644 --- a/arch/riscv/kernel/cpu.c +++ b/arch/riscv/kernel/cpu.c @@ -44,6 +44,22 @@ int riscv_of_processor_hartid(struct device_node *node) return hart; } +/* + * Find hart ID of the CPU DT node under which given DT node falls. + * + * To achieve this, we walk up the DT tree until we find an active + * RISC-V core (HART) node and extract the cpuid from it. + */ +int riscv_of_parent_hartid(struct device_node *node) +{ + for (; node; node = node->parent) { + if (of_device_is_compatible(node, "riscv")) + return riscv_of_processor_hartid(node); + } + + return -1; +} + #ifdef CONFIG_PROC_FS static void print_isa(struct seq_file *f, const char *isa) diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S index 56d071b2c0a1..524d918f3601 100644 --- a/arch/riscv/kernel/entry.S +++ b/arch/riscv/kernel/entry.S @@ -97,17 +97,36 @@ _save_context: la gp, __global_pointer$ .option pop - la ra, ret_from_exception +#ifdef CONFIG_TRACE_IRQFLAGS + call trace_hardirqs_off +#endif + +#ifdef CONFIG_CONTEXT_TRACKING + /* If previous state is in user mode, call context_tracking_user_exit. */ + li a0, SR_PP + and a0, s1, a0 + bnez a0, skip_context_tracking + call context_tracking_user_exit +skip_context_tracking: +#endif + /* * MSB of cause differentiates between * interrupts and exceptions */ bge s4, zero, 1f + la ra, ret_from_exception + /* Handle interrupts */ move a0, sp /* pt_regs */ - tail do_IRQ + la a1, handle_arch_irq + REG_L a1, (a1) + jr a1 1: +#ifdef CONFIG_TRACE_IRQFLAGS + call trace_hardirqs_on +#endif /* * Exceptions run with interrupts enabled or disabled depending on the * state of SR_PIE in m/sstatus. @@ -117,6 +136,7 @@ _save_context: csrs CSR_STATUS, SR_IE 1: + la ra, ret_from_exception /* Handle syscalls */ li t0, EXC_SYSCALL beq s4, t0, handle_syscall @@ -135,6 +155,17 @@ _save_context: tail do_trap_unknown handle_syscall: +#if defined(CONFIG_TRACE_IRQFLAGS) || defined(CONFIG_CONTEXT_TRACKING) + /* Recover a0 - a7 for system calls */ + REG_L a0, PT_A0(sp) + REG_L a1, PT_A1(sp) + REG_L a2, PT_A2(sp) + REG_L a3, PT_A3(sp) + REG_L a4, PT_A4(sp) + REG_L a5, PT_A5(sp) + REG_L a6, PT_A6(sp) + REG_L a7, PT_A7(sp) +#endif /* save the initial A0 value (needed in signal handlers) */ REG_S a0, PT_ORIG_A0(sp) /* @@ -188,6 +219,9 @@ ret_from_syscall_rejected: ret_from_exception: REG_L s0, PT_STATUS(sp) csrc CSR_STATUS, SR_IE +#ifdef CONFIG_TRACE_IRQFLAGS + call trace_hardirqs_off +#endif #ifdef CONFIG_RISCV_M_MODE /* the MPP value is too large to be used as an immediate arg for addi */ li t0, SR_MPP @@ -203,6 +237,10 @@ resume_userspace: andi s1, s0, _TIF_WORK_MASK bnez s1, work_pending +#ifdef CONFIG_CONTEXT_TRACKING + call context_tracking_user_enter +#endif + /* Save unwound kernel stack pointer in thread_info */ addi s0, sp, PT_SIZE_ON_STACK REG_S s0, TASK_TI_KERNEL_SP(tp) @@ -214,6 +252,16 @@ resume_userspace: csrw CSR_SCRATCH, tp restore_all: +#ifdef CONFIG_TRACE_IRQFLAGS + REG_L s1, PT_STATUS(sp) + andi t0, s1, SR_PIE + beqz t0, 1f + call trace_hardirqs_on + j 2f +1: + call trace_hardirqs_off +2: +#endif REG_L a0, PT_STATUS(sp) /* * The current load reservation is effectively part of the processor's @@ -387,12 +435,8 @@ ENTRY(__switch_to) lw a4, TASK_TI_CPU(a1) sw a3, TASK_TI_CPU(a1) sw a4, TASK_TI_CPU(a0) -#if TASK_TI != 0 -#error "TASK_TI != 0: tp will contain a 'struct thread_info', not a 'struct task_struct' so get_current() won't work." - addi tp, a1, TASK_TI -#else + /* The offset of thread_info in task_struct is zero. */ move tp, a1 -#endif ret ENDPROC(__switch_to) diff --git a/arch/riscv/kernel/ftrace.c b/arch/riscv/kernel/ftrace.c index ce69b34ff55d..99e12faa5498 100644 --- a/arch/riscv/kernel/ftrace.c +++ b/arch/riscv/kernel/ftrace.c @@ -7,10 +7,23 @@ #include <linux/ftrace.h> #include <linux/uaccess.h> +#include <linux/memory.h> #include <asm/cacheflush.h> #include <asm/patch.h> #ifdef CONFIG_DYNAMIC_FTRACE +int ftrace_arch_code_modify_prepare(void) __acquires(&text_mutex) +{ + mutex_lock(&text_mutex); + return 0; +} + +int ftrace_arch_code_modify_post_process(void) __releases(&text_mutex) +{ + mutex_unlock(&text_mutex); + return 0; +} + static int ftrace_check_current_call(unsigned long hook_pos, unsigned int *expected) { @@ -25,7 +38,8 @@ static int ftrace_check_current_call(unsigned long hook_pos, * Read the text we want to modify; * return must be -EFAULT on read error */ - if (probe_kernel_read(replaced, (void *)hook_pos, MCOUNT_INSN_SIZE)) + if (copy_from_kernel_nofault(replaced, (void *)hook_pos, + MCOUNT_INSN_SIZE)) return -EFAULT; /* @@ -51,7 +65,7 @@ static int __ftrace_modify_call(unsigned long hook_pos, unsigned long target, make_call(hook_pos, target, call); /* Replace the auipc-jalr pair at once. Return -EPERM on write error. */ - if (riscv_patch_text_nosync + if (patch_text_nosync ((void *)hook_pos, enable ? call : nops, MCOUNT_INSN_SIZE)) return -EPERM; @@ -83,6 +97,25 @@ int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, return __ftrace_modify_call(rec->ip, addr, false); } + +/* + * This is called early on, and isn't wrapped by + * ftrace_arch_code_modify_{prepare,post_process}() and therefor doesn't hold + * text_mutex, which triggers a lockdep failure. SMP isn't running so we could + * just directly poke the text, but it's simpler to just take the lock + * ourselves. + */ +int ftrace_init_nop(struct module *mod, struct dyn_ftrace *rec) +{ + int out; + + ftrace_arch_code_modify_prepare(); + out = ftrace_make_nop(mod, rec, MCOUNT_ADDR); + ftrace_arch_code_modify_post_process(); + + return out; +} + int ftrace_update_ftrace_func(ftrace_func_t func) { int ret = __ftrace_modify_call((unsigned long)&ftrace_call, diff --git a/arch/riscv/kernel/head.S b/arch/riscv/kernel/head.S index 98a406474e7d..0a4e81b8dc79 100644 --- a/arch/riscv/kernel/head.S +++ b/arch/riscv/kernel/head.S @@ -138,9 +138,23 @@ secondary_start_common: la a0, swapper_pg_dir call relocate #endif + call setup_trap_vector tail smp_callin #endif /* CONFIG_SMP */ +.align 2 +setup_trap_vector: + /* Set trap vector to exception handler */ + la a0, handle_exception + csrw CSR_TVEC, a0 + + /* + * Set sup0 scratch register to 0, indicating to exception vector that + * we are presently executing in kernel. + */ + csrw CSR_SCRATCH, zero + ret + .Lsecondary_park: /* We lack SMP support or have too many harts, so park this hart */ wfi @@ -161,11 +175,20 @@ ENTRY(_start_kernel) /* Reset all registers except ra, a0, a1 */ call reset_regs - /* Setup a PMP to permit access to all of memory. */ + /* + * Setup a PMP to permit access to all of memory. Some machines may + * not implement PMPs, so we set up a quick trap handler to just skip + * touching the PMPs on any trap. + */ + la a0, pmp_done + csrw CSR_TVEC, a0 + li a0, -1 csrw CSR_PMPADDR0, a0 li a0, (PMP_A_NAPOT | PMP_R | PMP_W | PMP_X) csrw CSR_PMPCFG0, a0 +.align 2 +pmp_done: /* * The hartid in a0 is expected later on, and we have no firmware @@ -225,6 +248,7 @@ clear_bss_done: call relocate #endif /* CONFIG_MMU */ + call setup_trap_vector /* Restore C environment */ la tp, init_task sw zero, TASK_TI_CPU(tp) diff --git a/arch/riscv/kernel/irq.c b/arch/riscv/kernel/irq.c index 345c4f2eba13..7207fa08d78f 100644 --- a/arch/riscv/kernel/irq.c +++ b/arch/riscv/kernel/irq.c @@ -7,7 +7,6 @@ #include <linux/interrupt.h> #include <linux/irqchip.h> -#include <linux/irqdomain.h> #include <linux/seq_file.h> #include <asm/smp.h> @@ -17,37 +16,9 @@ int arch_show_interrupts(struct seq_file *p, int prec) return 0; } -asmlinkage __visible void __irq_entry do_IRQ(struct pt_regs *regs) -{ - struct pt_regs *old_regs = set_irq_regs(regs); - - irq_enter(); - switch (regs->cause & ~CAUSE_IRQ_FLAG) { - case RV_IRQ_TIMER: - riscv_timer_interrupt(); - break; -#ifdef CONFIG_SMP - case RV_IRQ_SOFT: - /* - * We only use software interrupts to pass IPIs, so if a non-SMP - * system gets one, then we don't know what to do. - */ - riscv_software_interrupt(); - break; -#endif - case RV_IRQ_EXT: - handle_arch_irq(regs); - break; - default: - pr_alert("unexpected interrupt cause 0x%lx", regs->cause); - BUG(); - } - irq_exit(); - - set_irq_regs(old_regs); -} - void __init init_IRQ(void) { irqchip_init(); + if (!handle_arch_irq) + panic("No interrupt controller found."); } diff --git a/arch/riscv/kernel/jump_label.c b/arch/riscv/kernel/jump_label.c new file mode 100644 index 000000000000..20e09056d141 --- /dev/null +++ b/arch/riscv/kernel/jump_label.c @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2020 Emil Renner Berthing + * + * Based on arch/arm64/kernel/jump_label.c + */ +#include <linux/jump_label.h> +#include <linux/kernel.h> +#include <linux/memory.h> +#include <linux/mutex.h> +#include <asm/bug.h> +#include <asm/patch.h> + +#define RISCV_INSN_NOP 0x00000013U +#define RISCV_INSN_JAL 0x0000006fU + +void arch_jump_label_transform(struct jump_entry *entry, + enum jump_label_type type) +{ + void *addr = (void *)jump_entry_code(entry); + u32 insn; + + if (type == JUMP_LABEL_JMP) { + long offset = jump_entry_target(entry) - jump_entry_code(entry); + + if (WARN_ON(offset & 1 || offset < -524288 || offset >= 524288)) + return; + + insn = RISCV_INSN_JAL | + (((u32)offset & GENMASK(19, 12)) << (12 - 12)) | + (((u32)offset & GENMASK(11, 11)) << (20 - 11)) | + (((u32)offset & GENMASK(10, 1)) << (21 - 1)) | + (((u32)offset & GENMASK(20, 20)) << (31 - 20)); + } else { + insn = RISCV_INSN_NOP; + } + + mutex_lock(&text_mutex); + patch_text_nosync(addr, &insn, sizeof(insn)); + mutex_unlock(&text_mutex); +} + +void arch_jump_label_transform_static(struct jump_entry *entry, + enum jump_label_type type) +{ + /* + * We use the same instructions in the arch_static_branch and + * arch_static_branch_jump inline functions, so there's no + * need to patch them up here. + * The core will call arch_jump_label_transform when those + * instructions need to be replaced. + */ +} diff --git a/arch/riscv/kernel/kgdb.c b/arch/riscv/kernel/kgdb.c new file mode 100644 index 000000000000..963ed7edcff2 --- /dev/null +++ b/arch/riscv/kernel/kgdb.c @@ -0,0 +1,390 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2020 SiFive + */ + +#include <linux/ptrace.h> +#include <linux/kdebug.h> +#include <linux/bug.h> +#include <linux/kgdb.h> +#include <linux/irqflags.h> +#include <linux/string.h> +#include <asm/cacheflush.h> +#include <asm/gdb_xml.h> +#include <asm/parse_asm.h> + +enum { + NOT_KGDB_BREAK = 0, + KGDB_SW_BREAK, + KGDB_COMPILED_BREAK, + KGDB_SW_SINGLE_STEP +}; + +static unsigned long stepped_address; +static unsigned int stepped_opcode; + +#if __riscv_xlen == 32 +/* C.JAL is an RV32C-only instruction */ +DECLARE_INSN(c_jal, MATCH_C_JAL, MASK_C_JAL) +#else +#define is_c_jal_insn(opcode) 0 +#endif +DECLARE_INSN(jalr, MATCH_JALR, MASK_JALR) +DECLARE_INSN(jal, MATCH_JAL, MASK_JAL) +DECLARE_INSN(c_jr, MATCH_C_JR, MASK_C_JR) +DECLARE_INSN(c_jalr, MATCH_C_JALR, MASK_C_JALR) +DECLARE_INSN(c_j, MATCH_C_J, MASK_C_J) +DECLARE_INSN(beq, MATCH_BEQ, MASK_BEQ) +DECLARE_INSN(bne, MATCH_BNE, MASK_BNE) +DECLARE_INSN(blt, MATCH_BLT, MASK_BLT) +DECLARE_INSN(bge, MATCH_BGE, MASK_BGE) +DECLARE_INSN(bltu, MATCH_BLTU, MASK_BLTU) +DECLARE_INSN(bgeu, MATCH_BGEU, MASK_BGEU) +DECLARE_INSN(c_beqz, MATCH_C_BEQZ, MASK_C_BEQZ) +DECLARE_INSN(c_bnez, MATCH_C_BNEZ, MASK_C_BNEZ) +DECLARE_INSN(sret, MATCH_SRET, MASK_SRET) + +static int decode_register_index(unsigned long opcode, int offset) +{ + return (opcode >> offset) & 0x1F; +} + +static int decode_register_index_short(unsigned long opcode, int offset) +{ + return ((opcode >> offset) & 0x7) + 8; +} + +/* Calculate the new address for after a step */ +static int get_step_address(struct pt_regs *regs, unsigned long *next_addr) +{ + unsigned long pc = regs->epc; + unsigned long *regs_ptr = (unsigned long *)regs; + unsigned int rs1_num, rs2_num; + int op_code; + + if (get_kernel_nofault(op_code, (void *)pc)) + return -EINVAL; + if ((op_code & __INSN_LENGTH_MASK) != __INSN_LENGTH_GE_32) { + if (is_c_jalr_insn(op_code) || is_c_jr_insn(op_code)) { + rs1_num = decode_register_index(op_code, RVC_C2_RS1_OPOFF); + *next_addr = regs_ptr[rs1_num]; + } else if (is_c_j_insn(op_code) || is_c_jal_insn(op_code)) { + *next_addr = EXTRACT_RVC_J_IMM(op_code) + pc; + } else if (is_c_beqz_insn(op_code)) { + rs1_num = decode_register_index_short(op_code, + RVC_C1_RS1_OPOFF); + if (!rs1_num || regs_ptr[rs1_num] == 0) + *next_addr = EXTRACT_RVC_B_IMM(op_code) + pc; + else + *next_addr = pc + 2; + } else if (is_c_bnez_insn(op_code)) { + rs1_num = + decode_register_index_short(op_code, RVC_C1_RS1_OPOFF); + if (rs1_num && regs_ptr[rs1_num] != 0) + *next_addr = EXTRACT_RVC_B_IMM(op_code) + pc; + else + *next_addr = pc + 2; + } else { + *next_addr = pc + 2; + } + } else { + if ((op_code & __INSN_OPCODE_MASK) == __INSN_BRANCH_OPCODE) { + bool result = false; + long imm = EXTRACT_BTYPE_IMM(op_code); + unsigned long rs1_val = 0, rs2_val = 0; + + rs1_num = decode_register_index(op_code, RVG_RS1_OPOFF); + rs2_num = decode_register_index(op_code, RVG_RS2_OPOFF); + if (rs1_num) + rs1_val = regs_ptr[rs1_num]; + if (rs2_num) + rs2_val = regs_ptr[rs2_num]; + + if (is_beq_insn(op_code)) + result = (rs1_val == rs2_val) ? true : false; + else if (is_bne_insn(op_code)) + result = (rs1_val != rs2_val) ? true : false; + else if (is_blt_insn(op_code)) + result = + ((long)rs1_val < + (long)rs2_val) ? true : false; + else if (is_bge_insn(op_code)) + result = + ((long)rs1_val >= + (long)rs2_val) ? true : false; + else if (is_bltu_insn(op_code)) + result = (rs1_val < rs2_val) ? true : false; + else if (is_bgeu_insn(op_code)) + result = (rs1_val >= rs2_val) ? true : false; + if (result) + *next_addr = imm + pc; + else + *next_addr = pc + 4; + } else if (is_jal_insn(op_code)) { + *next_addr = EXTRACT_JTYPE_IMM(op_code) + pc; + } else if (is_jalr_insn(op_code)) { + rs1_num = decode_register_index(op_code, RVG_RS1_OPOFF); + if (rs1_num) + *next_addr = ((unsigned long *)regs)[rs1_num]; + *next_addr += EXTRACT_ITYPE_IMM(op_code); + } else if (is_sret_insn(op_code)) { + *next_addr = pc; + } else { + *next_addr = pc + 4; + } + } + return 0; +} + +static int do_single_step(struct pt_regs *regs) +{ + /* Determine where the target instruction will send us to */ + unsigned long addr = 0; + int error = get_step_address(regs, &addr); + + if (error) + return error; + + /* Store the op code in the stepped address */ + error = get_kernel_nofault(stepped_opcode, (void *)addr); + if (error) + return error; + + stepped_address = addr; + + /* Replace the op code with the break instruction */ + error = copy_to_kernel_nofault((void *)stepped_address, + arch_kgdb_ops.gdb_bpt_instr, + BREAK_INSTR_SIZE); + /* Flush and return */ + if (!error) { + flush_icache_range(addr, addr + BREAK_INSTR_SIZE); + kgdb_single_step = 1; + atomic_set(&kgdb_cpu_doing_single_step, + raw_smp_processor_id()); + } else { + stepped_address = 0; + stepped_opcode = 0; + } + return error; +} + +/* Undo a single step */ +static void undo_single_step(struct pt_regs *regs) +{ + if (stepped_opcode != 0) { + copy_to_kernel_nofault((void *)stepped_address, + (void *)&stepped_opcode, BREAK_INSTR_SIZE); + flush_icache_range(stepped_address, + stepped_address + BREAK_INSTR_SIZE); + } + stepped_address = 0; + stepped_opcode = 0; + kgdb_single_step = 0; + atomic_set(&kgdb_cpu_doing_single_step, -1); +} + +struct dbg_reg_def_t dbg_reg_def[DBG_MAX_REG_NUM] = { + {DBG_REG_ZERO, GDB_SIZEOF_REG, -1}, + {DBG_REG_RA, GDB_SIZEOF_REG, offsetof(struct pt_regs, ra)}, + {DBG_REG_SP, GDB_SIZEOF_REG, offsetof(struct pt_regs, sp)}, + {DBG_REG_GP, GDB_SIZEOF_REG, offsetof(struct pt_regs, gp)}, + {DBG_REG_TP, GDB_SIZEOF_REG, offsetof(struct pt_regs, tp)}, + {DBG_REG_T0, GDB_SIZEOF_REG, offsetof(struct pt_regs, t0)}, + {DBG_REG_T1, GDB_SIZEOF_REG, offsetof(struct pt_regs, t1)}, + {DBG_REG_T2, GDB_SIZEOF_REG, offsetof(struct pt_regs, t2)}, + {DBG_REG_FP, GDB_SIZEOF_REG, offsetof(struct pt_regs, s0)}, + {DBG_REG_S1, GDB_SIZEOF_REG, offsetof(struct pt_regs, a1)}, + {DBG_REG_A0, GDB_SIZEOF_REG, offsetof(struct pt_regs, a0)}, + {DBG_REG_A1, GDB_SIZEOF_REG, offsetof(struct pt_regs, a1)}, + {DBG_REG_A2, GDB_SIZEOF_REG, offsetof(struct pt_regs, a2)}, + {DBG_REG_A3, GDB_SIZEOF_REG, offsetof(struct pt_regs, a3)}, + {DBG_REG_A4, GDB_SIZEOF_REG, offsetof(struct pt_regs, a4)}, + {DBG_REG_A5, GDB_SIZEOF_REG, offsetof(struct pt_regs, a5)}, + {DBG_REG_A6, GDB_SIZEOF_REG, offsetof(struct pt_regs, a6)}, + {DBG_REG_A7, GDB_SIZEOF_REG, offsetof(struct pt_regs, a7)}, + {DBG_REG_S2, GDB_SIZEOF_REG, offsetof(struct pt_regs, s2)}, + {DBG_REG_S3, GDB_SIZEOF_REG, offsetof(struct pt_regs, s3)}, + {DBG_REG_S4, GDB_SIZEOF_REG, offsetof(struct pt_regs, s4)}, + {DBG_REG_S5, GDB_SIZEOF_REG, offsetof(struct pt_regs, s5)}, + {DBG_REG_S6, GDB_SIZEOF_REG, offsetof(struct pt_regs, s6)}, + {DBG_REG_S7, GDB_SIZEOF_REG, offsetof(struct pt_regs, s7)}, + {DBG_REG_S8, GDB_SIZEOF_REG, offsetof(struct pt_regs, s8)}, + {DBG_REG_S9, GDB_SIZEOF_REG, offsetof(struct pt_regs, s9)}, + {DBG_REG_S10, GDB_SIZEOF_REG, offsetof(struct pt_regs, s10)}, + {DBG_REG_S11, GDB_SIZEOF_REG, offsetof(struct pt_regs, s11)}, + {DBG_REG_T3, GDB_SIZEOF_REG, offsetof(struct pt_regs, t3)}, + {DBG_REG_T4, GDB_SIZEOF_REG, offsetof(struct pt_regs, t4)}, + {DBG_REG_T5, GDB_SIZEOF_REG, offsetof(struct pt_regs, t5)}, + {DBG_REG_T6, GDB_SIZEOF_REG, offsetof(struct pt_regs, t6)}, + {DBG_REG_EPC, GDB_SIZEOF_REG, offsetof(struct pt_regs, epc)}, + {DBG_REG_STATUS, GDB_SIZEOF_REG, offsetof(struct pt_regs, status)}, + {DBG_REG_BADADDR, GDB_SIZEOF_REG, offsetof(struct pt_regs, badaddr)}, + {DBG_REG_CAUSE, GDB_SIZEOF_REG, offsetof(struct pt_regs, cause)}, +}; + +char *dbg_get_reg(int regno, void *mem, struct pt_regs *regs) +{ + if (regno >= DBG_MAX_REG_NUM || regno < 0) + return NULL; + + if (dbg_reg_def[regno].offset != -1) + memcpy(mem, (void *)regs + dbg_reg_def[regno].offset, + dbg_reg_def[regno].size); + else + memset(mem, 0, dbg_reg_def[regno].size); + return dbg_reg_def[regno].name; +} + +int dbg_set_reg(int regno, void *mem, struct pt_regs *regs) +{ + if (regno >= DBG_MAX_REG_NUM || regno < 0) + return -EINVAL; + + if (dbg_reg_def[regno].offset != -1) + memcpy((void *)regs + dbg_reg_def[regno].offset, mem, + dbg_reg_def[regno].size); + return 0; +} + +void +sleeping_thread_to_gdb_regs(unsigned long *gdb_regs, struct task_struct *task) +{ + /* Initialize to zero */ + memset((char *)gdb_regs, 0, NUMREGBYTES); + + gdb_regs[DBG_REG_SP_OFF] = task->thread.sp; + gdb_regs[DBG_REG_FP_OFF] = task->thread.s[0]; + gdb_regs[DBG_REG_S1_OFF] = task->thread.s[1]; + gdb_regs[DBG_REG_S2_OFF] = task->thread.s[2]; + gdb_regs[DBG_REG_S3_OFF] = task->thread.s[3]; + gdb_regs[DBG_REG_S4_OFF] = task->thread.s[4]; + gdb_regs[DBG_REG_S5_OFF] = task->thread.s[5]; + gdb_regs[DBG_REG_S6_OFF] = task->thread.s[6]; + gdb_regs[DBG_REG_S7_OFF] = task->thread.s[7]; + gdb_regs[DBG_REG_S8_OFF] = task->thread.s[8]; + gdb_regs[DBG_REG_S9_OFF] = task->thread.s[10]; + gdb_regs[DBG_REG_S10_OFF] = task->thread.s[11]; + gdb_regs[DBG_REG_EPC_OFF] = task->thread.ra; +} + +void kgdb_arch_set_pc(struct pt_regs *regs, unsigned long pc) +{ + regs->epc = pc; +} + +void kgdb_arch_handle_qxfer_pkt(char *remcom_in_buffer, + char *remcom_out_buffer) +{ + if (!strncmp(remcom_in_buffer, gdb_xfer_read_target, + sizeof(gdb_xfer_read_target))) + strcpy(remcom_out_buffer, riscv_gdb_stub_target_desc); + else if (!strncmp(remcom_in_buffer, gdb_xfer_read_cpuxml, + sizeof(gdb_xfer_read_cpuxml))) + strcpy(remcom_out_buffer, riscv_gdb_stub_cpuxml); +} + +static inline void kgdb_arch_update_addr(struct pt_regs *regs, + char *remcom_in_buffer) +{ + unsigned long addr; + char *ptr; + + ptr = &remcom_in_buffer[1]; + if (kgdb_hex2long(&ptr, &addr)) + regs->epc = addr; +} + +int kgdb_arch_handle_exception(int vector, int signo, int err_code, + char *remcom_in_buffer, char *remcom_out_buffer, + struct pt_regs *regs) +{ + int err = 0; + + undo_single_step(regs); + + switch (remcom_in_buffer[0]) { + case 'c': + case 'D': + case 'k': + if (remcom_in_buffer[0] == 'c') + kgdb_arch_update_addr(regs, remcom_in_buffer); + break; + case 's': + kgdb_arch_update_addr(regs, remcom_in_buffer); + err = do_single_step(regs); + break; + default: + err = -1; + } + return err; +} + +static int kgdb_riscv_kgdbbreak(unsigned long addr) +{ + if (stepped_address == addr) + return KGDB_SW_SINGLE_STEP; + if (atomic_read(&kgdb_setting_breakpoint)) + if (addr == (unsigned long)&kgdb_compiled_break) + return KGDB_COMPILED_BREAK; + + return kgdb_has_hit_break(addr); +} + +static int kgdb_riscv_notify(struct notifier_block *self, unsigned long cmd, + void *ptr) +{ + struct die_args *args = (struct die_args *)ptr; + struct pt_regs *regs = args->regs; + unsigned long flags; + int type; + + if (user_mode(regs)) + return NOTIFY_DONE; + + type = kgdb_riscv_kgdbbreak(regs->epc); + if (type == NOT_KGDB_BREAK && cmd == DIE_TRAP) + return NOTIFY_DONE; + + local_irq_save(flags); + + if (kgdb_handle_exception(type == KGDB_SW_SINGLE_STEP ? 0 : 1, + args->signr, cmd, regs)) + return NOTIFY_DONE; + + if (type == KGDB_COMPILED_BREAK) + regs->epc += 4; + + local_irq_restore(flags); + + return NOTIFY_STOP; +} + +static struct notifier_block kgdb_notifier = { + .notifier_call = kgdb_riscv_notify, +}; + +int kgdb_arch_init(void) +{ + register_die_notifier(&kgdb_notifier); + + return 0; +} + +void kgdb_arch_exit(void) +{ + unregister_die_notifier(&kgdb_notifier); +} + +/* + * Global data + */ +#ifdef CONFIG_RISCV_ISA_C +const struct kgdb_arch arch_kgdb_ops = { + .gdb_bpt_instr = {0x02, 0x90}, /* c.ebreak */ +}; +#else +const struct kgdb_arch arch_kgdb_ops = { + .gdb_bpt_instr = {0x73, 0x00, 0x10, 0x00}, /* ebreak */ +}; +#endif diff --git a/arch/riscv/kernel/module.c b/arch/riscv/kernel/module.c index 8bbe5dbe1341..104fba889cf7 100644 --- a/arch/riscv/kernel/module.c +++ b/arch/riscv/kernel/module.c @@ -10,7 +10,7 @@ #include <linux/moduleloader.h> #include <linux/vmalloc.h> #include <linux/sizes.h> -#include <asm/pgtable.h> +#include <linux/pgtable.h> #include <asm/sections.h> static int apply_r_riscv_32_rela(struct module *me, u32 *location, Elf_Addr v) @@ -263,6 +263,13 @@ static int apply_r_riscv_add32_rela(struct module *me, u32 *location, return 0; } +static int apply_r_riscv_add64_rela(struct module *me, u32 *location, + Elf_Addr v) +{ + *(u64 *)location += (u64)v; + return 0; +} + static int apply_r_riscv_sub32_rela(struct module *me, u32 *location, Elf_Addr v) { @@ -270,6 +277,13 @@ static int apply_r_riscv_sub32_rela(struct module *me, u32 *location, return 0; } +static int apply_r_riscv_sub64_rela(struct module *me, u32 *location, + Elf_Addr v) +{ + *(u64 *)location -= (u64)v; + return 0; +} + static int (*reloc_handlers_rela[]) (struct module *me, u32 *location, Elf_Addr v) = { [R_RISCV_32] = apply_r_riscv_32_rela, @@ -290,7 +304,9 @@ static int (*reloc_handlers_rela[]) (struct module *me, u32 *location, [R_RISCV_RELAX] = apply_r_riscv_relax_rela, [R_RISCV_ALIGN] = apply_r_riscv_align_rela, [R_RISCV_ADD32] = apply_r_riscv_add32_rela, + [R_RISCV_ADD64] = apply_r_riscv_add64_rela, [R_RISCV_SUB32] = apply_r_riscv_sub32_rela, + [R_RISCV_SUB64] = apply_r_riscv_sub64_rela, }; int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab, diff --git a/arch/riscv/kernel/patch.c b/arch/riscv/kernel/patch.c index 8a4fc65ee022..3fe7a5296aa5 100644 --- a/arch/riscv/kernel/patch.c +++ b/arch/riscv/kernel/patch.c @@ -5,22 +5,22 @@ #include <linux/spinlock.h> #include <linux/mm.h> +#include <linux/memory.h> #include <linux/uaccess.h> #include <linux/stop_machine.h> #include <asm/kprobes.h> #include <asm/cacheflush.h> #include <asm/fixmap.h> +#include <asm/patch.h> -struct riscv_insn_patch { +struct patch_insn { void *addr; u32 insn; atomic_t cpu_count; }; #ifdef CONFIG_MMU -static DEFINE_RAW_SPINLOCK(patch_lock); - -static void __kprobes *patch_map(void *addr, int fixmap) +static void *patch_map(void *addr, int fixmap) { uintptr_t uintaddr = (uintptr_t) addr; struct page *page; @@ -37,65 +37,72 @@ static void __kprobes *patch_map(void *addr, int fixmap) return (void *)set_fixmap_offset(fixmap, page_to_phys(page) + (uintaddr & ~PAGE_MASK)); } +NOKPROBE_SYMBOL(patch_map); -static void __kprobes patch_unmap(int fixmap) +static void patch_unmap(int fixmap) { clear_fixmap(fixmap); } +NOKPROBE_SYMBOL(patch_unmap); -static int __kprobes riscv_insn_write(void *addr, const void *insn, size_t len) +static int patch_insn_write(void *addr, const void *insn, size_t len) { void *waddr = addr; bool across_pages = (((uintptr_t) addr & ~PAGE_MASK) + len) > PAGE_SIZE; - unsigned long flags = 0; int ret; - raw_spin_lock_irqsave(&patch_lock, flags); + /* + * Before reaching here, it was expected to lock the text_mutex + * already, so we don't need to give another lock here and could + * ensure that it was safe between each cores. + */ + lockdep_assert_held(&text_mutex); if (across_pages) patch_map(addr + len, FIX_TEXT_POKE1); waddr = patch_map(addr, FIX_TEXT_POKE0); - ret = probe_kernel_write(waddr, insn, len); + ret = copy_to_kernel_nofault(waddr, insn, len); patch_unmap(FIX_TEXT_POKE0); if (across_pages) patch_unmap(FIX_TEXT_POKE1); - raw_spin_unlock_irqrestore(&patch_lock, flags); - return ret; } +NOKPROBE_SYMBOL(patch_insn_write); #else -static int __kprobes riscv_insn_write(void *addr, const void *insn, size_t len) +static int patch_insn_write(void *addr, const void *insn, size_t len) { - return probe_kernel_write(addr, insn, len); + return copy_to_kernel_nofault(addr, insn, len); } +NOKPROBE_SYMBOL(patch_insn_write); #endif /* CONFIG_MMU */ -int __kprobes riscv_patch_text_nosync(void *addr, const void *insns, size_t len) +int patch_text_nosync(void *addr, const void *insns, size_t len) { u32 *tp = addr; int ret; - ret = riscv_insn_write(tp, insns, len); + ret = patch_insn_write(tp, insns, len); if (!ret) flush_icache_range((uintptr_t) tp, (uintptr_t) tp + len); return ret; } +NOKPROBE_SYMBOL(patch_text_nosync); -static int __kprobes riscv_patch_text_cb(void *data) +static int patch_text_cb(void *data) { - struct riscv_insn_patch *patch = data; + struct patch_insn *patch = data; int ret = 0; if (atomic_inc_return(&patch->cpu_count) == 1) { ret = - riscv_patch_text_nosync(patch->addr, &patch->insn, + patch_text_nosync(patch->addr, &patch->insn, GET_INSN_LENGTH(patch->insn)); atomic_inc(&patch->cpu_count); } else { @@ -106,15 +113,17 @@ static int __kprobes riscv_patch_text_cb(void *data) return ret; } +NOKPROBE_SYMBOL(patch_text_cb); -int __kprobes riscv_patch_text(void *addr, u32 insn) +int patch_text(void *addr, u32 insn) { - struct riscv_insn_patch patch = { + struct patch_insn patch = { .addr = addr, .insn = insn, .cpu_count = ATOMIC_INIT(0), }; - return stop_machine_cpuslocked(riscv_patch_text_cb, + return stop_machine_cpuslocked(patch_text_cb, &patch, cpu_online_mask); } +NOKPROBE_SYMBOL(patch_text); diff --git a/arch/riscv/kernel/process.c b/arch/riscv/kernel/process.c index 824d117cf202..2b97c493427c 100644 --- a/arch/riscv/kernel/process.c +++ b/arch/riscv/kernel/process.c @@ -24,6 +24,12 @@ register unsigned long gp_in_global __asm__("gp"); +#ifdef CONFIG_STACKPROTECTOR +#include <linux/stackprotector.h> +unsigned long __stack_chk_guard __read_mostly; +EXPORT_SYMBOL(__stack_chk_guard); +#endif + extern asmlinkage void ret_from_fork(void); extern asmlinkage void ret_from_kernel_thread(void); @@ -101,8 +107,8 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) return 0; } -int copy_thread_tls(unsigned long clone_flags, unsigned long usp, - unsigned long arg, struct task_struct *p, unsigned long tls) +int copy_thread(unsigned long clone_flags, unsigned long usp, unsigned long arg, + struct task_struct *p, unsigned long tls) { struct pt_regs *childregs = task_pt_regs(p); diff --git a/arch/riscv/kernel/ptrace.c b/arch/riscv/kernel/ptrace.c index 444dc7b0fd78..2d6395f5ad54 100644 --- a/arch/riscv/kernel/ptrace.c +++ b/arch/riscv/kernel/ptrace.c @@ -30,13 +30,10 @@ enum riscv_regset { static int riscv_gpr_get(struct task_struct *target, const struct user_regset *regset, - unsigned int pos, unsigned int count, - void *kbuf, void __user *ubuf) + struct membuf to) { - struct pt_regs *regs; - - regs = task_pt_regs(target); - return user_regset_copyout(&pos, &count, &kbuf, &ubuf, regs, 0, -1); + return membuf_write(&to, task_pt_regs(target), + sizeof(struct user_regs_struct)); } static int riscv_gpr_set(struct task_struct *target, @@ -55,21 +52,13 @@ static int riscv_gpr_set(struct task_struct *target, #ifdef CONFIG_FPU static int riscv_fpr_get(struct task_struct *target, const struct user_regset *regset, - unsigned int pos, unsigned int count, - void *kbuf, void __user *ubuf) + struct membuf to) { - int ret; struct __riscv_d_ext_state *fstate = &target->thread.fstate; - ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, fstate, 0, - offsetof(struct __riscv_d_ext_state, fcsr)); - if (!ret) { - ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, fstate, 0, - offsetof(struct __riscv_d_ext_state, fcsr) + - sizeof(fstate->fcsr)); - } - - return ret; + membuf_write(&to, fstate, offsetof(struct __riscv_d_ext_state, fcsr)); + membuf_store(&to, fstate->fcsr); + return membuf_zero(&to, 4); // explicitly pad } static int riscv_fpr_set(struct task_struct *target, @@ -98,8 +87,8 @@ static const struct user_regset riscv_user_regset[] = { .n = ELF_NGREG, .size = sizeof(elf_greg_t), .align = sizeof(elf_greg_t), - .get = &riscv_gpr_get, - .set = &riscv_gpr_set, + .regset_get = riscv_gpr_get, + .set = riscv_gpr_set, }, #ifdef CONFIG_FPU [REGSET_F] = { @@ -107,8 +96,8 @@ static const struct user_regset riscv_user_regset[] = { .n = ELF_NFPREG, .size = sizeof(elf_fpreg_t), .align = sizeof(elf_fpreg_t), - .get = &riscv_fpr_get, - .set = &riscv_fpr_set, + .regset_get = riscv_fpr_get, + .set = riscv_fpr_set, }, #endif }; diff --git a/arch/riscv/kernel/sbi.c b/arch/riscv/kernel/sbi.c index f383ef5672b2..226ccce0f9e0 100644 --- a/arch/riscv/kernel/sbi.c +++ b/arch/riscv/kernel/sbi.c @@ -547,6 +547,18 @@ static inline long sbi_get_firmware_version(void) return __sbi_base_ecall(SBI_EXT_BASE_GET_IMP_VERSION); } +static void sbi_send_cpumask_ipi(const struct cpumask *target) +{ + struct cpumask hartid_mask; + + riscv_cpuid_to_hartid_mask(target, &hartid_mask); + + sbi_send_ipi(cpumask_bits(&hartid_mask)); +} + +static struct riscv_ipi_ops sbi_ipi_ops = { + .ipi_inject = sbi_send_cpumask_ipi +}; int __init sbi_init(void) { @@ -587,5 +599,7 @@ int __init sbi_init(void) __sbi_rfence = __sbi_rfence_v01; } + riscv_set_ipi_ops(&sbi_ipi_ops); + return 0; } diff --git a/arch/riscv/kernel/setup.c b/arch/riscv/kernel/setup.c index 145128a7e560..2c6dd329312b 100644 --- a/arch/riscv/kernel/setup.c +++ b/arch/riscv/kernel/setup.c @@ -18,11 +18,9 @@ #include <linux/swiotlb.h> #include <linux/smp.h> -#include <asm/clint.h> #include <asm/cpu_ops.h> #include <asm/setup.h> #include <asm/sections.h> -#include <asm/pgtable.h> #include <asm/sbi.h> #include <asm/tlbflush.h> #include <asm/thread_info.h> @@ -75,8 +73,11 @@ void __init setup_arch(char **cmdline_p) setup_bootmem(); paging_init(); +#if IS_ENABLED(CONFIG_BUILTIN_DTB) + unflatten_and_copy_device_tree(); +#else unflatten_device_tree(); - clint_init_boot_cpu(); +#endif #ifdef CONFIG_SWIOTLB swiotlb_init(1); diff --git a/arch/riscv/kernel/signal.c b/arch/riscv/kernel/signal.c index 17ba190e84a5..e996e08f1061 100644 --- a/arch/riscv/kernel/signal.c +++ b/arch/riscv/kernel/signal.c @@ -250,7 +250,7 @@ static void handle_signal(struct ksignal *ksig, struct pt_regs *regs) regs->a0 = -EINTR; break; } - /* fallthrough */ + fallthrough; case -ERESTARTNOINTR: regs->a0 = regs->orig_a0; regs->epc -= 0x4; diff --git a/arch/riscv/kernel/smp.c b/arch/riscv/kernel/smp.c index a65a8fa0c22d..ea028d9e0d24 100644 --- a/arch/riscv/kernel/smp.c +++ b/arch/riscv/kernel/smp.c @@ -16,8 +16,8 @@ #include <linux/sched.h> #include <linux/seq_file.h> #include <linux/delay.h> +#include <linux/irq_work.h> -#include <asm/clint.h> #include <asm/sbi.h> #include <asm/tlbflush.h> #include <asm/cacheflush.h> @@ -26,6 +26,7 @@ enum ipi_message_type { IPI_RESCHEDULE, IPI_CALL_FUNC, IPI_CPU_STOP, + IPI_IRQ_WORK, IPI_MAX }; @@ -84,9 +85,25 @@ static void ipi_stop(void) wait_for_interrupt(); } +static struct riscv_ipi_ops *ipi_ops; + +void riscv_set_ipi_ops(struct riscv_ipi_ops *ops) +{ + ipi_ops = ops; +} +EXPORT_SYMBOL_GPL(riscv_set_ipi_ops); + +void riscv_clear_ipi(void) +{ + if (ipi_ops && ipi_ops->ipi_clear) + ipi_ops->ipi_clear(); + + csr_clear(CSR_IP, IE_SIE); +} +EXPORT_SYMBOL_GPL(riscv_clear_ipi); + static void send_ipi_mask(const struct cpumask *mask, enum ipi_message_type op) { - struct cpumask hartid_mask; int cpu; smp_mb__before_atomic(); @@ -94,41 +111,40 @@ static void send_ipi_mask(const struct cpumask *mask, enum ipi_message_type op) set_bit(op, &ipi_data[cpu].bits); smp_mb__after_atomic(); - riscv_cpuid_to_hartid_mask(mask, &hartid_mask); - if (IS_ENABLED(CONFIG_RISCV_SBI)) - sbi_send_ipi(cpumask_bits(&hartid_mask)); + if (ipi_ops && ipi_ops->ipi_inject) + ipi_ops->ipi_inject(mask); else - clint_send_ipi_mask(mask); + pr_warn("SMP: IPI inject method not available\n"); } static void send_ipi_single(int cpu, enum ipi_message_type op) { - int hartid = cpuid_to_hartid_map(cpu); - smp_mb__before_atomic(); set_bit(op, &ipi_data[cpu].bits); smp_mb__after_atomic(); - if (IS_ENABLED(CONFIG_RISCV_SBI)) - sbi_send_ipi(cpumask_bits(cpumask_of(hartid))); + if (ipi_ops && ipi_ops->ipi_inject) + ipi_ops->ipi_inject(cpumask_of(cpu)); else - clint_send_ipi_single(hartid); + pr_warn("SMP: IPI inject method not available\n"); } -static inline void clear_ipi(void) +#ifdef CONFIG_IRQ_WORK +void arch_irq_work_raise(void) { - if (IS_ENABLED(CONFIG_RISCV_SBI)) - csr_clear(CSR_IP, IE_SIE); - else - clint_clear_ipi(cpuid_to_hartid_map(smp_processor_id())); + send_ipi_single(smp_processor_id(), IPI_IRQ_WORK); } +#endif -void riscv_software_interrupt(void) +void handle_IPI(struct pt_regs *regs) { + struct pt_regs *old_regs = set_irq_regs(regs); unsigned long *pending_ipis = &ipi_data[smp_processor_id()].bits; unsigned long *stats = ipi_data[smp_processor_id()].stats; - clear_ipi(); + irq_enter(); + + riscv_clear_ipi(); while (true) { unsigned long ops; @@ -138,7 +154,7 @@ void riscv_software_interrupt(void) ops = xchg(pending_ipis, 0); if (ops == 0) - return; + goto done; if (ops & (1 << IPI_RESCHEDULE)) { stats[IPI_RESCHEDULE]++; @@ -155,17 +171,27 @@ void riscv_software_interrupt(void) ipi_stop(); } + if (ops & (1 << IPI_IRQ_WORK)) { + stats[IPI_IRQ_WORK]++; + irq_work_run(); + } + BUG_ON((ops >> IPI_MAX) != 0); /* Order data access and bit testing. */ mb(); } + +done: + irq_exit(); + set_irq_regs(old_regs); } static const char * const ipi_names[] = { [IPI_RESCHEDULE] = "Rescheduling interrupts", [IPI_CALL_FUNC] = "Function call interrupts", [IPI_CPU_STOP] = "CPU stop interrupts", + [IPI_IRQ_WORK] = "IRQ work interrupts", }; void show_ipi_stats(struct seq_file *p, int prec) diff --git a/arch/riscv/kernel/smpboot.c b/arch/riscv/kernel/smpboot.c index 4e9922790f6e..96167d55ed98 100644 --- a/arch/riscv/kernel/smpboot.c +++ b/arch/riscv/kernel/smpboot.c @@ -24,7 +24,6 @@ #include <linux/of.h> #include <linux/sched/task_stack.h> #include <linux/sched/mm.h> -#include <asm/clint.h> #include <asm/cpu_ops.h> #include <asm/irq.h> #include <asm/mmu_context.h> @@ -106,7 +105,7 @@ void __init setup_smp(void) } } -int start_secondary_cpu(int cpu, struct task_struct *tidle) +static int start_secondary_cpu(int cpu, struct task_struct *tidle) { if (cpu_ops[cpu]->cpu_start) return cpu_ops[cpu]->cpu_start(cpu, tidle); @@ -121,7 +120,6 @@ int __cpu_up(unsigned int cpu, struct task_struct *tidle) ret = start_secondary_cpu(cpu, tidle); if (!ret) { - lockdep_assert_held(&cpu_running); wait_for_completion_timeout(&cpu_running, msecs_to_jiffies(1000)); @@ -146,18 +144,18 @@ void __init smp_cpus_done(unsigned int max_cpus) asmlinkage __visible void smp_callin(void) { struct mm_struct *mm = &init_mm; + unsigned int curr_cpuid = smp_processor_id(); - if (!IS_ENABLED(CONFIG_RISCV_SBI)) - clint_clear_ipi(cpuid_to_hartid_map(smp_processor_id())); + riscv_clear_ipi(); /* All kernel threads share the same mm context. */ mmgrab(mm); current->active_mm = mm; - trap_init(); - notify_cpu_starting(smp_processor_id()); - update_siblings_masks(smp_processor_id()); - set_cpu_online(smp_processor_id(), 1); + notify_cpu_starting(curr_cpuid); + update_siblings_masks(curr_cpuid); + set_cpu_online(curr_cpuid, 1); + /* * Remote TLB flushes are ignored while the CPU is offline, so emit * a local TLB flush right now just in case. diff --git a/arch/riscv/kernel/soc.c b/arch/riscv/kernel/soc.c index 0b3b3dc9ad0f..c7b0a73e382e 100644 --- a/arch/riscv/kernel/soc.c +++ b/arch/riscv/kernel/soc.c @@ -4,7 +4,7 @@ */ #include <linux/init.h> #include <linux/libfdt.h> -#include <asm/pgtable.h> +#include <linux/pgtable.h> #include <asm/soc.h> /* @@ -26,3 +26,30 @@ void __init soc_early_init(void) } } } + +static bool soc_builtin_dtb_match(unsigned long vendor_id, + unsigned long arch_id, unsigned long imp_id, + const struct soc_builtin_dtb *entry) +{ + return entry->vendor_id == vendor_id && + entry->arch_id == arch_id && + entry->imp_id == imp_id; +} + +void * __init soc_lookup_builtin_dtb(void) +{ + unsigned long vendor_id, arch_id, imp_id; + const struct soc_builtin_dtb *s; + + __asm__ ("csrr %0, mvendorid" : "=r"(vendor_id)); + __asm__ ("csrr %0, marchid" : "=r"(arch_id)); + __asm__ ("csrr %0, mimpid" : "=r"(imp_id)); + + for (s = (void *)&__soc_builtin_dtb_table_start; + (void *)s < (void *)&__soc_builtin_dtb_table_end; s++) { + if (soc_builtin_dtb_match(vendor_id, arch_id, imp_id, s)) + return s->dtb_func(); + } + + return NULL; +} diff --git a/arch/riscv/kernel/stacktrace.c b/arch/riscv/kernel/stacktrace.c index 837b9b38f825..595342910c3f 100644 --- a/arch/riscv/kernel/stacktrace.c +++ b/arch/riscv/kernel/stacktrace.c @@ -99,17 +99,18 @@ void notrace walk_stackframe(struct task_struct *task, static bool print_trace_address(unsigned long pc, void *arg) { - print_ip_sym(pc); + const char *loglvl = arg; + + print_ip_sym(loglvl, pc); return false; } -void show_stack(struct task_struct *task, unsigned long *sp) +void show_stack(struct task_struct *task, unsigned long *sp, const char *loglvl) { pr_cont("Call Trace:\n"); - walk_stackframe(task, NULL, print_trace_address, NULL); + walk_stackframe(task, NULL, print_trace_address, (void *)loglvl); } - static bool save_wchan(unsigned long pc, void *arg) { if (!in_sched_functions(pc)) { diff --git a/arch/riscv/kernel/sys_riscv.c b/arch/riscv/kernel/sys_riscv.c index f3619f59d85c..12f8a7fce78b 100644 --- a/arch/riscv/kernel/sys_riscv.c +++ b/arch/riscv/kernel/sys_riscv.c @@ -8,6 +8,7 @@ #include <linux/syscalls.h> #include <asm/unistd.h> #include <asm/cacheflush.h> +#include <asm-generic/mman-common.h> static long riscv_sys_mmap(unsigned long addr, unsigned long len, unsigned long prot, unsigned long flags, @@ -16,6 +17,11 @@ static long riscv_sys_mmap(unsigned long addr, unsigned long len, { if (unlikely(offset & (~PAGE_MASK >> page_shift_offset))) return -EINVAL; + + if ((prot & PROT_WRITE) && (prot & PROT_EXEC)) + if (unlikely(!(prot & PROT_READ))) + return -EINVAL; + return ksys_mmap_pgoff(addr, len, prot, flags, fd, offset >> (PAGE_SHIFT - page_shift_offset)); } diff --git a/arch/riscv/kernel/time.c b/arch/riscv/kernel/time.c index 6a53c02e9c73..4d3a1048ad8b 100644 --- a/arch/riscv/kernel/time.c +++ b/arch/riscv/kernel/time.c @@ -26,3 +26,12 @@ void __init time_init(void) lpj_fine = riscv_timebase / HZ; timer_probe(); } + +void clocksource_arch_init(struct clocksource *cs) +{ +#ifdef CONFIG_GENERIC_GETTIMEOFDAY + cs->vdso_clock_mode = VDSO_CLOCKMODE_ARCHTIMER; +#else + cs->vdso_clock_mode = VDSO_CLOCKMODE_NONE; +#endif +} diff --git a/arch/riscv/kernel/traps.c b/arch/riscv/kernel/traps.c index 7f58fa53033f..ad14f4466d92 100644 --- a/arch/riscv/kernel/traps.c +++ b/arch/riscv/kernel/traps.c @@ -137,7 +137,7 @@ static inline unsigned long get_break_insn_length(unsigned long pc) { bug_insn_t insn; - if (probe_kernel_address((bug_insn_t *)pc, insn)) + if (get_kernel_nofault(insn, (bug_insn_t *)pc)) return 0; return GET_INSN_LENGTH(insn); @@ -147,6 +147,11 @@ asmlinkage __visible void do_trap_break(struct pt_regs *regs) { if (user_mode(regs)) force_sig_fault(SIGTRAP, TRAP_BRKPT, (void __user *)regs->epc); +#ifdef CONFIG_KGDB + else if (notify_die(DIE_TRAP, "EBREAK", regs, 0, regs->cause, SIGTRAP) + == NOTIFY_STOP) + return; +#endif else if (report_bug(regs->epc, regs) == BUG_TRAP_TYPE_WARN) regs->epc += get_break_insn_length(regs->epc); else @@ -160,7 +165,7 @@ int is_valid_bugaddr(unsigned long pc) if (pc < VMALLOC_START) return 0; - if (probe_kernel_address((bug_insn_t *)pc, insn)) + if (get_kernel_nofault(insn, (bug_insn_t *)pc)) return 0; if ((insn & __INSN_LENGTH_MASK) == __INSN_LENGTH_32) return (insn == __BUG_INSN_32); @@ -169,15 +174,7 @@ int is_valid_bugaddr(unsigned long pc) } #endif /* CONFIG_GENERIC_BUG */ +/* stvec & scratch is already set from head.S */ void trap_init(void) { - /* - * Set sup0 scratch register to 0, indicating to exception vector - * that we are presently executing in the kernel - */ - csr_write(CSR_SCRATCH, 0); - /* Set the exception vector address */ - csr_write(CSR_TVEC, &handle_exception); - /* Enable interrupts */ - csr_write(CSR_IE, IE_SIE); } diff --git a/arch/riscv/kernel/vdso.c b/arch/riscv/kernel/vdso.c index 484d95a70907..678204231700 100644 --- a/arch/riscv/kernel/vdso.c +++ b/arch/riscv/kernel/vdso.c @@ -11,8 +11,12 @@ #include <linux/slab.h> #include <linux/binfmts.h> #include <linux/err.h> - +#include <asm/page.h> +#ifdef GENERIC_TIME_VSYSCALL +#include <vdso/datapage.h> +#else #include <asm/vdso.h> +#endif extern char vdso_start[], vdso_end[]; @@ -26,7 +30,7 @@ static union { struct vdso_data data; u8 page[PAGE_SIZE]; } vdso_data_store __page_aligned_data; -static struct vdso_data *vdso_data = &vdso_data_store.data; +struct vdso_data *vdso_data = &vdso_data_store.data; static int __init vdso_init(void) { @@ -61,7 +65,7 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, vdso_len = (vdso_pages + 1) << PAGE_SHIFT; - down_write(&mm->mmap_sem); + mmap_write_lock(mm); vdso_base = get_unmapped_area(NULL, 0, vdso_len, 0, 0); if (IS_ERR_VALUE(vdso_base)) { ret = vdso_base; @@ -75,15 +79,24 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, */ mm->context.vdso = (void *)vdso_base; - ret = install_special_mapping(mm, vdso_base, vdso_len, + ret = + install_special_mapping(mm, vdso_base, vdso_pages << PAGE_SHIFT, (VM_READ | VM_EXEC | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC), vdso_pagelist); - if (unlikely(ret)) + if (unlikely(ret)) { mm->context.vdso = NULL; + goto end; + } + vdso_base += (vdso_pages << PAGE_SHIFT); + ret = install_special_mapping(mm, vdso_base, PAGE_SIZE, + (VM_READ | VM_MAYREAD), &vdso_pagelist[vdso_pages]); + + if (unlikely(ret)) + mm->context.vdso = NULL; end: - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); return ret; } @@ -91,5 +104,8 @@ const char *arch_vma_name(struct vm_area_struct *vma) { if (vma->vm_mm && (vma->vm_start == (long)vma->vm_mm->context.vdso)) return "[vdso]"; + if (vma->vm_mm && (vma->vm_start == + (long)vma->vm_mm->context.vdso + PAGE_SIZE)) + return "[vdso_data]"; return NULL; } diff --git a/arch/riscv/kernel/vdso/Makefile b/arch/riscv/kernel/vdso/Makefile index 4c8b2a4a6a70..478e7338ddc1 100644 --- a/arch/riscv/kernel/vdso/Makefile +++ b/arch/riscv/kernel/vdso/Makefile @@ -1,12 +1,14 @@ # SPDX-License-Identifier: GPL-2.0-only # Copied from arch/tile/kernel/vdso/Makefile +# Absolute relocation type $(ARCH_REL_TYPE_ABS) needs to be defined before +# the inclusion of generic Makefile. +ARCH_REL_TYPE_ABS := R_RISCV_32|R_RISCV_64|R_RISCV_JUMP_SLOT +include $(srctree)/lib/vdso/Makefile # Symbols present in the vdso vdso-syms = rt_sigreturn ifdef CONFIG_64BIT -vdso-syms += gettimeofday -vdso-syms += clock_gettime -vdso-syms += clock_getres +vdso-syms += vgettimeofday endif vdso-syms += getcpu vdso-syms += flush_icache @@ -14,6 +16,12 @@ vdso-syms += flush_icache # Files to link into the vdso obj-vdso = $(patsubst %, %.o, $(vdso-syms)) note.o +ccflags-y := -fno-stack-protector + +ifneq ($(c-gettimeofday-y),) + CFLAGS_vgettimeofday.o += -fPIC -include $(c-gettimeofday-y) +endif + # Build rules targets := $(obj-vdso) vdso.so vdso.so.dbg vdso.lds vdso-dummy.o obj-vdso := $(addprefix $(obj)/, $(obj-vdso)) @@ -21,8 +29,12 @@ obj-vdso := $(addprefix $(obj)/, $(obj-vdso)) obj-y += vdso.o vdso-syms.o CPPFLAGS_vdso.lds += -P -C -U$(ARCH) +# Disable -pg to prevent insert call site +CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os + # Disable gcov profiling for VDSO code GCOV_PROFILE := n +KCOV_INSTRUMENT := n # Force dependency $(obj)/vdso.o: $(obj)/vdso.so diff --git a/arch/riscv/kernel/vdso/clock_getres.S b/arch/riscv/kernel/vdso/clock_getres.S deleted file mode 100644 index 91378a52eb22..000000000000 --- a/arch/riscv/kernel/vdso/clock_getres.S +++ /dev/null @@ -1,18 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2017 SiFive - */ - -#include <linux/linkage.h> -#include <asm/unistd.h> - - .text -/* int __vdso_clock_getres(clockid_t clock_id, struct timespec *res); */ -ENTRY(__vdso_clock_getres) - .cfi_startproc - /* For now, just do the syscall. */ - li a7, __NR_clock_getres - ecall - ret - .cfi_endproc -ENDPROC(__vdso_clock_getres) diff --git a/arch/riscv/kernel/vdso/clock_gettime.S b/arch/riscv/kernel/vdso/clock_gettime.S deleted file mode 100644 index 5371fd9bc01f..000000000000 --- a/arch/riscv/kernel/vdso/clock_gettime.S +++ /dev/null @@ -1,18 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2017 SiFive - */ - -#include <linux/linkage.h> -#include <asm/unistd.h> - - .text -/* int __vdso_clock_gettime(clockid_t clock_id, struct timespec *tp); */ -ENTRY(__vdso_clock_gettime) - .cfi_startproc - /* For now, just do the syscall. */ - li a7, __NR_clock_gettime - ecall - ret - .cfi_endproc -ENDPROC(__vdso_clock_gettime) diff --git a/arch/riscv/kernel/vdso/gettimeofday.S b/arch/riscv/kernel/vdso/gettimeofday.S deleted file mode 100644 index e6fb8af88632..000000000000 --- a/arch/riscv/kernel/vdso/gettimeofday.S +++ /dev/null @@ -1,18 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2017 SiFive - */ - -#include <linux/linkage.h> -#include <asm/unistd.h> - - .text -/* int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz); */ -ENTRY(__vdso_gettimeofday) - .cfi_startproc - /* For now, just do the syscall. */ - li a7, __NR_gettimeofday - ecall - ret - .cfi_endproc -ENDPROC(__vdso_gettimeofday) diff --git a/arch/riscv/kernel/vdso/vdso.lds.S b/arch/riscv/kernel/vdso/vdso.lds.S index f66a091cb890..e6f558bca71b 100644 --- a/arch/riscv/kernel/vdso/vdso.lds.S +++ b/arch/riscv/kernel/vdso/vdso.lds.S @@ -2,11 +2,13 @@ /* * Copyright (C) 2012 Regents of the University of California */ +#include <asm/page.h> OUTPUT_ARCH(riscv) SECTIONS { + PROVIDE(_vdso_data = . + PAGE_SIZE); . = SIZEOF_HEADERS; .hash : { *(.hash) } :text diff --git a/arch/riscv/kernel/vdso/vgettimeofday.c b/arch/riscv/kernel/vdso/vgettimeofday.c new file mode 100644 index 000000000000..cc0d80699c31 --- /dev/null +++ b/arch/riscv/kernel/vdso/vgettimeofday.c @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copied from arch/arm64/kernel/vdso/vgettimeofday.c + * + * Copyright (C) 2018 ARM Ltd. + * Copyright (C) 2020 SiFive + */ + +#include <linux/time.h> +#include <linux/types.h> + +extern +int __vdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts); +int __vdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts) +{ + return __cvdso_clock_gettime(clock, ts); +} + +extern +int __vdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz); +int __vdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) +{ + return __cvdso_gettimeofday(tv, tz); +} + +extern +int __vdso_clock_getres(clockid_t clock_id, struct __kernel_timespec *res); +int __vdso_clock_getres(clockid_t clock_id, struct __kernel_timespec *res) +{ + return __cvdso_clock_getres(clock_id, res); +} diff --git a/arch/riscv/kernel/vmlinux.lds.S b/arch/riscv/kernel/vmlinux.lds.S index 0339b6bbe11a..34d00d9e6eac 100644 --- a/arch/riscv/kernel/vmlinux.lds.S +++ b/arch/riscv/kernel/vmlinux.lds.S @@ -27,13 +27,17 @@ SECTIONS __init_begin = .; INIT_TEXT_SECTION(PAGE_SIZE) - INIT_DATA_SECTION(16) . = ALIGN(8); __soc_early_init_table : { __soc_early_init_table_start = .; KEEP(*(__soc_early_init_table)) __soc_early_init_table_end = .; } + __soc_builtin_dtb_table : { + __soc_builtin_dtb_table_start = .; + KEEP(*(__soc_builtin_dtb_table)) + __soc_builtin_dtb_table_end = .; + } /* we have to discard exit text and such at runtime, not link time */ .exit.text : { @@ -62,6 +66,8 @@ SECTIONS _etext = .; } + INIT_DATA_SECTION(16) + /* Start of data section */ _sdata = .; RO_DATA(SECTION_ALIGN) diff --git a/arch/riscv/mm/Makefile b/arch/riscv/mm/Makefile index 363ef01c30b1..c0185e556ca5 100644 --- a/arch/riscv/mm/Makefile +++ b/arch/riscv/mm/Makefile @@ -5,6 +5,8 @@ ifdef CONFIG_FTRACE CFLAGS_REMOVE_init.o = -pg endif +KCOV_INSTRUMENT_init.o := n + obj-y += init.o obj-y += extable.o obj-$(CONFIG_MMU) += fault.o pageattr.o diff --git a/arch/riscv/mm/cacheflush.c b/arch/riscv/mm/cacheflush.c index 8930ab7278e6..094118663285 100644 --- a/arch/riscv/mm/cacheflush.c +++ b/arch/riscv/mm/cacheflush.c @@ -3,7 +3,6 @@ * Copyright (C) 2017 SiFive */ -#include <asm/pgtable.h> #include <asm/cacheflush.h> #ifdef CONFIG_SMP diff --git a/arch/riscv/mm/fault.c b/arch/riscv/mm/fault.c index be84e32adc4c..716d64e36f83 100644 --- a/arch/riscv/mm/fault.c +++ b/arch/riscv/mm/fault.c @@ -14,7 +14,6 @@ #include <linux/signal.h> #include <linux/uaccess.h> -#include <asm/pgalloc.h> #include <asm/ptrace.h> #include <asm/tlbflush.h> @@ -69,7 +68,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs) perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr); retry: - down_read(&mm->mmap_sem); + mmap_read_lock(mm); vma = find_vma(mm, addr); if (unlikely(!vma)) goto bad_area; @@ -110,11 +109,11 @@ good_area: * make sure we exit gracefully rather than endlessly redo * the fault. */ - fault = handle_mm_fault(vma, addr, flags); + fault = handle_mm_fault(vma, addr, flags, regs); /* * If we need to retry but a fatal signal is pending, handle the - * signal first. We do not need to release the mmap_sem because it + * signal first. We do not need to release the mmap_lock because it * would already be released in __lock_page_or_retry in mm/filemap.c. */ if (fault_signal_pending(fault, regs)) @@ -128,26 +127,12 @@ good_area: BUG(); } - /* - * Major/minor page fault accounting is only done on the - * initial attempt. If we go through a retry, it is extremely - * likely that the page will be found in page cache at that point. - */ if (flags & FAULT_FLAG_ALLOW_RETRY) { - if (fault & VM_FAULT_MAJOR) { - tsk->maj_flt++; - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, - 1, regs, addr); - } else { - tsk->min_flt++; - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, - 1, regs, addr); - } if (fault & VM_FAULT_RETRY) { flags |= FAULT_FLAG_TRIED; /* - * No need to up_read(&mm->mmap_sem) as we would + * No need to mmap_read_unlock(mm) as we would * have already released it in __lock_page_or_retry * in mm/filemap.c. */ @@ -155,7 +140,7 @@ good_area: } } - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); return; /* @@ -163,7 +148,7 @@ good_area: * Fix it, but check if it's kernel or user first. */ bad_area: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); /* User mode accesses just cause a SIGSEGV */ if (user_mode(regs)) { do_trap(regs, SIGSEGV, code, addr); @@ -191,14 +176,14 @@ no_context: * (which will retry the fault, or kill us if we got oom-killed). */ out_of_memory: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); if (!user_mode(regs)) goto no_context; pagefault_out_of_memory(); return; do_sigbus: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); /* Kernel mode? Handle exceptions or die */ if (!user_mode(regs)) goto no_context; diff --git a/arch/riscv/mm/hugetlbpage.c b/arch/riscv/mm/hugetlbpage.c index a6189ed36c5f..932dadfdca54 100644 --- a/arch/riscv/mm/hugetlbpage.c +++ b/arch/riscv/mm/hugetlbpage.c @@ -12,29 +12,21 @@ int pmd_huge(pmd_t pmd) return pmd_leaf(pmd); } -static __init int setup_hugepagesz(char *opt) +bool __init arch_hugetlb_valid_size(unsigned long size) { - unsigned long ps = memparse(opt, &opt); - - if (ps == HPAGE_SIZE) { - hugetlb_add_hstate(HPAGE_SHIFT - PAGE_SHIFT); - } else if (IS_ENABLED(CONFIG_64BIT) && ps == PUD_SIZE) { - hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT); - } else { - hugetlb_bad_size(); - pr_err("hugepagesz: Unsupported page size %lu M\n", ps >> 20); - return 0; - } - - return 1; + if (size == HPAGE_SIZE) + return true; + else if (IS_ENABLED(CONFIG_64BIT) && size == PUD_SIZE) + return true; + else + return false; } -__setup("hugepagesz=", setup_hugepagesz); #ifdef CONFIG_CONTIG_ALLOC static __init int gigantic_pages_init(void) { /* With CONTIG_ALLOC, we can allocate gigantic pages at runtime */ - if (IS_ENABLED(CONFIG_64BIT) && !size_to_hstate(1UL << PUD_SHIFT)) + if (IS_ENABLED(CONFIG_64BIT)) hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT); return 0; } diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c index 736de6c8739f..f750e012dbe5 100644 --- a/arch/riscv/mm/init.c +++ b/arch/riscv/mm/init.c @@ -17,8 +17,9 @@ #include <asm/fixmap.h> #include <asm/tlbflush.h> #include <asm/sections.h> -#include <asm/pgtable.h> +#include <asm/soc.h> #include <asm/io.h> +#include <asm/ptdump.h> #include "../kernel/head.h" @@ -39,7 +40,7 @@ static void __init zone_sizes_init(void) #endif max_zone_pfns[ZONE_NORMAL] = max_low_pfn; - free_area_init_nodes(max_zone_pfns); + free_area_init(max_zone_pfns); } static void setup_zero_page(void) @@ -94,19 +95,40 @@ void __init mem_init(void) #ifdef CONFIG_BLK_DEV_INITRD static void __init setup_initrd(void) { + phys_addr_t start; unsigned long size; - if (initrd_start >= initrd_end) { - pr_info("initrd not found or empty"); + /* Ignore the virtul address computed during device tree parsing */ + initrd_start = initrd_end = 0; + + if (!phys_initrd_size) + return; + /* + * Round the memory region to page boundaries as per free_initrd_mem() + * This allows us to detect whether the pages overlapping the initrd + * are in use, but more importantly, reserves the entire set of pages + * as we don't want these pages allocated for other purposes. + */ + start = round_down(phys_initrd_start, PAGE_SIZE); + size = phys_initrd_size + (phys_initrd_start - start); + size = round_up(size, PAGE_SIZE); + + if (!memblock_is_region_memory(start, size)) { + pr_err("INITRD: 0x%08llx+0x%08lx is not a memory region", + (u64)start, size); goto disable; } - if (__pa_symbol(initrd_end) > PFN_PHYS(max_low_pfn)) { - pr_err("initrd extends beyond end of memory"); + + if (memblock_is_region_reserved(start, size)) { + pr_err("INITRD: 0x%08llx+0x%08lx overlaps in-use memory region\n", + (u64)start, size); goto disable; } - size = initrd_end - initrd_start; - memblock_reserve(__pa_symbol(initrd_start), size); + memblock_reserve(start, size); + /* Now convert initrd to virtual addresses */ + initrd_start = (unsigned long)__va(phys_initrd_start); + initrd_end = initrd_start + phys_initrd_size; initrd_below_start_ok = 1; pr_info("Initial ramdisk at: 0x%p (%lu bytes)\n", @@ -125,33 +147,36 @@ void __init setup_bootmem(void) { struct memblock_region *reg; phys_addr_t mem_size = 0; + phys_addr_t total_mem = 0; + phys_addr_t mem_start, end = 0; phys_addr_t vmlinux_end = __pa_symbol(&_end); phys_addr_t vmlinux_start = __pa_symbol(&_start); /* Find the memory region containing the kernel */ for_each_memblock(memory, reg) { - phys_addr_t end = reg->base + reg->size; - - if (reg->base <= vmlinux_start && vmlinux_end <= end) { - mem_size = min(reg->size, (phys_addr_t)-PAGE_OFFSET); - - /* - * Remove memblock from the end of usable area to the - * end of region - */ - if (reg->base + mem_size < end) - memblock_remove(reg->base + mem_size, - end - reg->base - mem_size); - } + end = reg->base + reg->size; + if (!total_mem) + mem_start = reg->base; + if (reg->base <= vmlinux_start && vmlinux_end <= end) + BUG_ON(reg->size == 0); + total_mem = total_mem + reg->size; } - BUG_ON(mem_size == 0); + + /* + * Remove memblock from the end of usable area to the + * end of region + */ + mem_size = min(total_mem, (phys_addr_t)-PAGE_OFFSET); + if (mem_start + mem_size < end) + memblock_remove(mem_start + mem_size, + end - mem_start - mem_size); /* Reserve from the start of the kernel to the end of the kernel */ memblock_reserve(vmlinux_start, vmlinux_end - vmlinux_start); - set_max_mapnr(PFN_DOWN(mem_size)); max_pfn = PFN_DOWN(memblock_end_of_DRAM()); max_low_pfn = max_pfn; + set_max_mapnr(max_low_pfn); #ifdef CONFIG_BLK_DEV_INITRD setup_initrd(); @@ -201,12 +226,11 @@ void __set_fixmap(enum fixed_addresses idx, phys_addr_t phys, pgprot_t prot) ptep = &fixmap_pte[pte_index(addr)]; - if (pgprot_val(prot)) { + if (pgprot_val(prot)) set_pte(ptep, pfn_pte(phys >> PAGE_SHIFT, prot)); - } else { + else pte_clear(&init_mm, addr, ptep); - local_flush_tlb_page(addr); - } + local_flush_tlb_page(addr); } static pte_t *__init get_pte_virt(phys_addr_t pa) @@ -234,12 +258,12 @@ static void __init create_pte_mapping(pte_t *ptep, uintptr_t va, phys_addr_t pa, phys_addr_t sz, pgprot_t prot) { - uintptr_t pte_index = pte_index(va); + uintptr_t pte_idx = pte_index(va); BUG_ON(sz != PAGE_SIZE); - if (pte_none(ptep[pte_index])) - ptep[pte_index] = pfn_pte(PFN_DOWN(pa), prot); + if (pte_none(ptep[pte_idx])) + ptep[pte_idx] = pfn_pte(PFN_DOWN(pa), prot); } #ifndef __PAGETABLE_PMD_FOLDED @@ -282,21 +306,21 @@ static void __init create_pmd_mapping(pmd_t *pmdp, { pte_t *ptep; phys_addr_t pte_phys; - uintptr_t pmd_index = pmd_index(va); + uintptr_t pmd_idx = pmd_index(va); if (sz == PMD_SIZE) { - if (pmd_none(pmdp[pmd_index])) - pmdp[pmd_index] = pfn_pmd(PFN_DOWN(pa), prot); + if (pmd_none(pmdp[pmd_idx])) + pmdp[pmd_idx] = pfn_pmd(PFN_DOWN(pa), prot); return; } - if (pmd_none(pmdp[pmd_index])) { + if (pmd_none(pmdp[pmd_idx])) { pte_phys = alloc_pte(va); - pmdp[pmd_index] = pfn_pmd(PFN_DOWN(pte_phys), PAGE_TABLE); + pmdp[pmd_idx] = pfn_pmd(PFN_DOWN(pte_phys), PAGE_TABLE); ptep = get_pte_virt(pte_phys); memset(ptep, 0, PAGE_SIZE); } else { - pte_phys = PFN_PHYS(_pmd_pfn(pmdp[pmd_index])); + pte_phys = PFN_PHYS(_pmd_pfn(pmdp[pmd_idx])); ptep = get_pte_virt(pte_phys); } @@ -324,21 +348,21 @@ static void __init create_pgd_mapping(pgd_t *pgdp, { pgd_next_t *nextp; phys_addr_t next_phys; - uintptr_t pgd_index = pgd_index(va); + uintptr_t pgd_idx = pgd_index(va); if (sz == PGDIR_SIZE) { - if (pgd_val(pgdp[pgd_index]) == 0) - pgdp[pgd_index] = pfn_pgd(PFN_DOWN(pa), prot); + if (pgd_val(pgdp[pgd_idx]) == 0) + pgdp[pgd_idx] = pfn_pgd(PFN_DOWN(pa), prot); return; } - if (pgd_val(pgdp[pgd_index]) == 0) { + if (pgd_val(pgdp[pgd_idx]) == 0) { next_phys = alloc_pgd_next(va); - pgdp[pgd_index] = pfn_pgd(PFN_DOWN(next_phys), PAGE_TABLE); + pgdp[pgd_idx] = pfn_pgd(PFN_DOWN(next_phys), PAGE_TABLE); nextp = get_pgd_next_virt(next_phys); memset(nextp, 0, PAGE_SIZE); } else { - next_phys = PFN_PHYS(_pgd_pfn(pgdp[pgd_index])); + next_phys = PFN_PHYS(_pgd_pfn(pgdp[pgd_idx])); nextp = get_pgd_next_virt(next_phys); } @@ -479,21 +503,19 @@ static void __init setup_vm_final(void) csr_write(CSR_SATP, PFN_DOWN(__pa_symbol(swapper_pg_dir)) | SATP_MODE); local_flush_tlb_all(); } - -void free_initmem(void) -{ - unsigned long init_begin = (unsigned long)__init_begin; - unsigned long init_end = (unsigned long)__init_end; - - /* Make the region as non-execuatble. */ - set_memory_nx(init_begin, (init_end - init_begin) >> PAGE_SHIFT); - free_initmem_default(POISON_FREE_INITMEM); -} - #else asmlinkage void __init setup_vm(uintptr_t dtb_pa) { +#ifdef CONFIG_BUILTIN_DTB + dtb_early_va = soc_lookup_builtin_dtb(); + if (!dtb_early_va) { + /* Fallback to first available DTS */ + dtb_early_va = (void *) __dtb_start; + } +#else dtb_early_va = (void *)dtb_pa; +#endif + dtb_early_pa = dtb_pa; } static inline void setup_vm_final(void) @@ -514,22 +536,50 @@ void mark_rodata_ro(void) set_memory_ro(rodata_start, (data_start - rodata_start) >> PAGE_SHIFT); set_memory_nx(rodata_start, (data_start - rodata_start) >> PAGE_SHIFT); set_memory_nx(data_start, (max_low - data_start) >> PAGE_SHIFT); + + debug_checkwx(); } #endif +static void __init resource_init(void) +{ + struct memblock_region *region; + + for_each_memblock(memory, region) { + struct resource *res; + + res = memblock_alloc(sizeof(struct resource), SMP_CACHE_BYTES); + if (!res) + panic("%s: Failed to allocate %zu bytes\n", __func__, + sizeof(struct resource)); + + if (memblock_is_nomap(region)) { + res->name = "reserved"; + res->flags = IORESOURCE_MEM; + } else { + res->name = "System RAM"; + res->flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY; + } + res->start = __pfn_to_phys(memblock_region_memory_base_pfn(region)); + res->end = __pfn_to_phys(memblock_region_memory_end_pfn(region)) - 1; + + request_resource(&iomem_resource, res); + } +} + void __init paging_init(void) { setup_vm_final(); - memblocks_present(); sparse_init(); setup_zero_page(); zone_sizes_init(); + resource_init(); } #ifdef CONFIG_SPARSEMEM_VMEMMAP int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, struct vmem_altmap *altmap) { - return vmemmap_populate_basepages(start, end, node); + return vmemmap_populate_basepages(start, end, node, NULL); } #endif diff --git a/arch/riscv/mm/kasan_init.c b/arch/riscv/mm/kasan_init.c index ec0ca90dd900..87b4ab3d3c77 100644 --- a/arch/riscv/mm/kasan_init.c +++ b/arch/riscv/mm/kasan_init.c @@ -6,8 +6,8 @@ #include <linux/kasan.h> #include <linux/kernel.h> #include <linux/memblock.h> +#include <linux/pgtable.h> #include <asm/tlbflush.h> -#include <asm/pgtable.h> #include <asm/fixmap.h> extern pgd_t early_pg_dir[PTRS_PER_PGD]; @@ -44,7 +44,7 @@ asmlinkage void __init kasan_early_init(void) (__pa(((uintptr_t) kasan_early_shadow_pmd))), __pgprot(_PAGE_TABLE))); - flush_tlb_all(); + local_flush_tlb_all(); } static void __init populate(void *start, void *end) @@ -79,7 +79,7 @@ static void __init populate(void *start, void *end) pfn_pgd(PFN_DOWN(__pa(&pmd[offset])), __pgprot(_PAGE_TABLE))); - flush_tlb_all(); + local_flush_tlb_all(); memset(start, 0, end - start); } diff --git a/arch/riscv/mm/pageattr.c b/arch/riscv/mm/pageattr.c index 728759eb530a..19fecb362d81 100644 --- a/arch/riscv/mm/pageattr.c +++ b/arch/riscv/mm/pageattr.c @@ -4,9 +4,10 @@ */ #include <linux/pagewalk.h> -#include <asm/pgtable.h> +#include <linux/pgtable.h> #include <asm/tlbflush.h> #include <asm/bitops.h> +#include <asm/set_memory.h> struct pageattr_masks { pgprot_t set_mask; @@ -94,7 +95,7 @@ static int pageattr_pte_hole(unsigned long addr, unsigned long next, return 0; } -const static struct mm_walk_ops pageattr_ops = { +static const struct mm_walk_ops pageattr_ops = { .pgd_entry = pageattr_pgd_entry, .p4d_entry = pageattr_p4d_entry, .pud_entry = pageattr_pud_entry, @@ -117,10 +118,10 @@ static int __set_memory(unsigned long addr, int numpages, pgprot_t set_mask, if (!numpages) return 0; - down_read(&init_mm.mmap_sem); + mmap_read_lock(&init_mm); ret = walk_page_range_novma(&init_mm, start, end, &pageattr_ops, NULL, &masks); - up_read(&init_mm.mmap_sem); + mmap_read_unlock(&init_mm); flush_tlb_kernel_range(start, end); @@ -151,6 +152,7 @@ int set_memory_nx(unsigned long addr, int numpages) int set_direct_map_invalid_noflush(struct page *page) { + int ret; unsigned long start = (unsigned long)page_address(page); unsigned long end = start + PAGE_SIZE; struct pageattr_masks masks = { @@ -158,11 +160,16 @@ int set_direct_map_invalid_noflush(struct page *page) .clear_mask = __pgprot(_PAGE_PRESENT) }; - return walk_page_range(&init_mm, start, end, &pageattr_ops, &masks); + mmap_read_lock(&init_mm); + ret = walk_page_range(&init_mm, start, end, &pageattr_ops, &masks); + mmap_read_unlock(&init_mm); + + return ret; } int set_direct_map_default_noflush(struct page *page) { + int ret; unsigned long start = (unsigned long)page_address(page); unsigned long end = start + PAGE_SIZE; struct pageattr_masks masks = { @@ -170,7 +177,11 @@ int set_direct_map_default_noflush(struct page *page) .clear_mask = __pgprot(0) }; - return walk_page_range(&init_mm, start, end, &pageattr_ops, &masks); + mmap_read_lock(&init_mm); + ret = walk_page_range(&init_mm, start, end, &pageattr_ops, &masks); + mmap_read_unlock(&init_mm); + + return ret; } void __kernel_map_pages(struct page *page, int numpages, int enable) diff --git a/arch/riscv/mm/ptdump.c b/arch/riscv/mm/ptdump.c index 7eab76a93106..0831c2e61a8f 100644 --- a/arch/riscv/mm/ptdump.c +++ b/arch/riscv/mm/ptdump.c @@ -9,7 +9,7 @@ #include <linux/ptdump.h> #include <asm/ptdump.h> -#include <asm/pgtable.h> +#include <linux/pgtable.h> #include <asm/kasan.h> #define pt_dump_seq_printf(m, fmt, args...) \ @@ -204,7 +204,7 @@ static void note_prot_wx(struct pg_state *st, unsigned long addr) } static void note_page(struct ptdump_state *pt_st, unsigned long addr, - int level, unsigned long val) + int level, u64 val) { struct pg_state *st = container_of(pt_st, struct pg_state, ptdump); u64 pa = PFN_PHYS(pte_pfn(__pte(val))); diff --git a/arch/riscv/net/bpf_jit.h b/arch/riscv/net/bpf_jit.h index 20e235d06f66..75c1e9996867 100644 --- a/arch/riscv/net/bpf_jit.h +++ b/arch/riscv/net/bpf_jit.h @@ -13,6 +13,11 @@ #include <linux/filter.h> #include <asm/cacheflush.h> +static inline bool rvc_enabled(void) +{ + return IS_ENABLED(CONFIG_RISCV_ISA_C); +} + enum { RV_REG_ZERO = 0, /* The constant value 0 */ RV_REG_RA = 1, /* Return address */ @@ -48,9 +53,21 @@ enum { RV_REG_T6 = 31, }; +static inline bool is_creg(u8 reg) +{ + return (1 << reg) & (BIT(RV_REG_FP) | + BIT(RV_REG_S1) | + BIT(RV_REG_A0) | + BIT(RV_REG_A1) | + BIT(RV_REG_A2) | + BIT(RV_REG_A3) | + BIT(RV_REG_A4) | + BIT(RV_REG_A5)); +} + struct rv_jit_context { struct bpf_prog *prog; - u32 *insns; /* RV insns */ + u16 *insns; /* RV insns */ int ninsns; int epilogue_offset; int *offset; /* BPF to RV */ @@ -58,6 +75,12 @@ struct rv_jit_context { int stack_size; }; +/* Convert from ninsns to bytes. */ +static inline int ninsns_rvoff(int ninsns) +{ + return ninsns << 1; +} + struct rv_jit_data { struct bpf_binary_header *header; u8 *image; @@ -74,8 +97,22 @@ static inline void bpf_flush_icache(void *start, void *end) flush_icache_range((unsigned long)start, (unsigned long)end); } +/* Emit a 4-byte riscv instruction. */ static inline void emit(const u32 insn, struct rv_jit_context *ctx) { + if (ctx->insns) { + ctx->insns[ctx->ninsns] = insn; + ctx->insns[ctx->ninsns + 1] = (insn >> 16); + } + + ctx->ninsns += 2; +} + +/* Emit a 2-byte riscv compressed instruction. */ +static inline void emitc(const u16 insn, struct rv_jit_context *ctx) +{ + BUILD_BUG_ON(!rvc_enabled()); + if (ctx->insns) ctx->insns[ctx->ninsns] = insn; @@ -86,7 +123,7 @@ static inline int epilogue_offset(struct rv_jit_context *ctx) { int to = ctx->epilogue_offset, from = ctx->ninsns; - return (to - from) << 2; + return ninsns_rvoff(to - from); } /* Return -1 or inverted cond. */ @@ -117,6 +154,36 @@ static inline int invert_bpf_cond(u8 cond) return -1; } +static inline bool is_6b_int(long val) +{ + return -(1L << 5) <= val && val < (1L << 5); +} + +static inline bool is_7b_uint(unsigned long val) +{ + return val < (1UL << 7); +} + +static inline bool is_8b_uint(unsigned long val) +{ + return val < (1UL << 8); +} + +static inline bool is_9b_uint(unsigned long val) +{ + return val < (1UL << 9); +} + +static inline bool is_10b_int(long val) +{ + return -(1L << 9) <= val && val < (1L << 9); +} + +static inline bool is_10b_uint(unsigned long val) +{ + return val < (1UL << 10); +} + static inline bool is_12b_int(long val) { return -(1L << 11) <= val && val < (1L << 11); @@ -149,7 +216,7 @@ static inline int rv_offset(int insn, int off, struct rv_jit_context *ctx) off++; /* BPF branch is from PC+1, RV is from PC */ from = (insn > 0) ? ctx->offset[insn - 1] : 0; to = (insn + off > 0) ? ctx->offset[insn + off - 1] : 0; - return (to - from) << 2; + return ninsns_rvoff(to - from); } /* Instruction formats. */ @@ -207,6 +274,59 @@ static inline u32 rv_amo_insn(u8 funct5, u8 aq, u8 rl, u8 rs2, u8 rs1, return rv_r_insn(funct7, rs2, rs1, funct3, rd, opcode); } +/* RISC-V compressed instruction formats. */ + +static inline u16 rv_cr_insn(u8 funct4, u8 rd, u8 rs2, u8 op) +{ + return (funct4 << 12) | (rd << 7) | (rs2 << 2) | op; +} + +static inline u16 rv_ci_insn(u8 funct3, u32 imm6, u8 rd, u8 op) +{ + u32 imm; + + imm = ((imm6 & 0x20) << 7) | ((imm6 & 0x1f) << 2); + return (funct3 << 13) | (rd << 7) | op | imm; +} + +static inline u16 rv_css_insn(u8 funct3, u32 uimm, u8 rs2, u8 op) +{ + return (funct3 << 13) | (uimm << 7) | (rs2 << 2) | op; +} + +static inline u16 rv_ciw_insn(u8 funct3, u32 uimm, u8 rd, u8 op) +{ + return (funct3 << 13) | (uimm << 5) | ((rd & 0x7) << 2) | op; +} + +static inline u16 rv_cl_insn(u8 funct3, u32 imm_hi, u8 rs1, u32 imm_lo, u8 rd, + u8 op) +{ + return (funct3 << 13) | (imm_hi << 10) | ((rs1 & 0x7) << 7) | + (imm_lo << 5) | ((rd & 0x7) << 2) | op; +} + +static inline u16 rv_cs_insn(u8 funct3, u32 imm_hi, u8 rs1, u32 imm_lo, u8 rs2, + u8 op) +{ + return (funct3 << 13) | (imm_hi << 10) | ((rs1 & 0x7) << 7) | + (imm_lo << 5) | ((rs2 & 0x7) << 2) | op; +} + +static inline u16 rv_ca_insn(u8 funct6, u8 rd, u8 funct2, u8 rs2, u8 op) +{ + return (funct6 << 10) | ((rd & 0x7) << 7) | (funct2 << 5) | + ((rs2 & 0x7) << 2) | op; +} + +static inline u16 rv_cb_insn(u8 funct3, u32 imm6, u8 funct2, u8 rd, u8 op) +{ + u32 imm; + + imm = ((imm6 & 0x20) << 7) | ((imm6 & 0x1f) << 2); + return (funct3 << 13) | (funct2 << 10) | ((rd & 0x7) << 7) | op | imm; +} + /* Instructions shared by both RV32 and RV64. */ static inline u32 rv_addi(u8 rd, u8 rs1, u16 imm11_0) @@ -414,6 +534,135 @@ static inline u32 rv_amoadd_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) return rv_amo_insn(0, aq, rl, rs2, rs1, 2, rd, 0x2f); } +/* RVC instrutions. */ + +static inline u16 rvc_addi4spn(u8 rd, u32 imm10) +{ + u32 imm; + + imm = ((imm10 & 0x30) << 2) | ((imm10 & 0x3c0) >> 4) | + ((imm10 & 0x4) >> 1) | ((imm10 & 0x8) >> 3); + return rv_ciw_insn(0x0, imm, rd, 0x0); +} + +static inline u16 rvc_lw(u8 rd, u32 imm7, u8 rs1) +{ + u32 imm_hi, imm_lo; + + imm_hi = (imm7 & 0x38) >> 3; + imm_lo = ((imm7 & 0x4) >> 1) | ((imm7 & 0x40) >> 6); + return rv_cl_insn(0x2, imm_hi, rs1, imm_lo, rd, 0x0); +} + +static inline u16 rvc_sw(u8 rs1, u32 imm7, u8 rs2) +{ + u32 imm_hi, imm_lo; + + imm_hi = (imm7 & 0x38) >> 3; + imm_lo = ((imm7 & 0x4) >> 1) | ((imm7 & 0x40) >> 6); + return rv_cs_insn(0x6, imm_hi, rs1, imm_lo, rs2, 0x0); +} + +static inline u16 rvc_addi(u8 rd, u32 imm6) +{ + return rv_ci_insn(0, imm6, rd, 0x1); +} + +static inline u16 rvc_li(u8 rd, u32 imm6) +{ + return rv_ci_insn(0x2, imm6, rd, 0x1); +} + +static inline u16 rvc_addi16sp(u32 imm10) +{ + u32 imm; + + imm = ((imm10 & 0x200) >> 4) | (imm10 & 0x10) | ((imm10 & 0x40) >> 3) | + ((imm10 & 0x180) >> 6) | ((imm10 & 0x20) >> 5); + return rv_ci_insn(0x3, imm, RV_REG_SP, 0x1); +} + +static inline u16 rvc_lui(u8 rd, u32 imm6) +{ + return rv_ci_insn(0x3, imm6, rd, 0x1); +} + +static inline u16 rvc_srli(u8 rd, u32 imm6) +{ + return rv_cb_insn(0x4, imm6, 0, rd, 0x1); +} + +static inline u16 rvc_srai(u8 rd, u32 imm6) +{ + return rv_cb_insn(0x4, imm6, 0x1, rd, 0x1); +} + +static inline u16 rvc_andi(u8 rd, u32 imm6) +{ + return rv_cb_insn(0x4, imm6, 0x2, rd, 0x1); +} + +static inline u16 rvc_sub(u8 rd, u8 rs) +{ + return rv_ca_insn(0x23, rd, 0, rs, 0x1); +} + +static inline u16 rvc_xor(u8 rd, u8 rs) +{ + return rv_ca_insn(0x23, rd, 0x1, rs, 0x1); +} + +static inline u16 rvc_or(u8 rd, u8 rs) +{ + return rv_ca_insn(0x23, rd, 0x2, rs, 0x1); +} + +static inline u16 rvc_and(u8 rd, u8 rs) +{ + return rv_ca_insn(0x23, rd, 0x3, rs, 0x1); +} + +static inline u16 rvc_slli(u8 rd, u32 imm6) +{ + return rv_ci_insn(0, imm6, rd, 0x2); +} + +static inline u16 rvc_lwsp(u8 rd, u32 imm8) +{ + u32 imm; + + imm = ((imm8 & 0xc0) >> 6) | (imm8 & 0x3c); + return rv_ci_insn(0x2, imm, rd, 0x2); +} + +static inline u16 rvc_jr(u8 rs1) +{ + return rv_cr_insn(0x8, rs1, RV_REG_ZERO, 0x2); +} + +static inline u16 rvc_mv(u8 rd, u8 rs) +{ + return rv_cr_insn(0x8, rd, rs, 0x2); +} + +static inline u16 rvc_jalr(u8 rs1) +{ + return rv_cr_insn(0x9, rs1, RV_REG_ZERO, 0x2); +} + +static inline u16 rvc_add(u8 rd, u8 rs) +{ + return rv_cr_insn(0x9, rd, rs, 0x2); +} + +static inline u16 rvc_swsp(u32 imm8, u8 rs2) +{ + u32 imm; + + imm = (imm8 & 0x3c) | ((imm8 & 0xc0) >> 6); + return rv_css_insn(0x6, imm, rs2, 0x2); +} + /* * RV64-only instructions. * @@ -503,6 +752,234 @@ static inline u32 rv_amoadd_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) return rv_amo_insn(0, aq, rl, rs2, rs1, 3, rd, 0x2f); } +/* RV64-only RVC instructions. */ + +static inline u16 rvc_ld(u8 rd, u32 imm8, u8 rs1) +{ + u32 imm_hi, imm_lo; + + imm_hi = (imm8 & 0x38) >> 3; + imm_lo = (imm8 & 0xc0) >> 6; + return rv_cl_insn(0x3, imm_hi, rs1, imm_lo, rd, 0x0); +} + +static inline u16 rvc_sd(u8 rs1, u32 imm8, u8 rs2) +{ + u32 imm_hi, imm_lo; + + imm_hi = (imm8 & 0x38) >> 3; + imm_lo = (imm8 & 0xc0) >> 6; + return rv_cs_insn(0x7, imm_hi, rs1, imm_lo, rs2, 0x0); +} + +static inline u16 rvc_subw(u8 rd, u8 rs) +{ + return rv_ca_insn(0x27, rd, 0, rs, 0x1); +} + +static inline u16 rvc_addiw(u8 rd, u32 imm6) +{ + return rv_ci_insn(0x1, imm6, rd, 0x1); +} + +static inline u16 rvc_ldsp(u8 rd, u32 imm9) +{ + u32 imm; + + imm = ((imm9 & 0x1c0) >> 6) | (imm9 & 0x38); + return rv_ci_insn(0x3, imm, rd, 0x2); +} + +static inline u16 rvc_sdsp(u32 imm9, u8 rs2) +{ + u32 imm; + + imm = (imm9 & 0x38) | ((imm9 & 0x1c0) >> 6); + return rv_css_insn(0x7, imm, rs2, 0x2); +} + +#endif /* __riscv_xlen == 64 */ + +/* Helper functions that emit RVC instructions when possible. */ + +static inline void emit_jalr(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) +{ + if (rvc_enabled() && rd == RV_REG_RA && rs && !imm) + emitc(rvc_jalr(rs), ctx); + else if (rvc_enabled() && !rd && rs && !imm) + emitc(rvc_jr(rs), ctx); + else + emit(rv_jalr(rd, rs, imm), ctx); +} + +static inline void emit_mv(u8 rd, u8 rs, struct rv_jit_context *ctx) +{ + if (rvc_enabled() && rd && rs) + emitc(rvc_mv(rd, rs), ctx); + else + emit(rv_addi(rd, rs, 0), ctx); +} + +static inline void emit_add(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) +{ + if (rvc_enabled() && rd && rd == rs1 && rs2) + emitc(rvc_add(rd, rs2), ctx); + else + emit(rv_add(rd, rs1, rs2), ctx); +} + +static inline void emit_addi(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) +{ + if (rvc_enabled() && rd == RV_REG_SP && rd == rs && is_10b_int(imm) && imm && !(imm & 0xf)) + emitc(rvc_addi16sp(imm), ctx); + else if (rvc_enabled() && is_creg(rd) && rs == RV_REG_SP && is_10b_uint(imm) && + !(imm & 0x3) && imm) + emitc(rvc_addi4spn(rd, imm), ctx); + else if (rvc_enabled() && rd && rd == rs && imm && is_6b_int(imm)) + emitc(rvc_addi(rd, imm), ctx); + else + emit(rv_addi(rd, rs, imm), ctx); +} + +static inline void emit_li(u8 rd, s32 imm, struct rv_jit_context *ctx) +{ + if (rvc_enabled() && rd && is_6b_int(imm)) + emitc(rvc_li(rd, imm), ctx); + else + emit(rv_addi(rd, RV_REG_ZERO, imm), ctx); +} + +static inline void emit_lui(u8 rd, s32 imm, struct rv_jit_context *ctx) +{ + if (rvc_enabled() && rd && rd != RV_REG_SP && is_6b_int(imm) && imm) + emitc(rvc_lui(rd, imm), ctx); + else + emit(rv_lui(rd, imm), ctx); +} + +static inline void emit_slli(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) +{ + if (rvc_enabled() && rd && rd == rs && imm && (u32)imm < __riscv_xlen) + emitc(rvc_slli(rd, imm), ctx); + else + emit(rv_slli(rd, rs, imm), ctx); +} + +static inline void emit_andi(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) +{ + if (rvc_enabled() && is_creg(rd) && rd == rs && is_6b_int(imm)) + emitc(rvc_andi(rd, imm), ctx); + else + emit(rv_andi(rd, rs, imm), ctx); +} + +static inline void emit_srli(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) +{ + if (rvc_enabled() && is_creg(rd) && rd == rs && imm && (u32)imm < __riscv_xlen) + emitc(rvc_srli(rd, imm), ctx); + else + emit(rv_srli(rd, rs, imm), ctx); +} + +static inline void emit_srai(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) +{ + if (rvc_enabled() && is_creg(rd) && rd == rs && imm && (u32)imm < __riscv_xlen) + emitc(rvc_srai(rd, imm), ctx); + else + emit(rv_srai(rd, rs, imm), ctx); +} + +static inline void emit_sub(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) +{ + if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2)) + emitc(rvc_sub(rd, rs2), ctx); + else + emit(rv_sub(rd, rs1, rs2), ctx); +} + +static inline void emit_or(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) +{ + if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2)) + emitc(rvc_or(rd, rs2), ctx); + else + emit(rv_or(rd, rs1, rs2), ctx); +} + +static inline void emit_and(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) +{ + if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2)) + emitc(rvc_and(rd, rs2), ctx); + else + emit(rv_and(rd, rs1, rs2), ctx); +} + +static inline void emit_xor(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) +{ + if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2)) + emitc(rvc_xor(rd, rs2), ctx); + else + emit(rv_xor(rd, rs1, rs2), ctx); +} + +static inline void emit_lw(u8 rd, s32 off, u8 rs1, struct rv_jit_context *ctx) +{ + if (rvc_enabled() && rs1 == RV_REG_SP && rd && is_8b_uint(off) && !(off & 0x3)) + emitc(rvc_lwsp(rd, off), ctx); + else if (rvc_enabled() && is_creg(rd) && is_creg(rs1) && is_7b_uint(off) && !(off & 0x3)) + emitc(rvc_lw(rd, off, rs1), ctx); + else + emit(rv_lw(rd, off, rs1), ctx); +} + +static inline void emit_sw(u8 rs1, s32 off, u8 rs2, struct rv_jit_context *ctx) +{ + if (rvc_enabled() && rs1 == RV_REG_SP && is_8b_uint(off) && !(off & 0x3)) + emitc(rvc_swsp(off, rs2), ctx); + else if (rvc_enabled() && is_creg(rs1) && is_creg(rs2) && is_7b_uint(off) && !(off & 0x3)) + emitc(rvc_sw(rs1, off, rs2), ctx); + else + emit(rv_sw(rs1, off, rs2), ctx); +} + +/* RV64-only helper functions. */ +#if __riscv_xlen == 64 + +static inline void emit_addiw(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) +{ + if (rvc_enabled() && rd && rd == rs && is_6b_int(imm)) + emitc(rvc_addiw(rd, imm), ctx); + else + emit(rv_addiw(rd, rs, imm), ctx); +} + +static inline void emit_ld(u8 rd, s32 off, u8 rs1, struct rv_jit_context *ctx) +{ + if (rvc_enabled() && rs1 == RV_REG_SP && rd && is_9b_uint(off) && !(off & 0x7)) + emitc(rvc_ldsp(rd, off), ctx); + else if (rvc_enabled() && is_creg(rd) && is_creg(rs1) && is_8b_uint(off) && !(off & 0x7)) + emitc(rvc_ld(rd, off, rs1), ctx); + else + emit(rv_ld(rd, off, rs1), ctx); +} + +static inline void emit_sd(u8 rs1, s32 off, u8 rs2, struct rv_jit_context *ctx) +{ + if (rvc_enabled() && rs1 == RV_REG_SP && is_9b_uint(off) && !(off & 0x7)) + emitc(rvc_sdsp(off, rs2), ctx); + else if (rvc_enabled() && is_creg(rs1) && is_creg(rs2) && is_8b_uint(off) && !(off & 0x7)) + emitc(rvc_sd(rs1, off, rs2), ctx); + else + emit(rv_sd(rs1, off, rs2), ctx); +} + +static inline void emit_subw(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) +{ + if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2)) + emitc(rvc_subw(rd, rs2), ctx); + else + emit(rv_subw(rd, rs1, rs2), ctx); +} + #endif /* __riscv_xlen == 64 */ void bpf_jit_build_prologue(struct rv_jit_context *ctx); diff --git a/arch/riscv/net/bpf_jit_comp32.c b/arch/riscv/net/bpf_jit_comp32.c index 302934177760..579575f9cdae 100644 --- a/arch/riscv/net/bpf_jit_comp32.c +++ b/arch/riscv/net/bpf_jit_comp32.c @@ -13,8 +13,35 @@ #include <linux/filter.h> #include "bpf_jit.h" +/* + * Stack layout during BPF program execution: + * + * high + * RV32 fp => +----------+ + * | saved ra | + * | saved fp | RV32 callee-saved registers + * | ... | + * +----------+ <= (fp - 4 * NR_SAVED_REGISTERS) + * | hi(R6) | + * | lo(R6) | + * | hi(R7) | JIT scratch space for BPF registers + * | lo(R7) | + * | ... | + * BPF_REG_FP => +----------+ <= (fp - 4 * NR_SAVED_REGISTERS + * | | - 4 * BPF_JIT_SCRATCH_REGS) + * | | + * | ... | BPF program stack + * | | + * RV32 sp => +----------+ + * | | + * | ... | Function call stack + * | | + * +----------+ + * low + */ + enum { - /* Stack layout - these are offsets from (top of stack - 4). */ + /* Stack layout - these are offsets from top of JIT scratch space. */ BPF_R6_HI, BPF_R6_LO, BPF_R7_HI, @@ -29,7 +56,11 @@ enum { BPF_JIT_SCRATCH_REGS, }; -#define STACK_OFFSET(k) (-4 - ((k) * 4)) +/* Number of callee-saved registers stored to stack: ra, fp, s1--s7. */ +#define NR_SAVED_REGISTERS 9 + +/* Offset from fp for BPF registers stored on stack. */ +#define STACK_OFFSET(k) (-4 - (4 * NR_SAVED_REGISTERS) - (4 * (k))) #define TMP_REG_1 (MAX_BPF_JIT_REG + 0) #define TMP_REG_2 (MAX_BPF_JIT_REG + 1) @@ -111,11 +142,9 @@ static void emit_imm64(const s8 *rd, s32 imm_hi, s32 imm_lo, static void __build_epilogue(bool is_tail_call, struct rv_jit_context *ctx) { - int stack_adjust = ctx->stack_size, store_offset = stack_adjust - 4; + int stack_adjust = ctx->stack_size; const s8 *r0 = bpf2rv32[BPF_REG_0]; - store_offset -= 4 * BPF_JIT_SCRATCH_REGS; - /* Set return value if not tail call. */ if (!is_tail_call) { emit(rv_addi(RV_REG_A0, lo(r0), 0), ctx); @@ -123,15 +152,15 @@ static void __build_epilogue(bool is_tail_call, struct rv_jit_context *ctx) } /* Restore callee-saved registers. */ - emit(rv_lw(RV_REG_RA, store_offset - 0, RV_REG_SP), ctx); - emit(rv_lw(RV_REG_FP, store_offset - 4, RV_REG_SP), ctx); - emit(rv_lw(RV_REG_S1, store_offset - 8, RV_REG_SP), ctx); - emit(rv_lw(RV_REG_S2, store_offset - 12, RV_REG_SP), ctx); - emit(rv_lw(RV_REG_S3, store_offset - 16, RV_REG_SP), ctx); - emit(rv_lw(RV_REG_S4, store_offset - 20, RV_REG_SP), ctx); - emit(rv_lw(RV_REG_S5, store_offset - 24, RV_REG_SP), ctx); - emit(rv_lw(RV_REG_S6, store_offset - 28, RV_REG_SP), ctx); - emit(rv_lw(RV_REG_S7, store_offset - 32, RV_REG_SP), ctx); + emit(rv_lw(RV_REG_RA, stack_adjust - 4, RV_REG_SP), ctx); + emit(rv_lw(RV_REG_FP, stack_adjust - 8, RV_REG_SP), ctx); + emit(rv_lw(RV_REG_S1, stack_adjust - 12, RV_REG_SP), ctx); + emit(rv_lw(RV_REG_S2, stack_adjust - 16, RV_REG_SP), ctx); + emit(rv_lw(RV_REG_S3, stack_adjust - 20, RV_REG_SP), ctx); + emit(rv_lw(RV_REG_S4, stack_adjust - 24, RV_REG_SP), ctx); + emit(rv_lw(RV_REG_S5, stack_adjust - 28, RV_REG_SP), ctx); + emit(rv_lw(RV_REG_S6, stack_adjust - 32, RV_REG_SP), ctx); + emit(rv_lw(RV_REG_S7, stack_adjust - 36, RV_REG_SP), ctx); emit(rv_addi(RV_REG_SP, RV_REG_SP, stack_adjust), ctx); @@ -615,7 +644,7 @@ static int emit_branch_r64(const s8 *src1, const s8 *src2, s32 rvoff, e = ctx->ninsns; /* Adjust for extra insns. */ - rvoff -= (e - s) << 2; + rvoff -= ninsns_rvoff(e - s); emit_jump_and_link(RV_REG_ZERO, rvoff, true, ctx); return 0; } @@ -684,7 +713,7 @@ static int emit_bcc(u8 op, u8 rd, u8 rs, int rvoff, struct rv_jit_context *ctx) if (far) { e = ctx->ninsns; /* Adjust for extra insns. */ - rvoff -= (e - s) << 2; + rvoff -= ninsns_rvoff(e - s); emit_jump_and_link(RV_REG_ZERO, rvoff, true, ctx); } return 0; @@ -702,7 +731,7 @@ static int emit_branch_r32(const s8 *src1, const s8 *src2, s32 rvoff, e = ctx->ninsns; /* Adjust for extra insns. */ - rvoff -= (e - s) << 2; + rvoff -= ninsns_rvoff(e - s); if (emit_bcc(op, lo(rs1), lo(rs2), rvoff, ctx)) return -1; @@ -766,16 +795,17 @@ static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) * if (index >= max_entries) * goto out; */ - off = (tc_ninsn - (ctx->ninsns - start_insn)) << 2; + off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); emit_bcc(BPF_JGE, lo(idx_reg), RV_REG_T1, off, ctx); /* - * if ((temp_tcc = tcc - 1) < 0) + * temp_tcc = tcc - 1; + * if (tcc < 0) * goto out; */ emit(rv_addi(RV_REG_T1, RV_REG_TCC, -1), ctx); - off = (tc_ninsn - (ctx->ninsns - start_insn)) << 2; - emit_bcc(BPF_JSLT, RV_REG_T1, RV_REG_ZERO, off, ctx); + off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); + emit_bcc(BPF_JSLT, RV_REG_TCC, RV_REG_ZERO, off, ctx); /* * prog = array->ptrs[index]; @@ -788,7 +818,7 @@ static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) if (is_12b_check(off, insn)) return -1; emit(rv_lw(RV_REG_T0, off, RV_REG_T0), ctx); - off = (tc_ninsn - (ctx->ninsns - start_insn)) << 2; + off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); emit_bcc(BPF_JEQ, RV_REG_T0, RV_REG_ZERO, off, ctx); /* @@ -990,7 +1020,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, emit_zext64(dst, ctx); break; } - /* Fallthrough. */ + fallthrough; case BPF_ALU | BPF_ADD | BPF_X: case BPF_ALU | BPF_SUB | BPF_X: @@ -1049,7 +1079,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, case 16: emit(rv_slli(lo(rd), lo(rd), 16), ctx); emit(rv_srli(lo(rd), lo(rd), 16), ctx); - /* Fallthrough. */ + fallthrough; case 32: if (!ctx->prog->aux->verifier_zext) emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx); @@ -1184,7 +1214,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, emit_imm32(tmp2, imm, ctx); src = tmp2; e = ctx->ninsns; - rvoff -= (e - s) << 2; + rvoff -= ninsns_rvoff(e - s); } if (is64) @@ -1259,17 +1289,20 @@ notsupported: void bpf_jit_build_prologue(struct rv_jit_context *ctx) { - /* Make space to save 9 registers: ra, fp, s1--s7. */ - int stack_adjust = 9 * sizeof(u32), store_offset, bpf_stack_adjust; const s8 *fp = bpf2rv32[BPF_REG_FP]; const s8 *r1 = bpf2rv32[BPF_REG_1]; - - bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16); + int stack_adjust = 0; + int bpf_stack_adjust = + round_up(ctx->prog->aux->stack_depth, STACK_ALIGN); + + /* Make space for callee-saved registers. */ + stack_adjust += NR_SAVED_REGISTERS * sizeof(u32); + /* Make space for BPF registers on stack. */ + stack_adjust += BPF_JIT_SCRATCH_REGS * sizeof(u32); + /* Make space for BPF stack. */ stack_adjust += bpf_stack_adjust; - - store_offset = stack_adjust - 4; - - stack_adjust += 4 * BPF_JIT_SCRATCH_REGS; + /* Round up for stack alignment. */ + stack_adjust = round_up(stack_adjust, STACK_ALIGN); /* * The first instruction sets the tail-call-counter (TCC) register. @@ -1280,24 +1313,24 @@ void bpf_jit_build_prologue(struct rv_jit_context *ctx) emit(rv_addi(RV_REG_SP, RV_REG_SP, -stack_adjust), ctx); /* Save callee-save registers. */ - emit(rv_sw(RV_REG_SP, store_offset - 0, RV_REG_RA), ctx); - emit(rv_sw(RV_REG_SP, store_offset - 4, RV_REG_FP), ctx); - emit(rv_sw(RV_REG_SP, store_offset - 8, RV_REG_S1), ctx); - emit(rv_sw(RV_REG_SP, store_offset - 12, RV_REG_S2), ctx); - emit(rv_sw(RV_REG_SP, store_offset - 16, RV_REG_S3), ctx); - emit(rv_sw(RV_REG_SP, store_offset - 20, RV_REG_S4), ctx); - emit(rv_sw(RV_REG_SP, store_offset - 24, RV_REG_S5), ctx); - emit(rv_sw(RV_REG_SP, store_offset - 28, RV_REG_S6), ctx); - emit(rv_sw(RV_REG_SP, store_offset - 32, RV_REG_S7), ctx); + emit(rv_sw(RV_REG_SP, stack_adjust - 4, RV_REG_RA), ctx); + emit(rv_sw(RV_REG_SP, stack_adjust - 8, RV_REG_FP), ctx); + emit(rv_sw(RV_REG_SP, stack_adjust - 12, RV_REG_S1), ctx); + emit(rv_sw(RV_REG_SP, stack_adjust - 16, RV_REG_S2), ctx); + emit(rv_sw(RV_REG_SP, stack_adjust - 20, RV_REG_S3), ctx); + emit(rv_sw(RV_REG_SP, stack_adjust - 24, RV_REG_S4), ctx); + emit(rv_sw(RV_REG_SP, stack_adjust - 28, RV_REG_S5), ctx); + emit(rv_sw(RV_REG_SP, stack_adjust - 32, RV_REG_S6), ctx); + emit(rv_sw(RV_REG_SP, stack_adjust - 36, RV_REG_S7), ctx); /* Set fp: used as the base address for stacked BPF registers. */ emit(rv_addi(RV_REG_FP, RV_REG_SP, stack_adjust), ctx); - /* Set up BPF stack pointer. */ + /* Set up BPF frame pointer. */ emit(rv_addi(lo(fp), RV_REG_SP, bpf_stack_adjust), ctx); emit(rv_addi(hi(fp), RV_REG_ZERO, 0), ctx); - /* Set up context pointer. */ + /* Set up BPF context pointer. */ emit(rv_addi(lo(r1), RV_REG_A0, 0), ctx); emit(rv_addi(hi(r1), RV_REG_ZERO, 0), ctx); diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index d208a9fd6c52..8a56b5293117 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -132,19 +132,23 @@ static void emit_imm(u8 rd, s64 val, struct rv_jit_context *ctx) * * This also means that we need to process LSB to MSB. */ - s64 upper = (val + (1 << 11)) >> 12, lower = val & 0xfff; + s64 upper = (val + (1 << 11)) >> 12; + /* Sign-extend lower 12 bits to 64 bits since immediates for li, addiw, + * and addi are signed and RVC checks will perform signed comparisons. + */ + s64 lower = ((val & 0xfff) << 52) >> 52; int shift; if (is_32b_int(val)) { if (upper) - emit(rv_lui(rd, upper), ctx); + emit_lui(rd, upper, ctx); if (!upper) { - emit(rv_addi(rd, RV_REG_ZERO, lower), ctx); + emit_li(rd, lower, ctx); return; } - emit(rv_addiw(rd, rd, lower), ctx); + emit_addiw(rd, rd, lower, ctx); return; } @@ -154,9 +158,9 @@ static void emit_imm(u8 rd, s64 val, struct rv_jit_context *ctx) emit_imm(rd, upper, ctx); - emit(rv_slli(rd, rd, shift), ctx); + emit_slli(rd, rd, shift, ctx); if (lower) - emit(rv_addi(rd, rd, lower), ctx); + emit_addi(rd, rd, lower, ctx); } static void __build_epilogue(bool is_tail_call, struct rv_jit_context *ctx) @@ -164,43 +168,43 @@ static void __build_epilogue(bool is_tail_call, struct rv_jit_context *ctx) int stack_adjust = ctx->stack_size, store_offset = stack_adjust - 8; if (seen_reg(RV_REG_RA, ctx)) { - emit(rv_ld(RV_REG_RA, store_offset, RV_REG_SP), ctx); + emit_ld(RV_REG_RA, store_offset, RV_REG_SP, ctx); store_offset -= 8; } - emit(rv_ld(RV_REG_FP, store_offset, RV_REG_SP), ctx); + emit_ld(RV_REG_FP, store_offset, RV_REG_SP, ctx); store_offset -= 8; if (seen_reg(RV_REG_S1, ctx)) { - emit(rv_ld(RV_REG_S1, store_offset, RV_REG_SP), ctx); + emit_ld(RV_REG_S1, store_offset, RV_REG_SP, ctx); store_offset -= 8; } if (seen_reg(RV_REG_S2, ctx)) { - emit(rv_ld(RV_REG_S2, store_offset, RV_REG_SP), ctx); + emit_ld(RV_REG_S2, store_offset, RV_REG_SP, ctx); store_offset -= 8; } if (seen_reg(RV_REG_S3, ctx)) { - emit(rv_ld(RV_REG_S3, store_offset, RV_REG_SP), ctx); + emit_ld(RV_REG_S3, store_offset, RV_REG_SP, ctx); store_offset -= 8; } if (seen_reg(RV_REG_S4, ctx)) { - emit(rv_ld(RV_REG_S4, store_offset, RV_REG_SP), ctx); + emit_ld(RV_REG_S4, store_offset, RV_REG_SP, ctx); store_offset -= 8; } if (seen_reg(RV_REG_S5, ctx)) { - emit(rv_ld(RV_REG_S5, store_offset, RV_REG_SP), ctx); + emit_ld(RV_REG_S5, store_offset, RV_REG_SP, ctx); store_offset -= 8; } if (seen_reg(RV_REG_S6, ctx)) { - emit(rv_ld(RV_REG_S6, store_offset, RV_REG_SP), ctx); + emit_ld(RV_REG_S6, store_offset, RV_REG_SP, ctx); store_offset -= 8; } - emit(rv_addi(RV_REG_SP, RV_REG_SP, stack_adjust), ctx); + emit_addi(RV_REG_SP, RV_REG_SP, stack_adjust, ctx); /* Set return value. */ if (!is_tail_call) - emit(rv_addi(RV_REG_A0, RV_REG_A5, 0), ctx); - emit(rv_jalr(RV_REG_ZERO, is_tail_call ? RV_REG_T3 : RV_REG_RA, - is_tail_call ? 4 : 0), /* skip TCC init */ - ctx); + emit_mv(RV_REG_A0, RV_REG_A5, ctx); + emit_jalr(RV_REG_ZERO, is_tail_call ? RV_REG_T3 : RV_REG_RA, + is_tail_call ? 4 : 0, /* skip TCC init */ + ctx); } static void emit_bcc(u8 cond, u8 rd, u8 rs, int rvoff, @@ -280,8 +284,8 @@ static void emit_branch(u8 cond, u8 rd, u8 rs, int rvoff, static void emit_zext_32(u8 reg, struct rv_jit_context *ctx) { - emit(rv_slli(reg, reg, 32), ctx); - emit(rv_srli(reg, reg, 32), ctx); + emit_slli(reg, reg, 32, ctx); + emit_srli(reg, reg, 32, ctx); } static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) @@ -304,35 +308,35 @@ static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) if (is_12b_check(off, insn)) return -1; emit(rv_lwu(RV_REG_T1, off, RV_REG_A1), ctx); - off = (tc_ninsn - (ctx->ninsns - start_insn)) << 2; + off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); emit_branch(BPF_JGE, RV_REG_A2, RV_REG_T1, off, ctx); /* if (TCC-- < 0) * goto out; */ - emit(rv_addi(RV_REG_T1, tcc, -1), ctx); - off = (tc_ninsn - (ctx->ninsns - start_insn)) << 2; + emit_addi(RV_REG_T1, tcc, -1, ctx); + off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); emit_branch(BPF_JSLT, tcc, RV_REG_ZERO, off, ctx); /* prog = array->ptrs[index]; * if (!prog) * goto out; */ - emit(rv_slli(RV_REG_T2, RV_REG_A2, 3), ctx); - emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_A1), ctx); + emit_slli(RV_REG_T2, RV_REG_A2, 3, ctx); + emit_add(RV_REG_T2, RV_REG_T2, RV_REG_A1, ctx); off = offsetof(struct bpf_array, ptrs); if (is_12b_check(off, insn)) return -1; - emit(rv_ld(RV_REG_T2, off, RV_REG_T2), ctx); - off = (tc_ninsn - (ctx->ninsns - start_insn)) << 2; + emit_ld(RV_REG_T2, off, RV_REG_T2, ctx); + off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); emit_branch(BPF_JEQ, RV_REG_T2, RV_REG_ZERO, off, ctx); /* goto *(prog->bpf_func + 4); */ off = offsetof(struct bpf_prog, bpf_func); if (is_12b_check(off, insn)) return -1; - emit(rv_ld(RV_REG_T3, off, RV_REG_T2), ctx); - emit(rv_addi(RV_REG_TCC, RV_REG_T1, 0), ctx); + emit_ld(RV_REG_T3, off, RV_REG_T2, ctx); + emit_mv(RV_REG_TCC, RV_REG_T1, ctx); __build_epilogue(true, ctx); return 0; } @@ -360,9 +364,9 @@ static void init_regs(u8 *rd, u8 *rs, const struct bpf_insn *insn, static void emit_zext_32_rd_rs(u8 *rd, u8 *rs, struct rv_jit_context *ctx) { - emit(rv_addi(RV_REG_T2, *rd, 0), ctx); + emit_mv(RV_REG_T2, *rd, ctx); emit_zext_32(RV_REG_T2, ctx); - emit(rv_addi(RV_REG_T1, *rs, 0), ctx); + emit_mv(RV_REG_T1, *rs, ctx); emit_zext_32(RV_REG_T1, ctx); *rd = RV_REG_T2; *rs = RV_REG_T1; @@ -370,15 +374,15 @@ static void emit_zext_32_rd_rs(u8 *rd, u8 *rs, struct rv_jit_context *ctx) static void emit_sext_32_rd_rs(u8 *rd, u8 *rs, struct rv_jit_context *ctx) { - emit(rv_addiw(RV_REG_T2, *rd, 0), ctx); - emit(rv_addiw(RV_REG_T1, *rs, 0), ctx); + emit_addiw(RV_REG_T2, *rd, 0, ctx); + emit_addiw(RV_REG_T1, *rs, 0, ctx); *rd = RV_REG_T2; *rs = RV_REG_T1; } static void emit_zext_32_rd_t1(u8 *rd, struct rv_jit_context *ctx) { - emit(rv_addi(RV_REG_T2, *rd, 0), ctx); + emit_mv(RV_REG_T2, *rd, ctx); emit_zext_32(RV_REG_T2, ctx); emit_zext_32(RV_REG_T1, ctx); *rd = RV_REG_T2; @@ -386,7 +390,7 @@ static void emit_zext_32_rd_t1(u8 *rd, struct rv_jit_context *ctx) static void emit_sext_32_rd(u8 *rd, struct rv_jit_context *ctx) { - emit(rv_addiw(RV_REG_T2, *rd, 0), ctx); + emit_addiw(RV_REG_T2, *rd, 0, ctx); *rd = RV_REG_T2; } @@ -432,7 +436,7 @@ static int emit_call(bool fixed, u64 addr, struct rv_jit_context *ctx) if (ret) return ret; rd = bpf_to_rv_reg(BPF_REG_0, ctx); - emit(rv_addi(rd, RV_REG_A0, 0), ctx); + emit_mv(rd, RV_REG_A0, ctx); return 0; } @@ -458,7 +462,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, emit_zext_32(rd, ctx); break; } - emit(is64 ? rv_addi(rd, rs, 0) : rv_addiw(rd, rs, 0), ctx); + emit_mv(rd, rs, ctx); if (!is64 && !aux->verifier_zext) emit_zext_32(rd, ctx); break; @@ -466,31 +470,35 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, /* dst = dst OP src */ case BPF_ALU | BPF_ADD | BPF_X: case BPF_ALU64 | BPF_ADD | BPF_X: - emit(is64 ? rv_add(rd, rd, rs) : rv_addw(rd, rd, rs), ctx); + emit_add(rd, rd, rs, ctx); if (!is64 && !aux->verifier_zext) emit_zext_32(rd, ctx); break; case BPF_ALU | BPF_SUB | BPF_X: case BPF_ALU64 | BPF_SUB | BPF_X: - emit(is64 ? rv_sub(rd, rd, rs) : rv_subw(rd, rd, rs), ctx); + if (is64) + emit_sub(rd, rd, rs, ctx); + else + emit_subw(rd, rd, rs, ctx); + if (!is64 && !aux->verifier_zext) emit_zext_32(rd, ctx); break; case BPF_ALU | BPF_AND | BPF_X: case BPF_ALU64 | BPF_AND | BPF_X: - emit(rv_and(rd, rd, rs), ctx); + emit_and(rd, rd, rs, ctx); if (!is64 && !aux->verifier_zext) emit_zext_32(rd, ctx); break; case BPF_ALU | BPF_OR | BPF_X: case BPF_ALU64 | BPF_OR | BPF_X: - emit(rv_or(rd, rd, rs), ctx); + emit_or(rd, rd, rs, ctx); if (!is64 && !aux->verifier_zext) emit_zext_32(rd, ctx); break; case BPF_ALU | BPF_XOR | BPF_X: case BPF_ALU64 | BPF_XOR | BPF_X: - emit(rv_xor(rd, rd, rs), ctx); + emit_xor(rd, rd, rs, ctx); if (!is64 && !aux->verifier_zext) emit_zext_32(rd, ctx); break; @@ -515,7 +523,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, case BPF_ALU | BPF_LSH | BPF_X: case BPF_ALU64 | BPF_LSH | BPF_X: emit(is64 ? rv_sll(rd, rd, rs) : rv_sllw(rd, rd, rs), ctx); - if (!is64) + if (!is64 && !aux->verifier_zext) emit_zext_32(rd, ctx); break; case BPF_ALU | BPF_RSH | BPF_X: @@ -534,67 +542,74 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, /* dst = -dst */ case BPF_ALU | BPF_NEG: case BPF_ALU64 | BPF_NEG: - emit(is64 ? rv_sub(rd, RV_REG_ZERO, rd) : - rv_subw(rd, RV_REG_ZERO, rd), ctx); + emit_sub(rd, RV_REG_ZERO, rd, ctx); if (!is64 && !aux->verifier_zext) emit_zext_32(rd, ctx); break; /* dst = BSWAP##imm(dst) */ case BPF_ALU | BPF_END | BPF_FROM_LE: - { - int shift = 64 - imm; - - emit(rv_slli(rd, rd, shift), ctx); - emit(rv_srli(rd, rd, shift), ctx); + switch (imm) { + case 16: + emit_slli(rd, rd, 48, ctx); + emit_srli(rd, rd, 48, ctx); + break; + case 32: + if (!aux->verifier_zext) + emit_zext_32(rd, ctx); + break; + case 64: + /* Do nothing */ + break; + } break; - } + case BPF_ALU | BPF_END | BPF_FROM_BE: - emit(rv_addi(RV_REG_T2, RV_REG_ZERO, 0), ctx); + emit_li(RV_REG_T2, 0, ctx); - emit(rv_andi(RV_REG_T1, rd, 0xff), ctx); - emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx); - emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx); - emit(rv_srli(rd, rd, 8), ctx); + emit_andi(RV_REG_T1, rd, 0xff, ctx); + emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); + emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); + emit_srli(rd, rd, 8, ctx); if (imm == 16) goto out_be; - emit(rv_andi(RV_REG_T1, rd, 0xff), ctx); - emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx); - emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx); - emit(rv_srli(rd, rd, 8), ctx); + emit_andi(RV_REG_T1, rd, 0xff, ctx); + emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); + emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); + emit_srli(rd, rd, 8, ctx); - emit(rv_andi(RV_REG_T1, rd, 0xff), ctx); - emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx); - emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx); - emit(rv_srli(rd, rd, 8), ctx); + emit_andi(RV_REG_T1, rd, 0xff, ctx); + emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); + emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); + emit_srli(rd, rd, 8, ctx); if (imm == 32) goto out_be; - emit(rv_andi(RV_REG_T1, rd, 0xff), ctx); - emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx); - emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx); - emit(rv_srli(rd, rd, 8), ctx); - - emit(rv_andi(RV_REG_T1, rd, 0xff), ctx); - emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx); - emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx); - emit(rv_srli(rd, rd, 8), ctx); - - emit(rv_andi(RV_REG_T1, rd, 0xff), ctx); - emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx); - emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx); - emit(rv_srli(rd, rd, 8), ctx); - - emit(rv_andi(RV_REG_T1, rd, 0xff), ctx); - emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx); - emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx); - emit(rv_srli(rd, rd, 8), ctx); + emit_andi(RV_REG_T1, rd, 0xff, ctx); + emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); + emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); + emit_srli(rd, rd, 8, ctx); + + emit_andi(RV_REG_T1, rd, 0xff, ctx); + emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); + emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); + emit_srli(rd, rd, 8, ctx); + + emit_andi(RV_REG_T1, rd, 0xff, ctx); + emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); + emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); + emit_srli(rd, rd, 8, ctx); + + emit_andi(RV_REG_T1, rd, 0xff, ctx); + emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); + emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); + emit_srli(rd, rd, 8, ctx); out_be: - emit(rv_andi(RV_REG_T1, rd, 0xff), ctx); - emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx); + emit_andi(RV_REG_T1, rd, 0xff, ctx); + emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); - emit(rv_addi(rd, RV_REG_T2, 0), ctx); + emit_mv(rd, RV_REG_T2, ctx); break; /* dst = imm */ @@ -609,12 +624,10 @@ out_be: case BPF_ALU | BPF_ADD | BPF_K: case BPF_ALU64 | BPF_ADD | BPF_K: if (is_12b_int(imm)) { - emit(is64 ? rv_addi(rd, rd, imm) : - rv_addiw(rd, rd, imm), ctx); + emit_addi(rd, rd, imm, ctx); } else { emit_imm(RV_REG_T1, imm, ctx); - emit(is64 ? rv_add(rd, rd, RV_REG_T1) : - rv_addw(rd, rd, RV_REG_T1), ctx); + emit_add(rd, rd, RV_REG_T1, ctx); } if (!is64 && !aux->verifier_zext) emit_zext_32(rd, ctx); @@ -622,12 +635,10 @@ out_be: case BPF_ALU | BPF_SUB | BPF_K: case BPF_ALU64 | BPF_SUB | BPF_K: if (is_12b_int(-imm)) { - emit(is64 ? rv_addi(rd, rd, -imm) : - rv_addiw(rd, rd, -imm), ctx); + emit_addi(rd, rd, -imm, ctx); } else { emit_imm(RV_REG_T1, imm, ctx); - emit(is64 ? rv_sub(rd, rd, RV_REG_T1) : - rv_subw(rd, rd, RV_REG_T1), ctx); + emit_sub(rd, rd, RV_REG_T1, ctx); } if (!is64 && !aux->verifier_zext) emit_zext_32(rd, ctx); @@ -635,10 +646,10 @@ out_be: case BPF_ALU | BPF_AND | BPF_K: case BPF_ALU64 | BPF_AND | BPF_K: if (is_12b_int(imm)) { - emit(rv_andi(rd, rd, imm), ctx); + emit_andi(rd, rd, imm, ctx); } else { emit_imm(RV_REG_T1, imm, ctx); - emit(rv_and(rd, rd, RV_REG_T1), ctx); + emit_and(rd, rd, RV_REG_T1, ctx); } if (!is64 && !aux->verifier_zext) emit_zext_32(rd, ctx); @@ -649,7 +660,7 @@ out_be: emit(rv_ori(rd, rd, imm), ctx); } else { emit_imm(RV_REG_T1, imm, ctx); - emit(rv_or(rd, rd, RV_REG_T1), ctx); + emit_or(rd, rd, RV_REG_T1, ctx); } if (!is64 && !aux->verifier_zext) emit_zext_32(rd, ctx); @@ -660,7 +671,7 @@ out_be: emit(rv_xori(rd, rd, imm), ctx); } else { emit_imm(RV_REG_T1, imm, ctx); - emit(rv_xor(rd, rd, RV_REG_T1), ctx); + emit_xor(rd, rd, RV_REG_T1, ctx); } if (!is64 && !aux->verifier_zext) emit_zext_32(rd, ctx); @@ -691,20 +702,29 @@ out_be: break; case BPF_ALU | BPF_LSH | BPF_K: case BPF_ALU64 | BPF_LSH | BPF_K: - emit(is64 ? rv_slli(rd, rd, imm) : rv_slliw(rd, rd, imm), ctx); - if (!is64) + emit_slli(rd, rd, imm, ctx); + + if (!is64 && !aux->verifier_zext) emit_zext_32(rd, ctx); break; case BPF_ALU | BPF_RSH | BPF_K: case BPF_ALU64 | BPF_RSH | BPF_K: - emit(is64 ? rv_srli(rd, rd, imm) : rv_srliw(rd, rd, imm), ctx); - if (!is64) + if (is64) + emit_srli(rd, rd, imm, ctx); + else + emit(rv_srliw(rd, rd, imm), ctx); + + if (!is64 && !aux->verifier_zext) emit_zext_32(rd, ctx); break; case BPF_ALU | BPF_ARSH | BPF_K: case BPF_ALU64 | BPF_ARSH | BPF_K: - emit(is64 ? rv_srai(rd, rd, imm) : rv_sraiw(rd, rd, imm), ctx); - if (!is64) + if (is64) + emit_srai(rd, rd, imm, ctx); + else + emit(rv_sraiw(rd, rd, imm), ctx); + + if (!is64 && !aux->verifier_zext) emit_zext_32(rd, ctx); break; @@ -749,13 +769,13 @@ out_be: e = ctx->ninsns; /* Adjust for extra insns */ - rvoff -= (e - s) << 2; + rvoff -= ninsns_rvoff(e - s); } if (BPF_OP(code) == BPF_JSET) { /* Adjust for and */ rvoff -= 4; - emit(rv_and(RV_REG_T1, rd, rs), ctx); + emit_and(RV_REG_T1, rd, rs, ctx); emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff, ctx); } else { @@ -784,11 +804,15 @@ out_be: case BPF_JMP32 | BPF_JSGE | BPF_K: case BPF_JMP | BPF_JSLE | BPF_K: case BPF_JMP32 | BPF_JSLE | BPF_K: - case BPF_JMP | BPF_JSET | BPF_K: - case BPF_JMP32 | BPF_JSET | BPF_K: rvoff = rv_offset(i, off, ctx); s = ctx->ninsns; - emit_imm(RV_REG_T1, imm, ctx); + if (imm) { + emit_imm(RV_REG_T1, imm, ctx); + rs = RV_REG_T1; + } else { + /* If imm is 0, simply use zero register. */ + rs = RV_REG_ZERO; + } if (!is64) { if (is_signed_bpf_cond(BPF_OP(code))) emit_sext_32_rd(&rd, ctx); @@ -798,17 +822,29 @@ out_be: e = ctx->ninsns; /* Adjust for extra insns */ - rvoff -= (e - s) << 2; + rvoff -= ninsns_rvoff(e - s); + emit_branch(BPF_OP(code), rd, rs, rvoff, ctx); + break; - if (BPF_OP(code) == BPF_JSET) { - /* Adjust for and */ - rvoff -= 4; - emit(rv_and(RV_REG_T1, rd, RV_REG_T1), ctx); - emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff, - ctx); + case BPF_JMP | BPF_JSET | BPF_K: + case BPF_JMP32 | BPF_JSET | BPF_K: + rvoff = rv_offset(i, off, ctx); + s = ctx->ninsns; + if (is_12b_int(imm)) { + emit_andi(RV_REG_T1, rd, imm, ctx); } else { - emit_branch(BPF_OP(code), rd, RV_REG_T1, rvoff, ctx); + emit_imm(RV_REG_T1, imm, ctx); + emit_and(RV_REG_T1, rd, RV_REG_T1, ctx); } + /* For jset32, we should clear the upper 32 bits of t1, but + * sign-extension is sufficient here and saves one instruction, + * as t1 is used only in comparison against zero. + */ + if (!is64 && imm < 0) + emit_addiw(RV_REG_T1, RV_REG_T1, 0, ctx); + e = ctx->ninsns; + rvoff -= ninsns_rvoff(e - s); + emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff, ctx); break; /* function call */ @@ -863,7 +899,7 @@ out_be: } emit_imm(RV_REG_T1, off, ctx); - emit(rv_add(RV_REG_T1, RV_REG_T1, rs), ctx); + emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); emit(rv_lbu(rd, 0, RV_REG_T1), ctx); if (insn_is_zext(&insn[1])) return 1; @@ -875,7 +911,7 @@ out_be: } emit_imm(RV_REG_T1, off, ctx); - emit(rv_add(RV_REG_T1, RV_REG_T1, rs), ctx); + emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); emit(rv_lhu(rd, 0, RV_REG_T1), ctx); if (insn_is_zext(&insn[1])) return 1; @@ -887,20 +923,20 @@ out_be: } emit_imm(RV_REG_T1, off, ctx); - emit(rv_add(RV_REG_T1, RV_REG_T1, rs), ctx); + emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); emit(rv_lwu(rd, 0, RV_REG_T1), ctx); if (insn_is_zext(&insn[1])) return 1; break; case BPF_LDX | BPF_MEM | BPF_DW: if (is_12b_int(off)) { - emit(rv_ld(rd, off, rs), ctx); + emit_ld(rd, off, rs, ctx); break; } emit_imm(RV_REG_T1, off, ctx); - emit(rv_add(RV_REG_T1, RV_REG_T1, rs), ctx); - emit(rv_ld(rd, 0, RV_REG_T1), ctx); + emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); + emit_ld(rd, 0, RV_REG_T1, ctx); break; /* ST: *(size *)(dst + off) = imm */ @@ -912,7 +948,7 @@ out_be: } emit_imm(RV_REG_T2, off, ctx); - emit(rv_add(RV_REG_T2, RV_REG_T2, rd), ctx); + emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); emit(rv_sb(RV_REG_T2, 0, RV_REG_T1), ctx); break; @@ -924,30 +960,30 @@ out_be: } emit_imm(RV_REG_T2, off, ctx); - emit(rv_add(RV_REG_T2, RV_REG_T2, rd), ctx); + emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); emit(rv_sh(RV_REG_T2, 0, RV_REG_T1), ctx); break; case BPF_ST | BPF_MEM | BPF_W: emit_imm(RV_REG_T1, imm, ctx); if (is_12b_int(off)) { - emit(rv_sw(rd, off, RV_REG_T1), ctx); + emit_sw(rd, off, RV_REG_T1, ctx); break; } emit_imm(RV_REG_T2, off, ctx); - emit(rv_add(RV_REG_T2, RV_REG_T2, rd), ctx); - emit(rv_sw(RV_REG_T2, 0, RV_REG_T1), ctx); + emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); + emit_sw(RV_REG_T2, 0, RV_REG_T1, ctx); break; case BPF_ST | BPF_MEM | BPF_DW: emit_imm(RV_REG_T1, imm, ctx); if (is_12b_int(off)) { - emit(rv_sd(rd, off, RV_REG_T1), ctx); + emit_sd(rd, off, RV_REG_T1, ctx); break; } emit_imm(RV_REG_T2, off, ctx); - emit(rv_add(RV_REG_T2, RV_REG_T2, rd), ctx); - emit(rv_sd(RV_REG_T2, 0, RV_REG_T1), ctx); + emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); + emit_sd(RV_REG_T2, 0, RV_REG_T1, ctx); break; /* STX: *(size *)(dst + off) = src */ @@ -958,7 +994,7 @@ out_be: } emit_imm(RV_REG_T1, off, ctx); - emit(rv_add(RV_REG_T1, RV_REG_T1, rd), ctx); + emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); emit(rv_sb(RV_REG_T1, 0, rs), ctx); break; case BPF_STX | BPF_MEM | BPF_H: @@ -968,28 +1004,28 @@ out_be: } emit_imm(RV_REG_T1, off, ctx); - emit(rv_add(RV_REG_T1, RV_REG_T1, rd), ctx); + emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); emit(rv_sh(RV_REG_T1, 0, rs), ctx); break; case BPF_STX | BPF_MEM | BPF_W: if (is_12b_int(off)) { - emit(rv_sw(rd, off, rs), ctx); + emit_sw(rd, off, rs, ctx); break; } emit_imm(RV_REG_T1, off, ctx); - emit(rv_add(RV_REG_T1, RV_REG_T1, rd), ctx); - emit(rv_sw(RV_REG_T1, 0, rs), ctx); + emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); + emit_sw(RV_REG_T1, 0, rs, ctx); break; case BPF_STX | BPF_MEM | BPF_DW: if (is_12b_int(off)) { - emit(rv_sd(rd, off, rs), ctx); + emit_sd(rd, off, rs, ctx); break; } emit_imm(RV_REG_T1, off, ctx); - emit(rv_add(RV_REG_T1, RV_REG_T1, rd), ctx); - emit(rv_sd(RV_REG_T1, 0, rs), ctx); + emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); + emit_sd(RV_REG_T1, 0, rs, ctx); break; /* STX XADD: lock *(u32 *)(dst + off) += src */ case BPF_STX | BPF_XADD | BPF_W: @@ -997,10 +1033,10 @@ out_be: case BPF_STX | BPF_XADD | BPF_DW: if (off) { if (is_12b_int(off)) { - emit(rv_addi(RV_REG_T1, rd, off), ctx); + emit_addi(RV_REG_T1, rd, off, ctx); } else { emit_imm(RV_REG_T1, off, ctx); - emit(rv_add(RV_REG_T1, RV_REG_T1, rd), ctx); + emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); } rd = RV_REG_T1; @@ -1049,52 +1085,53 @@ void bpf_jit_build_prologue(struct rv_jit_context *ctx) /* First instruction is always setting the tail-call-counter * (TCC) register. This instruction is skipped for tail calls. + * Force using a 4-byte (non-compressed) instruction. */ emit(rv_addi(RV_REG_TCC, RV_REG_ZERO, MAX_TAIL_CALL_CNT), ctx); - emit(rv_addi(RV_REG_SP, RV_REG_SP, -stack_adjust), ctx); + emit_addi(RV_REG_SP, RV_REG_SP, -stack_adjust, ctx); if (seen_reg(RV_REG_RA, ctx)) { - emit(rv_sd(RV_REG_SP, store_offset, RV_REG_RA), ctx); + emit_sd(RV_REG_SP, store_offset, RV_REG_RA, ctx); store_offset -= 8; } - emit(rv_sd(RV_REG_SP, store_offset, RV_REG_FP), ctx); + emit_sd(RV_REG_SP, store_offset, RV_REG_FP, ctx); store_offset -= 8; if (seen_reg(RV_REG_S1, ctx)) { - emit(rv_sd(RV_REG_SP, store_offset, RV_REG_S1), ctx); + emit_sd(RV_REG_SP, store_offset, RV_REG_S1, ctx); store_offset -= 8; } if (seen_reg(RV_REG_S2, ctx)) { - emit(rv_sd(RV_REG_SP, store_offset, RV_REG_S2), ctx); + emit_sd(RV_REG_SP, store_offset, RV_REG_S2, ctx); store_offset -= 8; } if (seen_reg(RV_REG_S3, ctx)) { - emit(rv_sd(RV_REG_SP, store_offset, RV_REG_S3), ctx); + emit_sd(RV_REG_SP, store_offset, RV_REG_S3, ctx); store_offset -= 8; } if (seen_reg(RV_REG_S4, ctx)) { - emit(rv_sd(RV_REG_SP, store_offset, RV_REG_S4), ctx); + emit_sd(RV_REG_SP, store_offset, RV_REG_S4, ctx); store_offset -= 8; } if (seen_reg(RV_REG_S5, ctx)) { - emit(rv_sd(RV_REG_SP, store_offset, RV_REG_S5), ctx); + emit_sd(RV_REG_SP, store_offset, RV_REG_S5, ctx); store_offset -= 8; } if (seen_reg(RV_REG_S6, ctx)) { - emit(rv_sd(RV_REG_SP, store_offset, RV_REG_S6), ctx); + emit_sd(RV_REG_SP, store_offset, RV_REG_S6, ctx); store_offset -= 8; } - emit(rv_addi(RV_REG_FP, RV_REG_SP, stack_adjust), ctx); + emit_addi(RV_REG_FP, RV_REG_SP, stack_adjust, ctx); if (bpf_stack_adjust) - emit(rv_addi(RV_REG_S5, RV_REG_SP, bpf_stack_adjust), ctx); + emit_addi(RV_REG_S5, RV_REG_SP, bpf_stack_adjust, ctx); /* Program contains calls and tail calls, so RV_REG_TCC need * to be saved across calls. */ if (seen_tail_call(ctx) && seen_call(ctx)) - emit(rv_addi(RV_REG_TCC_SAVED, RV_REG_TCC, 0), ctx); + emit_mv(RV_REG_TCC_SAVED, RV_REG_TCC, ctx); ctx->stack_size = stack_adjust; } diff --git a/arch/riscv/net/bpf_jit_core.c b/arch/riscv/net/bpf_jit_core.c index 709b94ece3ed..3630d447352c 100644 --- a/arch/riscv/net/bpf_jit_core.c +++ b/arch/riscv/net/bpf_jit_core.c @@ -73,7 +73,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) if (ctx->offset) { extra_pass = true; - image_size = sizeof(u32) * ctx->ninsns; + image_size = sizeof(*ctx->insns) * ctx->ninsns; goto skip_init_ctx; } @@ -103,7 +103,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) if (jit_data->header) break; - image_size = sizeof(u32) * ctx->ninsns; + image_size = sizeof(*ctx->insns) * ctx->ninsns; jit_data->header = bpf_jit_binary_alloc(image_size, &jit_data->image, @@ -114,7 +114,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) goto out_offset; } - ctx->insns = (u32 *)jit_data->image; + ctx->insns = (u16 *)jit_data->image; /* * Now, when the image is allocated, the image can * potentially shrink more (auipc/jalr -> jal). |