summaryrefslogtreecommitdiffstats
path: root/arch/arm
diff options
context:
space:
mode:
Diffstat (limited to 'arch/arm')
-rw-r--r--arch/arm/Kconfig5
-rw-r--r--arch/arm/Kconfig.debug11
-rw-r--r--arch/arm/boot/dts/Makefile3
-rw-r--r--arch/arm/boot/dts/am335x-icev2.dts1
-rw-r--r--arch/arm/boot/dts/am33xx.dtsi4
-rw-r--r--arch/arm/boot/dts/am4372.dtsi1
-rw-r--r--arch/arm/boot/dts/bcm-nsp.dtsi2
-rw-r--r--arch/arm/boot/dts/da850-evm.dts1
-rw-r--r--arch/arm/boot/dts/dm814x.dtsi10
-rw-r--r--arch/arm/boot/dts/dm816x.dtsi6
-rw-r--r--arch/arm/boot/dts/dra7.dtsi2
-rw-r--r--arch/arm/boot/dts/dra72-evm-revc.dts2
-rw-r--r--arch/arm/boot/dts/imx1.dtsi8
-rw-r--r--arch/arm/boot/dts/imx23.dtsi8
-rw-r--r--arch/arm/boot/dts/imx25.dtsi8
-rw-r--r--arch/arm/boot/dts/imx27.dtsi8
-rw-r--r--arch/arm/boot/dts/imx28.dtsi8
-rw-r--r--arch/arm/boot/dts/imx31.dtsi8
-rw-r--r--arch/arm/boot/dts/imx35.dtsi8
-rw-r--r--arch/arm/boot/dts/imx50.dtsi8
-rw-r--r--arch/arm/boot/dts/imx51.dtsi8
-rw-r--r--arch/arm/boot/dts/imx53.dtsi8
-rw-r--r--arch/arm/boot/dts/imx6dl.dtsi2
-rw-r--r--arch/arm/boot/dts/imx6qdl-nitrogen6_max.dtsi4
-rw-r--r--arch/arm/boot/dts/imx6qdl-nitrogen6_som2.dtsi4
-rw-r--r--arch/arm/boot/dts/imx6qdl.dtsi10
-rw-r--r--arch/arm/boot/dts/imx6sl.dtsi8
-rw-r--r--arch/arm/boot/dts/imx6sx.dtsi8
-rw-r--r--arch/arm/boot/dts/imx6ul.dtsi8
-rw-r--r--arch/arm/boot/dts/imx7s.dtsi8
-rw-r--r--arch/arm/boot/dts/logicpd-som-lv-37xx-devkit.dts11
-rw-r--r--arch/arm/boot/dts/omap5.dtsi1
-rw-r--r--arch/arm/boot/dts/orion5x-linkstation-lschl.dts (renamed from arch/arm/boot/dts/orion5x-lschl.dts)4
-rw-r--r--arch/arm/boot/dts/qcom-mdm9615.dtsi2
-rw-r--r--arch/arm/boot/dts/stih407-family.dtsi14
-rw-r--r--arch/arm/boot/dts/stih407-pinctrl.dtsi12
-rw-r--r--arch/arm/boot/dts/stih410-b2260.dts5
-rw-r--r--arch/arm/boot/dts/sun6i-a31-hummingbird.dts4
-rw-r--r--arch/arm/boot/dts/sun6i-a31.dtsi1
-rw-r--r--arch/arm/boot/dts/sun7i-a20-olinuxino-lime2-emmc.dts2
-rw-r--r--arch/arm/configs/aspeed_g4_defconfig4
-rw-r--r--arch/arm/configs/aspeed_g5_defconfig4
-rw-r--r--arch/arm/configs/exynos_defconfig2
-rw-r--r--arch/arm/configs/ezx_defconfig4
-rw-r--r--arch/arm/configs/imote2_defconfig4
-rw-r--r--arch/arm/configs/multi_v5_defconfig2
-rw-r--r--arch/arm/configs/multi_v7_defconfig11
-rw-r--r--arch/arm/configs/mvebu_v5_defconfig2
-rw-r--r--arch/arm/configs/pxa_defconfig2
-rw-r--r--arch/arm/configs/s3c2410_defconfig6
-rw-r--r--arch/arm/configs/shmobile_defconfig2
-rw-r--r--arch/arm/crypto/Kconfig27
-rw-r--r--arch/arm/crypto/Makefile11
-rw-r--r--arch/arm/crypto/aes-armv4.S1089
-rw-r--r--arch/arm/crypto/aes-ce-core.S84
-rw-r--r--arch/arm/crypto/aes-ce-glue.c15
-rw-r--r--arch/arm/crypto/aes-cipher-core.S179
-rw-r--r--arch/arm/crypto/aes-cipher-glue.c74
-rw-r--r--arch/arm/crypto/aes-neonbs-core.S1023
-rw-r--r--arch/arm/crypto/aes-neonbs-glue.c406
-rw-r--r--arch/arm/crypto/aes_glue.c98
-rw-r--r--arch/arm/crypto/aes_glue.h19
-rw-r--r--arch/arm/crypto/aesbs-core.S_shipped2548
-rw-r--r--arch/arm/crypto/aesbs-glue.c367
-rw-r--r--arch/arm/crypto/bsaes-armv7.pl2471
-rw-r--r--arch/arm/crypto/chacha20-neon-core.S523
-rw-r--r--arch/arm/crypto/chacha20-neon-glue.c127
-rw-r--r--arch/arm/include/asm/Kbuild1
-rw-r--r--arch/arm/include/asm/cacheflush.h2
-rw-r--r--arch/arm/include/asm/cputype.h3
-rw-r--r--arch/arm/include/asm/efi.h1
-rw-r--r--arch/arm/include/asm/ftrace.h18
-rw-r--r--arch/arm/include/asm/kvm_host.h3
-rw-r--r--arch/arm/include/asm/kvm_mmu.h12
-rw-r--r--arch/arm/include/asm/uaccess.h44
-rw-r--r--arch/arm/include/asm/virt.h5
-rw-r--r--arch/arm/include/uapi/asm/kvm.h13
-rw-r--r--arch/arm/include/uapi/asm/types.h (renamed from arch/arm/include/asm/types.h)6
-rw-r--r--arch/arm/kernel/armksyms.c4
-rw-r--r--arch/arm/kernel/hw_breakpoint.c16
-rw-r--r--arch/arm/kernel/patch.c4
-rw-r--r--arch/arm/kernel/perf_event_v6.c6
-rw-r--r--arch/arm/kernel/perf_event_v7.c6
-rw-r--r--arch/arm/kernel/perf_event_xscale.c6
-rw-r--r--arch/arm/kernel/ptrace.c2
-rw-r--r--arch/arm/kernel/smccc-call.S14
-rw-r--r--arch/arm/kernel/smp_tlb.c7
-rw-r--r--arch/arm/kernel/vmlinux.lds.S8
-rw-r--r--arch/arm/kvm/Makefile5
-rw-r--r--arch/arm/kvm/arm.c11
-rw-r--r--arch/arm/kvm/mmu.c20
-rw-r--r--arch/arm/kvm/reset.c9
-rw-r--r--arch/arm/kvm/vgic-v3-coproc.c35
-rw-r--r--arch/arm/lib/getuser.S2
-rw-r--r--arch/arm/mach-davinci/board-da850-evm.c1
-rw-r--r--arch/arm/mach-davinci/board-dm644x-evm.c1
-rw-r--r--arch/arm/mach-davinci/board-neuros-osd2.c1
-rw-r--r--arch/arm/mach-davinci/board-omapl138-hawk.c1
-rw-r--r--arch/arm/mach-davinci/da850.c10
-rw-r--r--arch/arm/mach-davinci/da8xx-dt.c12
-rw-r--r--arch/arm/mach-ep93xx/edb93xx.c31
-rw-r--r--arch/arm/mach-ep93xx/simone.c63
-rw-r--r--arch/arm/mach-ep93xx/vision_ep9307.c88
-rw-r--r--arch/arm/mach-exynos/suspend.c64
-rw-r--r--arch/arm/mach-imx/mmdc.c2
-rw-r--r--arch/arm/mach-omap1/dma.c16
-rw-r--r--arch/arm/mach-omap2/id.c10
-rw-r--r--arch/arm/mach-omap2/omap_hwmod_54xx_data.c8
-rw-r--r--arch/arm/mach-omap2/pdata-quirks.c13
-rw-r--r--arch/arm/mach-omap2/pm.c5
-rw-r--r--arch/arm/mach-orion5x/common.c2
-rw-r--r--arch/arm/mach-orion5x/common.h4
-rw-r--r--arch/arm/mach-orion5x/rd88f5181l-fxo-setup.c7
-rw-r--r--arch/arm/mach-orion5x/rd88f5181l-ge-setup.c7
-rw-r--r--arch/arm/mach-orion5x/rd88f6183ap-ge-setup.c7
-rw-r--r--arch/arm/mach-orion5x/wnr854t-setup.c7
-rw-r--r--arch/arm/mach-orion5x/wrt350n-v2-setup.c7
-rw-r--r--arch/arm/mach-pxa/balloon3.c1
-rw-r--r--arch/arm/mach-pxa/colibri-pxa270-income.c1
-rw-r--r--arch/arm/mach-pxa/corgi.c1
-rw-r--r--arch/arm/mach-pxa/trizeps4.c1
-rw-r--r--arch/arm/mach-pxa/vpac270.c1
-rw-r--r--arch/arm/mach-pxa/zeus.c1
-rw-r--r--arch/arm/mach-pxa/zylonite.c1
-rw-r--r--arch/arm/mach-s3c64xx/dev-audio.c4
-rw-r--r--arch/arm/mach-s5pv210/pm.c7
-rw-r--r--arch/arm/mach-s5pv210/regs-clock.h4
-rw-r--r--arch/arm/mach-shmobile/Kconfig1
-rw-r--r--arch/arm/mach-ux500/pm.c4
-rw-r--r--arch/arm/mm/Kconfig14
-rw-r--r--arch/arm/mm/dma-mapping.c60
-rw-r--r--arch/arm/mm/extable.c2
-rw-r--r--arch/arm/mm/fault.c6
-rw-r--r--arch/arm/mm/fault.h4
-rw-r--r--arch/arm/mm/init.c4
-rw-r--r--arch/arm/plat-orion/common.c25
-rw-r--r--arch/arm/plat-orion/include/plat/common.h4
-rw-r--r--arch/arm/xen/enlighten.c1
-rw-r--r--arch/arm/xen/hypercall.S1
139 files changed, 2941 insertions, 7177 deletions
diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig
index 186c4c214e0a..fda6a46d27cf 100644
--- a/arch/arm/Kconfig
+++ b/arch/arm/Kconfig
@@ -4,10 +4,15 @@ config ARM
select ARCH_CLOCKSOURCE_DATA
select ARCH_HAS_DEVMEM_IS_ALLOWED
select ARCH_HAS_ELF_RANDOMIZE
+ select ARCH_HAS_SET_MEMORY
+ select ARCH_HAS_STRICT_KERNEL_RWX if MMU && !XIP_KERNEL
+ select ARCH_HAS_STRICT_MODULE_RWX if MMU
select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST
select ARCH_HAVE_CUSTOM_GPIO_H
select ARCH_HAS_GCOV_PROFILE_ALL
select ARCH_MIGHT_HAVE_PC_PARPORT
+ select ARCH_OPTIONAL_KERNEL_RWX if ARCH_HAS_STRICT_KERNEL_RWX
+ select ARCH_OPTIONAL_KERNEL_RWX_DEFAULT if CPU_V7
select ARCH_SUPPORTS_ATOMIC_RMW
select ARCH_USE_BUILTIN_BSWAP
select ARCH_USE_CMPXCHG_LOCKREF
diff --git a/arch/arm/Kconfig.debug b/arch/arm/Kconfig.debug
index d83f7c369e51..426d2716f55d 100644
--- a/arch/arm/Kconfig.debug
+++ b/arch/arm/Kconfig.debug
@@ -1738,17 +1738,6 @@ config PID_IN_CONTEXTIDR
additional instructions during context switch. Say Y here only if you
are planning to use hardware trace tools with this kernel.
-config DEBUG_SET_MODULE_RONX
- bool "Set loadable kernel module data as NX and text as RO"
- depends on MODULES && MMU
- ---help---
- This option helps catch unintended modifications to loadable
- kernel module's text and read-only data. It also prevents execution
- of module data. Such protection may interfere with run-time code
- patching and dynamic kernel tracing - and they might also protect
- against certain classes of kernel exploits.
- If in doubt, say "N".
-
source "drivers/hwtracing/coresight/Kconfig"
endmenu
diff --git a/arch/arm/boot/dts/Makefile b/arch/arm/boot/dts/Makefile
index 7327250f0bb6..01d178a2009f 100644
--- a/arch/arm/boot/dts/Makefile
+++ b/arch/arm/boot/dts/Makefile
@@ -617,7 +617,7 @@ dtb-$(CONFIG_ARCH_ORION5X) += \
orion5x-lacie-ethernet-disk-mini-v2.dtb \
orion5x-linkstation-lsgl.dtb \
orion5x-linkstation-lswtgl.dtb \
- orion5x-lschl.dtb \
+ orion5x-linkstation-lschl.dtb \
orion5x-lswsgl.dtb \
orion5x-maxtor-shared-storage-2.dtb \
orion5x-netgear-wnr854t.dtb \
@@ -846,6 +846,7 @@ dtb-$(CONFIG_MACH_SUN8I) += \
sun8i-a83t-allwinner-h8homlet-v2.dtb \
sun8i-a83t-cubietruck-plus.dtb \
sun8i-h3-bananapi-m2-plus.dtb \
+ sun8i-h3-nanopi-m1.dtb \
sun8i-h3-nanopi-neo.dtb \
sun8i-h3-orangepi-2.dtb \
sun8i-h3-orangepi-lite.dtb \
diff --git a/arch/arm/boot/dts/am335x-icev2.dts b/arch/arm/boot/dts/am335x-icev2.dts
index 1463df3b5b19..8ed46f9d79b7 100644
--- a/arch/arm/boot/dts/am335x-icev2.dts
+++ b/arch/arm/boot/dts/am335x-icev2.dts
@@ -170,7 +170,6 @@
AM33XX_IOPAD(0x8fc, PIN_INPUT_PULLUP | MUX_MODE0) /* (G16) mmc0_dat0.mmc0_dat0 */
AM33XX_IOPAD(0x900, PIN_INPUT_PULLUP | MUX_MODE0) /* (G17) mmc0_clk.mmc0_clk */
AM33XX_IOPAD(0x904, PIN_INPUT_PULLUP | MUX_MODE0) /* (G18) mmc0_cmd.mmc0_cmd */
- AM33XX_IOPAD(0x960, PIN_INPUT_PULLUP | MUX_MODE5) /* (C15) spi0_cs1.mmc0_sdcd */
>;
};
diff --git a/arch/arm/boot/dts/am33xx.dtsi b/arch/arm/boot/dts/am33xx.dtsi
index 18d72a245e88..9e96d60976b7 100644
--- a/arch/arm/boot/dts/am33xx.dtsi
+++ b/arch/arm/boot/dts/am33xx.dtsi
@@ -145,10 +145,11 @@
};
scm_conf: scm_conf@0 {
- compatible = "syscon";
+ compatible = "syscon", "simple-bus";
reg = <0x0 0x800>;
#address-cells = <1>;
#size-cells = <1>;
+ ranges = <0 0 0x800>;
scm_clocks: clocks {
#address-cells = <1>;
@@ -782,7 +783,6 @@
cpdma_channels = <8>;
ale_entries = <1024>;
bd_ram_size = <0x2000>;
- no_bd_ram = <0>;
mac_control = <0x20>;
slaves = <2>;
active_slave = <0>;
diff --git a/arch/arm/boot/dts/am4372.dtsi b/arch/arm/boot/dts/am4372.dtsi
index 2df9e6050c2f..97fcaf415de1 100644
--- a/arch/arm/boot/dts/am4372.dtsi
+++ b/arch/arm/boot/dts/am4372.dtsi
@@ -670,7 +670,6 @@
cpdma_channels = <8>;
ale_entries = <1024>;
bd_ram_size = <0x2000>;
- no_bd_ram = <0>;
mac_control = <0x20>;
slaves = <2>;
active_slave = <0>;
diff --git a/arch/arm/boot/dts/bcm-nsp.dtsi b/arch/arm/boot/dts/bcm-nsp.dtsi
index b6142bda661e..15f07f9af3b3 100644
--- a/arch/arm/boot/dts/bcm-nsp.dtsi
+++ b/arch/arm/boot/dts/bcm-nsp.dtsi
@@ -160,7 +160,7 @@
axi {
compatible = "simple-bus";
- ranges = <0x00000000 0x18000000 0x0011c40a>;
+ ranges = <0x00000000 0x18000000 0x0011c40c>;
#address-cells = <1>;
#size-cells = <1>;
diff --git a/arch/arm/boot/dts/da850-evm.dts b/arch/arm/boot/dts/da850-evm.dts
index 41de15fe15a2..78492a0bbbab 100644
--- a/arch/arm/boot/dts/da850-evm.dts
+++ b/arch/arm/boot/dts/da850-evm.dts
@@ -99,6 +99,7 @@
#size-cells = <1>;
compatible = "m25p64";
spi-max-frequency = <30000000>;
+ m25p,fast-read;
reg = <0>;
partition@0 {
label = "U-Boot-SPL";
diff --git a/arch/arm/boot/dts/dm814x.dtsi b/arch/arm/boot/dts/dm814x.dtsi
index 81b8cecb5820..9708157f5daf 100644
--- a/arch/arm/boot/dts/dm814x.dtsi
+++ b/arch/arm/boot/dts/dm814x.dtsi
@@ -252,7 +252,7 @@
};
uart1: uart@20000 {
- compatible = "ti,omap3-uart";
+ compatible = "ti,am3352-uart", "ti,omap3-uart";
ti,hwmods = "uart1";
reg = <0x20000 0x2000>;
clock-frequency = <48000000>;
@@ -262,7 +262,7 @@
};
uart2: uart@22000 {
- compatible = "ti,omap3-uart";
+ compatible = "ti,am3352-uart", "ti,omap3-uart";
ti,hwmods = "uart2";
reg = <0x22000 0x2000>;
clock-frequency = <48000000>;
@@ -272,7 +272,7 @@
};
uart3: uart@24000 {
- compatible = "ti,omap3-uart";
+ compatible = "ti,am3352-uart", "ti,omap3-uart";
ti,hwmods = "uart3";
reg = <0x24000 0x2000>;
clock-frequency = <48000000>;
@@ -332,10 +332,11 @@
ranges = <0 0x140000 0x20000>;
scm_conf: scm_conf@0 {
- compatible = "syscon";
+ compatible = "syscon", "simple-bus";
reg = <0x0 0x800>;
#address-cells = <1>;
#size-cells = <1>;
+ ranges = <0 0 0x800>;
scm_clocks: clocks {
#address-cells = <1>;
@@ -510,7 +511,6 @@
cpdma_channels = <8>;
ale_entries = <1024>;
bd_ram_size = <0x2000>;
- no_bd_ram = <0>;
mac_control = <0x20>;
slaves = <2>;
active_slave = <0>;
diff --git a/arch/arm/boot/dts/dm816x.dtsi b/arch/arm/boot/dts/dm816x.dtsi
index 6db652ae9bd5..276211e1ee53 100644
--- a/arch/arm/boot/dts/dm816x.dtsi
+++ b/arch/arm/boot/dts/dm816x.dtsi
@@ -373,7 +373,7 @@
};
uart1: uart@48020000 {
- compatible = "ti,omap3-uart";
+ compatible = "ti,am3352-uart", "ti,omap3-uart";
ti,hwmods = "uart1";
reg = <0x48020000 0x2000>;
clock-frequency = <48000000>;
@@ -383,7 +383,7 @@
};
uart2: uart@48022000 {
- compatible = "ti,omap3-uart";
+ compatible = "ti,am3352-uart", "ti,omap3-uart";
ti,hwmods = "uart2";
reg = <0x48022000 0x2000>;
clock-frequency = <48000000>;
@@ -393,7 +393,7 @@
};
uart3: uart@48024000 {
- compatible = "ti,omap3-uart";
+ compatible = "ti,am3352-uart", "ti,omap3-uart";
ti,hwmods = "uart3";
reg = <0x48024000 0x2000>;
clock-frequency = <48000000>;
diff --git a/arch/arm/boot/dts/dra7.dtsi b/arch/arm/boot/dts/dra7.dtsi
index 1faf24acd521..3e1f75026eac 100644
--- a/arch/arm/boot/dts/dra7.dtsi
+++ b/arch/arm/boot/dts/dra7.dtsi
@@ -1378,6 +1378,7 @@
phy-names = "sata-phy";
clocks = <&sata_ref_clk>;
ti,hwmods = "sata";
+ ports-implemented = <0x1>;
};
rtc: rtc@48838000 {
@@ -1708,7 +1709,6 @@
cpdma_channels = <8>;
ale_entries = <1024>;
bd_ram_size = <0x2000>;
- no_bd_ram = <0>;
mac_control = <0x20>;
slaves = <2>;
active_slave = <0>;
diff --git a/arch/arm/boot/dts/dra72-evm-revc.dts b/arch/arm/boot/dts/dra72-evm-revc.dts
index c3d939c9666c..3f808a47df03 100644
--- a/arch/arm/boot/dts/dra72-evm-revc.dts
+++ b/arch/arm/boot/dts/dra72-evm-revc.dts
@@ -75,6 +75,6 @@
ti,rx-internal-delay = <DP83867_RGMIIDCTL_2_25_NS>;
ti,tx-internal-delay = <DP83867_RGMIIDCTL_250_PS>;
ti,fifo-depth = <DP83867_PHYCR_FIFO_DEPTH_8_B_NIB>;
- ti,min-output-imepdance;
+ ti,min-output-impedance;
};
};
diff --git a/arch/arm/boot/dts/imx1.dtsi b/arch/arm/boot/dts/imx1.dtsi
index b792eee3899b..2ee40bc9ec21 100644
--- a/arch/arm/boot/dts/imx1.dtsi
+++ b/arch/arm/boot/dts/imx1.dtsi
@@ -18,6 +18,14 @@
/ {
#address-cells = <1>;
#size-cells = <1>;
+ /*
+ * The decompressor and also some bootloaders rely on a
+ * pre-existing /chosen node to be available to insert the
+ * command line and merge other ATAGS info.
+ * Also for U-Boot there must be a pre-existing /memory node.
+ */
+ chosen {};
+ memory { device_type = "memory"; reg = <0 0>; };
aliases {
gpio0 = &gpio1;
diff --git a/arch/arm/boot/dts/imx23.dtsi b/arch/arm/boot/dts/imx23.dtsi
index ac2a9da62b6c..43ccbbf754a3 100644
--- a/arch/arm/boot/dts/imx23.dtsi
+++ b/arch/arm/boot/dts/imx23.dtsi
@@ -16,6 +16,14 @@
#size-cells = <1>;
interrupt-parent = <&icoll>;
+ /*
+ * The decompressor and also some bootloaders rely on a
+ * pre-existing /chosen node to be available to insert the
+ * command line and merge other ATAGS info.
+ * Also for U-Boot there must be a pre-existing /memory node.
+ */
+ chosen {};
+ memory { device_type = "memory"; reg = <0 0>; };
aliases {
gpio0 = &gpio0;
diff --git a/arch/arm/boot/dts/imx25.dtsi b/arch/arm/boot/dts/imx25.dtsi
index 831d09a28155..acd475659156 100644
--- a/arch/arm/boot/dts/imx25.dtsi
+++ b/arch/arm/boot/dts/imx25.dtsi
@@ -14,6 +14,14 @@
/ {
#address-cells = <1>;
#size-cells = <1>;
+ /*
+ * The decompressor and also some bootloaders rely on a
+ * pre-existing /chosen node to be available to insert the
+ * command line and merge other ATAGS info.
+ * Also for U-Boot there must be a pre-existing /memory node.
+ */
+ chosen {};
+ memory { device_type = "memory"; reg = <0 0>; };
aliases {
ethernet0 = &fec;
diff --git a/arch/arm/boot/dts/imx27.dtsi b/arch/arm/boot/dts/imx27.dtsi
index 9d8b5969ee3b..b397384248f4 100644
--- a/arch/arm/boot/dts/imx27.dtsi
+++ b/arch/arm/boot/dts/imx27.dtsi
@@ -19,6 +19,14 @@
/ {
#address-cells = <1>;
#size-cells = <1>;
+ /*
+ * The decompressor and also some bootloaders rely on a
+ * pre-existing /chosen node to be available to insert the
+ * command line and merge other ATAGS info.
+ * Also for U-Boot there must be a pre-existing /memory node.
+ */
+ chosen {};
+ memory { device_type = "memory"; reg = <0 0>; };
aliases {
ethernet0 = &fec;
diff --git a/arch/arm/boot/dts/imx28.dtsi b/arch/arm/boot/dts/imx28.dtsi
index 3aabf65a6a52..d6a2190b60ef 100644
--- a/arch/arm/boot/dts/imx28.dtsi
+++ b/arch/arm/boot/dts/imx28.dtsi
@@ -17,6 +17,14 @@
#size-cells = <1>;
interrupt-parent = <&icoll>;
+ /*
+ * The decompressor and also some bootloaders rely on a
+ * pre-existing /chosen node to be available to insert the
+ * command line and merge other ATAGS info.
+ * Also for U-Boot there must be a pre-existing /memory node.
+ */
+ chosen {};
+ memory { device_type = "memory"; reg = <0 0>; };
aliases {
ethernet0 = &mac0;
diff --git a/arch/arm/boot/dts/imx31.dtsi b/arch/arm/boot/dts/imx31.dtsi
index 85cd8be22f71..23b0d2cf9acd 100644
--- a/arch/arm/boot/dts/imx31.dtsi
+++ b/arch/arm/boot/dts/imx31.dtsi
@@ -12,6 +12,14 @@
/ {
#address-cells = <1>;
#size-cells = <1>;
+ /*
+ * The decompressor and also some bootloaders rely on a
+ * pre-existing /chosen node to be available to insert the
+ * command line and merge other ATAGS info.
+ * Also for U-Boot there must be a pre-existing /memory node.
+ */
+ chosen {};
+ memory { device_type = "memory"; reg = <0 0>; };
aliases {
serial0 = &uart1;
diff --git a/arch/arm/boot/dts/imx35.dtsi b/arch/arm/boot/dts/imx35.dtsi
index 9f40e6229189..d0496c65cea2 100644
--- a/arch/arm/boot/dts/imx35.dtsi
+++ b/arch/arm/boot/dts/imx35.dtsi
@@ -13,6 +13,14 @@
/ {
#address-cells = <1>;
#size-cells = <1>;
+ /*
+ * The decompressor and also some bootloaders rely on a
+ * pre-existing /chosen node to be available to insert the
+ * command line and merge other ATAGS info.
+ * Also for U-Boot there must be a pre-existing /memory node.
+ */
+ chosen {};
+ memory { device_type = "memory"; reg = <0 0>; };
aliases {
ethernet0 = &fec;
diff --git a/arch/arm/boot/dts/imx50.dtsi b/arch/arm/boot/dts/imx50.dtsi
index fe0221e4cbf7..ceae909e2201 100644
--- a/arch/arm/boot/dts/imx50.dtsi
+++ b/arch/arm/boot/dts/imx50.dtsi
@@ -17,6 +17,14 @@
/ {
#address-cells = <1>;
#size-cells = <1>;
+ /*
+ * The decompressor and also some bootloaders rely on a
+ * pre-existing /chosen node to be available to insert the
+ * command line and merge other ATAGS info.
+ * Also for U-Boot there must be a pre-existing /memory node.
+ */
+ chosen {};
+ memory { device_type = "memory"; reg = <0 0>; };
aliases {
ethernet0 = &fec;
diff --git a/arch/arm/boot/dts/imx51.dtsi b/arch/arm/boot/dts/imx51.dtsi
index 33526cade735..1ee1d542d9ad 100644
--- a/arch/arm/boot/dts/imx51.dtsi
+++ b/arch/arm/boot/dts/imx51.dtsi
@@ -19,6 +19,14 @@
/ {
#address-cells = <1>;
#size-cells = <1>;
+ /*
+ * The decompressor and also some bootloaders rely on a
+ * pre-existing /chosen node to be available to insert the
+ * command line and merge other ATAGS info.
+ * Also for U-Boot there must be a pre-existing /memory node.
+ */
+ chosen {};
+ memory { device_type = "memory"; reg = <0 0>; };
aliases {
ethernet0 = &fec;
diff --git a/arch/arm/boot/dts/imx53.dtsi b/arch/arm/boot/dts/imx53.dtsi
index ca51dc03e327..2e516f4985e4 100644
--- a/arch/arm/boot/dts/imx53.dtsi
+++ b/arch/arm/boot/dts/imx53.dtsi
@@ -19,6 +19,14 @@
/ {
#address-cells = <1>;
#size-cells = <1>;
+ /*
+ * The decompressor and also some bootloaders rely on a
+ * pre-existing /chosen node to be available to insert the
+ * command line and merge other ATAGS info.
+ * Also for U-Boot there must be a pre-existing /memory node.
+ */
+ chosen {};
+ memory { device_type = "memory"; reg = <0 0>; };
aliases {
ethernet0 = &fec;
diff --git a/arch/arm/boot/dts/imx6dl.dtsi b/arch/arm/boot/dts/imx6dl.dtsi
index 1ade1951e620..7aa120fbdc71 100644
--- a/arch/arm/boot/dts/imx6dl.dtsi
+++ b/arch/arm/boot/dts/imx6dl.dtsi
@@ -137,7 +137,7 @@
&gpio4 {
gpio-ranges = <&iomuxc 5 136 1>, <&iomuxc 6 145 1>, <&iomuxc 7 150 1>,
<&iomuxc 8 146 1>, <&iomuxc 9 151 1>, <&iomuxc 10 147 1>,
- <&iomuxc 11 151 1>, <&iomuxc 12 148 1>, <&iomuxc 13 153 1>,
+ <&iomuxc 11 152 1>, <&iomuxc 12 148 1>, <&iomuxc 13 153 1>,
<&iomuxc 14 149 1>, <&iomuxc 15 154 1>, <&iomuxc 16 39 7>,
<&iomuxc 23 56 1>, <&iomuxc 24 61 7>, <&iomuxc 31 46 1>;
};
diff --git a/arch/arm/boot/dts/imx6qdl-nitrogen6_max.dtsi b/arch/arm/boot/dts/imx6qdl-nitrogen6_max.dtsi
index 34887a10c5f1..47ba97229a48 100644
--- a/arch/arm/boot/dts/imx6qdl-nitrogen6_max.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-nitrogen6_max.dtsi
@@ -319,8 +319,6 @@
compatible = "fsl,imx6q-nitrogen6_max-sgtl5000",
"fsl,imx-audio-sgtl5000";
model = "imx6q-nitrogen6_max-sgtl5000";
- pinctrl-names = "default";
- pinctrl-0 = <&pinctrl_sgtl5000>;
ssi-controller = <&ssi1>;
audio-codec = <&codec>;
audio-routing =
@@ -402,6 +400,8 @@
codec: sgtl5000@0a {
compatible = "fsl,sgtl5000";
+ pinctrl-names = "default";
+ pinctrl-0 = <&pinctrl_sgtl5000>;
reg = <0x0a>;
clocks = <&clks IMX6QDL_CLK_CKO>;
VDDA-supply = <&reg_2p5v>;
diff --git a/arch/arm/boot/dts/imx6qdl-nitrogen6_som2.dtsi b/arch/arm/boot/dts/imx6qdl-nitrogen6_som2.dtsi
index d80f21abea62..31d4cc62dbc7 100644
--- a/arch/arm/boot/dts/imx6qdl-nitrogen6_som2.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-nitrogen6_som2.dtsi
@@ -250,8 +250,6 @@
compatible = "fsl,imx6q-nitrogen6_som2-sgtl5000",
"fsl,imx-audio-sgtl5000";
model = "imx6q-nitrogen6_som2-sgtl5000";
- pinctrl-names = "default";
- pinctrl-0 = <&pinctrl_sgtl5000>;
ssi-controller = <&ssi1>;
audio-codec = <&codec>;
audio-routing =
@@ -320,6 +318,8 @@
codec: sgtl5000@0a {
compatible = "fsl,sgtl5000";
+ pinctrl-names = "default";
+ pinctrl-0 = <&pinctrl_sgtl5000>;
reg = <0x0a>;
clocks = <&clks IMX6QDL_CLK_CKO>;
VDDA-supply = <&reg_2p5v>;
diff --git a/arch/arm/boot/dts/imx6qdl.dtsi b/arch/arm/boot/dts/imx6qdl.dtsi
index 89b834f3fa17..6d37d9af5f1d 100644
--- a/arch/arm/boot/dts/imx6qdl.dtsi
+++ b/arch/arm/boot/dts/imx6qdl.dtsi
@@ -16,6 +16,14 @@
/ {
#address-cells = <1>;
#size-cells = <1>;
+ /*
+ * The decompressor and also some bootloaders rely on a
+ * pre-existing /chosen node to be available to insert the
+ * command line and merge other ATAGS info.
+ * Also for U-Boot there must be a pre-existing /memory node.
+ */
+ chosen {};
+ memory { device_type = "memory"; reg = <0 0>; };
aliases {
ethernet0 = &fec;
@@ -1158,8 +1166,10 @@
};
vdoa@021e4000 {
+ compatible = "fsl,imx6q-vdoa";
reg = <0x021e4000 0x4000>;
interrupts = <0 18 IRQ_TYPE_LEVEL_HIGH>;
+ clocks = <&clks IMX6QDL_CLK_VDOA>;
};
uart2: serial@021e8000 {
diff --git a/arch/arm/boot/dts/imx6sl.dtsi b/arch/arm/boot/dts/imx6sl.dtsi
index 19cbd879c448..cc9572ea2860 100644
--- a/arch/arm/boot/dts/imx6sl.dtsi
+++ b/arch/arm/boot/dts/imx6sl.dtsi
@@ -14,6 +14,14 @@
/ {
#address-cells = <1>;
#size-cells = <1>;
+ /*
+ * The decompressor and also some bootloaders rely on a
+ * pre-existing /chosen node to be available to insert the
+ * command line and merge other ATAGS info.
+ * Also for U-Boot there must be a pre-existing /memory node.
+ */
+ chosen {};
+ memory { device_type = "memory"; reg = <0 0>; };
aliases {
ethernet0 = &fec;
diff --git a/arch/arm/boot/dts/imx6sx.dtsi b/arch/arm/boot/dts/imx6sx.dtsi
index 10f333016197..dd4ec85ecbaa 100644
--- a/arch/arm/boot/dts/imx6sx.dtsi
+++ b/arch/arm/boot/dts/imx6sx.dtsi
@@ -15,6 +15,14 @@
/ {
#address-cells = <1>;
#size-cells = <1>;
+ /*
+ * The decompressor and also some bootloaders rely on a
+ * pre-existing /chosen node to be available to insert the
+ * command line and merge other ATAGS info.
+ * Also for U-Boot there must be a pre-existing /memory node.
+ */
+ chosen {};
+ memory { device_type = "memory"; reg = <0 0>; };
aliases {
can0 = &flexcan1;
diff --git a/arch/arm/boot/dts/imx6ul.dtsi b/arch/arm/boot/dts/imx6ul.dtsi
index 39845a7e0463..53d3f8e41e9b 100644
--- a/arch/arm/boot/dts/imx6ul.dtsi
+++ b/arch/arm/boot/dts/imx6ul.dtsi
@@ -15,6 +15,14 @@
/ {
#address-cells = <1>;
#size-cells = <1>;
+ /*
+ * The decompressor and also some bootloaders rely on a
+ * pre-existing /chosen node to be available to insert the
+ * command line and merge other ATAGS info.
+ * Also for U-Boot there must be a pre-existing /memory node.
+ */
+ chosen {};
+ memory { device_type = "memory"; reg = <0 0>; };
aliases {
ethernet0 = &fec1;
diff --git a/arch/arm/boot/dts/imx7s.dtsi b/arch/arm/boot/dts/imx7s.dtsi
index 8ff2cbdd8f0d..be33dfc86838 100644
--- a/arch/arm/boot/dts/imx7s.dtsi
+++ b/arch/arm/boot/dts/imx7s.dtsi
@@ -50,6 +50,14 @@
/ {
#address-cells = <1>;
#size-cells = <1>;
+ /*
+ * The decompressor and also some bootloaders rely on a
+ * pre-existing /chosen node to be available to insert the
+ * command line and merge other ATAGS info.
+ * Also for U-Boot there must be a pre-existing /memory node.
+ */
+ chosen {};
+ memory { device_type = "memory"; reg = <0 0>; };
aliases {
gpio0 = &gpio1;
diff --git a/arch/arm/boot/dts/logicpd-som-lv-37xx-devkit.dts b/arch/arm/boot/dts/logicpd-som-lv-37xx-devkit.dts
index da8598402ab8..38faa90007d7 100644
--- a/arch/arm/boot/dts/logicpd-som-lv-37xx-devkit.dts
+++ b/arch/arm/boot/dts/logicpd-som-lv-37xx-devkit.dts
@@ -158,7 +158,7 @@
&mmc1 {
interrupts-extended = <&intc 83 &omap3_pmx_core 0x11a>;
pinctrl-names = "default";
- pinctrl-0 = <&mmc1_pins &mmc1_cd>;
+ pinctrl-0 = <&mmc1_pins>;
wp-gpios = <&gpio4 30 GPIO_ACTIVE_HIGH>; /* gpio_126 */
cd-gpios = <&gpio4 14 IRQ_TYPE_LEVEL_LOW>; /* gpio_110 */
vmmc-supply = <&vmmc1>;
@@ -193,7 +193,8 @@
OMAP3_CORE1_IOPAD(0x214a, PIN_INPUT | MUX_MODE0) /* sdmmc1_dat1.sdmmc1_dat1 */
OMAP3_CORE1_IOPAD(0x214c, PIN_INPUT | MUX_MODE0) /* sdmmc1_dat2.sdmmc1_dat2 */
OMAP3_CORE1_IOPAD(0x214e, PIN_INPUT | MUX_MODE0) /* sdmmc1_dat3.sdmmc1_dat3 */
- OMAP3_CORE1_IOPAD(0x2132, PIN_INPUT_PULLUP | MUX_MODE4) /* cam_strobe.gpio_126 sdmmc1_wp*/
+ OMAP3_CORE1_IOPAD(0x2132, PIN_INPUT_PULLUP | MUX_MODE4) /* cam_strobe.gpio_126 */
+ OMAP3_CORE1_IOPAD(0x212c, PIN_INPUT_PULLUP | MUX_MODE4) /* cam_d11.gpio_110 */
>;
};
@@ -242,12 +243,6 @@
OMAP3_WKUP_IOPAD(0x2a16, PIN_OUTPUT | PIN_OFF_OUTPUT_LOW | MUX_MODE4) /* sys_boot6.gpio_8 */
>;
};
-
- mmc1_cd: pinmux_mmc1_cd {
- pinctrl-single,pins = <
- OMAP3_WKUP_IOPAD(0x212c, PIN_INPUT_PULLUP | MUX_MODE4) /* cam_d11.gpio_110 */
- >;
- };
};
diff --git a/arch/arm/boot/dts/omap5.dtsi b/arch/arm/boot/dts/omap5.dtsi
index 7cd92babc41a..0844737b72b2 100644
--- a/arch/arm/boot/dts/omap5.dtsi
+++ b/arch/arm/boot/dts/omap5.dtsi
@@ -988,6 +988,7 @@
phy-names = "sata-phy";
clocks = <&sata_ref_clk>;
ti,hwmods = "sata";
+ ports-implemented = <0x1>;
};
dss: dss@58000000 {
diff --git a/arch/arm/boot/dts/orion5x-lschl.dts b/arch/arm/boot/dts/orion5x-linkstation-lschl.dts
index 947409252845..ea6c881634b9 100644
--- a/arch/arm/boot/dts/orion5x-lschl.dts
+++ b/arch/arm/boot/dts/orion5x-linkstation-lschl.dts
@@ -2,7 +2,7 @@
* Device Tree file for Buffalo Linkstation LS-CHLv3
*
* Copyright (C) 2016 Ash Hughes <ashley.hughes@blueyonder.co.uk>
- * Copyright (C) 2015, 2016
+ * Copyright (C) 2015-2017
* Roger Shimizu <rogershimizu@gmail.com>
*
* This file is dual-licensed: you can use it either under the terms
@@ -52,7 +52,7 @@
#include <dt-bindings/gpio/gpio.h>
/ {
- model = "Buffalo Linkstation Live v3 (LS-CHL)";
+ model = "Buffalo Linkstation LiveV3 (LS-CHL)";
compatible = "buffalo,lschl", "marvell,orion5x-88f5182", "marvell,orion5x";
memory { /* 128 MB */
diff --git a/arch/arm/boot/dts/qcom-mdm9615.dtsi b/arch/arm/boot/dts/qcom-mdm9615.dtsi
index 5ae4ec59e6ea..c852b69229c9 100644
--- a/arch/arm/boot/dts/qcom-mdm9615.dtsi
+++ b/arch/arm/boot/dts/qcom-mdm9615.dtsi
@@ -357,7 +357,7 @@
};
amba {
- compatible = "arm,amba-bus";
+ compatible = "simple-bus";
#address-cells = <1>;
#size-cells = <1>;
ranges;
diff --git a/arch/arm/boot/dts/stih407-family.dtsi b/arch/arm/boot/dts/stih407-family.dtsi
index c8b2944e304a..0fe03cb88628 100644
--- a/arch/arm/boot/dts/stih407-family.dtsi
+++ b/arch/arm/boot/dts/stih407-family.dtsi
@@ -222,9 +222,8 @@
compatible = "st,asc";
reg = <0x9830000 0x2c>;
interrupts = <GIC_SPI 122 IRQ_TYPE_NONE>;
- pinctrl-names = "default";
- pinctrl-0 = <&pinctrl_serial0>;
clocks = <&clk_s_c0_flexgen CLK_EXT2F_A9>;
+ /* Pinctrl moved out to a per-board configuration */
status = "disabled";
};
@@ -680,6 +679,7 @@
phy-names = "usb2-phy", "usb3-phy";
phys = <&usb2_picophy0>,
<&phy_port2 PHY_TYPE_USB3>;
+ snps,dis_u3_susphy_quirk;
};
};
@@ -1002,5 +1002,15 @@
status = "disabled";
};
+
+ delta0 {
+ compatible = "st,st-delta";
+ clock-names = "delta",
+ "delta-st231",
+ "delta-flash-promip";
+ clocks = <&clk_s_c0_flexgen CLK_VID_DMU>,
+ <&clk_s_c0_flexgen CLK_ST231_DMU>,
+ <&clk_s_c0_flexgen CLK_FLASH_PROMIP>;
+ };
};
};
diff --git a/arch/arm/boot/dts/stih407-pinctrl.dtsi b/arch/arm/boot/dts/stih407-pinctrl.dtsi
index daab16b5ae64..bd1a82e8fffe 100644
--- a/arch/arm/boot/dts/stih407-pinctrl.dtsi
+++ b/arch/arm/boot/dts/stih407-pinctrl.dtsi
@@ -465,8 +465,16 @@
serial0 {
pinctrl_serial0: serial0-0 {
st,pins {
- tx = <&pio17 0 ALT1 OUT>;
- rx = <&pio17 1 ALT1 IN>;
+ tx = <&pio17 0 ALT1 OUT>;
+ rx = <&pio17 1 ALT1 IN>;
+ };
+ };
+ pinctrl_serial0_hw_flowctrl: serial0-0_hw_flowctrl {
+ st,pins {
+ tx = <&pio17 0 ALT1 OUT>;
+ rx = <&pio17 1 ALT1 IN>;
+ cts = <&pio17 2 ALT1 IN>;
+ rts = <&pio17 3 ALT1 OUT>;
};
};
};
diff --git a/arch/arm/boot/dts/stih410-b2260.dts b/arch/arm/boot/dts/stih410-b2260.dts
index 06b0696cb6b8..93c14d183e29 100644
--- a/arch/arm/boot/dts/stih410-b2260.dts
+++ b/arch/arm/boot/dts/stih410-b2260.dts
@@ -62,6 +62,11 @@
/* Low speed expansion connector */
uart0: serial@9830000 {
label = "LS-UART0";
+ pinctrl-names = "default", "no-hw-flowctrl";
+ pinctrl-0 = <&pinctrl_serial0_hw_flowctrl>;
+ pinctrl-1 = <&pinctrl_serial0>;
+ rts-gpios = <&pio17 3 GPIO_ACTIVE_LOW>;
+ uart-has-rtscts;
status = "okay";
};
diff --git a/arch/arm/boot/dts/sun6i-a31-hummingbird.dts b/arch/arm/boot/dts/sun6i-a31-hummingbird.dts
index 735914f6ae44..7cae328398b1 100644
--- a/arch/arm/boot/dts/sun6i-a31-hummingbird.dts
+++ b/arch/arm/boot/dts/sun6i-a31-hummingbird.dts
@@ -140,6 +140,10 @@
cpu-supply = <&reg_dcdc3>;
};
+&de {
+ status = "okay";
+};
+
&ehci0 {
status = "okay";
};
diff --git a/arch/arm/boot/dts/sun6i-a31.dtsi b/arch/arm/boot/dts/sun6i-a31.dtsi
index 2b26175d55d1..e78faaf9243c 100644
--- a/arch/arm/boot/dts/sun6i-a31.dtsi
+++ b/arch/arm/boot/dts/sun6i-a31.dtsi
@@ -234,6 +234,7 @@
de: display-engine {
compatible = "allwinner,sun6i-a31-display-engine";
allwinner,pipelines = <&fe0>;
+ status = "disabled";
};
soc@01c00000 {
diff --git a/arch/arm/boot/dts/sun7i-a20-olinuxino-lime2-emmc.dts b/arch/arm/boot/dts/sun7i-a20-olinuxino-lime2-emmc.dts
index 5ea4915f6d75..10d307408f23 100644
--- a/arch/arm/boot/dts/sun7i-a20-olinuxino-lime2-emmc.dts
+++ b/arch/arm/boot/dts/sun7i-a20-olinuxino-lime2-emmc.dts
@@ -56,7 +56,7 @@
};
&pio {
- mmc2_pins_nrst: mmc2@0 {
+ mmc2_pins_nrst: mmc2-rst-pin {
allwinner,pins = "PC16";
allwinner,function = "gpio_out";
allwinner,drive = <SUN4I_PINCTRL_10_MA>;
diff --git a/arch/arm/configs/aspeed_g4_defconfig b/arch/arm/configs/aspeed_g4_defconfig
index ca39c04fec6b..05b99bc1c1ce 100644
--- a/arch/arm/configs/aspeed_g4_defconfig
+++ b/arch/arm/configs/aspeed_g4_defconfig
@@ -25,7 +25,6 @@ CONFIG_MODULE_UNLOAD=y
# CONFIG_ARCH_MULTI_V7 is not set
CONFIG_ARCH_ASPEED=y
CONFIG_MACH_ASPEED_G4=y
-CONFIG_DEBUG_RODATA=y
CONFIG_AEABI=y
CONFIG_UACCESS_WITH_MEMCPY=y
CONFIG_SECCOMP=y
@@ -79,7 +78,8 @@ CONFIG_DEBUG_LL_UART_8250=y
CONFIG_DEBUG_UART_PHYS=0x1e784000
CONFIG_DEBUG_UART_VIRT=0xe8784000
CONFIG_EARLY_PRINTK=y
-CONFIG_DEBUG_SET_MODULE_RONX=y
+CONFIG_STRICT_MODULE_RWX=y
+CONFIG_STRICT_KERNEL_RWX=y
# CONFIG_XZ_DEC_X86 is not set
# CONFIG_XZ_DEC_POWERPC is not set
# CONFIG_XZ_DEC_IA64 is not set
diff --git a/arch/arm/configs/aspeed_g5_defconfig b/arch/arm/configs/aspeed_g5_defconfig
index 4f366b0370e9..05a16d53d03c 100644
--- a/arch/arm/configs/aspeed_g5_defconfig
+++ b/arch/arm/configs/aspeed_g5_defconfig
@@ -26,7 +26,6 @@ CONFIG_ARCH_MULTI_V6=y
# CONFIG_ARCH_MULTI_V7 is not set
CONFIG_ARCH_ASPEED=y
CONFIG_MACH_ASPEED_G5=y
-CONFIG_DEBUG_RODATA=y
CONFIG_AEABI=y
CONFIG_UACCESS_WITH_MEMCPY=y
CONFIG_SECCOMP=y
@@ -81,7 +80,8 @@ CONFIG_DEBUG_LL_UART_8250=y
CONFIG_DEBUG_UART_PHYS=0x1e784000
CONFIG_DEBUG_UART_VIRT=0xe8784000
CONFIG_EARLY_PRINTK=y
-CONFIG_DEBUG_SET_MODULE_RONX=y
+CONFIG_STRICT_MODULE_RWX=y
+CONFIG_STRICT_KERNEL_RWX=y
# CONFIG_XZ_DEC_X86 is not set
# CONFIG_XZ_DEC_POWERPC is not set
# CONFIG_XZ_DEC_IA64 is not set
diff --git a/arch/arm/configs/exynos_defconfig b/arch/arm/configs/exynos_defconfig
index 79c415c33f69..809f0bf3042a 100644
--- a/arch/arm/configs/exynos_defconfig
+++ b/arch/arm/configs/exynos_defconfig
@@ -24,7 +24,7 @@ CONFIG_ARM_APPENDED_DTB=y
CONFIG_ARM_ATAG_DTB_COMPAT=y
CONFIG_CMDLINE="root=/dev/ram0 rw ramdisk=8192 initrd=0x41000000,8M console=ttySAC1,115200 init=/linuxrc mem=256M"
CONFIG_CPU_FREQ=y
-CONFIG_CPU_FREQ_STAT_DETAILS=y
+CONFIG_CPU_FREQ_STAT=y
CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND=y
CONFIG_CPU_FREQ_GOV_POWERSAVE=m
CONFIG_CPU_FREQ_GOV_USERSPACE=m
diff --git a/arch/arm/configs/ezx_defconfig b/arch/arm/configs/ezx_defconfig
index ea316c4b890e..d3f1768840e2 100644
--- a/arch/arm/configs/ezx_defconfig
+++ b/arch/arm/configs/ezx_defconfig
@@ -64,8 +64,8 @@ CONFIG_NETFILTER=y
CONFIG_NETFILTER_NETLINK_QUEUE=m
CONFIG_NF_CONNTRACK=m
CONFIG_NF_CONNTRACK_EVENTS=y
-CONFIG_NF_CT_PROTO_SCTP=m
-CONFIG_NF_CT_PROTO_UDPLITE=m
+CONFIG_NF_CT_PROTO_SCTP=y
+CONFIG_NF_CT_PROTO_UDPLITE=y
CONFIG_NF_CONNTRACK_AMANDA=m
CONFIG_NF_CONNTRACK_FTP=m
CONFIG_NF_CONNTRACK_H323=m
diff --git a/arch/arm/configs/imote2_defconfig b/arch/arm/configs/imote2_defconfig
index 18e59feaa307..7f479cdb3479 100644
--- a/arch/arm/configs/imote2_defconfig
+++ b/arch/arm/configs/imote2_defconfig
@@ -56,8 +56,8 @@ CONFIG_NETFILTER=y
CONFIG_NETFILTER_NETLINK_QUEUE=m
CONFIG_NF_CONNTRACK=m
CONFIG_NF_CONNTRACK_EVENTS=y
-CONFIG_NF_CT_PROTO_SCTP=m
-CONFIG_NF_CT_PROTO_UDPLITE=m
+CONFIG_NF_CT_PROTO_SCTP=y
+CONFIG_NF_CT_PROTO_UDPLITE=y
CONFIG_NF_CONNTRACK_AMANDA=m
CONFIG_NF_CONNTRACK_FTP=m
CONFIG_NF_CONNTRACK_H323=m
diff --git a/arch/arm/configs/multi_v5_defconfig b/arch/arm/configs/multi_v5_defconfig
index 361686a362f1..69a4bd13eea5 100644
--- a/arch/arm/configs/multi_v5_defconfig
+++ b/arch/arm/configs/multi_v5_defconfig
@@ -58,7 +58,7 @@ CONFIG_ZBOOT_ROM_BSS=0x0
CONFIG_ARM_APPENDED_DTB=y
CONFIG_ARM_ATAG_DTB_COMPAT=y
CONFIG_CPU_FREQ=y
-CONFIG_CPU_FREQ_STAT_DETAILS=y
+CONFIG_CPU_FREQ_STAT=y
CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND=y
CONFIG_CPU_IDLE=y
CONFIG_ARM_KIRKWOOD_CPUIDLE=y
diff --git a/arch/arm/configs/multi_v7_defconfig b/arch/arm/configs/multi_v7_defconfig
index b01a43851294..78f7c91051ff 100644
--- a/arch/arm/configs/multi_v7_defconfig
+++ b/arch/arm/configs/multi_v7_defconfig
@@ -132,7 +132,7 @@ CONFIG_ARM_ATAG_DTB_COMPAT=y
CONFIG_KEXEC=y
CONFIG_EFI=y
CONFIG_CPU_FREQ=y
-CONFIG_CPU_FREQ_STAT_DETAILS=y
+CONFIG_CPU_FREQ_STAT=y
CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND=y
CONFIG_CPU_FREQ_GOV_POWERSAVE=m
CONFIG_CPU_FREQ_GOV_USERSPACE=m
@@ -253,7 +253,8 @@ CONFIG_R8169=y
CONFIG_SH_ETH=y
CONFIG_SMSC911X=y
CONFIG_STMMAC_ETH=y
-CONFIG_SYNOPSYS_DWC_ETH_QOS=y
+CONFIG_STMMAC_PLATFORM=y
+CONFIG_DWMAC_DWC_QOS_ETH=y
CONFIG_TI_CPSW=y
CONFIG_XILINX_EMACLITE=y
CONFIG_AT803X_PHY=y
@@ -471,7 +472,7 @@ CONFIG_MESON_WATCHDOG=y
CONFIG_DW_WATCHDOG=y
CONFIG_DIGICOLOR_WATCHDOG=y
CONFIG_BCM2835_WDT=y
-CONFIG_BCM47XX_WATCHDOG=y
+CONFIG_BCM47XX_WDT=y
CONFIG_BCM7038_WDT=m
CONFIG_BCM_KONA_WDT=y
CONFIG_MFD_ACT8945A=y
@@ -569,6 +570,7 @@ CONFIG_VIDEO_SAMSUNG_S5P_MFC=m
CONFIG_VIDEO_SAMSUNG_EXYNOS_GSC=m
CONFIG_VIDEO_STI_BDISP=m
CONFIG_VIDEO_STI_HVA=m
+CONFIG_VIDEO_STI_DELTA=m
CONFIG_VIDEO_RENESAS_JPU=m
CONFIG_VIDEO_RENESAS_VSP1=m
CONFIG_V4L_TEST_DRIVERS=y
@@ -824,6 +826,7 @@ CONFIG_QCOM_SMSM=y
CONFIG_QCOM_WCNSS_CTRL=m
CONFIG_ROCKCHIP_PM_DOMAINS=y
CONFIG_COMMON_CLK_QCOM=y
+CONFIG_QCOM_CLK_RPM=y
CONFIG_CHROME_PLATFORMS=y
CONFIG_STAGING_BOARD=y
CONFIG_CROS_EC_CHARDEV=m
@@ -893,7 +896,7 @@ CONFIG_BCM2835_MBOX=y
CONFIG_RASPBERRYPI_FIRMWARE=y
CONFIG_EFI_VARS=m
CONFIG_EFI_CAPSULE_LOADER=m
-CONFIG_CONFIG_BCM47XX_NVRAM=y
+CONFIG_BCM47XX_NVRAM=y
CONFIG_BCM47XX_SPROM=y
CONFIG_EXT4_FS=y
CONFIG_AUTOFS4_FS=y
diff --git a/arch/arm/configs/mvebu_v5_defconfig b/arch/arm/configs/mvebu_v5_defconfig
index f7f6039419aa..4b598da0d086 100644
--- a/arch/arm/configs/mvebu_v5_defconfig
+++ b/arch/arm/configs/mvebu_v5_defconfig
@@ -44,7 +44,7 @@ CONFIG_ZBOOT_ROM_BSS=0x0
CONFIG_ARM_APPENDED_DTB=y
CONFIG_ARM_ATAG_DTB_COMPAT=y
CONFIG_CPU_FREQ=y
-CONFIG_CPU_FREQ_STAT_DETAILS=y
+CONFIG_CPU_FREQ_STAT=y
CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND=y
CONFIG_CPU_IDLE=y
CONFIG_ARM_KIRKWOOD_CPUIDLE=y
diff --git a/arch/arm/configs/pxa_defconfig b/arch/arm/configs/pxa_defconfig
index e4314b1227a3..271dc7e78e43 100644
--- a/arch/arm/configs/pxa_defconfig
+++ b/arch/arm/configs/pxa_defconfig
@@ -97,7 +97,7 @@ CONFIG_ZBOOT_ROM_BSS=0x0
CONFIG_CMDLINE="root=/dev/ram0 ro"
CONFIG_KEXEC=y
CONFIG_CPU_FREQ=y
-CONFIG_CPU_FREQ_STAT_DETAILS=y
+CONFIG_CPU_FREQ_STAT=y
CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND=y
CONFIG_CPU_FREQ_GOV_POWERSAVE=m
CONFIG_CPU_FREQ_GOV_USERSPACE=m
diff --git a/arch/arm/configs/s3c2410_defconfig b/arch/arm/configs/s3c2410_defconfig
index 4364040ed696..1e6c48dd7b11 100644
--- a/arch/arm/configs/s3c2410_defconfig
+++ b/arch/arm/configs/s3c2410_defconfig
@@ -86,9 +86,9 @@ CONFIG_IPV6_TUNNEL=m
CONFIG_NETFILTER=y
CONFIG_NF_CONNTRACK=m
CONFIG_NF_CONNTRACK_EVENTS=y
-CONFIG_NF_CT_PROTO_DCCP=m
-CONFIG_NF_CT_PROTO_SCTP=m
-CONFIG_NF_CT_PROTO_UDPLITE=m
+CONFIG_NF_CT_PROTO_DCCP=y
+CONFIG_NF_CT_PROTO_SCTP=y
+CONFIG_NF_CT_PROTO_UDPLITE=y
CONFIG_NF_CONNTRACK_AMANDA=m
CONFIG_NF_CONNTRACK_FTP=m
CONFIG_NF_CONNTRACK_H323=m
diff --git a/arch/arm/configs/shmobile_defconfig b/arch/arm/configs/shmobile_defconfig
index 1b0f8ae36fb3..adeaecd831a4 100644
--- a/arch/arm/configs/shmobile_defconfig
+++ b/arch/arm/configs/shmobile_defconfig
@@ -38,7 +38,7 @@ CONFIG_ZBOOT_ROM_BSS=0x0
CONFIG_ARM_APPENDED_DTB=y
CONFIG_KEXEC=y
CONFIG_CPU_FREQ=y
-CONFIG_CPU_FREQ_STAT_DETAILS=y
+CONFIG_CPU_FREQ_STAT=y
CONFIG_CPU_FREQ_GOV_POWERSAVE=y
CONFIG_CPU_FREQ_GOV_USERSPACE=y
CONFIG_CPU_FREQ_GOV_ONDEMAND=y
diff --git a/arch/arm/crypto/Kconfig b/arch/arm/crypto/Kconfig
index 13f1b4c289d4..a8fce93137fb 100644
--- a/arch/arm/crypto/Kconfig
+++ b/arch/arm/crypto/Kconfig
@@ -62,35 +62,18 @@ config CRYPTO_SHA512_ARM
using optimized ARM assembler and NEON, when available.
config CRYPTO_AES_ARM
- tristate "AES cipher algorithms (ARM-asm)"
- depends on ARM
+ tristate "Scalar AES cipher for ARM"
select CRYPTO_ALGAPI
select CRYPTO_AES
help
Use optimized AES assembler routines for ARM platforms.
- AES cipher algorithms (FIPS-197). AES uses the Rijndael
- algorithm.
-
- Rijndael appears to be consistently a very good performer in
- both hardware and software across a wide range of computing
- environments regardless of its use in feedback or non-feedback
- modes. Its key setup time is excellent, and its key agility is
- good. Rijndael's very low memory requirements make it very well
- suited for restricted-space environments, in which it also
- demonstrates excellent performance. Rijndael's operations are
- among the easiest to defend against power and timing attacks.
-
- The AES specifies three key sizes: 128, 192 and 256 bits
-
- See <http://csrc.nist.gov/encryption/aes/> for more information.
-
config CRYPTO_AES_ARM_BS
tristate "Bit sliced AES using NEON instructions"
depends on KERNEL_MODE_NEON
- select CRYPTO_AES_ARM
select CRYPTO_BLKCIPHER
select CRYPTO_SIMD
+ select CRYPTO_AES_ARM
help
Use a faster and more secure NEON based implementation of AES in CBC,
CTR and XTS modes
@@ -130,4 +113,10 @@ config CRYPTO_CRC32_ARM_CE
depends on KERNEL_MODE_NEON && CRC32
select CRYPTO_HASH
+config CRYPTO_CHACHA20_NEON
+ tristate "NEON accelerated ChaCha20 symmetric cipher"
+ depends on KERNEL_MODE_NEON
+ select CRYPTO_BLKCIPHER
+ select CRYPTO_CHACHA20
+
endif
diff --git a/arch/arm/crypto/Makefile b/arch/arm/crypto/Makefile
index b578a1820ab1..1822c4697278 100644
--- a/arch/arm/crypto/Makefile
+++ b/arch/arm/crypto/Makefile
@@ -8,6 +8,7 @@ obj-$(CONFIG_CRYPTO_SHA1_ARM) += sha1-arm.o
obj-$(CONFIG_CRYPTO_SHA1_ARM_NEON) += sha1-arm-neon.o
obj-$(CONFIG_CRYPTO_SHA256_ARM) += sha256-arm.o
obj-$(CONFIG_CRYPTO_SHA512_ARM) += sha512-arm.o
+obj-$(CONFIG_CRYPTO_CHACHA20_NEON) += chacha20-neon.o
ce-obj-$(CONFIG_CRYPTO_AES_ARM_CE) += aes-arm-ce.o
ce-obj-$(CONFIG_CRYPTO_SHA1_ARM_CE) += sha1-arm-ce.o
@@ -26,8 +27,8 @@ $(warning $(ce-obj-y) $(ce-obj-m))
endif
endif
-aes-arm-y := aes-armv4.o aes_glue.o
-aes-arm-bs-y := aesbs-core.o aesbs-glue.o
+aes-arm-y := aes-cipher-core.o aes-cipher-glue.o
+aes-arm-bs-y := aes-neonbs-core.o aes-neonbs-glue.o
sha1-arm-y := sha1-armv4-large.o sha1_glue.o
sha1-arm-neon-y := sha1-armv7-neon.o sha1_neon_glue.o
sha256-arm-neon-$(CONFIG_KERNEL_MODE_NEON) := sha256_neon_glue.o
@@ -40,17 +41,15 @@ aes-arm-ce-y := aes-ce-core.o aes-ce-glue.o
ghash-arm-ce-y := ghash-ce-core.o ghash-ce-glue.o
crct10dif-arm-ce-y := crct10dif-ce-core.o crct10dif-ce-glue.o
crc32-arm-ce-y:= crc32-ce-core.o crc32-ce-glue.o
+chacha20-neon-y := chacha20-neon-core.o chacha20-neon-glue.o
quiet_cmd_perl = PERL $@
cmd_perl = $(PERL) $(<) > $(@)
-$(src)/aesbs-core.S_shipped: $(src)/bsaes-armv7.pl
- $(call cmd,perl)
-
$(src)/sha256-core.S_shipped: $(src)/sha256-armv4.pl
$(call cmd,perl)
$(src)/sha512-core.S_shipped: $(src)/sha512-armv4.pl
$(call cmd,perl)
-.PRECIOUS: $(obj)/aesbs-core.S $(obj)/sha256-core.S $(obj)/sha512-core.S
+.PRECIOUS: $(obj)/sha256-core.S $(obj)/sha512-core.S
diff --git a/arch/arm/crypto/aes-armv4.S b/arch/arm/crypto/aes-armv4.S
deleted file mode 100644
index ebb9761fb572..000000000000
--- a/arch/arm/crypto/aes-armv4.S
+++ /dev/null
@@ -1,1089 +0,0 @@
-#define __ARM_ARCH__ __LINUX_ARM_ARCH__
-@ ====================================================================
-@ Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
-@ project. The module is, however, dual licensed under OpenSSL and
-@ CRYPTOGAMS licenses depending on where you obtain it. For further
-@ details see http://www.openssl.org/~appro/cryptogams/.
-@ ====================================================================
-
-@ AES for ARMv4
-
-@ January 2007.
-@
-@ Code uses single 1K S-box and is >2 times faster than code generated
-@ by gcc-3.4.1. This is thanks to unique feature of ARMv4 ISA, which
-@ allows to merge logical or arithmetic operation with shift or rotate
-@ in one instruction and emit combined result every cycle. The module
-@ is endian-neutral. The performance is ~42 cycles/byte for 128-bit
-@ key [on single-issue Xscale PXA250 core].
-
-@ May 2007.
-@
-@ AES_set_[en|de]crypt_key is added.
-
-@ July 2010.
-@
-@ Rescheduling for dual-issue pipeline resulted in 12% improvement on
-@ Cortex A8 core and ~25 cycles per byte processed with 128-bit key.
-
-@ February 2011.
-@
-@ Profiler-assisted and platform-specific optimization resulted in 16%
-@ improvement on Cortex A8 core and ~21.5 cycles per byte.
-
-@ A little glue here to select the correct code below for the ARM CPU
-@ that is being targetted.
-
-#include <linux/linkage.h>
-#include <asm/assembler.h>
-
-.text
-
-.type AES_Te,%object
-.align 5
-AES_Te:
-.word 0xc66363a5, 0xf87c7c84, 0xee777799, 0xf67b7b8d
-.word 0xfff2f20d, 0xd66b6bbd, 0xde6f6fb1, 0x91c5c554
-.word 0x60303050, 0x02010103, 0xce6767a9, 0x562b2b7d
-.word 0xe7fefe19, 0xb5d7d762, 0x4dababe6, 0xec76769a
-.word 0x8fcaca45, 0x1f82829d, 0x89c9c940, 0xfa7d7d87
-.word 0xeffafa15, 0xb25959eb, 0x8e4747c9, 0xfbf0f00b
-.word 0x41adadec, 0xb3d4d467, 0x5fa2a2fd, 0x45afafea
-.word 0x239c9cbf, 0x53a4a4f7, 0xe4727296, 0x9bc0c05b
-.word 0x75b7b7c2, 0xe1fdfd1c, 0x3d9393ae, 0x4c26266a
-.word 0x6c36365a, 0x7e3f3f41, 0xf5f7f702, 0x83cccc4f
-.word 0x6834345c, 0x51a5a5f4, 0xd1e5e534, 0xf9f1f108
-.word 0xe2717193, 0xabd8d873, 0x62313153, 0x2a15153f
-.word 0x0804040c, 0x95c7c752, 0x46232365, 0x9dc3c35e
-.word 0x30181828, 0x379696a1, 0x0a05050f, 0x2f9a9ab5
-.word 0x0e070709, 0x24121236, 0x1b80809b, 0xdfe2e23d
-.word 0xcdebeb26, 0x4e272769, 0x7fb2b2cd, 0xea75759f
-.word 0x1209091b, 0x1d83839e, 0x582c2c74, 0x341a1a2e
-.word 0x361b1b2d, 0xdc6e6eb2, 0xb45a5aee, 0x5ba0a0fb
-.word 0xa45252f6, 0x763b3b4d, 0xb7d6d661, 0x7db3b3ce
-.word 0x5229297b, 0xdde3e33e, 0x5e2f2f71, 0x13848497
-.word 0xa65353f5, 0xb9d1d168, 0x00000000, 0xc1eded2c
-.word 0x40202060, 0xe3fcfc1f, 0x79b1b1c8, 0xb65b5bed
-.word 0xd46a6abe, 0x8dcbcb46, 0x67bebed9, 0x7239394b
-.word 0x944a4ade, 0x984c4cd4, 0xb05858e8, 0x85cfcf4a
-.word 0xbbd0d06b, 0xc5efef2a, 0x4faaaae5, 0xedfbfb16
-.word 0x864343c5, 0x9a4d4dd7, 0x66333355, 0x11858594
-.word 0x8a4545cf, 0xe9f9f910, 0x04020206, 0xfe7f7f81
-.word 0xa05050f0, 0x783c3c44, 0x259f9fba, 0x4ba8a8e3
-.word 0xa25151f3, 0x5da3a3fe, 0x804040c0, 0x058f8f8a
-.word 0x3f9292ad, 0x219d9dbc, 0x70383848, 0xf1f5f504
-.word 0x63bcbcdf, 0x77b6b6c1, 0xafdada75, 0x42212163
-.word 0x20101030, 0xe5ffff1a, 0xfdf3f30e, 0xbfd2d26d
-.word 0x81cdcd4c, 0x180c0c14, 0x26131335, 0xc3ecec2f
-.word 0xbe5f5fe1, 0x359797a2, 0x884444cc, 0x2e171739
-.word 0x93c4c457, 0x55a7a7f2, 0xfc7e7e82, 0x7a3d3d47
-.word 0xc86464ac, 0xba5d5de7, 0x3219192b, 0xe6737395
-.word 0xc06060a0, 0x19818198, 0x9e4f4fd1, 0xa3dcdc7f
-.word 0x44222266, 0x542a2a7e, 0x3b9090ab, 0x0b888883
-.word 0x8c4646ca, 0xc7eeee29, 0x6bb8b8d3, 0x2814143c
-.word 0xa7dede79, 0xbc5e5ee2, 0x160b0b1d, 0xaddbdb76
-.word 0xdbe0e03b, 0x64323256, 0x743a3a4e, 0x140a0a1e
-.word 0x924949db, 0x0c06060a, 0x4824246c, 0xb85c5ce4
-.word 0x9fc2c25d, 0xbdd3d36e, 0x43acacef, 0xc46262a6
-.word 0x399191a8, 0x319595a4, 0xd3e4e437, 0xf279798b
-.word 0xd5e7e732, 0x8bc8c843, 0x6e373759, 0xda6d6db7
-.word 0x018d8d8c, 0xb1d5d564, 0x9c4e4ed2, 0x49a9a9e0
-.word 0xd86c6cb4, 0xac5656fa, 0xf3f4f407, 0xcfeaea25
-.word 0xca6565af, 0xf47a7a8e, 0x47aeaee9, 0x10080818
-.word 0x6fbabad5, 0xf0787888, 0x4a25256f, 0x5c2e2e72
-.word 0x381c1c24, 0x57a6a6f1, 0x73b4b4c7, 0x97c6c651
-.word 0xcbe8e823, 0xa1dddd7c, 0xe874749c, 0x3e1f1f21
-.word 0x964b4bdd, 0x61bdbddc, 0x0d8b8b86, 0x0f8a8a85
-.word 0xe0707090, 0x7c3e3e42, 0x71b5b5c4, 0xcc6666aa
-.word 0x904848d8, 0x06030305, 0xf7f6f601, 0x1c0e0e12
-.word 0xc26161a3, 0x6a35355f, 0xae5757f9, 0x69b9b9d0
-.word 0x17868691, 0x99c1c158, 0x3a1d1d27, 0x279e9eb9
-.word 0xd9e1e138, 0xebf8f813, 0x2b9898b3, 0x22111133
-.word 0xd26969bb, 0xa9d9d970, 0x078e8e89, 0x339494a7
-.word 0x2d9b9bb6, 0x3c1e1e22, 0x15878792, 0xc9e9e920
-.word 0x87cece49, 0xaa5555ff, 0x50282878, 0xa5dfdf7a
-.word 0x038c8c8f, 0x59a1a1f8, 0x09898980, 0x1a0d0d17
-.word 0x65bfbfda, 0xd7e6e631, 0x844242c6, 0xd06868b8
-.word 0x824141c3, 0x299999b0, 0x5a2d2d77, 0x1e0f0f11
-.word 0x7bb0b0cb, 0xa85454fc, 0x6dbbbbd6, 0x2c16163a
-@ Te4[256]
-.byte 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5
-.byte 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76
-.byte 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0
-.byte 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0
-.byte 0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc
-.byte 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15
-.byte 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a
-.byte 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75
-.byte 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0
-.byte 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84
-.byte 0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b
-.byte 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf
-.byte 0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85
-.byte 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8
-.byte 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5
-.byte 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2
-.byte 0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17
-.byte 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73
-.byte 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88
-.byte 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb
-.byte 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c
-.byte 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79
-.byte 0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9
-.byte 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08
-.byte 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6
-.byte 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a
-.byte 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e
-.byte 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e
-.byte 0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94
-.byte 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf
-.byte 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68
-.byte 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16
-@ rcon[]
-.word 0x01000000, 0x02000000, 0x04000000, 0x08000000
-.word 0x10000000, 0x20000000, 0x40000000, 0x80000000
-.word 0x1B000000, 0x36000000, 0, 0, 0, 0, 0, 0
-.size AES_Te,.-AES_Te
-
-@ void AES_encrypt(const unsigned char *in, unsigned char *out,
-@ const AES_KEY *key) {
-.align 5
-ENTRY(AES_encrypt)
- adr r3,AES_encrypt
- stmdb sp!,{r1,r4-r12,lr}
- mov r12,r0 @ inp
- mov r11,r2
- sub r10,r3,#AES_encrypt-AES_Te @ Te
-#if __ARM_ARCH__<7
- ldrb r0,[r12,#3] @ load input data in endian-neutral
- ldrb r4,[r12,#2] @ manner...
- ldrb r5,[r12,#1]
- ldrb r6,[r12,#0]
- orr r0,r0,r4,lsl#8
- ldrb r1,[r12,#7]
- orr r0,r0,r5,lsl#16
- ldrb r4,[r12,#6]
- orr r0,r0,r6,lsl#24
- ldrb r5,[r12,#5]
- ldrb r6,[r12,#4]
- orr r1,r1,r4,lsl#8
- ldrb r2,[r12,#11]
- orr r1,r1,r5,lsl#16
- ldrb r4,[r12,#10]
- orr r1,r1,r6,lsl#24
- ldrb r5,[r12,#9]
- ldrb r6,[r12,#8]
- orr r2,r2,r4,lsl#8
- ldrb r3,[r12,#15]
- orr r2,r2,r5,lsl#16
- ldrb r4,[r12,#14]
- orr r2,r2,r6,lsl#24
- ldrb r5,[r12,#13]
- ldrb r6,[r12,#12]
- orr r3,r3,r4,lsl#8
- orr r3,r3,r5,lsl#16
- orr r3,r3,r6,lsl#24
-#else
- ldr r0,[r12,#0]
- ldr r1,[r12,#4]
- ldr r2,[r12,#8]
- ldr r3,[r12,#12]
-#ifdef __ARMEL__
- rev r0,r0
- rev r1,r1
- rev r2,r2
- rev r3,r3
-#endif
-#endif
- bl _armv4_AES_encrypt
-
- ldr r12,[sp],#4 @ pop out
-#if __ARM_ARCH__>=7
-#ifdef __ARMEL__
- rev r0,r0
- rev r1,r1
- rev r2,r2
- rev r3,r3
-#endif
- str r0,[r12,#0]
- str r1,[r12,#4]
- str r2,[r12,#8]
- str r3,[r12,#12]
-#else
- mov r4,r0,lsr#24 @ write output in endian-neutral
- mov r5,r0,lsr#16 @ manner...
- mov r6,r0,lsr#8
- strb r4,[r12,#0]
- strb r5,[r12,#1]
- mov r4,r1,lsr#24
- strb r6,[r12,#2]
- mov r5,r1,lsr#16
- strb r0,[r12,#3]
- mov r6,r1,lsr#8
- strb r4,[r12,#4]
- strb r5,[r12,#5]
- mov r4,r2,lsr#24
- strb r6,[r12,#6]
- mov r5,r2,lsr#16
- strb r1,[r12,#7]
- mov r6,r2,lsr#8
- strb r4,[r12,#8]
- strb r5,[r12,#9]
- mov r4,r3,lsr#24
- strb r6,[r12,#10]
- mov r5,r3,lsr#16
- strb r2,[r12,#11]
- mov r6,r3,lsr#8
- strb r4,[r12,#12]
- strb r5,[r12,#13]
- strb r6,[r12,#14]
- strb r3,[r12,#15]
-#endif
- ldmia sp!,{r4-r12,pc}
-ENDPROC(AES_encrypt)
-
-.type _armv4_AES_encrypt,%function
-.align 2
-_armv4_AES_encrypt:
- str lr,[sp,#-4]! @ push lr
- ldmia r11!,{r4-r7}
- eor r0,r0,r4
- ldr r12,[r11,#240-16]
- eor r1,r1,r5
- eor r2,r2,r6
- eor r3,r3,r7
- sub r12,r12,#1
- mov lr,#255
-
- and r7,lr,r0
- and r8,lr,r0,lsr#8
- and r9,lr,r0,lsr#16
- mov r0,r0,lsr#24
-.Lenc_loop:
- ldr r4,[r10,r7,lsl#2] @ Te3[s0>>0]
- and r7,lr,r1,lsr#16 @ i0
- ldr r5,[r10,r8,lsl#2] @ Te2[s0>>8]
- and r8,lr,r1
- ldr r6,[r10,r9,lsl#2] @ Te1[s0>>16]
- and r9,lr,r1,lsr#8
- ldr r0,[r10,r0,lsl#2] @ Te0[s0>>24]
- mov r1,r1,lsr#24
-
- ldr r7,[r10,r7,lsl#2] @ Te1[s1>>16]
- ldr r8,[r10,r8,lsl#2] @ Te3[s1>>0]
- ldr r9,[r10,r9,lsl#2] @ Te2[s1>>8]
- eor r0,r0,r7,ror#8
- ldr r1,[r10,r1,lsl#2] @ Te0[s1>>24]
- and r7,lr,r2,lsr#8 @ i0
- eor r5,r5,r8,ror#8
- and r8,lr,r2,lsr#16 @ i1
- eor r6,r6,r9,ror#8
- and r9,lr,r2
- ldr r7,[r10,r7,lsl#2] @ Te2[s2>>8]
- eor r1,r1,r4,ror#24
- ldr r8,[r10,r8,lsl#2] @ Te1[s2>>16]
- mov r2,r2,lsr#24
-
- ldr r9,[r10,r9,lsl#2] @ Te3[s2>>0]
- eor r0,r0,r7,ror#16
- ldr r2,[r10,r2,lsl#2] @ Te0[s2>>24]
- and r7,lr,r3 @ i0
- eor r1,r1,r8,ror#8
- and r8,lr,r3,lsr#8 @ i1
- eor r6,r6,r9,ror#16
- and r9,lr,r3,lsr#16 @ i2
- ldr r7,[r10,r7,lsl#2] @ Te3[s3>>0]
- eor r2,r2,r5,ror#16
- ldr r8,[r10,r8,lsl#2] @ Te2[s3>>8]
- mov r3,r3,lsr#24
-
- ldr r9,[r10,r9,lsl#2] @ Te1[s3>>16]
- eor r0,r0,r7,ror#24
- ldr r7,[r11],#16
- eor r1,r1,r8,ror#16
- ldr r3,[r10,r3,lsl#2] @ Te0[s3>>24]
- eor r2,r2,r9,ror#8
- ldr r4,[r11,#-12]
- eor r3,r3,r6,ror#8
-
- ldr r5,[r11,#-8]
- eor r0,r0,r7
- ldr r6,[r11,#-4]
- and r7,lr,r0
- eor r1,r1,r4
- and r8,lr,r0,lsr#8
- eor r2,r2,r5
- and r9,lr,r0,lsr#16
- eor r3,r3,r6
- mov r0,r0,lsr#24
-
- subs r12,r12,#1
- bne .Lenc_loop
-
- add r10,r10,#2
-
- ldrb r4,[r10,r7,lsl#2] @ Te4[s0>>0]
- and r7,lr,r1,lsr#16 @ i0
- ldrb r5,[r10,r8,lsl#2] @ Te4[s0>>8]
- and r8,lr,r1
- ldrb r6,[r10,r9,lsl#2] @ Te4[s0>>16]
- and r9,lr,r1,lsr#8
- ldrb r0,[r10,r0,lsl#2] @ Te4[s0>>24]
- mov r1,r1,lsr#24
-
- ldrb r7,[r10,r7,lsl#2] @ Te4[s1>>16]
- ldrb r8,[r10,r8,lsl#2] @ Te4[s1>>0]
- ldrb r9,[r10,r9,lsl#2] @ Te4[s1>>8]
- eor r0,r7,r0,lsl#8
- ldrb r1,[r10,r1,lsl#2] @ Te4[s1>>24]
- and r7,lr,r2,lsr#8 @ i0
- eor r5,r8,r5,lsl#8
- and r8,lr,r2,lsr#16 @ i1
- eor r6,r9,r6,lsl#8
- and r9,lr,r2
- ldrb r7,[r10,r7,lsl#2] @ Te4[s2>>8]
- eor r1,r4,r1,lsl#24
- ldrb r8,[r10,r8,lsl#2] @ Te4[s2>>16]
- mov r2,r2,lsr#24
-
- ldrb r9,[r10,r9,lsl#2] @ Te4[s2>>0]
- eor r0,r7,r0,lsl#8
- ldrb r2,[r10,r2,lsl#2] @ Te4[s2>>24]
- and r7,lr,r3 @ i0
- eor r1,r1,r8,lsl#16
- and r8,lr,r3,lsr#8 @ i1
- eor r6,r9,r6,lsl#8
- and r9,lr,r3,lsr#16 @ i2
- ldrb r7,[r10,r7,lsl#2] @ Te4[s3>>0]
- eor r2,r5,r2,lsl#24
- ldrb r8,[r10,r8,lsl#2] @ Te4[s3>>8]
- mov r3,r3,lsr#24
-
- ldrb r9,[r10,r9,lsl#2] @ Te4[s3>>16]
- eor r0,r7,r0,lsl#8
- ldr r7,[r11,#0]
- ldrb r3,[r10,r3,lsl#2] @ Te4[s3>>24]
- eor r1,r1,r8,lsl#8
- ldr r4,[r11,#4]
- eor r2,r2,r9,lsl#16
- ldr r5,[r11,#8]
- eor r3,r6,r3,lsl#24
- ldr r6,[r11,#12]
-
- eor r0,r0,r7
- eor r1,r1,r4
- eor r2,r2,r5
- eor r3,r3,r6
-
- sub r10,r10,#2
- ldr pc,[sp],#4 @ pop and return
-.size _armv4_AES_encrypt,.-_armv4_AES_encrypt
-
-.align 5
-ENTRY(private_AES_set_encrypt_key)
-_armv4_AES_set_encrypt_key:
- adr r3,_armv4_AES_set_encrypt_key
- teq r0,#0
- moveq r0,#-1
- beq .Labrt
- teq r2,#0
- moveq r0,#-1
- beq .Labrt
-
- teq r1,#128
- beq .Lok
- teq r1,#192
- beq .Lok
- teq r1,#256
- movne r0,#-1
- bne .Labrt
-
-.Lok: stmdb sp!,{r4-r12,lr}
- sub r10,r3,#_armv4_AES_set_encrypt_key-AES_Te-1024 @ Te4
-
- mov r12,r0 @ inp
- mov lr,r1 @ bits
- mov r11,r2 @ key
-
-#if __ARM_ARCH__<7
- ldrb r0,[r12,#3] @ load input data in endian-neutral
- ldrb r4,[r12,#2] @ manner...
- ldrb r5,[r12,#1]
- ldrb r6,[r12,#0]
- orr r0,r0,r4,lsl#8
- ldrb r1,[r12,#7]
- orr r0,r0,r5,lsl#16
- ldrb r4,[r12,#6]
- orr r0,r0,r6,lsl#24
- ldrb r5,[r12,#5]
- ldrb r6,[r12,#4]
- orr r1,r1,r4,lsl#8
- ldrb r2,[r12,#11]
- orr r1,r1,r5,lsl#16
- ldrb r4,[r12,#10]
- orr r1,r1,r6,lsl#24
- ldrb r5,[r12,#9]
- ldrb r6,[r12,#8]
- orr r2,r2,r4,lsl#8
- ldrb r3,[r12,#15]
- orr r2,r2,r5,lsl#16
- ldrb r4,[r12,#14]
- orr r2,r2,r6,lsl#24
- ldrb r5,[r12,#13]
- ldrb r6,[r12,#12]
- orr r3,r3,r4,lsl#8
- str r0,[r11],#16
- orr r3,r3,r5,lsl#16
- str r1,[r11,#-12]
- orr r3,r3,r6,lsl#24
- str r2,[r11,#-8]
- str r3,[r11,#-4]
-#else
- ldr r0,[r12,#0]
- ldr r1,[r12,#4]
- ldr r2,[r12,#8]
- ldr r3,[r12,#12]
-#ifdef __ARMEL__
- rev r0,r0
- rev r1,r1
- rev r2,r2
- rev r3,r3
-#endif
- str r0,[r11],#16
- str r1,[r11,#-12]
- str r2,[r11,#-8]
- str r3,[r11,#-4]
-#endif
-
- teq lr,#128
- bne .Lnot128
- mov r12,#10
- str r12,[r11,#240-16]
- add r6,r10,#256 @ rcon
- mov lr,#255
-
-.L128_loop:
- and r5,lr,r3,lsr#24
- and r7,lr,r3,lsr#16
- ldrb r5,[r10,r5]
- and r8,lr,r3,lsr#8
- ldrb r7,[r10,r7]
- and r9,lr,r3
- ldrb r8,[r10,r8]
- orr r5,r5,r7,lsl#24
- ldrb r9,[r10,r9]
- orr r5,r5,r8,lsl#16
- ldr r4,[r6],#4 @ rcon[i++]
- orr r5,r5,r9,lsl#8
- eor r5,r5,r4
- eor r0,r0,r5 @ rk[4]=rk[0]^...
- eor r1,r1,r0 @ rk[5]=rk[1]^rk[4]
- str r0,[r11],#16
- eor r2,r2,r1 @ rk[6]=rk[2]^rk[5]
- str r1,[r11,#-12]
- eor r3,r3,r2 @ rk[7]=rk[3]^rk[6]
- str r2,[r11,#-8]
- subs r12,r12,#1
- str r3,[r11,#-4]
- bne .L128_loop
- sub r2,r11,#176
- b .Ldone
-
-.Lnot128:
-#if __ARM_ARCH__<7
- ldrb r8,[r12,#19]
- ldrb r4,[r12,#18]
- ldrb r5,[r12,#17]
- ldrb r6,[r12,#16]
- orr r8,r8,r4,lsl#8
- ldrb r9,[r12,#23]
- orr r8,r8,r5,lsl#16
- ldrb r4,[r12,#22]
- orr r8,r8,r6,lsl#24
- ldrb r5,[r12,#21]
- ldrb r6,[r12,#20]
- orr r9,r9,r4,lsl#8
- orr r9,r9,r5,lsl#16
- str r8,[r11],#8
- orr r9,r9,r6,lsl#24
- str r9,[r11,#-4]
-#else
- ldr r8,[r12,#16]
- ldr r9,[r12,#20]
-#ifdef __ARMEL__
- rev r8,r8
- rev r9,r9
-#endif
- str r8,[r11],#8
- str r9,[r11,#-4]
-#endif
-
- teq lr,#192
- bne .Lnot192
- mov r12,#12
- str r12,[r11,#240-24]
- add r6,r10,#256 @ rcon
- mov lr,#255
- mov r12,#8
-
-.L192_loop:
- and r5,lr,r9,lsr#24
- and r7,lr,r9,lsr#16
- ldrb r5,[r10,r5]
- and r8,lr,r9,lsr#8
- ldrb r7,[r10,r7]
- and r9,lr,r9
- ldrb r8,[r10,r8]
- orr r5,r5,r7,lsl#24
- ldrb r9,[r10,r9]
- orr r5,r5,r8,lsl#16
- ldr r4,[r6],#4 @ rcon[i++]
- orr r5,r5,r9,lsl#8
- eor r9,r5,r4
- eor r0,r0,r9 @ rk[6]=rk[0]^...
- eor r1,r1,r0 @ rk[7]=rk[1]^rk[6]
- str r0,[r11],#24
- eor r2,r2,r1 @ rk[8]=rk[2]^rk[7]
- str r1,[r11,#-20]
- eor r3,r3,r2 @ rk[9]=rk[3]^rk[8]
- str r2,[r11,#-16]
- subs r12,r12,#1
- str r3,[r11,#-12]
- subeq r2,r11,#216
- beq .Ldone
-
- ldr r7,[r11,#-32]
- ldr r8,[r11,#-28]
- eor r7,r7,r3 @ rk[10]=rk[4]^rk[9]
- eor r9,r8,r7 @ rk[11]=rk[5]^rk[10]
- str r7,[r11,#-8]
- str r9,[r11,#-4]
- b .L192_loop
-
-.Lnot192:
-#if __ARM_ARCH__<7
- ldrb r8,[r12,#27]
- ldrb r4,[r12,#26]
- ldrb r5,[r12,#25]
- ldrb r6,[r12,#24]
- orr r8,r8,r4,lsl#8
- ldrb r9,[r12,#31]
- orr r8,r8,r5,lsl#16
- ldrb r4,[r12,#30]
- orr r8,r8,r6,lsl#24
- ldrb r5,[r12,#29]
- ldrb r6,[r12,#28]
- orr r9,r9,r4,lsl#8
- orr r9,r9,r5,lsl#16
- str r8,[r11],#8
- orr r9,r9,r6,lsl#24
- str r9,[r11,#-4]
-#else
- ldr r8,[r12,#24]
- ldr r9,[r12,#28]
-#ifdef __ARMEL__
- rev r8,r8
- rev r9,r9
-#endif
- str r8,[r11],#8
- str r9,[r11,#-4]
-#endif
-
- mov r12,#14
- str r12,[r11,#240-32]
- add r6,r10,#256 @ rcon
- mov lr,#255
- mov r12,#7
-
-.L256_loop:
- and r5,lr,r9,lsr#24
- and r7,lr,r9,lsr#16
- ldrb r5,[r10,r5]
- and r8,lr,r9,lsr#8
- ldrb r7,[r10,r7]
- and r9,lr,r9
- ldrb r8,[r10,r8]
- orr r5,r5,r7,lsl#24
- ldrb r9,[r10,r9]
- orr r5,r5,r8,lsl#16
- ldr r4,[r6],#4 @ rcon[i++]
- orr r5,r5,r9,lsl#8
- eor r9,r5,r4
- eor r0,r0,r9 @ rk[8]=rk[0]^...
- eor r1,r1,r0 @ rk[9]=rk[1]^rk[8]
- str r0,[r11],#32
- eor r2,r2,r1 @ rk[10]=rk[2]^rk[9]
- str r1,[r11,#-28]
- eor r3,r3,r2 @ rk[11]=rk[3]^rk[10]
- str r2,[r11,#-24]
- subs r12,r12,#1
- str r3,[r11,#-20]
- subeq r2,r11,#256
- beq .Ldone
-
- and r5,lr,r3
- and r7,lr,r3,lsr#8
- ldrb r5,[r10,r5]
- and r8,lr,r3,lsr#16
- ldrb r7,[r10,r7]
- and r9,lr,r3,lsr#24
- ldrb r8,[r10,r8]
- orr r5,r5,r7,lsl#8
- ldrb r9,[r10,r9]
- orr r5,r5,r8,lsl#16
- ldr r4,[r11,#-48]
- orr r5,r5,r9,lsl#24
-
- ldr r7,[r11,#-44]
- ldr r8,[r11,#-40]
- eor r4,r4,r5 @ rk[12]=rk[4]^...
- ldr r9,[r11,#-36]
- eor r7,r7,r4 @ rk[13]=rk[5]^rk[12]
- str r4,[r11,#-16]
- eor r8,r8,r7 @ rk[14]=rk[6]^rk[13]
- str r7,[r11,#-12]
- eor r9,r9,r8 @ rk[15]=rk[7]^rk[14]
- str r8,[r11,#-8]
- str r9,[r11,#-4]
- b .L256_loop
-
-.Ldone: mov r0,#0
- ldmia sp!,{r4-r12,lr}
-.Labrt: ret lr
-ENDPROC(private_AES_set_encrypt_key)
-
-.align 5
-ENTRY(private_AES_set_decrypt_key)
- str lr,[sp,#-4]! @ push lr
-#if 0
- @ kernel does both of these in setkey so optimise this bit out by
- @ expecting the key to already have the enc_key work done (see aes_glue.c)
- bl _armv4_AES_set_encrypt_key
-#else
- mov r0,#0
-#endif
- teq r0,#0
- ldrne lr,[sp],#4 @ pop lr
- bne .Labrt
-
- stmdb sp!,{r4-r12}
-
- ldr r12,[r2,#240] @ AES_set_encrypt_key preserves r2,
- mov r11,r2 @ which is AES_KEY *key
- mov r7,r2
- add r8,r2,r12,lsl#4
-
-.Linv: ldr r0,[r7]
- ldr r1,[r7,#4]
- ldr r2,[r7,#8]
- ldr r3,[r7,#12]
- ldr r4,[r8]
- ldr r5,[r8,#4]
- ldr r6,[r8,#8]
- ldr r9,[r8,#12]
- str r0,[r8],#-16
- str r1,[r8,#16+4]
- str r2,[r8,#16+8]
- str r3,[r8,#16+12]
- str r4,[r7],#16
- str r5,[r7,#-12]
- str r6,[r7,#-8]
- str r9,[r7,#-4]
- teq r7,r8
- bne .Linv
- ldr r0,[r11,#16]! @ prefetch tp1
- mov r7,#0x80
- mov r8,#0x1b
- orr r7,r7,#0x8000
- orr r8,r8,#0x1b00
- orr r7,r7,r7,lsl#16
- orr r8,r8,r8,lsl#16
- sub r12,r12,#1
- mvn r9,r7
- mov r12,r12,lsl#2 @ (rounds-1)*4
-
-.Lmix: and r4,r0,r7
- and r1,r0,r9
- sub r4,r4,r4,lsr#7
- and r4,r4,r8
- eor r1,r4,r1,lsl#1 @ tp2
-
- and r4,r1,r7
- and r2,r1,r9
- sub r4,r4,r4,lsr#7
- and r4,r4,r8
- eor r2,r4,r2,lsl#1 @ tp4
-
- and r4,r2,r7
- and r3,r2,r9
- sub r4,r4,r4,lsr#7
- and r4,r4,r8
- eor r3,r4,r3,lsl#1 @ tp8
-
- eor r4,r1,r2
- eor r5,r0,r3 @ tp9
- eor r4,r4,r3 @ tpe
- eor r4,r4,r1,ror#24
- eor r4,r4,r5,ror#24 @ ^= ROTATE(tpb=tp9^tp2,8)
- eor r4,r4,r2,ror#16
- eor r4,r4,r5,ror#16 @ ^= ROTATE(tpd=tp9^tp4,16)
- eor r4,r4,r5,ror#8 @ ^= ROTATE(tp9,24)
-
- ldr r0,[r11,#4] @ prefetch tp1
- str r4,[r11],#4
- subs r12,r12,#1
- bne .Lmix
-
- mov r0,#0
- ldmia sp!,{r4-r12,pc}
-ENDPROC(private_AES_set_decrypt_key)
-
-.type AES_Td,%object
-.align 5
-AES_Td:
-.word 0x51f4a750, 0x7e416553, 0x1a17a4c3, 0x3a275e96
-.word 0x3bab6bcb, 0x1f9d45f1, 0xacfa58ab, 0x4be30393
-.word 0x2030fa55, 0xad766df6, 0x88cc7691, 0xf5024c25
-.word 0x4fe5d7fc, 0xc52acbd7, 0x26354480, 0xb562a38f
-.word 0xdeb15a49, 0x25ba1b67, 0x45ea0e98, 0x5dfec0e1
-.word 0xc32f7502, 0x814cf012, 0x8d4697a3, 0x6bd3f9c6
-.word 0x038f5fe7, 0x15929c95, 0xbf6d7aeb, 0x955259da
-.word 0xd4be832d, 0x587421d3, 0x49e06929, 0x8ec9c844
-.word 0x75c2896a, 0xf48e7978, 0x99583e6b, 0x27b971dd
-.word 0xbee14fb6, 0xf088ad17, 0xc920ac66, 0x7dce3ab4
-.word 0x63df4a18, 0xe51a3182, 0x97513360, 0x62537f45
-.word 0xb16477e0, 0xbb6bae84, 0xfe81a01c, 0xf9082b94
-.word 0x70486858, 0x8f45fd19, 0x94de6c87, 0x527bf8b7
-.word 0xab73d323, 0x724b02e2, 0xe31f8f57, 0x6655ab2a
-.word 0xb2eb2807, 0x2fb5c203, 0x86c57b9a, 0xd33708a5
-.word 0x302887f2, 0x23bfa5b2, 0x02036aba, 0xed16825c
-.word 0x8acf1c2b, 0xa779b492, 0xf307f2f0, 0x4e69e2a1
-.word 0x65daf4cd, 0x0605bed5, 0xd134621f, 0xc4a6fe8a
-.word 0x342e539d, 0xa2f355a0, 0x058ae132, 0xa4f6eb75
-.word 0x0b83ec39, 0x4060efaa, 0x5e719f06, 0xbd6e1051
-.word 0x3e218af9, 0x96dd063d, 0xdd3e05ae, 0x4de6bd46
-.word 0x91548db5, 0x71c45d05, 0x0406d46f, 0x605015ff
-.word 0x1998fb24, 0xd6bde997, 0x894043cc, 0x67d99e77
-.word 0xb0e842bd, 0x07898b88, 0xe7195b38, 0x79c8eedb
-.word 0xa17c0a47, 0x7c420fe9, 0xf8841ec9, 0x00000000
-.word 0x09808683, 0x322bed48, 0x1e1170ac, 0x6c5a724e
-.word 0xfd0efffb, 0x0f853856, 0x3daed51e, 0x362d3927
-.word 0x0a0fd964, 0x685ca621, 0x9b5b54d1, 0x24362e3a
-.word 0x0c0a67b1, 0x9357e70f, 0xb4ee96d2, 0x1b9b919e
-.word 0x80c0c54f, 0x61dc20a2, 0x5a774b69, 0x1c121a16
-.word 0xe293ba0a, 0xc0a02ae5, 0x3c22e043, 0x121b171d
-.word 0x0e090d0b, 0xf28bc7ad, 0x2db6a8b9, 0x141ea9c8
-.word 0x57f11985, 0xaf75074c, 0xee99ddbb, 0xa37f60fd
-.word 0xf701269f, 0x5c72f5bc, 0x44663bc5, 0x5bfb7e34
-.word 0x8b432976, 0xcb23c6dc, 0xb6edfc68, 0xb8e4f163
-.word 0xd731dcca, 0x42638510, 0x13972240, 0x84c61120
-.word 0x854a247d, 0xd2bb3df8, 0xaef93211, 0xc729a16d
-.word 0x1d9e2f4b, 0xdcb230f3, 0x0d8652ec, 0x77c1e3d0
-.word 0x2bb3166c, 0xa970b999, 0x119448fa, 0x47e96422
-.word 0xa8fc8cc4, 0xa0f03f1a, 0x567d2cd8, 0x223390ef
-.word 0x87494ec7, 0xd938d1c1, 0x8ccaa2fe, 0x98d40b36
-.word 0xa6f581cf, 0xa57ade28, 0xdab78e26, 0x3fadbfa4
-.word 0x2c3a9de4, 0x5078920d, 0x6a5fcc9b, 0x547e4662
-.word 0xf68d13c2, 0x90d8b8e8, 0x2e39f75e, 0x82c3aff5
-.word 0x9f5d80be, 0x69d0937c, 0x6fd52da9, 0xcf2512b3
-.word 0xc8ac993b, 0x10187da7, 0xe89c636e, 0xdb3bbb7b
-.word 0xcd267809, 0x6e5918f4, 0xec9ab701, 0x834f9aa8
-.word 0xe6956e65, 0xaaffe67e, 0x21bccf08, 0xef15e8e6
-.word 0xbae79bd9, 0x4a6f36ce, 0xea9f09d4, 0x29b07cd6
-.word 0x31a4b2af, 0x2a3f2331, 0xc6a59430, 0x35a266c0
-.word 0x744ebc37, 0xfc82caa6, 0xe090d0b0, 0x33a7d815
-.word 0xf104984a, 0x41ecdaf7, 0x7fcd500e, 0x1791f62f
-.word 0x764dd68d, 0x43efb04d, 0xccaa4d54, 0xe49604df
-.word 0x9ed1b5e3, 0x4c6a881b, 0xc12c1fb8, 0x4665517f
-.word 0x9d5eea04, 0x018c355d, 0xfa877473, 0xfb0b412e
-.word 0xb3671d5a, 0x92dbd252, 0xe9105633, 0x6dd64713
-.word 0x9ad7618c, 0x37a10c7a, 0x59f8148e, 0xeb133c89
-.word 0xcea927ee, 0xb761c935, 0xe11ce5ed, 0x7a47b13c
-.word 0x9cd2df59, 0x55f2733f, 0x1814ce79, 0x73c737bf
-.word 0x53f7cdea, 0x5ffdaa5b, 0xdf3d6f14, 0x7844db86
-.word 0xcaaff381, 0xb968c43e, 0x3824342c, 0xc2a3405f
-.word 0x161dc372, 0xbce2250c, 0x283c498b, 0xff0d9541
-.word 0x39a80171, 0x080cb3de, 0xd8b4e49c, 0x6456c190
-.word 0x7bcb8461, 0xd532b670, 0x486c5c74, 0xd0b85742
-@ Td4[256]
-.byte 0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38
-.byte 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb
-.byte 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87
-.byte 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb
-.byte 0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d
-.byte 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e
-.byte 0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2
-.byte 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25
-.byte 0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16
-.byte 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92
-.byte 0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda
-.byte 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84
-.byte 0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a
-.byte 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06
-.byte 0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02
-.byte 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b
-.byte 0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea
-.byte 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73
-.byte 0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85
-.byte 0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e
-.byte 0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89
-.byte 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b
-.byte 0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20
-.byte 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4
-.byte 0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31
-.byte 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f
-.byte 0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d
-.byte 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef
-.byte 0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0
-.byte 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61
-.byte 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26
-.byte 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d
-.size AES_Td,.-AES_Td
-
-@ void AES_decrypt(const unsigned char *in, unsigned char *out,
-@ const AES_KEY *key) {
-.align 5
-ENTRY(AES_decrypt)
- adr r3,AES_decrypt
- stmdb sp!,{r1,r4-r12,lr}
- mov r12,r0 @ inp
- mov r11,r2
- sub r10,r3,#AES_decrypt-AES_Td @ Td
-#if __ARM_ARCH__<7
- ldrb r0,[r12,#3] @ load input data in endian-neutral
- ldrb r4,[r12,#2] @ manner...
- ldrb r5,[r12,#1]
- ldrb r6,[r12,#0]
- orr r0,r0,r4,lsl#8
- ldrb r1,[r12,#7]
- orr r0,r0,r5,lsl#16
- ldrb r4,[r12,#6]
- orr r0,r0,r6,lsl#24
- ldrb r5,[r12,#5]
- ldrb r6,[r12,#4]
- orr r1,r1,r4,lsl#8
- ldrb r2,[r12,#11]
- orr r1,r1,r5,lsl#16
- ldrb r4,[r12,#10]
- orr r1,r1,r6,lsl#24
- ldrb r5,[r12,#9]
- ldrb r6,[r12,#8]
- orr r2,r2,r4,lsl#8
- ldrb r3,[r12,#15]
- orr r2,r2,r5,lsl#16
- ldrb r4,[r12,#14]
- orr r2,r2,r6,lsl#24
- ldrb r5,[r12,#13]
- ldrb r6,[r12,#12]
- orr r3,r3,r4,lsl#8
- orr r3,r3,r5,lsl#16
- orr r3,r3,r6,lsl#24
-#else
- ldr r0,[r12,#0]
- ldr r1,[r12,#4]
- ldr r2,[r12,#8]
- ldr r3,[r12,#12]
-#ifdef __ARMEL__
- rev r0,r0
- rev r1,r1
- rev r2,r2
- rev r3,r3
-#endif
-#endif
- bl _armv4_AES_decrypt
-
- ldr r12,[sp],#4 @ pop out
-#if __ARM_ARCH__>=7
-#ifdef __ARMEL__
- rev r0,r0
- rev r1,r1
- rev r2,r2
- rev r3,r3
-#endif
- str r0,[r12,#0]
- str r1,[r12,#4]
- str r2,[r12,#8]
- str r3,[r12,#12]
-#else
- mov r4,r0,lsr#24 @ write output in endian-neutral
- mov r5,r0,lsr#16 @ manner...
- mov r6,r0,lsr#8
- strb r4,[r12,#0]
- strb r5,[r12,#1]
- mov r4,r1,lsr#24
- strb r6,[r12,#2]
- mov r5,r1,lsr#16
- strb r0,[r12,#3]
- mov r6,r1,lsr#8
- strb r4,[r12,#4]
- strb r5,[r12,#5]
- mov r4,r2,lsr#24
- strb r6,[r12,#6]
- mov r5,r2,lsr#16
- strb r1,[r12,#7]
- mov r6,r2,lsr#8
- strb r4,[r12,#8]
- strb r5,[r12,#9]
- mov r4,r3,lsr#24
- strb r6,[r12,#10]
- mov r5,r3,lsr#16
- strb r2,[r12,#11]
- mov r6,r3,lsr#8
- strb r4,[r12,#12]
- strb r5,[r12,#13]
- strb r6,[r12,#14]
- strb r3,[r12,#15]
-#endif
- ldmia sp!,{r4-r12,pc}
-ENDPROC(AES_decrypt)
-
-.type _armv4_AES_decrypt,%function
-.align 2
-_armv4_AES_decrypt:
- str lr,[sp,#-4]! @ push lr
- ldmia r11!,{r4-r7}
- eor r0,r0,r4
- ldr r12,[r11,#240-16]
- eor r1,r1,r5
- eor r2,r2,r6
- eor r3,r3,r7
- sub r12,r12,#1
- mov lr,#255
-
- and r7,lr,r0,lsr#16
- and r8,lr,r0,lsr#8
- and r9,lr,r0
- mov r0,r0,lsr#24
-.Ldec_loop:
- ldr r4,[r10,r7,lsl#2] @ Td1[s0>>16]
- and r7,lr,r1 @ i0
- ldr r5,[r10,r8,lsl#2] @ Td2[s0>>8]
- and r8,lr,r1,lsr#16
- ldr r6,[r10,r9,lsl#2] @ Td3[s0>>0]
- and r9,lr,r1,lsr#8
- ldr r0,[r10,r0,lsl#2] @ Td0[s0>>24]
- mov r1,r1,lsr#24
-
- ldr r7,[r10,r7,lsl#2] @ Td3[s1>>0]
- ldr r8,[r10,r8,lsl#2] @ Td1[s1>>16]
- ldr r9,[r10,r9,lsl#2] @ Td2[s1>>8]
- eor r0,r0,r7,ror#24
- ldr r1,[r10,r1,lsl#2] @ Td0[s1>>24]
- and r7,lr,r2,lsr#8 @ i0
- eor r5,r8,r5,ror#8
- and r8,lr,r2 @ i1
- eor r6,r9,r6,ror#8
- and r9,lr,r2,lsr#16
- ldr r7,[r10,r7,lsl#2] @ Td2[s2>>8]
- eor r1,r1,r4,ror#8
- ldr r8,[r10,r8,lsl#2] @ Td3[s2>>0]
- mov r2,r2,lsr#24
-
- ldr r9,[r10,r9,lsl#2] @ Td1[s2>>16]
- eor r0,r0,r7,ror#16
- ldr r2,[r10,r2,lsl#2] @ Td0[s2>>24]
- and r7,lr,r3,lsr#16 @ i0
- eor r1,r1,r8,ror#24
- and r8,lr,r3,lsr#8 @ i1
- eor r6,r9,r6,ror#8
- and r9,lr,r3 @ i2
- ldr r7,[r10,r7,lsl#2] @ Td1[s3>>16]
- eor r2,r2,r5,ror#8
- ldr r8,[r10,r8,lsl#2] @ Td2[s3>>8]
- mov r3,r3,lsr#24
-
- ldr r9,[r10,r9,lsl#2] @ Td3[s3>>0]
- eor r0,r0,r7,ror#8
- ldr r7,[r11],#16
- eor r1,r1,r8,ror#16
- ldr r3,[r10,r3,lsl#2] @ Td0[s3>>24]
- eor r2,r2,r9,ror#24
-
- ldr r4,[r11,#-12]
- eor r0,r0,r7
- ldr r5,[r11,#-8]
- eor r3,r3,r6,ror#8
- ldr r6,[r11,#-4]
- and r7,lr,r0,lsr#16
- eor r1,r1,r4
- and r8,lr,r0,lsr#8
- eor r2,r2,r5
- and r9,lr,r0
- eor r3,r3,r6
- mov r0,r0,lsr#24
-
- subs r12,r12,#1
- bne .Ldec_loop
-
- add r10,r10,#1024
-
- ldr r5,[r10,#0] @ prefetch Td4
- ldr r6,[r10,#32]
- ldr r4,[r10,#64]
- ldr r5,[r10,#96]
- ldr r6,[r10,#128]
- ldr r4,[r10,#160]
- ldr r5,[r10,#192]
- ldr r6,[r10,#224]
-
- ldrb r0,[r10,r0] @ Td4[s0>>24]
- ldrb r4,[r10,r7] @ Td4[s0>>16]
- and r7,lr,r1 @ i0
- ldrb r5,[r10,r8] @ Td4[s0>>8]
- and r8,lr,r1,lsr#16
- ldrb r6,[r10,r9] @ Td4[s0>>0]
- and r9,lr,r1,lsr#8
-
- ldrb r7,[r10,r7] @ Td4[s1>>0]
- ARM( ldrb r1,[r10,r1,lsr#24] ) @ Td4[s1>>24]
- THUMB( add r1,r10,r1,lsr#24 ) @ Td4[s1>>24]
- THUMB( ldrb r1,[r1] )
- ldrb r8,[r10,r8] @ Td4[s1>>16]
- eor r0,r7,r0,lsl#24
- ldrb r9,[r10,r9] @ Td4[s1>>8]
- eor r1,r4,r1,lsl#8
- and r7,lr,r2,lsr#8 @ i0
- eor r5,r5,r8,lsl#8
- and r8,lr,r2 @ i1
- ldrb r7,[r10,r7] @ Td4[s2>>8]
- eor r6,r6,r9,lsl#8
- ldrb r8,[r10,r8] @ Td4[s2>>0]
- and r9,lr,r2,lsr#16
-
- ARM( ldrb r2,[r10,r2,lsr#24] ) @ Td4[s2>>24]
- THUMB( add r2,r10,r2,lsr#24 ) @ Td4[s2>>24]
- THUMB( ldrb r2,[r2] )
- eor r0,r0,r7,lsl#8
- ldrb r9,[r10,r9] @ Td4[s2>>16]
- eor r1,r8,r1,lsl#16
- and r7,lr,r3,lsr#16 @ i0
- eor r2,r5,r2,lsl#16
- and r8,lr,r3,lsr#8 @ i1
- ldrb r7,[r10,r7] @ Td4[s3>>16]
- eor r6,r6,r9,lsl#16
- ldrb r8,[r10,r8] @ Td4[s3>>8]
- and r9,lr,r3 @ i2
-
- ldrb r9,[r10,r9] @ Td4[s3>>0]
- ARM( ldrb r3,[r10,r3,lsr#24] ) @ Td4[s3>>24]
- THUMB( add r3,r10,r3,lsr#24 ) @ Td4[s3>>24]
- THUMB( ldrb r3,[r3] )
- eor r0,r0,r7,lsl#16
- ldr r7,[r11,#0]
- eor r1,r1,r8,lsl#8
- ldr r4,[r11,#4]
- eor r2,r9,r2,lsl#8
- ldr r5,[r11,#8]
- eor r3,r6,r3,lsl#24
- ldr r6,[r11,#12]
-
- eor r0,r0,r7
- eor r1,r1,r4
- eor r2,r2,r5
- eor r3,r3,r6
-
- sub r10,r10,#1024
- ldr pc,[sp],#4 @ pop and return
-.size _armv4_AES_decrypt,.-_armv4_AES_decrypt
-.asciz "AES for ARMv4, CRYPTOGAMS by <appro@openssl.org>"
-.align 2
diff --git a/arch/arm/crypto/aes-ce-core.S b/arch/arm/crypto/aes-ce-core.S
index 987aa632c9f0..ba8e6a32fdc9 100644
--- a/arch/arm/crypto/aes-ce-core.S
+++ b/arch/arm/crypto/aes-ce-core.S
@@ -169,19 +169,19 @@ ENTRY(ce_aes_ecb_encrypt)
.Lecbencloop3x:
subs r4, r4, #3
bmi .Lecbenc1x
- vld1.8 {q0-q1}, [r1, :64]!
- vld1.8 {q2}, [r1, :64]!
+ vld1.8 {q0-q1}, [r1]!
+ vld1.8 {q2}, [r1]!
bl aes_encrypt_3x
- vst1.8 {q0-q1}, [r0, :64]!
- vst1.8 {q2}, [r0, :64]!
+ vst1.8 {q0-q1}, [r0]!
+ vst1.8 {q2}, [r0]!
b .Lecbencloop3x
.Lecbenc1x:
adds r4, r4, #3
beq .Lecbencout
.Lecbencloop:
- vld1.8 {q0}, [r1, :64]!
+ vld1.8 {q0}, [r1]!
bl aes_encrypt
- vst1.8 {q0}, [r0, :64]!
+ vst1.8 {q0}, [r0]!
subs r4, r4, #1
bne .Lecbencloop
.Lecbencout:
@@ -195,19 +195,19 @@ ENTRY(ce_aes_ecb_decrypt)
.Lecbdecloop3x:
subs r4, r4, #3
bmi .Lecbdec1x
- vld1.8 {q0-q1}, [r1, :64]!
- vld1.8 {q2}, [r1, :64]!
+ vld1.8 {q0-q1}, [r1]!
+ vld1.8 {q2}, [r1]!
bl aes_decrypt_3x
- vst1.8 {q0-q1}, [r0, :64]!
- vst1.8 {q2}, [r0, :64]!
+ vst1.8 {q0-q1}, [r0]!
+ vst1.8 {q2}, [r0]!
b .Lecbdecloop3x
.Lecbdec1x:
adds r4, r4, #3
beq .Lecbdecout
.Lecbdecloop:
- vld1.8 {q0}, [r1, :64]!
+ vld1.8 {q0}, [r1]!
bl aes_decrypt
- vst1.8 {q0}, [r0, :64]!
+ vst1.8 {q0}, [r0]!
subs r4, r4, #1
bne .Lecbdecloop
.Lecbdecout:
@@ -226,10 +226,10 @@ ENTRY(ce_aes_cbc_encrypt)
vld1.8 {q0}, [r5]
prepare_key r2, r3
.Lcbcencloop:
- vld1.8 {q1}, [r1, :64]! @ get next pt block
+ vld1.8 {q1}, [r1]! @ get next pt block
veor q0, q0, q1 @ ..and xor with iv
bl aes_encrypt
- vst1.8 {q0}, [r0, :64]!
+ vst1.8 {q0}, [r0]!
subs r4, r4, #1
bne .Lcbcencloop
vst1.8 {q0}, [r5]
@@ -244,8 +244,8 @@ ENTRY(ce_aes_cbc_decrypt)
.Lcbcdecloop3x:
subs r4, r4, #3
bmi .Lcbcdec1x
- vld1.8 {q0-q1}, [r1, :64]!
- vld1.8 {q2}, [r1, :64]!
+ vld1.8 {q0-q1}, [r1]!
+ vld1.8 {q2}, [r1]!
vmov q3, q0
vmov q4, q1
vmov q5, q2
@@ -254,19 +254,19 @@ ENTRY(ce_aes_cbc_decrypt)
veor q1, q1, q3
veor q2, q2, q4
vmov q6, q5
- vst1.8 {q0-q1}, [r0, :64]!
- vst1.8 {q2}, [r0, :64]!
+ vst1.8 {q0-q1}, [r0]!
+ vst1.8 {q2}, [r0]!
b .Lcbcdecloop3x
.Lcbcdec1x:
adds r4, r4, #3
beq .Lcbcdecout
vmov q15, q14 @ preserve last round key
.Lcbcdecloop:
- vld1.8 {q0}, [r1, :64]! @ get next ct block
+ vld1.8 {q0}, [r1]! @ get next ct block
veor q14, q15, q6 @ combine prev ct with last key
vmov q6, q0
bl aes_decrypt
- vst1.8 {q0}, [r0, :64]!
+ vst1.8 {q0}, [r0]!
subs r4, r4, #1
bne .Lcbcdecloop
.Lcbcdecout:
@@ -300,15 +300,15 @@ ENTRY(ce_aes_ctr_encrypt)
rev ip, r6
add r6, r6, #1
vmov s11, ip
- vld1.8 {q3-q4}, [r1, :64]!
- vld1.8 {q5}, [r1, :64]!
+ vld1.8 {q3-q4}, [r1]!
+ vld1.8 {q5}, [r1]!
bl aes_encrypt_3x
veor q0, q0, q3
veor q1, q1, q4
veor q2, q2, q5
rev ip, r6
- vst1.8 {q0-q1}, [r0, :64]!
- vst1.8 {q2}, [r0, :64]!
+ vst1.8 {q0-q1}, [r0]!
+ vst1.8 {q2}, [r0]!
vmov s27, ip
b .Lctrloop3x
.Lctr1x:
@@ -318,10 +318,10 @@ ENTRY(ce_aes_ctr_encrypt)
vmov q0, q6
bl aes_encrypt
subs r4, r4, #1
- bmi .Lctrhalfblock @ blocks < 0 means 1/2 block
- vld1.8 {q3}, [r1, :64]!
+ bmi .Lctrtailblock @ blocks < 0 means tail block
+ vld1.8 {q3}, [r1]!
veor q3, q0, q3
- vst1.8 {q3}, [r0, :64]!
+ vst1.8 {q3}, [r0]!
adds r6, r6, #1 @ increment BE ctr
rev ip, r6
@@ -333,10 +333,8 @@ ENTRY(ce_aes_ctr_encrypt)
vst1.8 {q6}, [r5]
pop {r4-r6, pc}
-.Lctrhalfblock:
- vld1.8 {d1}, [r1, :64]
- veor d0, d0, d1
- vst1.8 {d0}, [r0, :64]
+.Lctrtailblock:
+ vst1.8 {q0}, [r0, :64] @ return just the key stream
pop {r4-r6, pc}
.Lctrcarry:
@@ -405,8 +403,8 @@ ENTRY(ce_aes_xts_encrypt)
.Lxtsenc3x:
subs r4, r4, #3
bmi .Lxtsenc1x
- vld1.8 {q0-q1}, [r1, :64]! @ get 3 pt blocks
- vld1.8 {q2}, [r1, :64]!
+ vld1.8 {q0-q1}, [r1]! @ get 3 pt blocks
+ vld1.8 {q2}, [r1]!
next_tweak q4, q3, q7, q6
veor q0, q0, q3
next_tweak q5, q4, q7, q6
@@ -416,8 +414,8 @@ ENTRY(ce_aes_xts_encrypt)
veor q0, q0, q3
veor q1, q1, q4
veor q2, q2, q5
- vst1.8 {q0-q1}, [r0, :64]! @ write 3 ct blocks
- vst1.8 {q2}, [r0, :64]!
+ vst1.8 {q0-q1}, [r0]! @ write 3 ct blocks
+ vst1.8 {q2}, [r0]!
vmov q3, q5
teq r4, #0
beq .Lxtsencout
@@ -426,11 +424,11 @@ ENTRY(ce_aes_xts_encrypt)
adds r4, r4, #3
beq .Lxtsencout
.Lxtsencloop:
- vld1.8 {q0}, [r1, :64]!
+ vld1.8 {q0}, [r1]!
veor q0, q0, q3
bl aes_encrypt
veor q0, q0, q3
- vst1.8 {q0}, [r0, :64]!
+ vst1.8 {q0}, [r0]!
subs r4, r4, #1
beq .Lxtsencout
next_tweak q3, q3, q7, q6
@@ -456,8 +454,8 @@ ENTRY(ce_aes_xts_decrypt)
.Lxtsdec3x:
subs r4, r4, #3
bmi .Lxtsdec1x
- vld1.8 {q0-q1}, [r1, :64]! @ get 3 ct blocks
- vld1.8 {q2}, [r1, :64]!
+ vld1.8 {q0-q1}, [r1]! @ get 3 ct blocks
+ vld1.8 {q2}, [r1]!
next_tweak q4, q3, q7, q6
veor q0, q0, q3
next_tweak q5, q4, q7, q6
@@ -467,8 +465,8 @@ ENTRY(ce_aes_xts_decrypt)
veor q0, q0, q3
veor q1, q1, q4
veor q2, q2, q5
- vst1.8 {q0-q1}, [r0, :64]! @ write 3 pt blocks
- vst1.8 {q2}, [r0, :64]!
+ vst1.8 {q0-q1}, [r0]! @ write 3 pt blocks
+ vst1.8 {q2}, [r0]!
vmov q3, q5
teq r4, #0
beq .Lxtsdecout
@@ -477,12 +475,12 @@ ENTRY(ce_aes_xts_decrypt)
adds r4, r4, #3
beq .Lxtsdecout
.Lxtsdecloop:
- vld1.8 {q0}, [r1, :64]!
+ vld1.8 {q0}, [r1]!
veor q0, q0, q3
add ip, r2, #32 @ 3rd round key
bl aes_decrypt
veor q0, q0, q3
- vst1.8 {q0}, [r0, :64]!
+ vst1.8 {q0}, [r0]!
subs r4, r4, #1
beq .Lxtsdecout
next_tweak q3, q3, q7, q6
diff --git a/arch/arm/crypto/aes-ce-glue.c b/arch/arm/crypto/aes-ce-glue.c
index 8857531915bf..883b84d828c5 100644
--- a/arch/arm/crypto/aes-ce-glue.c
+++ b/arch/arm/crypto/aes-ce-glue.c
@@ -278,14 +278,15 @@ static int ctr_encrypt(struct skcipher_request *req)
u8 *tsrc = walk.src.virt.addr;
/*
- * Minimum alignment is 8 bytes, so if nbytes is <= 8, we need
- * to tell aes_ctr_encrypt() to only read half a block.
+ * Tell aes_ctr_encrypt() to process a tail block.
*/
- blocks = (nbytes <= 8) ? -1 : 1;
+ blocks = -1;
- ce_aes_ctr_encrypt(tail, tsrc, (u8 *)ctx->key_enc,
+ ce_aes_ctr_encrypt(tail, NULL, (u8 *)ctx->key_enc,
num_rounds(ctx), blocks, walk.iv);
- memcpy(tdst, tail, nbytes);
+ if (tdst != tsrc)
+ memcpy(tdst, tsrc, nbytes);
+ crypto_xor(tdst, tail, nbytes);
err = skcipher_walk_done(&walk, 0);
}
kernel_neon_end();
@@ -345,7 +346,6 @@ static struct skcipher_alg aes_algs[] = { {
.cra_flags = CRYPTO_ALG_INTERNAL,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct crypto_aes_ctx),
- .cra_alignmask = 7,
.cra_module = THIS_MODULE,
},
.min_keysize = AES_MIN_KEY_SIZE,
@@ -361,7 +361,6 @@ static struct skcipher_alg aes_algs[] = { {
.cra_flags = CRYPTO_ALG_INTERNAL,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct crypto_aes_ctx),
- .cra_alignmask = 7,
.cra_module = THIS_MODULE,
},
.min_keysize = AES_MIN_KEY_SIZE,
@@ -378,7 +377,6 @@ static struct skcipher_alg aes_algs[] = { {
.cra_flags = CRYPTO_ALG_INTERNAL,
.cra_blocksize = 1,
.cra_ctxsize = sizeof(struct crypto_aes_ctx),
- .cra_alignmask = 7,
.cra_module = THIS_MODULE,
},
.min_keysize = AES_MIN_KEY_SIZE,
@@ -396,7 +394,6 @@ static struct skcipher_alg aes_algs[] = { {
.cra_flags = CRYPTO_ALG_INTERNAL,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct crypto_aes_xts_ctx),
- .cra_alignmask = 7,
.cra_module = THIS_MODULE,
},
.min_keysize = 2 * AES_MIN_KEY_SIZE,
diff --git a/arch/arm/crypto/aes-cipher-core.S b/arch/arm/crypto/aes-cipher-core.S
new file mode 100644
index 000000000000..c817a86c4ca8
--- /dev/null
+++ b/arch/arm/crypto/aes-cipher-core.S
@@ -0,0 +1,179 @@
+/*
+ * Scalar AES core transform
+ *
+ * Copyright (C) 2017 Linaro Ltd.
+ * Author: Ard Biesheuvel <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/linkage.h>
+
+ .text
+ .align 5
+
+ rk .req r0
+ rounds .req r1
+ in .req r2
+ out .req r3
+ ttab .req ip
+
+ t0 .req lr
+ t1 .req r2
+ t2 .req r3
+
+ .macro __select, out, in, idx
+ .if __LINUX_ARM_ARCH__ < 7
+ and \out, \in, #0xff << (8 * \idx)
+ .else
+ ubfx \out, \in, #(8 * \idx), #8
+ .endif
+ .endm
+
+ .macro __load, out, in, idx
+ .if __LINUX_ARM_ARCH__ < 7 && \idx > 0
+ ldr \out, [ttab, \in, lsr #(8 * \idx) - 2]
+ .else
+ ldr \out, [ttab, \in, lsl #2]
+ .endif
+ .endm
+
+ .macro __hround, out0, out1, in0, in1, in2, in3, t3, t4, enc
+ __select \out0, \in0, 0
+ __select t0, \in1, 1
+ __load \out0, \out0, 0
+ __load t0, t0, 1
+
+ .if \enc
+ __select \out1, \in1, 0
+ __select t1, \in2, 1
+ .else
+ __select \out1, \in3, 0
+ __select t1, \in0, 1
+ .endif
+ __load \out1, \out1, 0
+ __select t2, \in2, 2
+ __load t1, t1, 1
+ __load t2, t2, 2
+
+ eor \out0, \out0, t0, ror #24
+
+ __select t0, \in3, 3
+ .if \enc
+ __select \t3, \in3, 2
+ __select \t4, \in0, 3
+ .else
+ __select \t3, \in1, 2
+ __select \t4, \in2, 3
+ .endif
+ __load \t3, \t3, 2
+ __load t0, t0, 3
+ __load \t4, \t4, 3
+
+ eor \out1, \out1, t1, ror #24
+ eor \out0, \out0, t2, ror #16
+ ldm rk!, {t1, t2}
+ eor \out1, \out1, \t3, ror #16
+ eor \out0, \out0, t0, ror #8
+ eor \out1, \out1, \t4, ror #8
+ eor \out0, \out0, t1
+ eor \out1, \out1, t2
+ .endm
+
+ .macro fround, out0, out1, out2, out3, in0, in1, in2, in3
+ __hround \out0, \out1, \in0, \in1, \in2, \in3, \out2, \out3, 1
+ __hround \out2, \out3, \in2, \in3, \in0, \in1, \in1, \in2, 1
+ .endm
+
+ .macro iround, out0, out1, out2, out3, in0, in1, in2, in3
+ __hround \out0, \out1, \in0, \in3, \in2, \in1, \out2, \out3, 0
+ __hround \out2, \out3, \in2, \in1, \in0, \in3, \in1, \in0, 0
+ .endm
+
+ .macro __rev, out, in
+ .if __LINUX_ARM_ARCH__ < 6
+ lsl t0, \in, #24
+ and t1, \in, #0xff00
+ and t2, \in, #0xff0000
+ orr \out, t0, \in, lsr #24
+ orr \out, \out, t1, lsl #8
+ orr \out, \out, t2, lsr #8
+ .else
+ rev \out, \in
+ .endif
+ .endm
+
+ .macro __adrl, out, sym, c
+ .if __LINUX_ARM_ARCH__ < 7
+ ldr\c \out, =\sym
+ .else
+ movw\c \out, #:lower16:\sym
+ movt\c \out, #:upper16:\sym
+ .endif
+ .endm
+
+ .macro do_crypt, round, ttab, ltab
+ push {r3-r11, lr}
+
+ ldr r4, [in]
+ ldr r5, [in, #4]
+ ldr r6, [in, #8]
+ ldr r7, [in, #12]
+
+ ldm rk!, {r8-r11}
+
+#ifdef CONFIG_CPU_BIG_ENDIAN
+ __rev r4, r4
+ __rev r5, r5
+ __rev r6, r6
+ __rev r7, r7
+#endif
+
+ eor r4, r4, r8
+ eor r5, r5, r9
+ eor r6, r6, r10
+ eor r7, r7, r11
+
+ __adrl ttab, \ttab
+
+ tst rounds, #2
+ bne 1f
+
+0: \round r8, r9, r10, r11, r4, r5, r6, r7
+ \round r4, r5, r6, r7, r8, r9, r10, r11
+
+1: subs rounds, rounds, #4
+ \round r8, r9, r10, r11, r4, r5, r6, r7
+ __adrl ttab, \ltab, ls
+ \round r4, r5, r6, r7, r8, r9, r10, r11
+ bhi 0b
+
+#ifdef CONFIG_CPU_BIG_ENDIAN
+ __rev r4, r4
+ __rev r5, r5
+ __rev r6, r6
+ __rev r7, r7
+#endif
+
+ ldr out, [sp]
+
+ str r4, [out]
+ str r5, [out, #4]
+ str r6, [out, #8]
+ str r7, [out, #12]
+
+ pop {r3-r11, pc}
+
+ .align 3
+ .ltorg
+ .endm
+
+ENTRY(__aes_arm_encrypt)
+ do_crypt fround, crypto_ft_tab, crypto_fl_tab
+ENDPROC(__aes_arm_encrypt)
+
+ENTRY(__aes_arm_decrypt)
+ do_crypt iround, crypto_it_tab, crypto_il_tab
+ENDPROC(__aes_arm_decrypt)
diff --git a/arch/arm/crypto/aes-cipher-glue.c b/arch/arm/crypto/aes-cipher-glue.c
new file mode 100644
index 000000000000..c222f6e072ad
--- /dev/null
+++ b/arch/arm/crypto/aes-cipher-glue.c
@@ -0,0 +1,74 @@
+/*
+ * Scalar AES core transform
+ *
+ * Copyright (C) 2017 Linaro Ltd.
+ * Author: Ard Biesheuvel <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <crypto/aes.h>
+#include <linux/crypto.h>
+#include <linux/module.h>
+
+asmlinkage void __aes_arm_encrypt(u32 *rk, int rounds, const u8 *in, u8 *out);
+EXPORT_SYMBOL(__aes_arm_encrypt);
+
+asmlinkage void __aes_arm_decrypt(u32 *rk, int rounds, const u8 *in, u8 *out);
+EXPORT_SYMBOL(__aes_arm_decrypt);
+
+static void aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
+{
+ struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
+ int rounds = 6 + ctx->key_length / 4;
+
+ __aes_arm_encrypt(ctx->key_enc, rounds, in, out);
+}
+
+static void aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
+{
+ struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
+ int rounds = 6 + ctx->key_length / 4;
+
+ __aes_arm_decrypt(ctx->key_dec, rounds, in, out);
+}
+
+static struct crypto_alg aes_alg = {
+ .cra_name = "aes",
+ .cra_driver_name = "aes-arm",
+ .cra_priority = 200,
+ .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct crypto_aes_ctx),
+ .cra_module = THIS_MODULE,
+
+ .cra_cipher.cia_min_keysize = AES_MIN_KEY_SIZE,
+ .cra_cipher.cia_max_keysize = AES_MAX_KEY_SIZE,
+ .cra_cipher.cia_setkey = crypto_aes_set_key,
+ .cra_cipher.cia_encrypt = aes_encrypt,
+ .cra_cipher.cia_decrypt = aes_decrypt,
+
+#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
+ .cra_alignmask = 3,
+#endif
+};
+
+static int __init aes_init(void)
+{
+ return crypto_register_alg(&aes_alg);
+}
+
+static void __exit aes_fini(void)
+{
+ crypto_unregister_alg(&aes_alg);
+}
+
+module_init(aes_init);
+module_exit(aes_fini);
+
+MODULE_DESCRIPTION("Scalar AES cipher for ARM");
+MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS_CRYPTO("aes");
diff --git a/arch/arm/crypto/aes-neonbs-core.S b/arch/arm/crypto/aes-neonbs-core.S
new file mode 100644
index 000000000000..2b625c6d4712
--- /dev/null
+++ b/arch/arm/crypto/aes-neonbs-core.S
@@ -0,0 +1,1023 @@
+/*
+ * Bit sliced AES using NEON instructions
+ *
+ * Copyright (C) 2017 Linaro Ltd.
+ * Author: Ard Biesheuvel <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+/*
+ * The algorithm implemented here is described in detail by the paper
+ * 'Faster and Timing-Attack Resistant AES-GCM' by Emilia Kaesper and
+ * Peter Schwabe (https://eprint.iacr.org/2009/129.pdf)
+ *
+ * This implementation is based primarily on the OpenSSL implementation
+ * for 32-bit ARM written by Andy Polyakov <appro@openssl.org>
+ */
+
+#include <linux/linkage.h>
+#include <asm/assembler.h>
+
+ .text
+ .fpu neon
+
+ rounds .req ip
+ bskey .req r4
+
+ q0l .req d0
+ q0h .req d1
+ q1l .req d2
+ q1h .req d3
+ q2l .req d4
+ q2h .req d5
+ q3l .req d6
+ q3h .req d7
+ q4l .req d8
+ q4h .req d9
+ q5l .req d10
+ q5h .req d11
+ q6l .req d12
+ q6h .req d13
+ q7l .req d14
+ q7h .req d15
+ q8l .req d16
+ q8h .req d17
+ q9l .req d18
+ q9h .req d19
+ q10l .req d20
+ q10h .req d21
+ q11l .req d22
+ q11h .req d23
+ q12l .req d24
+ q12h .req d25
+ q13l .req d26
+ q13h .req d27
+ q14l .req d28
+ q14h .req d29
+ q15l .req d30
+ q15h .req d31
+
+ .macro __tbl, out, tbl, in, tmp
+ .ifc \out, \tbl
+ .ifb \tmp
+ .error __tbl needs temp register if out == tbl
+ .endif
+ vmov \tmp, \out
+ .endif
+ vtbl.8 \out\()l, {\tbl}, \in\()l
+ .ifc \out, \tbl
+ vtbl.8 \out\()h, {\tmp}, \in\()h
+ .else
+ vtbl.8 \out\()h, {\tbl}, \in\()h
+ .endif
+ .endm
+
+ .macro __ldr, out, sym
+ vldr \out\()l, \sym
+ vldr \out\()h, \sym + 8
+ .endm
+
+ .macro __adr, reg, lbl
+ adr \reg, \lbl
+THUMB( orr \reg, \reg, #1 )
+ .endm
+
+ .macro in_bs_ch, b0, b1, b2, b3, b4, b5, b6, b7
+ veor \b2, \b2, \b1
+ veor \b5, \b5, \b6
+ veor \b3, \b3, \b0
+ veor \b6, \b6, \b2
+ veor \b5, \b5, \b0
+ veor \b6, \b6, \b3
+ veor \b3, \b3, \b7
+ veor \b7, \b7, \b5
+ veor \b3, \b3, \b4
+ veor \b4, \b4, \b5
+ veor \b2, \b2, \b7
+ veor \b3, \b3, \b1
+ veor \b1, \b1, \b5
+ .endm
+
+ .macro out_bs_ch, b0, b1, b2, b3, b4, b5, b6, b7
+ veor \b0, \b0, \b6
+ veor \b1, \b1, \b4
+ veor \b4, \b4, \b6
+ veor \b2, \b2, \b0
+ veor \b6, \b6, \b1
+ veor \b1, \b1, \b5
+ veor \b5, \b5, \b3
+ veor \b3, \b3, \b7
+ veor \b7, \b7, \b5
+ veor \b2, \b2, \b5
+ veor \b4, \b4, \b7
+ .endm
+
+ .macro inv_in_bs_ch, b6, b1, b2, b4, b7, b0, b3, b5
+ veor \b1, \b1, \b7
+ veor \b4, \b4, \b7
+ veor \b7, \b7, \b5
+ veor \b1, \b1, \b3
+ veor \b2, \b2, \b5
+ veor \b3, \b3, \b7
+ veor \b6, \b6, \b1
+ veor \b2, \b2, \b0
+ veor \b5, \b5, \b3
+ veor \b4, \b4, \b6
+ veor \b0, \b0, \b6
+ veor \b1, \b1, \b4
+ .endm
+
+ .macro inv_out_bs_ch, b6, b5, b0, b3, b7, b1, b4, b2
+ veor \b1, \b1, \b5
+ veor \b2, \b2, \b7
+ veor \b3, \b3, \b1
+ veor \b4, \b4, \b5
+ veor \b7, \b7, \b5
+ veor \b3, \b3, \b4
+ veor \b5, \b5, \b0
+ veor \b3, \b3, \b7
+ veor \b6, \b6, \b2
+ veor \b2, \b2, \b1
+ veor \b6, \b6, \b3
+ veor \b3, \b3, \b0
+ veor \b5, \b5, \b6
+ .endm
+
+ .macro mul_gf4, x0, x1, y0, y1, t0, t1
+ veor \t0, \y0, \y1
+ vand \t0, \t0, \x0
+ veor \x0, \x0, \x1
+ vand \t1, \x1, \y0
+ vand \x0, \x0, \y1
+ veor \x1, \t1, \t0
+ veor \x0, \x0, \t1
+ .endm
+
+ .macro mul_gf4_n_gf4, x0, x1, y0, y1, t0, x2, x3, y2, y3, t1
+ veor \t0, \y0, \y1
+ veor \t1, \y2, \y3
+ vand \t0, \t0, \x0
+ vand \t1, \t1, \x2
+ veor \x0, \x0, \x1
+ veor \x2, \x2, \x3
+ vand \x1, \x1, \y0
+ vand \x3, \x3, \y2
+ vand \x0, \x0, \y1
+ vand \x2, \x2, \y3
+ veor \x1, \x1, \x0
+ veor \x2, \x2, \x3
+ veor \x0, \x0, \t0
+ veor \x3, \x3, \t1
+ .endm
+
+ .macro mul_gf16_2, x0, x1, x2, x3, x4, x5, x6, x7, \
+ y0, y1, y2, y3, t0, t1, t2, t3
+ veor \t0, \x0, \x2
+ veor \t1, \x1, \x3
+ mul_gf4 \x0, \x1, \y0, \y1, \t2, \t3
+ veor \y0, \y0, \y2
+ veor \y1, \y1, \y3
+ mul_gf4_n_gf4 \t0, \t1, \y0, \y1, \t3, \x2, \x3, \y2, \y3, \t2
+ veor \x0, \x0, \t0
+ veor \x2, \x2, \t0
+ veor \x1, \x1, \t1
+ veor \x3, \x3, \t1
+ veor \t0, \x4, \x6
+ veor \t1, \x5, \x7
+ mul_gf4_n_gf4 \t0, \t1, \y0, \y1, \t3, \x6, \x7, \y2, \y3, \t2
+ veor \y0, \y0, \y2
+ veor \y1, \y1, \y3
+ mul_gf4 \x4, \x5, \y0, \y1, \t2, \t3
+ veor \x4, \x4, \t0
+ veor \x6, \x6, \t0
+ veor \x5, \x5, \t1
+ veor \x7, \x7, \t1
+ .endm
+
+ .macro inv_gf256, x0, x1, x2, x3, x4, x5, x6, x7, \
+ t0, t1, t2, t3, s0, s1, s2, s3
+ veor \t3, \x4, \x6
+ veor \t0, \x5, \x7
+ veor \t1, \x1, \x3
+ veor \s1, \x7, \x6
+ veor \s0, \x0, \x2
+ veor \s3, \t3, \t0
+ vorr \t2, \t0, \t1
+ vand \s2, \t3, \s0
+ vorr \t3, \t3, \s0
+ veor \s0, \s0, \t1
+ vand \t0, \t0, \t1
+ veor \t1, \x3, \x2
+ vand \s3, \s3, \s0
+ vand \s1, \s1, \t1
+ veor \t1, \x4, \x5
+ veor \s0, \x1, \x0
+ veor \t3, \t3, \s1
+ veor \t2, \t2, \s1
+ vand \s1, \t1, \s0
+ vorr \t1, \t1, \s0
+ veor \t3, \t3, \s3
+ veor \t0, \t0, \s1
+ veor \t2, \t2, \s2
+ veor \t1, \t1, \s3
+ veor \t0, \t0, \s2
+ vand \s0, \x7, \x3
+ veor \t1, \t1, \s2
+ vand \s1, \x6, \x2
+ vand \s2, \x5, \x1
+ vorr \s3, \x4, \x0
+ veor \t3, \t3, \s0
+ veor \t1, \t1, \s2
+ veor \s0, \t0, \s3
+ veor \t2, \t2, \s1
+ vand \s2, \t3, \t1
+ veor \s1, \t2, \s2
+ veor \s3, \s0, \s2
+ vbsl \s1, \t1, \s0
+ vmvn \t0, \s0
+ vbsl \s0, \s1, \s3
+ vbsl \t0, \s1, \s3
+ vbsl \s3, \t3, \t2
+ veor \t3, \t3, \t2
+ vand \s2, \s0, \s3
+ veor \t1, \t1, \t0
+ veor \s2, \s2, \t3
+ mul_gf16_2 \x0, \x1, \x2, \x3, \x4, \x5, \x6, \x7, \
+ \s3, \s2, \s1, \t1, \s0, \t0, \t2, \t3
+ .endm
+
+ .macro sbox, b0, b1, b2, b3, b4, b5, b6, b7, \
+ t0, t1, t2, t3, s0, s1, s2, s3
+ in_bs_ch \b0, \b1, \b2, \b3, \b4, \b5, \b6, \b7
+ inv_gf256 \b6, \b5, \b0, \b3, \b7, \b1, \b4, \b2, \
+ \t0, \t1, \t2, \t3, \s0, \s1, \s2, \s3
+ out_bs_ch \b7, \b1, \b4, \b2, \b6, \b5, \b0, \b3
+ .endm
+
+ .macro inv_sbox, b0, b1, b2, b3, b4, b5, b6, b7, \
+ t0, t1, t2, t3, s0, s1, s2, s3
+ inv_in_bs_ch \b0, \b1, \b2, \b3, \b4, \b5, \b6, \b7
+ inv_gf256 \b5, \b1, \b2, \b6, \b3, \b7, \b0, \b4, \
+ \t0, \t1, \t2, \t3, \s0, \s1, \s2, \s3
+ inv_out_bs_ch \b3, \b7, \b0, \b4, \b5, \b1, \b2, \b6
+ .endm
+
+ .macro shift_rows, x0, x1, x2, x3, x4, x5, x6, x7, \
+ t0, t1, t2, t3, mask
+ vld1.8 {\t0-\t1}, [bskey, :256]!
+ veor \t0, \t0, \x0
+ vld1.8 {\t2-\t3}, [bskey, :256]!
+ veor \t1, \t1, \x1
+ __tbl \x0, \t0, \mask
+ veor \t2, \t2, \x2
+ __tbl \x1, \t1, \mask
+ vld1.8 {\t0-\t1}, [bskey, :256]!
+ veor \t3, \t3, \x3
+ __tbl \x2, \t2, \mask
+ __tbl \x3, \t3, \mask
+ vld1.8 {\t2-\t3}, [bskey, :256]!
+ veor \t0, \t0, \x4
+ veor \t1, \t1, \x5
+ __tbl \x4, \t0, \mask
+ veor \t2, \t2, \x6
+ __tbl \x5, \t1, \mask
+ veor \t3, \t3, \x7
+ __tbl \x6, \t2, \mask
+ __tbl \x7, \t3, \mask
+ .endm
+
+ .macro inv_shift_rows, x0, x1, x2, x3, x4, x5, x6, x7, \
+ t0, t1, t2, t3, mask
+ __tbl \x0, \x0, \mask, \t0
+ __tbl \x1, \x1, \mask, \t1
+ __tbl \x2, \x2, \mask, \t2
+ __tbl \x3, \x3, \mask, \t3
+ __tbl \x4, \x4, \mask, \t0
+ __tbl \x5, \x5, \mask, \t1
+ __tbl \x6, \x6, \mask, \t2
+ __tbl \x7, \x7, \mask, \t3
+ .endm
+
+ .macro mix_cols, x0, x1, x2, x3, x4, x5, x6, x7, \
+ t0, t1, t2, t3, t4, t5, t6, t7, inv
+ vext.8 \t0, \x0, \x0, #12
+ vext.8 \t1, \x1, \x1, #12
+ veor \x0, \x0, \t0
+ vext.8 \t2, \x2, \x2, #12
+ veor \x1, \x1, \t1
+ vext.8 \t3, \x3, \x3, #12
+ veor \x2, \x2, \t2
+ vext.8 \t4, \x4, \x4, #12
+ veor \x3, \x3, \t3
+ vext.8 \t5, \x5, \x5, #12
+ veor \x4, \x4, \t4
+ vext.8 \t6, \x6, \x6, #12
+ veor \x5, \x5, \t5
+ vext.8 \t7, \x7, \x7, #12
+ veor \x6, \x6, \t6
+ veor \t1, \t1, \x0
+ veor.8 \x7, \x7, \t7
+ vext.8 \x0, \x0, \x0, #8
+ veor \t2, \t2, \x1
+ veor \t0, \t0, \x7
+ veor \t1, \t1, \x7
+ vext.8 \x1, \x1, \x1, #8
+ veor \t5, \t5, \x4
+ veor \x0, \x0, \t0
+ veor \t6, \t6, \x5
+ veor \x1, \x1, \t1
+ vext.8 \t0, \x4, \x4, #8
+ veor \t4, \t4, \x3
+ vext.8 \t1, \x5, \x5, #8
+ veor \t7, \t7, \x6
+ vext.8 \x4, \x3, \x3, #8
+ veor \t3, \t3, \x2
+ vext.8 \x5, \x7, \x7, #8
+ veor \t4, \t4, \x7
+ vext.8 \x3, \x6, \x6, #8
+ veor \t3, \t3, \x7
+ vext.8 \x6, \x2, \x2, #8
+ veor \x7, \t1, \t5
+ .ifb \inv
+ veor \x2, \t0, \t4
+ veor \x4, \x4, \t3
+ veor \x5, \x5, \t7
+ veor \x3, \x3, \t6
+ veor \x6, \x6, \t2
+ .else
+ veor \t3, \t3, \x4
+ veor \x5, \x5, \t7
+ veor \x2, \x3, \t6
+ veor \x3, \t0, \t4
+ veor \x4, \x6, \t2
+ vmov \x6, \t3
+ .endif
+ .endm
+
+ .macro inv_mix_cols, x0, x1, x2, x3, x4, x5, x6, x7, \
+ t0, t1, t2, t3, t4, t5, t6, t7
+ vld1.8 {\t0-\t1}, [bskey, :256]!
+ veor \x0, \x0, \t0
+ vld1.8 {\t2-\t3}, [bskey, :256]!
+ veor \x1, \x1, \t1
+ vld1.8 {\t4-\t5}, [bskey, :256]!
+ veor \x2, \x2, \t2
+ vld1.8 {\t6-\t7}, [bskey, :256]
+ sub bskey, bskey, #224
+ veor \x3, \x3, \t3
+ veor \x4, \x4, \t4
+ veor \x5, \x5, \t5
+ veor \x6, \x6, \t6
+ veor \x7, \x7, \t7
+ vext.8 \t0, \x0, \x0, #8
+ vext.8 \t6, \x6, \x6, #8
+ vext.8 \t7, \x7, \x7, #8
+ veor \t0, \t0, \x0
+ vext.8 \t1, \x1, \x1, #8
+ veor \t6, \t6, \x6
+ vext.8 \t2, \x2, \x2, #8
+ veor \t7, \t7, \x7
+ vext.8 \t3, \x3, \x3, #8
+ veor \t1, \t1, \x1
+ vext.8 \t4, \x4, \x4, #8
+ veor \t2, \t2, \x2
+ vext.8 \t5, \x5, \x5, #8
+ veor \t3, \t3, \x3
+ veor \t4, \t4, \x4
+ veor \t5, \t5, \x5
+ veor \x0, \x0, \t6
+ veor \x1, \x1, \t6
+ veor \x2, \x2, \t0
+ veor \x4, \x4, \t2
+ veor \x3, \x3, \t1
+ veor \x1, \x1, \t7
+ veor \x2, \x2, \t7
+ veor \x4, \x4, \t6
+ veor \x5, \x5, \t3
+ veor \x3, \x3, \t6
+ veor \x6, \x6, \t4
+ veor \x4, \x4, \t7
+ veor \x5, \x5, \t7
+ veor \x7, \x7, \t5
+ mix_cols \x0, \x1, \x2, \x3, \x4, \x5, \x6, \x7, \
+ \t0, \t1, \t2, \t3, \t4, \t5, \t6, \t7, 1
+ .endm
+
+ .macro swapmove_2x, a0, b0, a1, b1, n, mask, t0, t1
+ vshr.u64 \t0, \b0, #\n
+ vshr.u64 \t1, \b1, #\n
+ veor \t0, \t0, \a0
+ veor \t1, \t1, \a1
+ vand \t0, \t0, \mask
+ vand \t1, \t1, \mask
+ veor \a0, \a0, \t0
+ vshl.s64 \t0, \t0, #\n
+ veor \a1, \a1, \t1
+ vshl.s64 \t1, \t1, #\n
+ veor \b0, \b0, \t0
+ veor \b1, \b1, \t1
+ .endm
+
+ .macro bitslice, x7, x6, x5, x4, x3, x2, x1, x0, t0, t1, t2, t3
+ vmov.i8 \t0, #0x55
+ vmov.i8 \t1, #0x33
+ swapmove_2x \x0, \x1, \x2, \x3, 1, \t0, \t2, \t3
+ swapmove_2x \x4, \x5, \x6, \x7, 1, \t0, \t2, \t3
+ vmov.i8 \t0, #0x0f
+ swapmove_2x \x0, \x2, \x1, \x3, 2, \t1, \t2, \t3
+ swapmove_2x \x4, \x6, \x5, \x7, 2, \t1, \t2, \t3
+ swapmove_2x \x0, \x4, \x1, \x5, 4, \t0, \t2, \t3
+ swapmove_2x \x2, \x6, \x3, \x7, 4, \t0, \t2, \t3
+ .endm
+
+ .align 4
+M0: .quad 0x02060a0e03070b0f, 0x0004080c0105090d
+
+ /*
+ * void aesbs_convert_key(u8 out[], u32 const rk[], int rounds)
+ */
+ENTRY(aesbs_convert_key)
+ vld1.32 {q7}, [r1]! // load round 0 key
+ vld1.32 {q15}, [r1]! // load round 1 key
+
+ vmov.i8 q8, #0x01 // bit masks
+ vmov.i8 q9, #0x02
+ vmov.i8 q10, #0x04
+ vmov.i8 q11, #0x08
+ vmov.i8 q12, #0x10
+ vmov.i8 q13, #0x20
+ __ldr q14, M0
+
+ sub r2, r2, #1
+ vst1.8 {q7}, [r0, :128]! // save round 0 key
+
+.Lkey_loop:
+ __tbl q7, q15, q14
+ vmov.i8 q6, #0x40
+ vmov.i8 q15, #0x80
+
+ vtst.8 q0, q7, q8
+ vtst.8 q1, q7, q9
+ vtst.8 q2, q7, q10
+ vtst.8 q3, q7, q11
+ vtst.8 q4, q7, q12
+ vtst.8 q5, q7, q13
+ vtst.8 q6, q7, q6
+ vtst.8 q7, q7, q15
+ vld1.32 {q15}, [r1]! // load next round key
+ vmvn q0, q0
+ vmvn q1, q1
+ vmvn q5, q5
+ vmvn q6, q6
+
+ subs r2, r2, #1
+ vst1.8 {q0-q1}, [r0, :256]!
+ vst1.8 {q2-q3}, [r0, :256]!
+ vst1.8 {q4-q5}, [r0, :256]!
+ vst1.8 {q6-q7}, [r0, :256]!
+ bne .Lkey_loop
+
+ vmov.i8 q7, #0x63 // compose .L63
+ veor q15, q15, q7
+ vst1.8 {q15}, [r0, :128]
+ bx lr
+ENDPROC(aesbs_convert_key)
+
+ .align 4
+M0SR: .quad 0x0a0e02060f03070b, 0x0004080c05090d01
+
+aesbs_encrypt8:
+ vld1.8 {q9}, [bskey, :128]! // round 0 key
+ __ldr q8, M0SR
+
+ veor q10, q0, q9 // xor with round0 key
+ veor q11, q1, q9
+ __tbl q0, q10, q8
+ veor q12, q2, q9
+ __tbl q1, q11, q8
+ veor q13, q3, q9
+ __tbl q2, q12, q8
+ veor q14, q4, q9
+ __tbl q3, q13, q8
+ veor q15, q5, q9
+ __tbl q4, q14, q8
+ veor q10, q6, q9
+ __tbl q5, q15, q8
+ veor q11, q7, q9
+ __tbl q6, q10, q8
+ __tbl q7, q11, q8
+
+ bitslice q0, q1, q2, q3, q4, q5, q6, q7, q8, q9, q10, q11
+
+ sub rounds, rounds, #1
+ b .Lenc_sbox
+
+ .align 5
+SR: .quad 0x0504070600030201, 0x0f0e0d0c0a09080b
+SRM0: .quad 0x0304090e00050a0f, 0x01060b0c0207080d
+
+.Lenc_last:
+ __ldr q12, SRM0
+.Lenc_loop:
+ shift_rows q0, q1, q2, q3, q4, q5, q6, q7, q8, q9, q10, q11, q12
+.Lenc_sbox:
+ sbox q0, q1, q2, q3, q4, q5, q6, q7, q8, q9, q10, q11, q12, \
+ q13, q14, q15
+ subs rounds, rounds, #1
+ bcc .Lenc_done
+
+ mix_cols q0, q1, q4, q6, q3, q7, q2, q5, q8, q9, q10, q11, q12, \
+ q13, q14, q15
+
+ beq .Lenc_last
+ __ldr q12, SR
+ b .Lenc_loop
+
+.Lenc_done:
+ vld1.8 {q12}, [bskey, :128] // last round key
+
+ bitslice q0, q1, q4, q6, q3, q7, q2, q5, q8, q9, q10, q11
+
+ veor q0, q0, q12
+ veor q1, q1, q12
+ veor q4, q4, q12
+ veor q6, q6, q12
+ veor q3, q3, q12
+ veor q7, q7, q12
+ veor q2, q2, q12
+ veor q5, q5, q12
+ bx lr
+ENDPROC(aesbs_encrypt8)
+
+ .align 4
+M0ISR: .quad 0x0a0e0206070b0f03, 0x0004080c0d010509
+
+aesbs_decrypt8:
+ add bskey, bskey, rounds, lsl #7
+ sub bskey, bskey, #112
+ vld1.8 {q9}, [bskey, :128] // round 0 key
+ sub bskey, bskey, #128
+ __ldr q8, M0ISR
+
+ veor q10, q0, q9 // xor with round0 key
+ veor q11, q1, q9
+ __tbl q0, q10, q8
+ veor q12, q2, q9
+ __tbl q1, q11, q8
+ veor q13, q3, q9
+ __tbl q2, q12, q8
+ veor q14, q4, q9
+ __tbl q3, q13, q8
+ veor q15, q5, q9
+ __tbl q4, q14, q8
+ veor q10, q6, q9
+ __tbl q5, q15, q8
+ veor q11, q7, q9
+ __tbl q6, q10, q8
+ __tbl q7, q11, q8
+
+ bitslice q0, q1, q2, q3, q4, q5, q6, q7, q8, q9, q10, q11
+
+ sub rounds, rounds, #1
+ b .Ldec_sbox
+
+ .align 5
+ISR: .quad 0x0504070602010003, 0x0f0e0d0c080b0a09
+ISRM0: .quad 0x01040b0e0205080f, 0x0306090c00070a0d
+
+.Ldec_last:
+ __ldr q12, ISRM0
+.Ldec_loop:
+ inv_shift_rows q0, q1, q2, q3, q4, q5, q6, q7, q8, q9, q10, q11, q12
+.Ldec_sbox:
+ inv_sbox q0, q1, q2, q3, q4, q5, q6, q7, q8, q9, q10, q11, q12, \
+ q13, q14, q15
+ subs rounds, rounds, #1
+ bcc .Ldec_done
+
+ inv_mix_cols q0, q1, q6, q4, q2, q7, q3, q5, q8, q9, q10, q11, q12, \
+ q13, q14, q15
+
+ beq .Ldec_last
+ __ldr q12, ISR
+ b .Ldec_loop
+
+.Ldec_done:
+ add bskey, bskey, #112
+ vld1.8 {q12}, [bskey, :128] // last round key
+
+ bitslice q0, q1, q6, q4, q2, q7, q3, q5, q8, q9, q10, q11
+
+ veor q0, q0, q12
+ veor q1, q1, q12
+ veor q6, q6, q12
+ veor q4, q4, q12
+ veor q2, q2, q12
+ veor q7, q7, q12
+ veor q3, q3, q12
+ veor q5, q5, q12
+ bx lr
+ENDPROC(aesbs_decrypt8)
+
+ /*
+ * aesbs_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
+ * int blocks)
+ * aesbs_ecb_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
+ * int blocks)
+ */
+ .macro __ecb_crypt, do8, o0, o1, o2, o3, o4, o5, o6, o7
+ push {r4-r6, lr}
+ ldr r5, [sp, #16] // number of blocks
+
+99: __adr ip, 0f
+ and lr, r5, #7
+ cmp r5, #8
+ sub ip, ip, lr, lsl #2
+ bxlt ip // computed goto if blocks < 8
+
+ vld1.8 {q0}, [r1]!
+ vld1.8 {q1}, [r1]!
+ vld1.8 {q2}, [r1]!
+ vld1.8 {q3}, [r1]!
+ vld1.8 {q4}, [r1]!
+ vld1.8 {q5}, [r1]!
+ vld1.8 {q6}, [r1]!
+ vld1.8 {q7}, [r1]!
+
+0: mov bskey, r2
+ mov rounds, r3
+ bl \do8
+
+ __adr ip, 1f
+ and lr, r5, #7
+ cmp r5, #8
+ sub ip, ip, lr, lsl #2
+ bxlt ip // computed goto if blocks < 8
+
+ vst1.8 {\o0}, [r0]!
+ vst1.8 {\o1}, [r0]!
+ vst1.8 {\o2}, [r0]!
+ vst1.8 {\o3}, [r0]!
+ vst1.8 {\o4}, [r0]!
+ vst1.8 {\o5}, [r0]!
+ vst1.8 {\o6}, [r0]!
+ vst1.8 {\o7}, [r0]!
+
+1: subs r5, r5, #8
+ bgt 99b
+
+ pop {r4-r6, pc}
+ .endm
+
+ .align 4
+ENTRY(aesbs_ecb_encrypt)
+ __ecb_crypt aesbs_encrypt8, q0, q1, q4, q6, q3, q7, q2, q5
+ENDPROC(aesbs_ecb_encrypt)
+
+ .align 4
+ENTRY(aesbs_ecb_decrypt)
+ __ecb_crypt aesbs_decrypt8, q0, q1, q6, q4, q2, q7, q3, q5
+ENDPROC(aesbs_ecb_decrypt)
+
+ /*
+ * aesbs_cbc_decrypt(u8 out[], u8 const in[], u8 const rk[],
+ * int rounds, int blocks, u8 iv[])
+ */
+ .align 4
+ENTRY(aesbs_cbc_decrypt)
+ mov ip, sp
+ push {r4-r6, lr}
+ ldm ip, {r5-r6} // load args 4-5
+
+99: __adr ip, 0f
+ and lr, r5, #7
+ cmp r5, #8
+ sub ip, ip, lr, lsl #2
+ mov lr, r1
+ bxlt ip // computed goto if blocks < 8
+
+ vld1.8 {q0}, [lr]!
+ vld1.8 {q1}, [lr]!
+ vld1.8 {q2}, [lr]!
+ vld1.8 {q3}, [lr]!
+ vld1.8 {q4}, [lr]!
+ vld1.8 {q5}, [lr]!
+ vld1.8 {q6}, [lr]!
+ vld1.8 {q7}, [lr]
+
+0: mov bskey, r2
+ mov rounds, r3
+ bl aesbs_decrypt8
+
+ vld1.8 {q8}, [r6]
+ vmov q9, q8
+ vmov q10, q8
+ vmov q11, q8
+ vmov q12, q8
+ vmov q13, q8
+ vmov q14, q8
+ vmov q15, q8
+
+ __adr ip, 1f
+ and lr, r5, #7
+ cmp r5, #8
+ sub ip, ip, lr, lsl #2
+ bxlt ip // computed goto if blocks < 8
+
+ vld1.8 {q9}, [r1]!
+ vld1.8 {q10}, [r1]!
+ vld1.8 {q11}, [r1]!
+ vld1.8 {q12}, [r1]!
+ vld1.8 {q13}, [r1]!
+ vld1.8 {q14}, [r1]!
+ vld1.8 {q15}, [r1]!
+ W(nop)
+
+1: __adr ip, 2f
+ sub ip, ip, lr, lsl #3
+ bxlt ip // computed goto if blocks < 8
+
+ veor q0, q0, q8
+ vst1.8 {q0}, [r0]!
+ veor q1, q1, q9
+ vst1.8 {q1}, [r0]!
+ veor q6, q6, q10
+ vst1.8 {q6}, [r0]!
+ veor q4, q4, q11
+ vst1.8 {q4}, [r0]!
+ veor q2, q2, q12
+ vst1.8 {q2}, [r0]!
+ veor q7, q7, q13
+ vst1.8 {q7}, [r0]!
+ veor q3, q3, q14
+ vst1.8 {q3}, [r0]!
+ veor q5, q5, q15
+ vld1.8 {q8}, [r1]! // load next round's iv
+2: vst1.8 {q5}, [r0]!
+
+ subs r5, r5, #8
+ vst1.8 {q8}, [r6] // store next round's iv
+ bgt 99b
+
+ pop {r4-r6, pc}
+ENDPROC(aesbs_cbc_decrypt)
+
+ .macro next_ctr, q
+ vmov.32 \q\()h[1], r10
+ adds r10, r10, #1
+ vmov.32 \q\()h[0], r9
+ adcs r9, r9, #0
+ vmov.32 \q\()l[1], r8
+ adcs r8, r8, #0
+ vmov.32 \q\()l[0], r7
+ adc r7, r7, #0
+ vrev32.8 \q, \q
+ .endm
+
+ /*
+ * aesbs_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[],
+ * int rounds, int blocks, u8 ctr[], u8 final[])
+ */
+ENTRY(aesbs_ctr_encrypt)
+ mov ip, sp
+ push {r4-r10, lr}
+
+ ldm ip, {r5-r7} // load args 4-6
+ teq r7, #0
+ addne r5, r5, #1 // one extra block if final != 0
+
+ vld1.8 {q0}, [r6] // load counter
+ vrev32.8 q1, q0
+ vmov r9, r10, d3
+ vmov r7, r8, d2
+
+ adds r10, r10, #1
+ adcs r9, r9, #0
+ adcs r8, r8, #0
+ adc r7, r7, #0
+
+99: vmov q1, q0
+ vmov q2, q0
+ vmov q3, q0
+ vmov q4, q0
+ vmov q5, q0
+ vmov q6, q0
+ vmov q7, q0
+
+ __adr ip, 0f
+ sub lr, r5, #1
+ and lr, lr, #7
+ cmp r5, #8
+ sub ip, ip, lr, lsl #5
+ sub ip, ip, lr, lsl #2
+ bxlt ip // computed goto if blocks < 8
+
+ next_ctr q1
+ next_ctr q2
+ next_ctr q3
+ next_ctr q4
+ next_ctr q5
+ next_ctr q6
+ next_ctr q7
+
+0: mov bskey, r2
+ mov rounds, r3
+ bl aesbs_encrypt8
+
+ __adr ip, 1f
+ and lr, r5, #7
+ cmp r5, #8
+ movgt r4, #0
+ ldrle r4, [sp, #40] // load final in the last round
+ sub ip, ip, lr, lsl #2
+ bxlt ip // computed goto if blocks < 8
+
+ vld1.8 {q8}, [r1]!
+ vld1.8 {q9}, [r1]!
+ vld1.8 {q10}, [r1]!
+ vld1.8 {q11}, [r1]!
+ vld1.8 {q12}, [r1]!
+ vld1.8 {q13}, [r1]!
+ vld1.8 {q14}, [r1]!
+ teq r4, #0 // skip last block if 'final'
+1: bne 2f
+ vld1.8 {q15}, [r1]!
+
+2: __adr ip, 3f
+ cmp r5, #8
+ sub ip, ip, lr, lsl #3
+ bxlt ip // computed goto if blocks < 8
+
+ veor q0, q0, q8
+ vst1.8 {q0}, [r0]!
+ veor q1, q1, q9
+ vst1.8 {q1}, [r0]!
+ veor q4, q4, q10
+ vst1.8 {q4}, [r0]!
+ veor q6, q6, q11
+ vst1.8 {q6}, [r0]!
+ veor q3, q3, q12
+ vst1.8 {q3}, [r0]!
+ veor q7, q7, q13
+ vst1.8 {q7}, [r0]!
+ veor q2, q2, q14
+ vst1.8 {q2}, [r0]!
+ teq r4, #0 // skip last block if 'final'
+ W(bne) 5f
+3: veor q5, q5, q15
+ vst1.8 {q5}, [r0]!
+
+4: next_ctr q0
+
+ subs r5, r5, #8
+ bgt 99b
+
+ vst1.8 {q0}, [r6]
+ pop {r4-r10, pc}
+
+5: vst1.8 {q5}, [r4]
+ b 4b
+ENDPROC(aesbs_ctr_encrypt)
+
+ .macro next_tweak, out, in, const, tmp
+ vshr.s64 \tmp, \in, #63
+ vand \tmp, \tmp, \const
+ vadd.u64 \out, \in, \in
+ vext.8 \tmp, \tmp, \tmp, #8
+ veor \out, \out, \tmp
+ .endm
+
+ .align 4
+.Lxts_mul_x:
+ .quad 1, 0x87
+
+ /*
+ * aesbs_xts_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
+ * int blocks, u8 iv[])
+ * aesbs_xts_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
+ * int blocks, u8 iv[])
+ */
+__xts_prepare8:
+ vld1.8 {q14}, [r7] // load iv
+ __ldr q15, .Lxts_mul_x // load tweak mask
+ vmov q12, q14
+
+ __adr ip, 0f
+ and r4, r6, #7
+ cmp r6, #8
+ sub ip, ip, r4, lsl #5
+ mov r4, sp
+ bxlt ip // computed goto if blocks < 8
+
+ vld1.8 {q0}, [r1]!
+ next_tweak q12, q14, q15, q13
+ veor q0, q0, q14
+ vst1.8 {q14}, [r4, :128]!
+
+ vld1.8 {q1}, [r1]!
+ next_tweak q14, q12, q15, q13
+ veor q1, q1, q12
+ vst1.8 {q12}, [r4, :128]!
+
+ vld1.8 {q2}, [r1]!
+ next_tweak q12, q14, q15, q13
+ veor q2, q2, q14
+ vst1.8 {q14}, [r4, :128]!
+
+ vld1.8 {q3}, [r1]!
+ next_tweak q14, q12, q15, q13
+ veor q3, q3, q12
+ vst1.8 {q12}, [r4, :128]!
+
+ vld1.8 {q4}, [r1]!
+ next_tweak q12, q14, q15, q13
+ veor q4, q4, q14
+ vst1.8 {q14}, [r4, :128]!
+
+ vld1.8 {q5}, [r1]!
+ next_tweak q14, q12, q15, q13
+ veor q5, q5, q12
+ vst1.8 {q12}, [r4, :128]!
+
+ vld1.8 {q6}, [r1]!
+ next_tweak q12, q14, q15, q13
+ veor q6, q6, q14
+ vst1.8 {q14}, [r4, :128]!
+
+ vld1.8 {q7}, [r1]!
+ next_tweak q14, q12, q15, q13
+ veor q7, q7, q12
+ vst1.8 {q12}, [r4, :128]
+
+0: vst1.8 {q14}, [r7] // store next iv
+ bx lr
+ENDPROC(__xts_prepare8)
+
+ .macro __xts_crypt, do8, o0, o1, o2, o3, o4, o5, o6, o7
+ push {r4-r8, lr}
+ mov r5, sp // preserve sp
+ ldrd r6, r7, [sp, #24] // get blocks and iv args
+ sub ip, sp, #128 // make room for 8x tweak
+ bic ip, ip, #0xf // align sp to 16 bytes
+ mov sp, ip
+
+99: bl __xts_prepare8
+
+ mov bskey, r2
+ mov rounds, r3
+ bl \do8
+
+ __adr ip, 0f
+ and lr, r6, #7
+ cmp r6, #8
+ sub ip, ip, lr, lsl #2
+ mov r4, sp
+ bxlt ip // computed goto if blocks < 8
+
+ vld1.8 {q8}, [r4, :128]!
+ vld1.8 {q9}, [r4, :128]!
+ vld1.8 {q10}, [r4, :128]!
+ vld1.8 {q11}, [r4, :128]!
+ vld1.8 {q12}, [r4, :128]!
+ vld1.8 {q13}, [r4, :128]!
+ vld1.8 {q14}, [r4, :128]!
+ vld1.8 {q15}, [r4, :128]
+
+0: __adr ip, 1f
+ sub ip, ip, lr, lsl #3
+ bxlt ip // computed goto if blocks < 8
+
+ veor \o0, \o0, q8
+ vst1.8 {\o0}, [r0]!
+ veor \o1, \o1, q9
+ vst1.8 {\o1}, [r0]!
+ veor \o2, \o2, q10
+ vst1.8 {\o2}, [r0]!
+ veor \o3, \o3, q11
+ vst1.8 {\o3}, [r0]!
+ veor \o4, \o4, q12
+ vst1.8 {\o4}, [r0]!
+ veor \o5, \o5, q13
+ vst1.8 {\o5}, [r0]!
+ veor \o6, \o6, q14
+ vst1.8 {\o6}, [r0]!
+ veor \o7, \o7, q15
+ vst1.8 {\o7}, [r0]!
+
+1: subs r6, r6, #8
+ bgt 99b
+
+ mov sp, r5
+ pop {r4-r8, pc}
+ .endm
+
+ENTRY(aesbs_xts_encrypt)
+ __xts_crypt aesbs_encrypt8, q0, q1, q4, q6, q3, q7, q2, q5
+ENDPROC(aesbs_xts_encrypt)
+
+ENTRY(aesbs_xts_decrypt)
+ __xts_crypt aesbs_decrypt8, q0, q1, q6, q4, q2, q7, q3, q5
+ENDPROC(aesbs_xts_decrypt)
diff --git a/arch/arm/crypto/aes-neonbs-glue.c b/arch/arm/crypto/aes-neonbs-glue.c
new file mode 100644
index 000000000000..2920b96dbd36
--- /dev/null
+++ b/arch/arm/crypto/aes-neonbs-glue.c
@@ -0,0 +1,406 @@
+/*
+ * Bit sliced AES using NEON instructions
+ *
+ * Copyright (C) 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <asm/neon.h>
+#include <crypto/aes.h>
+#include <crypto/cbc.h>
+#include <crypto/internal/simd.h>
+#include <crypto/internal/skcipher.h>
+#include <crypto/xts.h>
+#include <linux/module.h>
+
+MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
+MODULE_LICENSE("GPL v2");
+
+MODULE_ALIAS_CRYPTO("ecb(aes)");
+MODULE_ALIAS_CRYPTO("cbc(aes)");
+MODULE_ALIAS_CRYPTO("ctr(aes)");
+MODULE_ALIAS_CRYPTO("xts(aes)");
+
+asmlinkage void aesbs_convert_key(u8 out[], u32 const rk[], int rounds);
+
+asmlinkage void aesbs_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[],
+ int rounds, int blocks);
+asmlinkage void aesbs_ecb_decrypt(u8 out[], u8 const in[], u8 const rk[],
+ int rounds, int blocks);
+
+asmlinkage void aesbs_cbc_decrypt(u8 out[], u8 const in[], u8 const rk[],
+ int rounds, int blocks, u8 iv[]);
+
+asmlinkage void aesbs_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[],
+ int rounds, int blocks, u8 ctr[], u8 final[]);
+
+asmlinkage void aesbs_xts_encrypt(u8 out[], u8 const in[], u8 const rk[],
+ int rounds, int blocks, u8 iv[]);
+asmlinkage void aesbs_xts_decrypt(u8 out[], u8 const in[], u8 const rk[],
+ int rounds, int blocks, u8 iv[]);
+
+asmlinkage void __aes_arm_encrypt(const u32 rk[], int rounds, const u8 in[],
+ u8 out[]);
+
+struct aesbs_ctx {
+ int rounds;
+ u8 rk[13 * (8 * AES_BLOCK_SIZE) + 32] __aligned(AES_BLOCK_SIZE);
+};
+
+struct aesbs_cbc_ctx {
+ struct aesbs_ctx key;
+ u32 enc[AES_MAX_KEYLENGTH_U32];
+};
+
+struct aesbs_xts_ctx {
+ struct aesbs_ctx key;
+ u32 twkey[AES_MAX_KEYLENGTH_U32];
+};
+
+static int aesbs_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
+ unsigned int key_len)
+{
+ struct aesbs_ctx *ctx = crypto_skcipher_ctx(tfm);
+ struct crypto_aes_ctx rk;
+ int err;
+
+ err = crypto_aes_expand_key(&rk, in_key, key_len);
+ if (err)
+ return err;
+
+ ctx->rounds = 6 + key_len / 4;
+
+ kernel_neon_begin();
+ aesbs_convert_key(ctx->rk, rk.key_enc, ctx->rounds);
+ kernel_neon_end();
+
+ return 0;
+}
+
+static int __ecb_crypt(struct skcipher_request *req,
+ void (*fn)(u8 out[], u8 const in[], u8 const rk[],
+ int rounds, int blocks))
+{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+ struct aesbs_ctx *ctx = crypto_skcipher_ctx(tfm);
+ struct skcipher_walk walk;
+ int err;
+
+ err = skcipher_walk_virt(&walk, req, true);
+
+ kernel_neon_begin();
+ while (walk.nbytes >= AES_BLOCK_SIZE) {
+ unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE;
+
+ if (walk.nbytes < walk.total)
+ blocks = round_down(blocks,
+ walk.stride / AES_BLOCK_SIZE);
+
+ fn(walk.dst.virt.addr, walk.src.virt.addr, ctx->rk,
+ ctx->rounds, blocks);
+ err = skcipher_walk_done(&walk,
+ walk.nbytes - blocks * AES_BLOCK_SIZE);
+ }
+ kernel_neon_end();
+
+ return err;
+}
+
+static int ecb_encrypt(struct skcipher_request *req)
+{
+ return __ecb_crypt(req, aesbs_ecb_encrypt);
+}
+
+static int ecb_decrypt(struct skcipher_request *req)
+{
+ return __ecb_crypt(req, aesbs_ecb_decrypt);
+}
+
+static int aesbs_cbc_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
+ unsigned int key_len)
+{
+ struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
+ struct crypto_aes_ctx rk;
+ int err;
+
+ err = crypto_aes_expand_key(&rk, in_key, key_len);
+ if (err)
+ return err;
+
+ ctx->key.rounds = 6 + key_len / 4;
+
+ memcpy(ctx->enc, rk.key_enc, sizeof(ctx->enc));
+
+ kernel_neon_begin();
+ aesbs_convert_key(ctx->key.rk, rk.key_enc, ctx->key.rounds);
+ kernel_neon_end();
+
+ return 0;
+}
+
+static void cbc_encrypt_one(struct crypto_skcipher *tfm, const u8 *src, u8 *dst)
+{
+ struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
+
+ __aes_arm_encrypt(ctx->enc, ctx->key.rounds, src, dst);
+}
+
+static int cbc_encrypt(struct skcipher_request *req)
+{
+ return crypto_cbc_encrypt_walk(req, cbc_encrypt_one);
+}
+
+static int cbc_decrypt(struct skcipher_request *req)
+{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+ struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
+ struct skcipher_walk walk;
+ int err;
+
+ err = skcipher_walk_virt(&walk, req, true);
+
+ kernel_neon_begin();
+ while (walk.nbytes >= AES_BLOCK_SIZE) {
+ unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE;
+
+ if (walk.nbytes < walk.total)
+ blocks = round_down(blocks,
+ walk.stride / AES_BLOCK_SIZE);
+
+ aesbs_cbc_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
+ ctx->key.rk, ctx->key.rounds, blocks,
+ walk.iv);
+ err = skcipher_walk_done(&walk,
+ walk.nbytes - blocks * AES_BLOCK_SIZE);
+ }
+ kernel_neon_end();
+
+ return err;
+}
+
+static int ctr_encrypt(struct skcipher_request *req)
+{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+ struct aesbs_ctx *ctx = crypto_skcipher_ctx(tfm);
+ struct skcipher_walk walk;
+ u8 buf[AES_BLOCK_SIZE];
+ int err;
+
+ err = skcipher_walk_virt(&walk, req, true);
+
+ kernel_neon_begin();
+ while (walk.nbytes > 0) {
+ unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE;
+ u8 *final = (walk.total % AES_BLOCK_SIZE) ? buf : NULL;
+
+ if (walk.nbytes < walk.total) {
+ blocks = round_down(blocks,
+ walk.stride / AES_BLOCK_SIZE);
+ final = NULL;
+ }
+
+ aesbs_ctr_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
+ ctx->rk, ctx->rounds, blocks, walk.iv, final);
+
+ if (final) {
+ u8 *dst = walk.dst.virt.addr + blocks * AES_BLOCK_SIZE;
+ u8 *src = walk.src.virt.addr + blocks * AES_BLOCK_SIZE;
+
+ if (dst != src)
+ memcpy(dst, src, walk.total % AES_BLOCK_SIZE);
+ crypto_xor(dst, final, walk.total % AES_BLOCK_SIZE);
+
+ err = skcipher_walk_done(&walk, 0);
+ break;
+ }
+ err = skcipher_walk_done(&walk,
+ walk.nbytes - blocks * AES_BLOCK_SIZE);
+ }
+ kernel_neon_end();
+
+ return err;
+}
+
+static int aesbs_xts_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
+ unsigned int key_len)
+{
+ struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
+ struct crypto_aes_ctx rk;
+ int err;
+
+ err = xts_verify_key(tfm, in_key, key_len);
+ if (err)
+ return err;
+
+ key_len /= 2;
+ err = crypto_aes_expand_key(&rk, in_key + key_len, key_len);
+ if (err)
+ return err;
+
+ memcpy(ctx->twkey, rk.key_enc, sizeof(ctx->twkey));
+
+ return aesbs_setkey(tfm, in_key, key_len);
+}
+
+static int __xts_crypt(struct skcipher_request *req,
+ void (*fn)(u8 out[], u8 const in[], u8 const rk[],
+ int rounds, int blocks, u8 iv[]))
+{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+ struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
+ struct skcipher_walk walk;
+ int err;
+
+ err = skcipher_walk_virt(&walk, req, true);
+
+ __aes_arm_encrypt(ctx->twkey, ctx->key.rounds, walk.iv, walk.iv);
+
+ kernel_neon_begin();
+ while (walk.nbytes >= AES_BLOCK_SIZE) {
+ unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE;
+
+ if (walk.nbytes < walk.total)
+ blocks = round_down(blocks,
+ walk.stride / AES_BLOCK_SIZE);
+
+ fn(walk.dst.virt.addr, walk.src.virt.addr, ctx->key.rk,
+ ctx->key.rounds, blocks, walk.iv);
+ err = skcipher_walk_done(&walk,
+ walk.nbytes - blocks * AES_BLOCK_SIZE);
+ }
+ kernel_neon_end();
+
+ return err;
+}
+
+static int xts_encrypt(struct skcipher_request *req)
+{
+ return __xts_crypt(req, aesbs_xts_encrypt);
+}
+
+static int xts_decrypt(struct skcipher_request *req)
+{
+ return __xts_crypt(req, aesbs_xts_decrypt);
+}
+
+static struct skcipher_alg aes_algs[] = { {
+ .base.cra_name = "__ecb(aes)",
+ .base.cra_driver_name = "__ecb-aes-neonbs",
+ .base.cra_priority = 250,
+ .base.cra_blocksize = AES_BLOCK_SIZE,
+ .base.cra_ctxsize = sizeof(struct aesbs_ctx),
+ .base.cra_module = THIS_MODULE,
+ .base.cra_flags = CRYPTO_ALG_INTERNAL,
+
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .walksize = 8 * AES_BLOCK_SIZE,
+ .setkey = aesbs_setkey,
+ .encrypt = ecb_encrypt,
+ .decrypt = ecb_decrypt,
+}, {
+ .base.cra_name = "__cbc(aes)",
+ .base.cra_driver_name = "__cbc-aes-neonbs",
+ .base.cra_priority = 250,
+ .base.cra_blocksize = AES_BLOCK_SIZE,
+ .base.cra_ctxsize = sizeof(struct aesbs_cbc_ctx),
+ .base.cra_module = THIS_MODULE,
+ .base.cra_flags = CRYPTO_ALG_INTERNAL,
+
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .walksize = 8 * AES_BLOCK_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .setkey = aesbs_cbc_setkey,
+ .encrypt = cbc_encrypt,
+ .decrypt = cbc_decrypt,
+}, {
+ .base.cra_name = "__ctr(aes)",
+ .base.cra_driver_name = "__ctr-aes-neonbs",
+ .base.cra_priority = 250,
+ .base.cra_blocksize = 1,
+ .base.cra_ctxsize = sizeof(struct aesbs_ctx),
+ .base.cra_module = THIS_MODULE,
+ .base.cra_flags = CRYPTO_ALG_INTERNAL,
+
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .chunksize = AES_BLOCK_SIZE,
+ .walksize = 8 * AES_BLOCK_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .setkey = aesbs_setkey,
+ .encrypt = ctr_encrypt,
+ .decrypt = ctr_encrypt,
+}, {
+ .base.cra_name = "__xts(aes)",
+ .base.cra_driver_name = "__xts-aes-neonbs",
+ .base.cra_priority = 250,
+ .base.cra_blocksize = AES_BLOCK_SIZE,
+ .base.cra_ctxsize = sizeof(struct aesbs_xts_ctx),
+ .base.cra_module = THIS_MODULE,
+ .base.cra_flags = CRYPTO_ALG_INTERNAL,
+
+ .min_keysize = 2 * AES_MIN_KEY_SIZE,
+ .max_keysize = 2 * AES_MAX_KEY_SIZE,
+ .walksize = 8 * AES_BLOCK_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .setkey = aesbs_xts_setkey,
+ .encrypt = xts_encrypt,
+ .decrypt = xts_decrypt,
+} };
+
+static struct simd_skcipher_alg *aes_simd_algs[ARRAY_SIZE(aes_algs)];
+
+static void aes_exit(void)
+{
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(aes_simd_algs); i++)
+ if (aes_simd_algs[i])
+ simd_skcipher_free(aes_simd_algs[i]);
+
+ crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
+}
+
+static int __init aes_init(void)
+{
+ struct simd_skcipher_alg *simd;
+ const char *basename;
+ const char *algname;
+ const char *drvname;
+ int err;
+ int i;
+
+ if (!(elf_hwcap & HWCAP_NEON))
+ return -ENODEV;
+
+ err = crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
+ if (err)
+ return err;
+
+ for (i = 0; i < ARRAY_SIZE(aes_algs); i++) {
+ if (!(aes_algs[i].base.cra_flags & CRYPTO_ALG_INTERNAL))
+ continue;
+
+ algname = aes_algs[i].base.cra_name + 2;
+ drvname = aes_algs[i].base.cra_driver_name + 2;
+ basename = aes_algs[i].base.cra_driver_name;
+ simd = simd_skcipher_create_compat(algname, drvname, basename);
+ err = PTR_ERR(simd);
+ if (IS_ERR(simd))
+ goto unregister_simds;
+
+ aes_simd_algs[i] = simd;
+ }
+ return 0;
+
+unregister_simds:
+ aes_exit();
+ return err;
+}
+
+module_init(aes_init);
+module_exit(aes_exit);
diff --git a/arch/arm/crypto/aes_glue.c b/arch/arm/crypto/aes_glue.c
deleted file mode 100644
index 0409b8f89782..000000000000
--- a/arch/arm/crypto/aes_glue.c
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Glue Code for the asm optimized version of the AES Cipher Algorithm
- */
-
-#include <linux/module.h>
-#include <linux/crypto.h>
-#include <crypto/aes.h>
-
-#include "aes_glue.h"
-
-EXPORT_SYMBOL(AES_encrypt);
-EXPORT_SYMBOL(AES_decrypt);
-EXPORT_SYMBOL(private_AES_set_encrypt_key);
-EXPORT_SYMBOL(private_AES_set_decrypt_key);
-
-static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
-{
- struct AES_CTX *ctx = crypto_tfm_ctx(tfm);
- AES_encrypt(src, dst, &ctx->enc_key);
-}
-
-static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
-{
- struct AES_CTX *ctx = crypto_tfm_ctx(tfm);
- AES_decrypt(src, dst, &ctx->dec_key);
-}
-
-static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
- unsigned int key_len)
-{
- struct AES_CTX *ctx = crypto_tfm_ctx(tfm);
-
- switch (key_len) {
- case AES_KEYSIZE_128:
- key_len = 128;
- break;
- case AES_KEYSIZE_192:
- key_len = 192;
- break;
- case AES_KEYSIZE_256:
- key_len = 256;
- break;
- default:
- tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
- return -EINVAL;
- }
-
- if (private_AES_set_encrypt_key(in_key, key_len, &ctx->enc_key) == -1) {
- tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
- return -EINVAL;
- }
- /* private_AES_set_decrypt_key expects an encryption key as input */
- ctx->dec_key = ctx->enc_key;
- if (private_AES_set_decrypt_key(in_key, key_len, &ctx->dec_key) == -1) {
- tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
- return -EINVAL;
- }
- return 0;
-}
-
-static struct crypto_alg aes_alg = {
- .cra_name = "aes",
- .cra_driver_name = "aes-asm",
- .cra_priority = 200,
- .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
- .cra_blocksize = AES_BLOCK_SIZE,
- .cra_ctxsize = sizeof(struct AES_CTX),
- .cra_module = THIS_MODULE,
- .cra_list = LIST_HEAD_INIT(aes_alg.cra_list),
- .cra_u = {
- .cipher = {
- .cia_min_keysize = AES_MIN_KEY_SIZE,
- .cia_max_keysize = AES_MAX_KEY_SIZE,
- .cia_setkey = aes_set_key,
- .cia_encrypt = aes_encrypt,
- .cia_decrypt = aes_decrypt
- }
- }
-};
-
-static int __init aes_init(void)
-{
- return crypto_register_alg(&aes_alg);
-}
-
-static void __exit aes_fini(void)
-{
- crypto_unregister_alg(&aes_alg);
-}
-
-module_init(aes_init);
-module_exit(aes_fini);
-
-MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm (ASM)");
-MODULE_LICENSE("GPL");
-MODULE_ALIAS_CRYPTO("aes");
-MODULE_ALIAS_CRYPTO("aes-asm");
-MODULE_AUTHOR("David McCullough <ucdevel@gmail.com>");
diff --git a/arch/arm/crypto/aes_glue.h b/arch/arm/crypto/aes_glue.h
deleted file mode 100644
index cca3e51eb606..000000000000
--- a/arch/arm/crypto/aes_glue.h
+++ /dev/null
@@ -1,19 +0,0 @@
-
-#define AES_MAXNR 14
-
-struct AES_KEY {
- unsigned int rd_key[4 * (AES_MAXNR + 1)];
- int rounds;
-};
-
-struct AES_CTX {
- struct AES_KEY enc_key;
- struct AES_KEY dec_key;
-};
-
-asmlinkage void AES_encrypt(const u8 *in, u8 *out, struct AES_KEY *ctx);
-asmlinkage void AES_decrypt(const u8 *in, u8 *out, struct AES_KEY *ctx);
-asmlinkage int private_AES_set_decrypt_key(const unsigned char *userKey,
- const int bits, struct AES_KEY *key);
-asmlinkage int private_AES_set_encrypt_key(const unsigned char *userKey,
- const int bits, struct AES_KEY *key);
diff --git a/arch/arm/crypto/aesbs-core.S_shipped b/arch/arm/crypto/aesbs-core.S_shipped
deleted file mode 100644
index 1d1800f71c5b..000000000000
--- a/arch/arm/crypto/aesbs-core.S_shipped
+++ /dev/null
@@ -1,2548 +0,0 @@
-
-@ ====================================================================
-@ Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
-@ project. The module is, however, dual licensed under OpenSSL and
-@ CRYPTOGAMS licenses depending on where you obtain it. For further
-@ details see http://www.openssl.org/~appro/cryptogams/.
-@
-@ Specific modes and adaptation for Linux kernel by Ard Biesheuvel
-@ <ard.biesheuvel@linaro.org>. Permission to use under GPL terms is
-@ granted.
-@ ====================================================================
-
-@ Bit-sliced AES for ARM NEON
-@
-@ February 2012.
-@
-@ This implementation is direct adaptation of bsaes-x86_64 module for
-@ ARM NEON. Except that this module is endian-neutral [in sense that
-@ it can be compiled for either endianness] by courtesy of vld1.8's
-@ neutrality. Initial version doesn't implement interface to OpenSSL,
-@ only low-level primitives and unsupported entry points, just enough
-@ to collect performance results, which for Cortex-A8 core are:
-@
-@ encrypt 19.5 cycles per byte processed with 128-bit key
-@ decrypt 22.1 cycles per byte processed with 128-bit key
-@ key conv. 440 cycles per 128-bit key/0.18 of 8x block
-@
-@ Snapdragon S4 encrypts byte in 17.6 cycles and decrypts in 19.7,
-@ which is [much] worse than anticipated (for further details see
-@ http://www.openssl.org/~appro/Snapdragon-S4.html).
-@
-@ Cortex-A15 manages in 14.2/16.1 cycles [when integer-only code
-@ manages in 20.0 cycles].
-@
-@ When comparing to x86_64 results keep in mind that NEON unit is
-@ [mostly] single-issue and thus can't [fully] benefit from
-@ instruction-level parallelism. And when comparing to aes-armv4
-@ results keep in mind key schedule conversion overhead (see
-@ bsaes-x86_64.pl for further details)...
-@
-@ <appro@openssl.org>
-
-@ April-August 2013
-@
-@ Add CBC, CTR and XTS subroutines, adapt for kernel use.
-@
-@ <ard.biesheuvel@linaro.org>
-
-#ifndef __KERNEL__
-# include "arm_arch.h"
-
-# define VFP_ABI_PUSH vstmdb sp!,{d8-d15}
-# define VFP_ABI_POP vldmia sp!,{d8-d15}
-# define VFP_ABI_FRAME 0x40
-#else
-# define VFP_ABI_PUSH
-# define VFP_ABI_POP
-# define VFP_ABI_FRAME 0
-# define BSAES_ASM_EXTENDED_KEY
-# define XTS_CHAIN_TWEAK
-# define __ARM_ARCH__ __LINUX_ARM_ARCH__
-# define __ARM_MAX_ARCH__ 7
-#endif
-
-#ifdef __thumb__
-# define adrl adr
-#endif
-
-#if __ARM_MAX_ARCH__>=7
-.arch armv7-a
-.fpu neon
-
-.text
-.syntax unified @ ARMv7-capable assembler is expected to handle this
-#ifdef __thumb2__
-.thumb
-#else
-.code 32
-#endif
-
-.type _bsaes_decrypt8,%function
-.align 4
-_bsaes_decrypt8:
- adr r6,_bsaes_decrypt8
- vldmia r4!, {q9} @ round 0 key
- add r6,r6,#.LM0ISR-_bsaes_decrypt8
-
- vldmia r6!, {q8} @ .LM0ISR
- veor q10, q0, q9 @ xor with round0 key
- veor q11, q1, q9
- vtbl.8 d0, {q10}, d16
- vtbl.8 d1, {q10}, d17
- veor q12, q2, q9
- vtbl.8 d2, {q11}, d16
- vtbl.8 d3, {q11}, d17
- veor q13, q3, q9
- vtbl.8 d4, {q12}, d16
- vtbl.8 d5, {q12}, d17
- veor q14, q4, q9
- vtbl.8 d6, {q13}, d16
- vtbl.8 d7, {q13}, d17
- veor q15, q5, q9
- vtbl.8 d8, {q14}, d16
- vtbl.8 d9, {q14}, d17
- veor q10, q6, q9
- vtbl.8 d10, {q15}, d16
- vtbl.8 d11, {q15}, d17
- veor q11, q7, q9
- vtbl.8 d12, {q10}, d16
- vtbl.8 d13, {q10}, d17
- vtbl.8 d14, {q11}, d16
- vtbl.8 d15, {q11}, d17
- vmov.i8 q8,#0x55 @ compose .LBS0
- vmov.i8 q9,#0x33 @ compose .LBS1
- vshr.u64 q10, q6, #1
- vshr.u64 q11, q4, #1
- veor q10, q10, q7
- veor q11, q11, q5
- vand q10, q10, q8
- vand q11, q11, q8
- veor q7, q7, q10
- vshl.u64 q10, q10, #1
- veor q5, q5, q11
- vshl.u64 q11, q11, #1
- veor q6, q6, q10
- veor q4, q4, q11
- vshr.u64 q10, q2, #1
- vshr.u64 q11, q0, #1
- veor q10, q10, q3
- veor q11, q11, q1
- vand q10, q10, q8
- vand q11, q11, q8
- veor q3, q3, q10
- vshl.u64 q10, q10, #1
- veor q1, q1, q11
- vshl.u64 q11, q11, #1
- veor q2, q2, q10
- veor q0, q0, q11
- vmov.i8 q8,#0x0f @ compose .LBS2
- vshr.u64 q10, q5, #2
- vshr.u64 q11, q4, #2
- veor q10, q10, q7
- veor q11, q11, q6
- vand q10, q10, q9
- vand q11, q11, q9
- veor q7, q7, q10
- vshl.u64 q10, q10, #2
- veor q6, q6, q11
- vshl.u64 q11, q11, #2
- veor q5, q5, q10
- veor q4, q4, q11
- vshr.u64 q10, q1, #2
- vshr.u64 q11, q0, #2
- veor q10, q10, q3
- veor q11, q11, q2
- vand q10, q10, q9
- vand q11, q11, q9
- veor q3, q3, q10
- vshl.u64 q10, q10, #2
- veor q2, q2, q11
- vshl.u64 q11, q11, #2
- veor q1, q1, q10
- veor q0, q0, q11
- vshr.u64 q10, q3, #4
- vshr.u64 q11, q2, #4
- veor q10, q10, q7
- veor q11, q11, q6
- vand q10, q10, q8
- vand q11, q11, q8
- veor q7, q7, q10
- vshl.u64 q10, q10, #4
- veor q6, q6, q11
- vshl.u64 q11, q11, #4
- veor q3, q3, q10
- veor q2, q2, q11
- vshr.u64 q10, q1, #4
- vshr.u64 q11, q0, #4
- veor q10, q10, q5
- veor q11, q11, q4
- vand q10, q10, q8
- vand q11, q11, q8
- veor q5, q5, q10
- vshl.u64 q10, q10, #4
- veor q4, q4, q11
- vshl.u64 q11, q11, #4
- veor q1, q1, q10
- veor q0, q0, q11
- sub r5,r5,#1
- b .Ldec_sbox
-.align 4
-.Ldec_loop:
- vldmia r4!, {q8-q11}
- veor q8, q8, q0
- veor q9, q9, q1
- vtbl.8 d0, {q8}, d24
- vtbl.8 d1, {q8}, d25
- vldmia r4!, {q8}
- veor q10, q10, q2
- vtbl.8 d2, {q9}, d24
- vtbl.8 d3, {q9}, d25
- vldmia r4!, {q9}
- veor q11, q11, q3
- vtbl.8 d4, {q10}, d24
- vtbl.8 d5, {q10}, d25
- vldmia r4!, {q10}
- vtbl.8 d6, {q11}, d24
- vtbl.8 d7, {q11}, d25
- vldmia r4!, {q11}
- veor q8, q8, q4
- veor q9, q9, q5
- vtbl.8 d8, {q8}, d24
- vtbl.8 d9, {q8}, d25
- veor q10, q10, q6
- vtbl.8 d10, {q9}, d24
- vtbl.8 d11, {q9}, d25
- veor q11, q11, q7
- vtbl.8 d12, {q10}, d24
- vtbl.8 d13, {q10}, d25
- vtbl.8 d14, {q11}, d24
- vtbl.8 d15, {q11}, d25
-.Ldec_sbox:
- veor q1, q1, q4
- veor q3, q3, q4
-
- veor q4, q4, q7
- veor q1, q1, q6
- veor q2, q2, q7
- veor q6, q6, q4
-
- veor q0, q0, q1
- veor q2, q2, q5
- veor q7, q7, q6
- veor q3, q3, q0
- veor q5, q5, q0
- veor q1, q1, q3
- veor q11, q3, q0
- veor q10, q7, q4
- veor q9, q1, q6
- veor q13, q4, q0
- vmov q8, q10
- veor q12, q5, q2
-
- vorr q10, q10, q9
- veor q15, q11, q8
- vand q14, q11, q12
- vorr q11, q11, q12
- veor q12, q12, q9
- vand q8, q8, q9
- veor q9, q6, q2
- vand q15, q15, q12
- vand q13, q13, q9
- veor q9, q3, q7
- veor q12, q1, q5
- veor q11, q11, q13
- veor q10, q10, q13
- vand q13, q9, q12
- vorr q9, q9, q12
- veor q11, q11, q15
- veor q8, q8, q13
- veor q10, q10, q14
- veor q9, q9, q15
- veor q8, q8, q14
- vand q12, q4, q6
- veor q9, q9, q14
- vand q13, q0, q2
- vand q14, q7, q1
- vorr q15, q3, q5
- veor q11, q11, q12
- veor q9, q9, q14
- veor q8, q8, q15
- veor q10, q10, q13
-
- @ Inv_GF16 0, 1, 2, 3, s0, s1, s2, s3
-
- @ new smaller inversion
-
- vand q14, q11, q9
- vmov q12, q8
-
- veor q13, q10, q14
- veor q15, q8, q14
- veor q14, q8, q14 @ q14=q15
-
- vbsl q13, q9, q8
- vbsl q15, q11, q10
- veor q11, q11, q10
-
- vbsl q12, q13, q14
- vbsl q8, q14, q13
-
- vand q14, q12, q15
- veor q9, q9, q8
-
- veor q14, q14, q11
- veor q12, q5, q2
- veor q8, q1, q6
- veor q10, q15, q14
- vand q10, q10, q5
- veor q5, q5, q1
- vand q11, q1, q15
- vand q5, q5, q14
- veor q1, q11, q10
- veor q5, q5, q11
- veor q15, q15, q13
- veor q14, q14, q9
- veor q11, q15, q14
- veor q10, q13, q9
- vand q11, q11, q12
- vand q10, q10, q2
- veor q12, q12, q8
- veor q2, q2, q6
- vand q8, q8, q15
- vand q6, q6, q13
- vand q12, q12, q14
- vand q2, q2, q9
- veor q8, q8, q12
- veor q2, q2, q6
- veor q12, q12, q11
- veor q6, q6, q10
- veor q5, q5, q12
- veor q2, q2, q12
- veor q1, q1, q8
- veor q6, q6, q8
-
- veor q12, q3, q0
- veor q8, q7, q4
- veor q11, q15, q14
- veor q10, q13, q9
- vand q11, q11, q12
- vand q10, q10, q0
- veor q12, q12, q8
- veor q0, q0, q4
- vand q8, q8, q15
- vand q4, q4, q13
- vand q12, q12, q14
- vand q0, q0, q9
- veor q8, q8, q12
- veor q0, q0, q4
- veor q12, q12, q11
- veor q4, q4, q10
- veor q15, q15, q13
- veor q14, q14, q9
- veor q10, q15, q14
- vand q10, q10, q3
- veor q3, q3, q7
- vand q11, q7, q15
- vand q3, q3, q14
- veor q7, q11, q10
- veor q3, q3, q11
- veor q3, q3, q12
- veor q0, q0, q12
- veor q7, q7, q8
- veor q4, q4, q8
- veor q1, q1, q7
- veor q6, q6, q5
-
- veor q4, q4, q1
- veor q2, q2, q7
- veor q5, q5, q7
- veor q4, q4, q2
- veor q7, q7, q0
- veor q4, q4, q5
- veor q3, q3, q6
- veor q6, q6, q1
- veor q3, q3, q4
-
- veor q4, q4, q0
- veor q7, q7, q3
- subs r5,r5,#1
- bcc .Ldec_done
- @ multiplication by 0x05-0x00-0x04-0x00
- vext.8 q8, q0, q0, #8
- vext.8 q14, q3, q3, #8
- vext.8 q15, q5, q5, #8
- veor q8, q8, q0
- vext.8 q9, q1, q1, #8
- veor q14, q14, q3
- vext.8 q10, q6, q6, #8
- veor q15, q15, q5
- vext.8 q11, q4, q4, #8
- veor q9, q9, q1
- vext.8 q12, q2, q2, #8
- veor q10, q10, q6
- vext.8 q13, q7, q7, #8
- veor q11, q11, q4
- veor q12, q12, q2
- veor q13, q13, q7
-
- veor q0, q0, q14
- veor q1, q1, q14
- veor q6, q6, q8
- veor q2, q2, q10
- veor q4, q4, q9
- veor q1, q1, q15
- veor q6, q6, q15
- veor q2, q2, q14
- veor q7, q7, q11
- veor q4, q4, q14
- veor q3, q3, q12
- veor q2, q2, q15
- veor q7, q7, q15
- veor q5, q5, q13
- vext.8 q8, q0, q0, #12 @ x0 <<< 32
- vext.8 q9, q1, q1, #12
- veor q0, q0, q8 @ x0 ^ (x0 <<< 32)
- vext.8 q10, q6, q6, #12
- veor q1, q1, q9
- vext.8 q11, q4, q4, #12
- veor q6, q6, q10
- vext.8 q12, q2, q2, #12
- veor q4, q4, q11
- vext.8 q13, q7, q7, #12
- veor q2, q2, q12
- vext.8 q14, q3, q3, #12
- veor q7, q7, q13
- vext.8 q15, q5, q5, #12
- veor q3, q3, q14
-
- veor q9, q9, q0
- veor q5, q5, q15
- vext.8 q0, q0, q0, #8 @ (x0 ^ (x0 <<< 32)) <<< 64)
- veor q10, q10, q1
- veor q8, q8, q5
- veor q9, q9, q5
- vext.8 q1, q1, q1, #8
- veor q13, q13, q2
- veor q0, q0, q8
- veor q14, q14, q7
- veor q1, q1, q9
- vext.8 q8, q2, q2, #8
- veor q12, q12, q4
- vext.8 q9, q7, q7, #8
- veor q15, q15, q3
- vext.8 q2, q4, q4, #8
- veor q11, q11, q6
- vext.8 q7, q5, q5, #8
- veor q12, q12, q5
- vext.8 q4, q3, q3, #8
- veor q11, q11, q5
- vext.8 q3, q6, q6, #8
- veor q5, q9, q13
- veor q11, q11, q2
- veor q7, q7, q15
- veor q6, q4, q14
- veor q4, q8, q12
- veor q2, q3, q10
- vmov q3, q11
- @ vmov q5, q9
- vldmia r6, {q12} @ .LISR
- ite eq @ Thumb2 thing, sanity check in ARM
- addeq r6,r6,#0x10
- bne .Ldec_loop
- vldmia r6, {q12} @ .LISRM0
- b .Ldec_loop
-.align 4
-.Ldec_done:
- vmov.i8 q8,#0x55 @ compose .LBS0
- vmov.i8 q9,#0x33 @ compose .LBS1
- vshr.u64 q10, q3, #1
- vshr.u64 q11, q2, #1
- veor q10, q10, q5
- veor q11, q11, q7
- vand q10, q10, q8
- vand q11, q11, q8
- veor q5, q5, q10
- vshl.u64 q10, q10, #1
- veor q7, q7, q11
- vshl.u64 q11, q11, #1
- veor q3, q3, q10
- veor q2, q2, q11
- vshr.u64 q10, q6, #1
- vshr.u64 q11, q0, #1
- veor q10, q10, q4
- veor q11, q11, q1
- vand q10, q10, q8
- vand q11, q11, q8
- veor q4, q4, q10
- vshl.u64 q10, q10, #1
- veor q1, q1, q11
- vshl.u64 q11, q11, #1
- veor q6, q6, q10
- veor q0, q0, q11
- vmov.i8 q8,#0x0f @ compose .LBS2
- vshr.u64 q10, q7, #2
- vshr.u64 q11, q2, #2
- veor q10, q10, q5
- veor q11, q11, q3
- vand q10, q10, q9
- vand q11, q11, q9
- veor q5, q5, q10
- vshl.u64 q10, q10, #2
- veor q3, q3, q11
- vshl.u64 q11, q11, #2
- veor q7, q7, q10
- veor q2, q2, q11
- vshr.u64 q10, q1, #2
- vshr.u64 q11, q0, #2
- veor q10, q10, q4
- veor q11, q11, q6
- vand q10, q10, q9
- vand q11, q11, q9
- veor q4, q4, q10
- vshl.u64 q10, q10, #2
- veor q6, q6, q11
- vshl.u64 q11, q11, #2
- veor q1, q1, q10
- veor q0, q0, q11
- vshr.u64 q10, q4, #4
- vshr.u64 q11, q6, #4
- veor q10, q10, q5
- veor q11, q11, q3
- vand q10, q10, q8
- vand q11, q11, q8
- veor q5, q5, q10
- vshl.u64 q10, q10, #4
- veor q3, q3, q11
- vshl.u64 q11, q11, #4
- veor q4, q4, q10
- veor q6, q6, q11
- vshr.u64 q10, q1, #4
- vshr.u64 q11, q0, #4
- veor q10, q10, q7
- veor q11, q11, q2
- vand q10, q10, q8
- vand q11, q11, q8
- veor q7, q7, q10
- vshl.u64 q10, q10, #4
- veor q2, q2, q11
- vshl.u64 q11, q11, #4
- veor q1, q1, q10
- veor q0, q0, q11
- vldmia r4, {q8} @ last round key
- veor q6, q6, q8
- veor q4, q4, q8
- veor q2, q2, q8
- veor q7, q7, q8
- veor q3, q3, q8
- veor q5, q5, q8
- veor q0, q0, q8
- veor q1, q1, q8
- bx lr
-.size _bsaes_decrypt8,.-_bsaes_decrypt8
-
-.type _bsaes_const,%object
-.align 6
-_bsaes_const:
-.LM0ISR: @ InvShiftRows constants
- .quad 0x0a0e0206070b0f03, 0x0004080c0d010509
-.LISR:
- .quad 0x0504070602010003, 0x0f0e0d0c080b0a09
-.LISRM0:
- .quad 0x01040b0e0205080f, 0x0306090c00070a0d
-.LM0SR: @ ShiftRows constants
- .quad 0x0a0e02060f03070b, 0x0004080c05090d01
-.LSR:
- .quad 0x0504070600030201, 0x0f0e0d0c0a09080b
-.LSRM0:
- .quad 0x0304090e00050a0f, 0x01060b0c0207080d
-.LM0:
- .quad 0x02060a0e03070b0f, 0x0004080c0105090d
-.LREVM0SR:
- .quad 0x090d01050c000408, 0x03070b0f060a0e02
-.asciz "Bit-sliced AES for NEON, CRYPTOGAMS by <appro@openssl.org>"
-.align 6
-.size _bsaes_const,.-_bsaes_const
-
-.type _bsaes_encrypt8,%function
-.align 4
-_bsaes_encrypt8:
- adr r6,_bsaes_encrypt8
- vldmia r4!, {q9} @ round 0 key
- sub r6,r6,#_bsaes_encrypt8-.LM0SR
-
- vldmia r6!, {q8} @ .LM0SR
-_bsaes_encrypt8_alt:
- veor q10, q0, q9 @ xor with round0 key
- veor q11, q1, q9
- vtbl.8 d0, {q10}, d16
- vtbl.8 d1, {q10}, d17
- veor q12, q2, q9
- vtbl.8 d2, {q11}, d16
- vtbl.8 d3, {q11}, d17
- veor q13, q3, q9
- vtbl.8 d4, {q12}, d16
- vtbl.8 d5, {q12}, d17
- veor q14, q4, q9
- vtbl.8 d6, {q13}, d16
- vtbl.8 d7, {q13}, d17
- veor q15, q5, q9
- vtbl.8 d8, {q14}, d16
- vtbl.8 d9, {q14}, d17
- veor q10, q6, q9
- vtbl.8 d10, {q15}, d16
- vtbl.8 d11, {q15}, d17
- veor q11, q7, q9
- vtbl.8 d12, {q10}, d16
- vtbl.8 d13, {q10}, d17
- vtbl.8 d14, {q11}, d16
- vtbl.8 d15, {q11}, d17
-_bsaes_encrypt8_bitslice:
- vmov.i8 q8,#0x55 @ compose .LBS0
- vmov.i8 q9,#0x33 @ compose .LBS1
- vshr.u64 q10, q6, #1
- vshr.u64 q11, q4, #1
- veor q10, q10, q7
- veor q11, q11, q5
- vand q10, q10, q8
- vand q11, q11, q8
- veor q7, q7, q10
- vshl.u64 q10, q10, #1
- veor q5, q5, q11
- vshl.u64 q11, q11, #1
- veor q6, q6, q10
- veor q4, q4, q11
- vshr.u64 q10, q2, #1
- vshr.u64 q11, q0, #1
- veor q10, q10, q3
- veor q11, q11, q1
- vand q10, q10, q8
- vand q11, q11, q8
- veor q3, q3, q10
- vshl.u64 q10, q10, #1
- veor q1, q1, q11
- vshl.u64 q11, q11, #1
- veor q2, q2, q10
- veor q0, q0, q11
- vmov.i8 q8,#0x0f @ compose .LBS2
- vshr.u64 q10, q5, #2
- vshr.u64 q11, q4, #2
- veor q10, q10, q7
- veor q11, q11, q6
- vand q10, q10, q9
- vand q11, q11, q9
- veor q7, q7, q10
- vshl.u64 q10, q10, #2
- veor q6, q6, q11
- vshl.u64 q11, q11, #2
- veor q5, q5, q10
- veor q4, q4, q11
- vshr.u64 q10, q1, #2
- vshr.u64 q11, q0, #2
- veor q10, q10, q3
- veor q11, q11, q2
- vand q10, q10, q9
- vand q11, q11, q9
- veor q3, q3, q10
- vshl.u64 q10, q10, #2
- veor q2, q2, q11
- vshl.u64 q11, q11, #2
- veor q1, q1, q10
- veor q0, q0, q11
- vshr.u64 q10, q3, #4
- vshr.u64 q11, q2, #4
- veor q10, q10, q7
- veor q11, q11, q6
- vand q10, q10, q8
- vand q11, q11, q8
- veor q7, q7, q10
- vshl.u64 q10, q10, #4
- veor q6, q6, q11
- vshl.u64 q11, q11, #4
- veor q3, q3, q10
- veor q2, q2, q11
- vshr.u64 q10, q1, #4
- vshr.u64 q11, q0, #4
- veor q10, q10, q5
- veor q11, q11, q4
- vand q10, q10, q8
- vand q11, q11, q8
- veor q5, q5, q10
- vshl.u64 q10, q10, #4
- veor q4, q4, q11
- vshl.u64 q11, q11, #4
- veor q1, q1, q10
- veor q0, q0, q11
- sub r5,r5,#1
- b .Lenc_sbox
-.align 4
-.Lenc_loop:
- vldmia r4!, {q8-q11}
- veor q8, q8, q0
- veor q9, q9, q1
- vtbl.8 d0, {q8}, d24
- vtbl.8 d1, {q8}, d25
- vldmia r4!, {q8}
- veor q10, q10, q2
- vtbl.8 d2, {q9}, d24
- vtbl.8 d3, {q9}, d25
- vldmia r4!, {q9}
- veor q11, q11, q3
- vtbl.8 d4, {q10}, d24
- vtbl.8 d5, {q10}, d25
- vldmia r4!, {q10}
- vtbl.8 d6, {q11}, d24
- vtbl.8 d7, {q11}, d25
- vldmia r4!, {q11}
- veor q8, q8, q4
- veor q9, q9, q5
- vtbl.8 d8, {q8}, d24
- vtbl.8 d9, {q8}, d25
- veor q10, q10, q6
- vtbl.8 d10, {q9}, d24
- vtbl.8 d11, {q9}, d25
- veor q11, q11, q7
- vtbl.8 d12, {q10}, d24
- vtbl.8 d13, {q10}, d25
- vtbl.8 d14, {q11}, d24
- vtbl.8 d15, {q11}, d25
-.Lenc_sbox:
- veor q2, q2, q1
- veor q5, q5, q6
- veor q3, q3, q0
- veor q6, q6, q2
- veor q5, q5, q0
-
- veor q6, q6, q3
- veor q3, q3, q7
- veor q7, q7, q5
- veor q3, q3, q4
- veor q4, q4, q5
-
- veor q2, q2, q7
- veor q3, q3, q1
- veor q1, q1, q5
- veor q11, q7, q4
- veor q10, q1, q2
- veor q9, q5, q3
- veor q13, q2, q4
- vmov q8, q10
- veor q12, q6, q0
-
- vorr q10, q10, q9
- veor q15, q11, q8
- vand q14, q11, q12
- vorr q11, q11, q12
- veor q12, q12, q9
- vand q8, q8, q9
- veor q9, q3, q0
- vand q15, q15, q12
- vand q13, q13, q9
- veor q9, q7, q1
- veor q12, q5, q6
- veor q11, q11, q13
- veor q10, q10, q13
- vand q13, q9, q12
- vorr q9, q9, q12
- veor q11, q11, q15
- veor q8, q8, q13
- veor q10, q10, q14
- veor q9, q9, q15
- veor q8, q8, q14
- vand q12, q2, q3
- veor q9, q9, q14
- vand q13, q4, q0
- vand q14, q1, q5
- vorr q15, q7, q6
- veor q11, q11, q12
- veor q9, q9, q14
- veor q8, q8, q15
- veor q10, q10, q13
-
- @ Inv_GF16 0, 1, 2, 3, s0, s1, s2, s3
-
- @ new smaller inversion
-
- vand q14, q11, q9
- vmov q12, q8
-
- veor q13, q10, q14
- veor q15, q8, q14
- veor q14, q8, q14 @ q14=q15
-
- vbsl q13, q9, q8
- vbsl q15, q11, q10
- veor q11, q11, q10
-
- vbsl q12, q13, q14
- vbsl q8, q14, q13
-
- vand q14, q12, q15
- veor q9, q9, q8
-
- veor q14, q14, q11
- veor q12, q6, q0
- veor q8, q5, q3
- veor q10, q15, q14
- vand q10, q10, q6
- veor q6, q6, q5
- vand q11, q5, q15
- vand q6, q6, q14
- veor q5, q11, q10
- veor q6, q6, q11
- veor q15, q15, q13
- veor q14, q14, q9
- veor q11, q15, q14
- veor q10, q13, q9
- vand q11, q11, q12
- vand q10, q10, q0
- veor q12, q12, q8
- veor q0, q0, q3
- vand q8, q8, q15
- vand q3, q3, q13
- vand q12, q12, q14
- vand q0, q0, q9
- veor q8, q8, q12
- veor q0, q0, q3
- veor q12, q12, q11
- veor q3, q3, q10
- veor q6, q6, q12
- veor q0, q0, q12
- veor q5, q5, q8
- veor q3, q3, q8
-
- veor q12, q7, q4
- veor q8, q1, q2
- veor q11, q15, q14
- veor q10, q13, q9
- vand q11, q11, q12
- vand q10, q10, q4
- veor q12, q12, q8
- veor q4, q4, q2
- vand q8, q8, q15
- vand q2, q2, q13
- vand q12, q12, q14
- vand q4, q4, q9
- veor q8, q8, q12
- veor q4, q4, q2
- veor q12, q12, q11
- veor q2, q2, q10
- veor q15, q15, q13
- veor q14, q14, q9
- veor q10, q15, q14
- vand q10, q10, q7
- veor q7, q7, q1
- vand q11, q1, q15
- vand q7, q7, q14
- veor q1, q11, q10
- veor q7, q7, q11
- veor q7, q7, q12
- veor q4, q4, q12
- veor q1, q1, q8
- veor q2, q2, q8
- veor q7, q7, q0
- veor q1, q1, q6
- veor q6, q6, q0
- veor q4, q4, q7
- veor q0, q0, q1
-
- veor q1, q1, q5
- veor q5, q5, q2
- veor q2, q2, q3
- veor q3, q3, q5
- veor q4, q4, q5
-
- veor q6, q6, q3
- subs r5,r5,#1
- bcc .Lenc_done
- vext.8 q8, q0, q0, #12 @ x0 <<< 32
- vext.8 q9, q1, q1, #12
- veor q0, q0, q8 @ x0 ^ (x0 <<< 32)
- vext.8 q10, q4, q4, #12
- veor q1, q1, q9
- vext.8 q11, q6, q6, #12
- veor q4, q4, q10
- vext.8 q12, q3, q3, #12
- veor q6, q6, q11
- vext.8 q13, q7, q7, #12
- veor q3, q3, q12
- vext.8 q14, q2, q2, #12
- veor q7, q7, q13
- vext.8 q15, q5, q5, #12
- veor q2, q2, q14
-
- veor q9, q9, q0
- veor q5, q5, q15
- vext.8 q0, q0, q0, #8 @ (x0 ^ (x0 <<< 32)) <<< 64)
- veor q10, q10, q1
- veor q8, q8, q5
- veor q9, q9, q5
- vext.8 q1, q1, q1, #8
- veor q13, q13, q3
- veor q0, q0, q8
- veor q14, q14, q7
- veor q1, q1, q9
- vext.8 q8, q3, q3, #8
- veor q12, q12, q6
- vext.8 q9, q7, q7, #8
- veor q15, q15, q2
- vext.8 q3, q6, q6, #8
- veor q11, q11, q4
- vext.8 q7, q5, q5, #8
- veor q12, q12, q5
- vext.8 q6, q2, q2, #8
- veor q11, q11, q5
- vext.8 q2, q4, q4, #8
- veor q5, q9, q13
- veor q4, q8, q12
- veor q3, q3, q11
- veor q7, q7, q15
- veor q6, q6, q14
- @ vmov q4, q8
- veor q2, q2, q10
- @ vmov q5, q9
- vldmia r6, {q12} @ .LSR
- ite eq @ Thumb2 thing, samity check in ARM
- addeq r6,r6,#0x10
- bne .Lenc_loop
- vldmia r6, {q12} @ .LSRM0
- b .Lenc_loop
-.align 4
-.Lenc_done:
- vmov.i8 q8,#0x55 @ compose .LBS0
- vmov.i8 q9,#0x33 @ compose .LBS1
- vshr.u64 q10, q2, #1
- vshr.u64 q11, q3, #1
- veor q10, q10, q5
- veor q11, q11, q7
- vand q10, q10, q8
- vand q11, q11, q8
- veor q5, q5, q10
- vshl.u64 q10, q10, #1
- veor q7, q7, q11
- vshl.u64 q11, q11, #1
- veor q2, q2, q10
- veor q3, q3, q11
- vshr.u64 q10, q4, #1
- vshr.u64 q11, q0, #1
- veor q10, q10, q6
- veor q11, q11, q1
- vand q10, q10, q8
- vand q11, q11, q8
- veor q6, q6, q10
- vshl.u64 q10, q10, #1
- veor q1, q1, q11
- vshl.u64 q11, q11, #1
- veor q4, q4, q10
- veor q0, q0, q11
- vmov.i8 q8,#0x0f @ compose .LBS2
- vshr.u64 q10, q7, #2
- vshr.u64 q11, q3, #2
- veor q10, q10, q5
- veor q11, q11, q2
- vand q10, q10, q9
- vand q11, q11, q9
- veor q5, q5, q10
- vshl.u64 q10, q10, #2
- veor q2, q2, q11
- vshl.u64 q11, q11, #2
- veor q7, q7, q10
- veor q3, q3, q11
- vshr.u64 q10, q1, #2
- vshr.u64 q11, q0, #2
- veor q10, q10, q6
- veor q11, q11, q4
- vand q10, q10, q9
- vand q11, q11, q9
- veor q6, q6, q10
- vshl.u64 q10, q10, #2
- veor q4, q4, q11
- vshl.u64 q11, q11, #2
- veor q1, q1, q10
- veor q0, q0, q11
- vshr.u64 q10, q6, #4
- vshr.u64 q11, q4, #4
- veor q10, q10, q5
- veor q11, q11, q2
- vand q10, q10, q8
- vand q11, q11, q8
- veor q5, q5, q10
- vshl.u64 q10, q10, #4
- veor q2, q2, q11
- vshl.u64 q11, q11, #4
- veor q6, q6, q10
- veor q4, q4, q11
- vshr.u64 q10, q1, #4
- vshr.u64 q11, q0, #4
- veor q10, q10, q7
- veor q11, q11, q3
- vand q10, q10, q8
- vand q11, q11, q8
- veor q7, q7, q10
- vshl.u64 q10, q10, #4
- veor q3, q3, q11
- vshl.u64 q11, q11, #4
- veor q1, q1, q10
- veor q0, q0, q11
- vldmia r4, {q8} @ last round key
- veor q4, q4, q8
- veor q6, q6, q8
- veor q3, q3, q8
- veor q7, q7, q8
- veor q2, q2, q8
- veor q5, q5, q8
- veor q0, q0, q8
- veor q1, q1, q8
- bx lr
-.size _bsaes_encrypt8,.-_bsaes_encrypt8
-.type _bsaes_key_convert,%function
-.align 4
-_bsaes_key_convert:
- adr r6,_bsaes_key_convert
- vld1.8 {q7}, [r4]! @ load round 0 key
- sub r6,r6,#_bsaes_key_convert-.LM0
- vld1.8 {q15}, [r4]! @ load round 1 key
-
- vmov.i8 q8, #0x01 @ bit masks
- vmov.i8 q9, #0x02
- vmov.i8 q10, #0x04
- vmov.i8 q11, #0x08
- vmov.i8 q12, #0x10
- vmov.i8 q13, #0x20
- vldmia r6, {q14} @ .LM0
-
-#ifdef __ARMEL__
- vrev32.8 q7, q7
- vrev32.8 q15, q15
-#endif
- sub r5,r5,#1
- vstmia r12!, {q7} @ save round 0 key
- b .Lkey_loop
-
-.align 4
-.Lkey_loop:
- vtbl.8 d14,{q15},d28
- vtbl.8 d15,{q15},d29
- vmov.i8 q6, #0x40
- vmov.i8 q15, #0x80
-
- vtst.8 q0, q7, q8
- vtst.8 q1, q7, q9
- vtst.8 q2, q7, q10
- vtst.8 q3, q7, q11
- vtst.8 q4, q7, q12
- vtst.8 q5, q7, q13
- vtst.8 q6, q7, q6
- vtst.8 q7, q7, q15
- vld1.8 {q15}, [r4]! @ load next round key
- vmvn q0, q0 @ "pnot"
- vmvn q1, q1
- vmvn q5, q5
- vmvn q6, q6
-#ifdef __ARMEL__
- vrev32.8 q15, q15
-#endif
- subs r5,r5,#1
- vstmia r12!,{q0-q7} @ write bit-sliced round key
- bne .Lkey_loop
-
- vmov.i8 q7,#0x63 @ compose .L63
- @ don't save last round key
- bx lr
-.size _bsaes_key_convert,.-_bsaes_key_convert
-.extern AES_cbc_encrypt
-.extern AES_decrypt
-
-.global bsaes_cbc_encrypt
-.type bsaes_cbc_encrypt,%function
-.align 5
-bsaes_cbc_encrypt:
-#ifndef __KERNEL__
- cmp r2, #128
-#ifndef __thumb__
- blo AES_cbc_encrypt
-#else
- bhs 1f
- b AES_cbc_encrypt
-1:
-#endif
-#endif
-
- @ it is up to the caller to make sure we are called with enc == 0
-
- mov ip, sp
- stmdb sp!, {r4-r10, lr}
- VFP_ABI_PUSH
- ldr r8, [ip] @ IV is 1st arg on the stack
- mov r2, r2, lsr#4 @ len in 16 byte blocks
- sub sp, #0x10 @ scratch space to carry over the IV
- mov r9, sp @ save sp
-
- ldr r10, [r3, #240] @ get # of rounds
-#ifndef BSAES_ASM_EXTENDED_KEY
- @ allocate the key schedule on the stack
- sub r12, sp, r10, lsl#7 @ 128 bytes per inner round key
- add r12, #96 @ sifze of bit-slices key schedule
-
- @ populate the key schedule
- mov r4, r3 @ pass key
- mov r5, r10 @ pass # of rounds
- mov sp, r12 @ sp is sp
- bl _bsaes_key_convert
- vldmia sp, {q6}
- vstmia r12, {q15} @ save last round key
- veor q7, q7, q6 @ fix up round 0 key
- vstmia sp, {q7}
-#else
- ldr r12, [r3, #244]
- eors r12, #1
- beq 0f
-
- @ populate the key schedule
- str r12, [r3, #244]
- mov r4, r3 @ pass key
- mov r5, r10 @ pass # of rounds
- add r12, r3, #248 @ pass key schedule
- bl _bsaes_key_convert
- add r4, r3, #248
- vldmia r4, {q6}
- vstmia r12, {q15} @ save last round key
- veor q7, q7, q6 @ fix up round 0 key
- vstmia r4, {q7}
-
-.align 2
-0:
-#endif
-
- vld1.8 {q15}, [r8] @ load IV
- b .Lcbc_dec_loop
-
-.align 4
-.Lcbc_dec_loop:
- subs r2, r2, #0x8
- bmi .Lcbc_dec_loop_finish
-
- vld1.8 {q0-q1}, [r0]! @ load input
- vld1.8 {q2-q3}, [r0]!
-#ifndef BSAES_ASM_EXTENDED_KEY
- mov r4, sp @ pass the key
-#else
- add r4, r3, #248
-#endif
- vld1.8 {q4-q5}, [r0]!
- mov r5, r10
- vld1.8 {q6-q7}, [r0]
- sub r0, r0, #0x60
- vstmia r9, {q15} @ put aside IV
-
- bl _bsaes_decrypt8
-
- vldmia r9, {q14} @ reload IV
- vld1.8 {q8-q9}, [r0]! @ reload input
- veor q0, q0, q14 @ ^= IV
- vld1.8 {q10-q11}, [r0]!
- veor q1, q1, q8
- veor q6, q6, q9
- vld1.8 {q12-q13}, [r0]!
- veor q4, q4, q10
- veor q2, q2, q11
- vld1.8 {q14-q15}, [r0]!
- veor q7, q7, q12
- vst1.8 {q0-q1}, [r1]! @ write output
- veor q3, q3, q13
- vst1.8 {q6}, [r1]!
- veor q5, q5, q14
- vst1.8 {q4}, [r1]!
- vst1.8 {q2}, [r1]!
- vst1.8 {q7}, [r1]!
- vst1.8 {q3}, [r1]!
- vst1.8 {q5}, [r1]!
-
- b .Lcbc_dec_loop
-
-.Lcbc_dec_loop_finish:
- adds r2, r2, #8
- beq .Lcbc_dec_done
-
- vld1.8 {q0}, [r0]! @ load input
- cmp r2, #2
- blo .Lcbc_dec_one
- vld1.8 {q1}, [r0]!
-#ifndef BSAES_ASM_EXTENDED_KEY
- mov r4, sp @ pass the key
-#else
- add r4, r3, #248
-#endif
- mov r5, r10
- vstmia r9, {q15} @ put aside IV
- beq .Lcbc_dec_two
- vld1.8 {q2}, [r0]!
- cmp r2, #4
- blo .Lcbc_dec_three
- vld1.8 {q3}, [r0]!
- beq .Lcbc_dec_four
- vld1.8 {q4}, [r0]!
- cmp r2, #6
- blo .Lcbc_dec_five
- vld1.8 {q5}, [r0]!
- beq .Lcbc_dec_six
- vld1.8 {q6}, [r0]!
- sub r0, r0, #0x70
-
- bl _bsaes_decrypt8
-
- vldmia r9, {q14} @ reload IV
- vld1.8 {q8-q9}, [r0]! @ reload input
- veor q0, q0, q14 @ ^= IV
- vld1.8 {q10-q11}, [r0]!
- veor q1, q1, q8
- veor q6, q6, q9
- vld1.8 {q12-q13}, [r0]!
- veor q4, q4, q10
- veor q2, q2, q11
- vld1.8 {q15}, [r0]!
- veor q7, q7, q12
- vst1.8 {q0-q1}, [r1]! @ write output
- veor q3, q3, q13
- vst1.8 {q6}, [r1]!
- vst1.8 {q4}, [r1]!
- vst1.8 {q2}, [r1]!
- vst1.8 {q7}, [r1]!
- vst1.8 {q3}, [r1]!
- b .Lcbc_dec_done
-.align 4
-.Lcbc_dec_six:
- sub r0, r0, #0x60
- bl _bsaes_decrypt8
- vldmia r9,{q14} @ reload IV
- vld1.8 {q8-q9}, [r0]! @ reload input
- veor q0, q0, q14 @ ^= IV
- vld1.8 {q10-q11}, [r0]!
- veor q1, q1, q8
- veor q6, q6, q9
- vld1.8 {q12}, [r0]!
- veor q4, q4, q10
- veor q2, q2, q11
- vld1.8 {q15}, [r0]!
- veor q7, q7, q12
- vst1.8 {q0-q1}, [r1]! @ write output
- vst1.8 {q6}, [r1]!
- vst1.8 {q4}, [r1]!
- vst1.8 {q2}, [r1]!
- vst1.8 {q7}, [r1]!
- b .Lcbc_dec_done
-.align 4
-.Lcbc_dec_five:
- sub r0, r0, #0x50
- bl _bsaes_decrypt8
- vldmia r9, {q14} @ reload IV
- vld1.8 {q8-q9}, [r0]! @ reload input
- veor q0, q0, q14 @ ^= IV
- vld1.8 {q10-q11}, [r0]!
- veor q1, q1, q8
- veor q6, q6, q9
- vld1.8 {q15}, [r0]!
- veor q4, q4, q10
- vst1.8 {q0-q1}, [r1]! @ write output
- veor q2, q2, q11
- vst1.8 {q6}, [r1]!
- vst1.8 {q4}, [r1]!
- vst1.8 {q2}, [r1]!
- b .Lcbc_dec_done
-.align 4
-.Lcbc_dec_four:
- sub r0, r0, #0x40
- bl _bsaes_decrypt8
- vldmia r9, {q14} @ reload IV
- vld1.8 {q8-q9}, [r0]! @ reload input
- veor q0, q0, q14 @ ^= IV
- vld1.8 {q10}, [r0]!
- veor q1, q1, q8
- veor q6, q6, q9
- vld1.8 {q15}, [r0]!
- veor q4, q4, q10
- vst1.8 {q0-q1}, [r1]! @ write output
- vst1.8 {q6}, [r1]!
- vst1.8 {q4}, [r1]!
- b .Lcbc_dec_done
-.align 4
-.Lcbc_dec_three:
- sub r0, r0, #0x30
- bl _bsaes_decrypt8
- vldmia r9, {q14} @ reload IV
- vld1.8 {q8-q9}, [r0]! @ reload input
- veor q0, q0, q14 @ ^= IV
- vld1.8 {q15}, [r0]!
- veor q1, q1, q8
- veor q6, q6, q9
- vst1.8 {q0-q1}, [r1]! @ write output
- vst1.8 {q6}, [r1]!
- b .Lcbc_dec_done
-.align 4
-.Lcbc_dec_two:
- sub r0, r0, #0x20
- bl _bsaes_decrypt8
- vldmia r9, {q14} @ reload IV
- vld1.8 {q8}, [r0]! @ reload input
- veor q0, q0, q14 @ ^= IV
- vld1.8 {q15}, [r0]! @ reload input
- veor q1, q1, q8
- vst1.8 {q0-q1}, [r1]! @ write output
- b .Lcbc_dec_done
-.align 4
-.Lcbc_dec_one:
- sub r0, r0, #0x10
- mov r10, r1 @ save original out pointer
- mov r1, r9 @ use the iv scratch space as out buffer
- mov r2, r3
- vmov q4,q15 @ just in case ensure that IV
- vmov q5,q0 @ and input are preserved
- bl AES_decrypt
- vld1.8 {q0}, [r9,:64] @ load result
- veor q0, q0, q4 @ ^= IV
- vmov q15, q5 @ q5 holds input
- vst1.8 {q0}, [r10] @ write output
-
-.Lcbc_dec_done:
-#ifndef BSAES_ASM_EXTENDED_KEY
- vmov.i32 q0, #0
- vmov.i32 q1, #0
-.Lcbc_dec_bzero: @ wipe key schedule [if any]
- vstmia sp!, {q0-q1}
- cmp sp, r9
- bne .Lcbc_dec_bzero
-#endif
-
- mov sp, r9
- add sp, #0x10 @ add sp,r9,#0x10 is no good for thumb
- vst1.8 {q15}, [r8] @ return IV
- VFP_ABI_POP
- ldmia sp!, {r4-r10, pc}
-.size bsaes_cbc_encrypt,.-bsaes_cbc_encrypt
-.extern AES_encrypt
-.global bsaes_ctr32_encrypt_blocks
-.type bsaes_ctr32_encrypt_blocks,%function
-.align 5
-bsaes_ctr32_encrypt_blocks:
- cmp r2, #8 @ use plain AES for
- blo .Lctr_enc_short @ small sizes
-
- mov ip, sp
- stmdb sp!, {r4-r10, lr}
- VFP_ABI_PUSH
- ldr r8, [ip] @ ctr is 1st arg on the stack
- sub sp, sp, #0x10 @ scratch space to carry over the ctr
- mov r9, sp @ save sp
-
- ldr r10, [r3, #240] @ get # of rounds
-#ifndef BSAES_ASM_EXTENDED_KEY
- @ allocate the key schedule on the stack
- sub r12, sp, r10, lsl#7 @ 128 bytes per inner round key
- add r12, #96 @ size of bit-sliced key schedule
-
- @ populate the key schedule
- mov r4, r3 @ pass key
- mov r5, r10 @ pass # of rounds
- mov sp, r12 @ sp is sp
- bl _bsaes_key_convert
- veor q7,q7,q15 @ fix up last round key
- vstmia r12, {q7} @ save last round key
-
- vld1.8 {q0}, [r8] @ load counter
- add r8, r6, #.LREVM0SR-.LM0 @ borrow r8
- vldmia sp, {q4} @ load round0 key
-#else
- ldr r12, [r3, #244]
- eors r12, #1
- beq 0f
-
- @ populate the key schedule
- str r12, [r3, #244]
- mov r4, r3 @ pass key
- mov r5, r10 @ pass # of rounds
- add r12, r3, #248 @ pass key schedule
- bl _bsaes_key_convert
- veor q7,q7,q15 @ fix up last round key
- vstmia r12, {q7} @ save last round key
-
-.align 2
-0: add r12, r3, #248
- vld1.8 {q0}, [r8] @ load counter
- adrl r8, .LREVM0SR @ borrow r8
- vldmia r12, {q4} @ load round0 key
- sub sp, #0x10 @ place for adjusted round0 key
-#endif
-
- vmov.i32 q8,#1 @ compose 1<<96
- veor q9,q9,q9
- vrev32.8 q0,q0
- vext.8 q8,q9,q8,#4
- vrev32.8 q4,q4
- vadd.u32 q9,q8,q8 @ compose 2<<96
- vstmia sp, {q4} @ save adjusted round0 key
- b .Lctr_enc_loop
-
-.align 4
-.Lctr_enc_loop:
- vadd.u32 q10, q8, q9 @ compose 3<<96
- vadd.u32 q1, q0, q8 @ +1
- vadd.u32 q2, q0, q9 @ +2
- vadd.u32 q3, q0, q10 @ +3
- vadd.u32 q4, q1, q10
- vadd.u32 q5, q2, q10
- vadd.u32 q6, q3, q10
- vadd.u32 q7, q4, q10
- vadd.u32 q10, q5, q10 @ next counter
-
- @ Borrow prologue from _bsaes_encrypt8 to use the opportunity
- @ to flip byte order in 32-bit counter
-
- vldmia sp, {q9} @ load round0 key
-#ifndef BSAES_ASM_EXTENDED_KEY
- add r4, sp, #0x10 @ pass next round key
-#else
- add r4, r3, #264
-#endif
- vldmia r8, {q8} @ .LREVM0SR
- mov r5, r10 @ pass rounds
- vstmia r9, {q10} @ save next counter
- sub r6, r8, #.LREVM0SR-.LSR @ pass constants
-
- bl _bsaes_encrypt8_alt
-
- subs r2, r2, #8
- blo .Lctr_enc_loop_done
-
- vld1.8 {q8-q9}, [r0]! @ load input
- vld1.8 {q10-q11}, [r0]!
- veor q0, q8
- veor q1, q9
- vld1.8 {q12-q13}, [r0]!
- veor q4, q10
- veor q6, q11
- vld1.8 {q14-q15}, [r0]!
- veor q3, q12
- vst1.8 {q0-q1}, [r1]! @ write output
- veor q7, q13
- veor q2, q14
- vst1.8 {q4}, [r1]!
- veor q5, q15
- vst1.8 {q6}, [r1]!
- vmov.i32 q8, #1 @ compose 1<<96
- vst1.8 {q3}, [r1]!
- veor q9, q9, q9
- vst1.8 {q7}, [r1]!
- vext.8 q8, q9, q8, #4
- vst1.8 {q2}, [r1]!
- vadd.u32 q9,q8,q8 @ compose 2<<96
- vst1.8 {q5}, [r1]!
- vldmia r9, {q0} @ load counter
-
- bne .Lctr_enc_loop
- b .Lctr_enc_done
-
-.align 4
-.Lctr_enc_loop_done:
- add r2, r2, #8
- vld1.8 {q8}, [r0]! @ load input
- veor q0, q8
- vst1.8 {q0}, [r1]! @ write output
- cmp r2, #2
- blo .Lctr_enc_done
- vld1.8 {q9}, [r0]!
- veor q1, q9
- vst1.8 {q1}, [r1]!
- beq .Lctr_enc_done
- vld1.8 {q10}, [r0]!
- veor q4, q10
- vst1.8 {q4}, [r1]!
- cmp r2, #4
- blo .Lctr_enc_done
- vld1.8 {q11}, [r0]!
- veor q6, q11
- vst1.8 {q6}, [r1]!
- beq .Lctr_enc_done
- vld1.8 {q12}, [r0]!
- veor q3, q12
- vst1.8 {q3}, [r1]!
- cmp r2, #6
- blo .Lctr_enc_done
- vld1.8 {q13}, [r0]!
- veor q7, q13
- vst1.8 {q7}, [r1]!
- beq .Lctr_enc_done
- vld1.8 {q14}, [r0]
- veor q2, q14
- vst1.8 {q2}, [r1]!
-
-.Lctr_enc_done:
- vmov.i32 q0, #0
- vmov.i32 q1, #0
-#ifndef BSAES_ASM_EXTENDED_KEY
-.Lctr_enc_bzero: @ wipe key schedule [if any]
- vstmia sp!, {q0-q1}
- cmp sp, r9
- bne .Lctr_enc_bzero
-#else
- vstmia sp, {q0-q1}
-#endif
-
- mov sp, r9
- add sp, #0x10 @ add sp,r9,#0x10 is no good for thumb
- VFP_ABI_POP
- ldmia sp!, {r4-r10, pc} @ return
-
-.align 4
-.Lctr_enc_short:
- ldr ip, [sp] @ ctr pointer is passed on stack
- stmdb sp!, {r4-r8, lr}
-
- mov r4, r0 @ copy arguments
- mov r5, r1
- mov r6, r2
- mov r7, r3
- ldr r8, [ip, #12] @ load counter LSW
- vld1.8 {q1}, [ip] @ load whole counter value
-#ifdef __ARMEL__
- rev r8, r8
-#endif
- sub sp, sp, #0x10
- vst1.8 {q1}, [sp,:64] @ copy counter value
- sub sp, sp, #0x10
-
-.Lctr_enc_short_loop:
- add r0, sp, #0x10 @ input counter value
- mov r1, sp @ output on the stack
- mov r2, r7 @ key
-
- bl AES_encrypt
-
- vld1.8 {q0}, [r4]! @ load input
- vld1.8 {q1}, [sp,:64] @ load encrypted counter
- add r8, r8, #1
-#ifdef __ARMEL__
- rev r0, r8
- str r0, [sp, #0x1c] @ next counter value
-#else
- str r8, [sp, #0x1c] @ next counter value
-#endif
- veor q0,q0,q1
- vst1.8 {q0}, [r5]! @ store output
- subs r6, r6, #1
- bne .Lctr_enc_short_loop
-
- vmov.i32 q0, #0
- vmov.i32 q1, #0
- vstmia sp!, {q0-q1}
-
- ldmia sp!, {r4-r8, pc}
-.size bsaes_ctr32_encrypt_blocks,.-bsaes_ctr32_encrypt_blocks
-.globl bsaes_xts_encrypt
-.type bsaes_xts_encrypt,%function
-.align 4
-bsaes_xts_encrypt:
- mov ip, sp
- stmdb sp!, {r4-r10, lr} @ 0x20
- VFP_ABI_PUSH
- mov r6, sp @ future r3
-
- mov r7, r0
- mov r8, r1
- mov r9, r2
- mov r10, r3
-
- sub r0, sp, #0x10 @ 0x10
- bic r0, #0xf @ align at 16 bytes
- mov sp, r0
-
-#ifdef XTS_CHAIN_TWEAK
- ldr r0, [ip] @ pointer to input tweak
-#else
- @ generate initial tweak
- ldr r0, [ip, #4] @ iv[]
- mov r1, sp
- ldr r2, [ip, #0] @ key2
- bl AES_encrypt
- mov r0,sp @ pointer to initial tweak
-#endif
-
- ldr r1, [r10, #240] @ get # of rounds
- mov r3, r6
-#ifndef BSAES_ASM_EXTENDED_KEY
- @ allocate the key schedule on the stack
- sub r12, sp, r1, lsl#7 @ 128 bytes per inner round key
- @ add r12, #96 @ size of bit-sliced key schedule
- sub r12, #48 @ place for tweak[9]
-
- @ populate the key schedule
- mov r4, r10 @ pass key
- mov r5, r1 @ pass # of rounds
- mov sp, r12
- add r12, #0x90 @ pass key schedule
- bl _bsaes_key_convert
- veor q7, q7, q15 @ fix up last round key
- vstmia r12, {q7} @ save last round key
-#else
- ldr r12, [r10, #244]
- eors r12, #1
- beq 0f
-
- str r12, [r10, #244]
- mov r4, r10 @ pass key
- mov r5, r1 @ pass # of rounds
- add r12, r10, #248 @ pass key schedule
- bl _bsaes_key_convert
- veor q7, q7, q15 @ fix up last round key
- vstmia r12, {q7}
-
-.align 2
-0: sub sp, #0x90 @ place for tweak[9]
-#endif
-
- vld1.8 {q8}, [r0] @ initial tweak
- adr r2, .Lxts_magic
-
- subs r9, #0x80
- blo .Lxts_enc_short
- b .Lxts_enc_loop
-
-.align 4
-.Lxts_enc_loop:
- vldmia r2, {q5} @ load XTS magic
- vshr.s64 q6, q8, #63
- mov r0, sp
- vand q6, q6, q5
- vadd.u64 q9, q8, q8
- vst1.64 {q8}, [r0,:128]!
- vswp d13,d12
- vshr.s64 q7, q9, #63
- veor q9, q9, q6
- vand q7, q7, q5
- vadd.u64 q10, q9, q9
- vst1.64 {q9}, [r0,:128]!
- vswp d15,d14
- vshr.s64 q6, q10, #63
- veor q10, q10, q7
- vand q6, q6, q5
- vld1.8 {q0}, [r7]!
- vadd.u64 q11, q10, q10
- vst1.64 {q10}, [r0,:128]!
- vswp d13,d12
- vshr.s64 q7, q11, #63
- veor q11, q11, q6
- vand q7, q7, q5
- vld1.8 {q1}, [r7]!
- veor q0, q0, q8
- vadd.u64 q12, q11, q11
- vst1.64 {q11}, [r0,:128]!
- vswp d15,d14
- vshr.s64 q6, q12, #63
- veor q12, q12, q7
- vand q6, q6, q5
- vld1.8 {q2}, [r7]!
- veor q1, q1, q9
- vadd.u64 q13, q12, q12
- vst1.64 {q12}, [r0,:128]!
- vswp d13,d12
- vshr.s64 q7, q13, #63
- veor q13, q13, q6
- vand q7, q7, q5
- vld1.8 {q3}, [r7]!
- veor q2, q2, q10
- vadd.u64 q14, q13, q13
- vst1.64 {q13}, [r0,:128]!
- vswp d15,d14
- vshr.s64 q6, q14, #63
- veor q14, q14, q7
- vand q6, q6, q5
- vld1.8 {q4}, [r7]!
- veor q3, q3, q11
- vadd.u64 q15, q14, q14
- vst1.64 {q14}, [r0,:128]!
- vswp d13,d12
- vshr.s64 q7, q15, #63
- veor q15, q15, q6
- vand q7, q7, q5
- vld1.8 {q5}, [r7]!
- veor q4, q4, q12
- vadd.u64 q8, q15, q15
- vst1.64 {q15}, [r0,:128]!
- vswp d15,d14
- veor q8, q8, q7
- vst1.64 {q8}, [r0,:128] @ next round tweak
-
- vld1.8 {q6-q7}, [r7]!
- veor q5, q5, q13
-#ifndef BSAES_ASM_EXTENDED_KEY
- add r4, sp, #0x90 @ pass key schedule
-#else
- add r4, r10, #248 @ pass key schedule
-#endif
- veor q6, q6, q14
- mov r5, r1 @ pass rounds
- veor q7, q7, q15
- mov r0, sp
-
- bl _bsaes_encrypt8
-
- vld1.64 {q8-q9}, [r0,:128]!
- vld1.64 {q10-q11}, [r0,:128]!
- veor q0, q0, q8
- vld1.64 {q12-q13}, [r0,:128]!
- veor q1, q1, q9
- veor q8, q4, q10
- vst1.8 {q0-q1}, [r8]!
- veor q9, q6, q11
- vld1.64 {q14-q15}, [r0,:128]!
- veor q10, q3, q12
- vst1.8 {q8-q9}, [r8]!
- veor q11, q7, q13
- veor q12, q2, q14
- vst1.8 {q10-q11}, [r8]!
- veor q13, q5, q15
- vst1.8 {q12-q13}, [r8]!
-
- vld1.64 {q8}, [r0,:128] @ next round tweak
-
- subs r9, #0x80
- bpl .Lxts_enc_loop
-
-.Lxts_enc_short:
- adds r9, #0x70
- bmi .Lxts_enc_done
-
- vldmia r2, {q5} @ load XTS magic
- vshr.s64 q7, q8, #63
- mov r0, sp
- vand q7, q7, q5
- vadd.u64 q9, q8, q8
- vst1.64 {q8}, [r0,:128]!
- vswp d15,d14
- vshr.s64 q6, q9, #63
- veor q9, q9, q7
- vand q6, q6, q5
- vadd.u64 q10, q9, q9
- vst1.64 {q9}, [r0,:128]!
- vswp d13,d12
- vshr.s64 q7, q10, #63
- veor q10, q10, q6
- vand q7, q7, q5
- vld1.8 {q0}, [r7]!
- subs r9, #0x10
- bmi .Lxts_enc_1
- vadd.u64 q11, q10, q10
- vst1.64 {q10}, [r0,:128]!
- vswp d15,d14
- vshr.s64 q6, q11, #63
- veor q11, q11, q7
- vand q6, q6, q5
- vld1.8 {q1}, [r7]!
- subs r9, #0x10
- bmi .Lxts_enc_2
- veor q0, q0, q8
- vadd.u64 q12, q11, q11
- vst1.64 {q11}, [r0,:128]!
- vswp d13,d12
- vshr.s64 q7, q12, #63
- veor q12, q12, q6
- vand q7, q7, q5
- vld1.8 {q2}, [r7]!
- subs r9, #0x10
- bmi .Lxts_enc_3
- veor q1, q1, q9
- vadd.u64 q13, q12, q12
- vst1.64 {q12}, [r0,:128]!
- vswp d15,d14
- vshr.s64 q6, q13, #63
- veor q13, q13, q7
- vand q6, q6, q5
- vld1.8 {q3}, [r7]!
- subs r9, #0x10
- bmi .Lxts_enc_4
- veor q2, q2, q10
- vadd.u64 q14, q13, q13
- vst1.64 {q13}, [r0,:128]!
- vswp d13,d12
- vshr.s64 q7, q14, #63
- veor q14, q14, q6
- vand q7, q7, q5
- vld1.8 {q4}, [r7]!
- subs r9, #0x10
- bmi .Lxts_enc_5
- veor q3, q3, q11
- vadd.u64 q15, q14, q14
- vst1.64 {q14}, [r0,:128]!
- vswp d15,d14
- vshr.s64 q6, q15, #63
- veor q15, q15, q7
- vand q6, q6, q5
- vld1.8 {q5}, [r7]!
- subs r9, #0x10
- bmi .Lxts_enc_6
- veor q4, q4, q12
- sub r9, #0x10
- vst1.64 {q15}, [r0,:128] @ next round tweak
-
- vld1.8 {q6}, [r7]!
- veor q5, q5, q13
-#ifndef BSAES_ASM_EXTENDED_KEY
- add r4, sp, #0x90 @ pass key schedule
-#else
- add r4, r10, #248 @ pass key schedule
-#endif
- veor q6, q6, q14
- mov r5, r1 @ pass rounds
- mov r0, sp
-
- bl _bsaes_encrypt8
-
- vld1.64 {q8-q9}, [r0,:128]!
- vld1.64 {q10-q11}, [r0,:128]!
- veor q0, q0, q8
- vld1.64 {q12-q13}, [r0,:128]!
- veor q1, q1, q9
- veor q8, q4, q10
- vst1.8 {q0-q1}, [r8]!
- veor q9, q6, q11
- vld1.64 {q14}, [r0,:128]!
- veor q10, q3, q12
- vst1.8 {q8-q9}, [r8]!
- veor q11, q7, q13
- veor q12, q2, q14
- vst1.8 {q10-q11}, [r8]!
- vst1.8 {q12}, [r8]!
-
- vld1.64 {q8}, [r0,:128] @ next round tweak
- b .Lxts_enc_done
-.align 4
-.Lxts_enc_6:
- vst1.64 {q14}, [r0,:128] @ next round tweak
-
- veor q4, q4, q12
-#ifndef BSAES_ASM_EXTENDED_KEY
- add r4, sp, #0x90 @ pass key schedule
-#else
- add r4, r10, #248 @ pass key schedule
-#endif
- veor q5, q5, q13
- mov r5, r1 @ pass rounds
- mov r0, sp
-
- bl _bsaes_encrypt8
-
- vld1.64 {q8-q9}, [r0,:128]!
- vld1.64 {q10-q11}, [r0,:128]!
- veor q0, q0, q8
- vld1.64 {q12-q13}, [r0,:128]!
- veor q1, q1, q9
- veor q8, q4, q10
- vst1.8 {q0-q1}, [r8]!
- veor q9, q6, q11
- veor q10, q3, q12
- vst1.8 {q8-q9}, [r8]!
- veor q11, q7, q13
- vst1.8 {q10-q11}, [r8]!
-
- vld1.64 {q8}, [r0,:128] @ next round tweak
- b .Lxts_enc_done
-
-@ put this in range for both ARM and Thumb mode adr instructions
-.align 5
-.Lxts_magic:
- .quad 1, 0x87
-
-.align 5
-.Lxts_enc_5:
- vst1.64 {q13}, [r0,:128] @ next round tweak
-
- veor q3, q3, q11
-#ifndef BSAES_ASM_EXTENDED_KEY
- add r4, sp, #0x90 @ pass key schedule
-#else
- add r4, r10, #248 @ pass key schedule
-#endif
- veor q4, q4, q12
- mov r5, r1 @ pass rounds
- mov r0, sp
-
- bl _bsaes_encrypt8
-
- vld1.64 {q8-q9}, [r0,:128]!
- vld1.64 {q10-q11}, [r0,:128]!
- veor q0, q0, q8
- vld1.64 {q12}, [r0,:128]!
- veor q1, q1, q9
- veor q8, q4, q10
- vst1.8 {q0-q1}, [r8]!
- veor q9, q6, q11
- veor q10, q3, q12
- vst1.8 {q8-q9}, [r8]!
- vst1.8 {q10}, [r8]!
-
- vld1.64 {q8}, [r0,:128] @ next round tweak
- b .Lxts_enc_done
-.align 4
-.Lxts_enc_4:
- vst1.64 {q12}, [r0,:128] @ next round tweak
-
- veor q2, q2, q10
-#ifndef BSAES_ASM_EXTENDED_KEY
- add r4, sp, #0x90 @ pass key schedule
-#else
- add r4, r10, #248 @ pass key schedule
-#endif
- veor q3, q3, q11
- mov r5, r1 @ pass rounds
- mov r0, sp
-
- bl _bsaes_encrypt8
-
- vld1.64 {q8-q9}, [r0,:128]!
- vld1.64 {q10-q11}, [r0,:128]!
- veor q0, q0, q8
- veor q1, q1, q9
- veor q8, q4, q10
- vst1.8 {q0-q1}, [r8]!
- veor q9, q6, q11
- vst1.8 {q8-q9}, [r8]!
-
- vld1.64 {q8}, [r0,:128] @ next round tweak
- b .Lxts_enc_done
-.align 4
-.Lxts_enc_3:
- vst1.64 {q11}, [r0,:128] @ next round tweak
-
- veor q1, q1, q9
-#ifndef BSAES_ASM_EXTENDED_KEY
- add r4, sp, #0x90 @ pass key schedule
-#else
- add r4, r10, #248 @ pass key schedule
-#endif
- veor q2, q2, q10
- mov r5, r1 @ pass rounds
- mov r0, sp
-
- bl _bsaes_encrypt8
-
- vld1.64 {q8-q9}, [r0,:128]!
- vld1.64 {q10}, [r0,:128]!
- veor q0, q0, q8
- veor q1, q1, q9
- veor q8, q4, q10
- vst1.8 {q0-q1}, [r8]!
- vst1.8 {q8}, [r8]!
-
- vld1.64 {q8}, [r0,:128] @ next round tweak
- b .Lxts_enc_done
-.align 4
-.Lxts_enc_2:
- vst1.64 {q10}, [r0,:128] @ next round tweak
-
- veor q0, q0, q8
-#ifndef BSAES_ASM_EXTENDED_KEY
- add r4, sp, #0x90 @ pass key schedule
-#else
- add r4, r10, #248 @ pass key schedule
-#endif
- veor q1, q1, q9
- mov r5, r1 @ pass rounds
- mov r0, sp
-
- bl _bsaes_encrypt8
-
- vld1.64 {q8-q9}, [r0,:128]!
- veor q0, q0, q8
- veor q1, q1, q9
- vst1.8 {q0-q1}, [r8]!
-
- vld1.64 {q8}, [r0,:128] @ next round tweak
- b .Lxts_enc_done
-.align 4
-.Lxts_enc_1:
- mov r0, sp
- veor q0, q8
- mov r1, sp
- vst1.8 {q0}, [sp,:128]
- mov r2, r10
- mov r4, r3 @ preserve fp
-
- bl AES_encrypt
-
- vld1.8 {q0}, [sp,:128]
- veor q0, q0, q8
- vst1.8 {q0}, [r8]!
- mov r3, r4
-
- vmov q8, q9 @ next round tweak
-
-.Lxts_enc_done:
-#ifndef XTS_CHAIN_TWEAK
- adds r9, #0x10
- beq .Lxts_enc_ret
- sub r6, r8, #0x10
-
-.Lxts_enc_steal:
- ldrb r0, [r7], #1
- ldrb r1, [r8, #-0x10]
- strb r0, [r8, #-0x10]
- strb r1, [r8], #1
-
- subs r9, #1
- bhi .Lxts_enc_steal
-
- vld1.8 {q0}, [r6]
- mov r0, sp
- veor q0, q0, q8
- mov r1, sp
- vst1.8 {q0}, [sp,:128]
- mov r2, r10
- mov r4, r3 @ preserve fp
-
- bl AES_encrypt
-
- vld1.8 {q0}, [sp,:128]
- veor q0, q0, q8
- vst1.8 {q0}, [r6]
- mov r3, r4
-#endif
-
-.Lxts_enc_ret:
- bic r0, r3, #0xf
- vmov.i32 q0, #0
- vmov.i32 q1, #0
-#ifdef XTS_CHAIN_TWEAK
- ldr r1, [r3, #0x20+VFP_ABI_FRAME] @ chain tweak
-#endif
-.Lxts_enc_bzero: @ wipe key schedule [if any]
- vstmia sp!, {q0-q1}
- cmp sp, r0
- bne .Lxts_enc_bzero
-
- mov sp, r3
-#ifdef XTS_CHAIN_TWEAK
- vst1.8 {q8}, [r1]
-#endif
- VFP_ABI_POP
- ldmia sp!, {r4-r10, pc} @ return
-
-.size bsaes_xts_encrypt,.-bsaes_xts_encrypt
-
-.globl bsaes_xts_decrypt
-.type bsaes_xts_decrypt,%function
-.align 4
-bsaes_xts_decrypt:
- mov ip, sp
- stmdb sp!, {r4-r10, lr} @ 0x20
- VFP_ABI_PUSH
- mov r6, sp @ future r3
-
- mov r7, r0
- mov r8, r1
- mov r9, r2
- mov r10, r3
-
- sub r0, sp, #0x10 @ 0x10
- bic r0, #0xf @ align at 16 bytes
- mov sp, r0
-
-#ifdef XTS_CHAIN_TWEAK
- ldr r0, [ip] @ pointer to input tweak
-#else
- @ generate initial tweak
- ldr r0, [ip, #4] @ iv[]
- mov r1, sp
- ldr r2, [ip, #0] @ key2
- bl AES_encrypt
- mov r0, sp @ pointer to initial tweak
-#endif
-
- ldr r1, [r10, #240] @ get # of rounds
- mov r3, r6
-#ifndef BSAES_ASM_EXTENDED_KEY
- @ allocate the key schedule on the stack
- sub r12, sp, r1, lsl#7 @ 128 bytes per inner round key
- @ add r12, #96 @ size of bit-sliced key schedule
- sub r12, #48 @ place for tweak[9]
-
- @ populate the key schedule
- mov r4, r10 @ pass key
- mov r5, r1 @ pass # of rounds
- mov sp, r12
- add r12, #0x90 @ pass key schedule
- bl _bsaes_key_convert
- add r4, sp, #0x90
- vldmia r4, {q6}
- vstmia r12, {q15} @ save last round key
- veor q7, q7, q6 @ fix up round 0 key
- vstmia r4, {q7}
-#else
- ldr r12, [r10, #244]
- eors r12, #1
- beq 0f
-
- str r12, [r10, #244]
- mov r4, r10 @ pass key
- mov r5, r1 @ pass # of rounds
- add r12, r10, #248 @ pass key schedule
- bl _bsaes_key_convert
- add r4, r10, #248
- vldmia r4, {q6}
- vstmia r12, {q15} @ save last round key
- veor q7, q7, q6 @ fix up round 0 key
- vstmia r4, {q7}
-
-.align 2
-0: sub sp, #0x90 @ place for tweak[9]
-#endif
- vld1.8 {q8}, [r0] @ initial tweak
- adr r2, .Lxts_magic
-
-#ifndef XTS_CHAIN_TWEAK
- tst r9, #0xf @ if not multiple of 16
- it ne @ Thumb2 thing, sanity check in ARM
- subne r9, #0x10 @ subtract another 16 bytes
-#endif
- subs r9, #0x80
-
- blo .Lxts_dec_short
- b .Lxts_dec_loop
-
-.align 4
-.Lxts_dec_loop:
- vldmia r2, {q5} @ load XTS magic
- vshr.s64 q6, q8, #63
- mov r0, sp
- vand q6, q6, q5
- vadd.u64 q9, q8, q8
- vst1.64 {q8}, [r0,:128]!
- vswp d13,d12
- vshr.s64 q7, q9, #63
- veor q9, q9, q6
- vand q7, q7, q5
- vadd.u64 q10, q9, q9
- vst1.64 {q9}, [r0,:128]!
- vswp d15,d14
- vshr.s64 q6, q10, #63
- veor q10, q10, q7
- vand q6, q6, q5
- vld1.8 {q0}, [r7]!
- vadd.u64 q11, q10, q10
- vst1.64 {q10}, [r0,:128]!
- vswp d13,d12
- vshr.s64 q7, q11, #63
- veor q11, q11, q6
- vand q7, q7, q5
- vld1.8 {q1}, [r7]!
- veor q0, q0, q8
- vadd.u64 q12, q11, q11
- vst1.64 {q11}, [r0,:128]!
- vswp d15,d14
- vshr.s64 q6, q12, #63
- veor q12, q12, q7
- vand q6, q6, q5
- vld1.8 {q2}, [r7]!
- veor q1, q1, q9
- vadd.u64 q13, q12, q12
- vst1.64 {q12}, [r0,:128]!
- vswp d13,d12
- vshr.s64 q7, q13, #63
- veor q13, q13, q6
- vand q7, q7, q5
- vld1.8 {q3}, [r7]!
- veor q2, q2, q10
- vadd.u64 q14, q13, q13
- vst1.64 {q13}, [r0,:128]!
- vswp d15,d14
- vshr.s64 q6, q14, #63
- veor q14, q14, q7
- vand q6, q6, q5
- vld1.8 {q4}, [r7]!
- veor q3, q3, q11
- vadd.u64 q15, q14, q14
- vst1.64 {q14}, [r0,:128]!
- vswp d13,d12
- vshr.s64 q7, q15, #63
- veor q15, q15, q6
- vand q7, q7, q5
- vld1.8 {q5}, [r7]!
- veor q4, q4, q12
- vadd.u64 q8, q15, q15
- vst1.64 {q15}, [r0,:128]!
- vswp d15,d14
- veor q8, q8, q7
- vst1.64 {q8}, [r0,:128] @ next round tweak
-
- vld1.8 {q6-q7}, [r7]!
- veor q5, q5, q13
-#ifndef BSAES_ASM_EXTENDED_KEY
- add r4, sp, #0x90 @ pass key schedule
-#else
- add r4, r10, #248 @ pass key schedule
-#endif
- veor q6, q6, q14
- mov r5, r1 @ pass rounds
- veor q7, q7, q15
- mov r0, sp
-
- bl _bsaes_decrypt8
-
- vld1.64 {q8-q9}, [r0,:128]!
- vld1.64 {q10-q11}, [r0,:128]!
- veor q0, q0, q8
- vld1.64 {q12-q13}, [r0,:128]!
- veor q1, q1, q9
- veor q8, q6, q10
- vst1.8 {q0-q1}, [r8]!
- veor q9, q4, q11
- vld1.64 {q14-q15}, [r0,:128]!
- veor q10, q2, q12
- vst1.8 {q8-q9}, [r8]!
- veor q11, q7, q13
- veor q12, q3, q14
- vst1.8 {q10-q11}, [r8]!
- veor q13, q5, q15
- vst1.8 {q12-q13}, [r8]!
-
- vld1.64 {q8}, [r0,:128] @ next round tweak
-
- subs r9, #0x80
- bpl .Lxts_dec_loop
-
-.Lxts_dec_short:
- adds r9, #0x70
- bmi .Lxts_dec_done
-
- vldmia r2, {q5} @ load XTS magic
- vshr.s64 q7, q8, #63
- mov r0, sp
- vand q7, q7, q5
- vadd.u64 q9, q8, q8
- vst1.64 {q8}, [r0,:128]!
- vswp d15,d14
- vshr.s64 q6, q9, #63
- veor q9, q9, q7
- vand q6, q6, q5
- vadd.u64 q10, q9, q9
- vst1.64 {q9}, [r0,:128]!
- vswp d13,d12
- vshr.s64 q7, q10, #63
- veor q10, q10, q6
- vand q7, q7, q5
- vld1.8 {q0}, [r7]!
- subs r9, #0x10
- bmi .Lxts_dec_1
- vadd.u64 q11, q10, q10
- vst1.64 {q10}, [r0,:128]!
- vswp d15,d14
- vshr.s64 q6, q11, #63
- veor q11, q11, q7
- vand q6, q6, q5
- vld1.8 {q1}, [r7]!
- subs r9, #0x10
- bmi .Lxts_dec_2
- veor q0, q0, q8
- vadd.u64 q12, q11, q11
- vst1.64 {q11}, [r0,:128]!
- vswp d13,d12
- vshr.s64 q7, q12, #63
- veor q12, q12, q6
- vand q7, q7, q5
- vld1.8 {q2}, [r7]!
- subs r9, #0x10
- bmi .Lxts_dec_3
- veor q1, q1, q9
- vadd.u64 q13, q12, q12
- vst1.64 {q12}, [r0,:128]!
- vswp d15,d14
- vshr.s64 q6, q13, #63
- veor q13, q13, q7
- vand q6, q6, q5
- vld1.8 {q3}, [r7]!
- subs r9, #0x10
- bmi .Lxts_dec_4
- veor q2, q2, q10
- vadd.u64 q14, q13, q13
- vst1.64 {q13}, [r0,:128]!
- vswp d13,d12
- vshr.s64 q7, q14, #63
- veor q14, q14, q6
- vand q7, q7, q5
- vld1.8 {q4}, [r7]!
- subs r9, #0x10
- bmi .Lxts_dec_5
- veor q3, q3, q11
- vadd.u64 q15, q14, q14
- vst1.64 {q14}, [r0,:128]!
- vswp d15,d14
- vshr.s64 q6, q15, #63
- veor q15, q15, q7
- vand q6, q6, q5
- vld1.8 {q5}, [r7]!
- subs r9, #0x10
- bmi .Lxts_dec_6
- veor q4, q4, q12
- sub r9, #0x10
- vst1.64 {q15}, [r0,:128] @ next round tweak
-
- vld1.8 {q6}, [r7]!
- veor q5, q5, q13
-#ifndef BSAES_ASM_EXTENDED_KEY
- add r4, sp, #0x90 @ pass key schedule
-#else
- add r4, r10, #248 @ pass key schedule
-#endif
- veor q6, q6, q14
- mov r5, r1 @ pass rounds
- mov r0, sp
-
- bl _bsaes_decrypt8
-
- vld1.64 {q8-q9}, [r0,:128]!
- vld1.64 {q10-q11}, [r0,:128]!
- veor q0, q0, q8
- vld1.64 {q12-q13}, [r0,:128]!
- veor q1, q1, q9
- veor q8, q6, q10
- vst1.8 {q0-q1}, [r8]!
- veor q9, q4, q11
- vld1.64 {q14}, [r0,:128]!
- veor q10, q2, q12
- vst1.8 {q8-q9}, [r8]!
- veor q11, q7, q13
- veor q12, q3, q14
- vst1.8 {q10-q11}, [r8]!
- vst1.8 {q12}, [r8]!
-
- vld1.64 {q8}, [r0,:128] @ next round tweak
- b .Lxts_dec_done
-.align 4
-.Lxts_dec_6:
- vst1.64 {q14}, [r0,:128] @ next round tweak
-
- veor q4, q4, q12
-#ifndef BSAES_ASM_EXTENDED_KEY
- add r4, sp, #0x90 @ pass key schedule
-#else
- add r4, r10, #248 @ pass key schedule
-#endif
- veor q5, q5, q13
- mov r5, r1 @ pass rounds
- mov r0, sp
-
- bl _bsaes_decrypt8
-
- vld1.64 {q8-q9}, [r0,:128]!
- vld1.64 {q10-q11}, [r0,:128]!
- veor q0, q0, q8
- vld1.64 {q12-q13}, [r0,:128]!
- veor q1, q1, q9
- veor q8, q6, q10
- vst1.8 {q0-q1}, [r8]!
- veor q9, q4, q11
- veor q10, q2, q12
- vst1.8 {q8-q9}, [r8]!
- veor q11, q7, q13
- vst1.8 {q10-q11}, [r8]!
-
- vld1.64 {q8}, [r0,:128] @ next round tweak
- b .Lxts_dec_done
-.align 4
-.Lxts_dec_5:
- vst1.64 {q13}, [r0,:128] @ next round tweak
-
- veor q3, q3, q11
-#ifndef BSAES_ASM_EXTENDED_KEY
- add r4, sp, #0x90 @ pass key schedule
-#else
- add r4, r10, #248 @ pass key schedule
-#endif
- veor q4, q4, q12
- mov r5, r1 @ pass rounds
- mov r0, sp
-
- bl _bsaes_decrypt8
-
- vld1.64 {q8-q9}, [r0,:128]!
- vld1.64 {q10-q11}, [r0,:128]!
- veor q0, q0, q8
- vld1.64 {q12}, [r0,:128]!
- veor q1, q1, q9
- veor q8, q6, q10
- vst1.8 {q0-q1}, [r8]!
- veor q9, q4, q11
- veor q10, q2, q12
- vst1.8 {q8-q9}, [r8]!
- vst1.8 {q10}, [r8]!
-
- vld1.64 {q8}, [r0,:128] @ next round tweak
- b .Lxts_dec_done
-.align 4
-.Lxts_dec_4:
- vst1.64 {q12}, [r0,:128] @ next round tweak
-
- veor q2, q2, q10
-#ifndef BSAES_ASM_EXTENDED_KEY
- add r4, sp, #0x90 @ pass key schedule
-#else
- add r4, r10, #248 @ pass key schedule
-#endif
- veor q3, q3, q11
- mov r5, r1 @ pass rounds
- mov r0, sp
-
- bl _bsaes_decrypt8
-
- vld1.64 {q8-q9}, [r0,:128]!
- vld1.64 {q10-q11}, [r0,:128]!
- veor q0, q0, q8
- veor q1, q1, q9
- veor q8, q6, q10
- vst1.8 {q0-q1}, [r8]!
- veor q9, q4, q11
- vst1.8 {q8-q9}, [r8]!
-
- vld1.64 {q8}, [r0,:128] @ next round tweak
- b .Lxts_dec_done
-.align 4
-.Lxts_dec_3:
- vst1.64 {q11}, [r0,:128] @ next round tweak
-
- veor q1, q1, q9
-#ifndef BSAES_ASM_EXTENDED_KEY
- add r4, sp, #0x90 @ pass key schedule
-#else
- add r4, r10, #248 @ pass key schedule
-#endif
- veor q2, q2, q10
- mov r5, r1 @ pass rounds
- mov r0, sp
-
- bl _bsaes_decrypt8
-
- vld1.64 {q8-q9}, [r0,:128]!
- vld1.64 {q10}, [r0,:128]!
- veor q0, q0, q8
- veor q1, q1, q9
- veor q8, q6, q10
- vst1.8 {q0-q1}, [r8]!
- vst1.8 {q8}, [r8]!
-
- vld1.64 {q8}, [r0,:128] @ next round tweak
- b .Lxts_dec_done
-.align 4
-.Lxts_dec_2:
- vst1.64 {q10}, [r0,:128] @ next round tweak
-
- veor q0, q0, q8
-#ifndef BSAES_ASM_EXTENDED_KEY
- add r4, sp, #0x90 @ pass key schedule
-#else
- add r4, r10, #248 @ pass key schedule
-#endif
- veor q1, q1, q9
- mov r5, r1 @ pass rounds
- mov r0, sp
-
- bl _bsaes_decrypt8
-
- vld1.64 {q8-q9}, [r0,:128]!
- veor q0, q0, q8
- veor q1, q1, q9
- vst1.8 {q0-q1}, [r8]!
-
- vld1.64 {q8}, [r0,:128] @ next round tweak
- b .Lxts_dec_done
-.align 4
-.Lxts_dec_1:
- mov r0, sp
- veor q0, q8
- mov r1, sp
- vst1.8 {q0}, [sp,:128]
- mov r2, r10
- mov r4, r3 @ preserve fp
- mov r5, r2 @ preserve magic
-
- bl AES_decrypt
-
- vld1.8 {q0}, [sp,:128]
- veor q0, q0, q8
- vst1.8 {q0}, [r8]!
- mov r3, r4
- mov r2, r5
-
- vmov q8, q9 @ next round tweak
-
-.Lxts_dec_done:
-#ifndef XTS_CHAIN_TWEAK
- adds r9, #0x10
- beq .Lxts_dec_ret
-
- @ calculate one round of extra tweak for the stolen ciphertext
- vldmia r2, {q5}
- vshr.s64 q6, q8, #63
- vand q6, q6, q5
- vadd.u64 q9, q8, q8
- vswp d13,d12
- veor q9, q9, q6
-
- @ perform the final decryption with the last tweak value
- vld1.8 {q0}, [r7]!
- mov r0, sp
- veor q0, q0, q9
- mov r1, sp
- vst1.8 {q0}, [sp,:128]
- mov r2, r10
- mov r4, r3 @ preserve fp
-
- bl AES_decrypt
-
- vld1.8 {q0}, [sp,:128]
- veor q0, q0, q9
- vst1.8 {q0}, [r8]
-
- mov r6, r8
-.Lxts_dec_steal:
- ldrb r1, [r8]
- ldrb r0, [r7], #1
- strb r1, [r8, #0x10]
- strb r0, [r8], #1
-
- subs r9, #1
- bhi .Lxts_dec_steal
-
- vld1.8 {q0}, [r6]
- mov r0, sp
- veor q0, q8
- mov r1, sp
- vst1.8 {q0}, [sp,:128]
- mov r2, r10
-
- bl AES_decrypt
-
- vld1.8 {q0}, [sp,:128]
- veor q0, q0, q8
- vst1.8 {q0}, [r6]
- mov r3, r4
-#endif
-
-.Lxts_dec_ret:
- bic r0, r3, #0xf
- vmov.i32 q0, #0
- vmov.i32 q1, #0
-#ifdef XTS_CHAIN_TWEAK
- ldr r1, [r3, #0x20+VFP_ABI_FRAME] @ chain tweak
-#endif
-.Lxts_dec_bzero: @ wipe key schedule [if any]
- vstmia sp!, {q0-q1}
- cmp sp, r0
- bne .Lxts_dec_bzero
-
- mov sp, r3
-#ifdef XTS_CHAIN_TWEAK
- vst1.8 {q8}, [r1]
-#endif
- VFP_ABI_POP
- ldmia sp!, {r4-r10, pc} @ return
-
-.size bsaes_xts_decrypt,.-bsaes_xts_decrypt
-#endif
diff --git a/arch/arm/crypto/aesbs-glue.c b/arch/arm/crypto/aesbs-glue.c
deleted file mode 100644
index d8e06de72ef3..000000000000
--- a/arch/arm/crypto/aesbs-glue.c
+++ /dev/null
@@ -1,367 +0,0 @@
-/*
- * linux/arch/arm/crypto/aesbs-glue.c - glue code for NEON bit sliced AES
- *
- * Copyright (C) 2013 Linaro Ltd <ard.biesheuvel@linaro.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-
-#include <asm/neon.h>
-#include <crypto/aes.h>
-#include <crypto/cbc.h>
-#include <crypto/internal/simd.h>
-#include <crypto/internal/skcipher.h>
-#include <linux/module.h>
-#include <crypto/xts.h>
-
-#include "aes_glue.h"
-
-#define BIT_SLICED_KEY_MAXSIZE (128 * (AES_MAXNR - 1) + 2 * AES_BLOCK_SIZE)
-
-struct BS_KEY {
- struct AES_KEY rk;
- int converted;
- u8 __aligned(8) bs[BIT_SLICED_KEY_MAXSIZE];
-} __aligned(8);
-
-asmlinkage void bsaes_enc_key_convert(u8 out[], struct AES_KEY const *in);
-asmlinkage void bsaes_dec_key_convert(u8 out[], struct AES_KEY const *in);
-
-asmlinkage void bsaes_cbc_encrypt(u8 const in[], u8 out[], u32 bytes,
- struct BS_KEY *key, u8 iv[]);
-
-asmlinkage void bsaes_ctr32_encrypt_blocks(u8 const in[], u8 out[], u32 blocks,
- struct BS_KEY *key, u8 const iv[]);
-
-asmlinkage void bsaes_xts_encrypt(u8 const in[], u8 out[], u32 bytes,
- struct BS_KEY *key, u8 tweak[]);
-
-asmlinkage void bsaes_xts_decrypt(u8 const in[], u8 out[], u32 bytes,
- struct BS_KEY *key, u8 tweak[]);
-
-struct aesbs_cbc_ctx {
- struct AES_KEY enc;
- struct BS_KEY dec;
-};
-
-struct aesbs_ctr_ctx {
- struct BS_KEY enc;
-};
-
-struct aesbs_xts_ctx {
- struct BS_KEY enc;
- struct BS_KEY dec;
- struct AES_KEY twkey;
-};
-
-static int aesbs_cbc_set_key(struct crypto_skcipher *tfm, const u8 *in_key,
- unsigned int key_len)
-{
- struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
- int bits = key_len * 8;
-
- if (private_AES_set_encrypt_key(in_key, bits, &ctx->enc)) {
- crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
- return -EINVAL;
- }
- ctx->dec.rk = ctx->enc;
- private_AES_set_decrypt_key(in_key, bits, &ctx->dec.rk);
- ctx->dec.converted = 0;
- return 0;
-}
-
-static int aesbs_ctr_set_key(struct crypto_skcipher *tfm, const u8 *in_key,
- unsigned int key_len)
-{
- struct aesbs_ctr_ctx *ctx = crypto_skcipher_ctx(tfm);
- int bits = key_len * 8;
-
- if (private_AES_set_encrypt_key(in_key, bits, &ctx->enc.rk)) {
- crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
- return -EINVAL;
- }
- ctx->enc.converted = 0;
- return 0;
-}
-
-static int aesbs_xts_set_key(struct crypto_skcipher *tfm, const u8 *in_key,
- unsigned int key_len)
-{
- struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
- int bits = key_len * 4;
- int err;
-
- err = xts_verify_key(tfm, in_key, key_len);
- if (err)
- return err;
-
- if (private_AES_set_encrypt_key(in_key, bits, &ctx->enc.rk)) {
- crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
- return -EINVAL;
- }
- ctx->dec.rk = ctx->enc.rk;
- private_AES_set_decrypt_key(in_key, bits, &ctx->dec.rk);
- private_AES_set_encrypt_key(in_key + key_len / 2, bits, &ctx->twkey);
- ctx->enc.converted = ctx->dec.converted = 0;
- return 0;
-}
-
-static inline void aesbs_encrypt_one(struct crypto_skcipher *tfm,
- const u8 *src, u8 *dst)
-{
- struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
-
- AES_encrypt(src, dst, &ctx->enc);
-}
-
-static int aesbs_cbc_encrypt(struct skcipher_request *req)
-{
- return crypto_cbc_encrypt_walk(req, aesbs_encrypt_one);
-}
-
-static inline void aesbs_decrypt_one(struct crypto_skcipher *tfm,
- const u8 *src, u8 *dst)
-{
- struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
-
- AES_decrypt(src, dst, &ctx->dec.rk);
-}
-
-static int aesbs_cbc_decrypt(struct skcipher_request *req)
-{
- struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
- struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
- struct skcipher_walk walk;
- unsigned int nbytes;
- int err;
-
- for (err = skcipher_walk_virt(&walk, req, false);
- (nbytes = walk.nbytes); err = skcipher_walk_done(&walk, nbytes)) {
- u32 blocks = nbytes / AES_BLOCK_SIZE;
- u8 *dst = walk.dst.virt.addr;
- u8 *src = walk.src.virt.addr;
- u8 *iv = walk.iv;
-
- if (blocks >= 8) {
- kernel_neon_begin();
- bsaes_cbc_encrypt(src, dst, nbytes, &ctx->dec, iv);
- kernel_neon_end();
- nbytes %= AES_BLOCK_SIZE;
- continue;
- }
-
- nbytes = crypto_cbc_decrypt_blocks(&walk, tfm,
- aesbs_decrypt_one);
- }
- return err;
-}
-
-static void inc_be128_ctr(__be32 ctr[], u32 addend)
-{
- int i;
-
- for (i = 3; i >= 0; i--, addend = 1) {
- u32 n = be32_to_cpu(ctr[i]) + addend;
-
- ctr[i] = cpu_to_be32(n);
- if (n >= addend)
- break;
- }
-}
-
-static int aesbs_ctr_encrypt(struct skcipher_request *req)
-{
- struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
- struct aesbs_ctr_ctx *ctx = crypto_skcipher_ctx(tfm);
- struct skcipher_walk walk;
- u32 blocks;
- int err;
-
- err = skcipher_walk_virt(&walk, req, false);
-
- while ((blocks = walk.nbytes / AES_BLOCK_SIZE)) {
- u32 tail = walk.nbytes % AES_BLOCK_SIZE;
- __be32 *ctr = (__be32 *)walk.iv;
- u32 headroom = UINT_MAX - be32_to_cpu(ctr[3]);
-
- /* avoid 32 bit counter overflow in the NEON code */
- if (unlikely(headroom < blocks)) {
- blocks = headroom + 1;
- tail = walk.nbytes - blocks * AES_BLOCK_SIZE;
- }
- kernel_neon_begin();
- bsaes_ctr32_encrypt_blocks(walk.src.virt.addr,
- walk.dst.virt.addr, blocks,
- &ctx->enc, walk.iv);
- kernel_neon_end();
- inc_be128_ctr(ctr, blocks);
-
- err = skcipher_walk_done(&walk, tail);
- }
- if (walk.nbytes) {
- u8 *tdst = walk.dst.virt.addr + blocks * AES_BLOCK_SIZE;
- u8 *tsrc = walk.src.virt.addr + blocks * AES_BLOCK_SIZE;
- u8 ks[AES_BLOCK_SIZE];
-
- AES_encrypt(walk.iv, ks, &ctx->enc.rk);
- if (tdst != tsrc)
- memcpy(tdst, tsrc, walk.nbytes);
- crypto_xor(tdst, ks, walk.nbytes);
- err = skcipher_walk_done(&walk, 0);
- }
- return err;
-}
-
-static int aesbs_xts_encrypt(struct skcipher_request *req)
-{
- struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
- struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
- struct skcipher_walk walk;
- int err;
-
- err = skcipher_walk_virt(&walk, req, false);
-
- /* generate the initial tweak */
- AES_encrypt(walk.iv, walk.iv, &ctx->twkey);
-
- while (walk.nbytes) {
- kernel_neon_begin();
- bsaes_xts_encrypt(walk.src.virt.addr, walk.dst.virt.addr,
- walk.nbytes, &ctx->enc, walk.iv);
- kernel_neon_end();
- err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
- }
- return err;
-}
-
-static int aesbs_xts_decrypt(struct skcipher_request *req)
-{
- struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
- struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
- struct skcipher_walk walk;
- int err;
-
- err = skcipher_walk_virt(&walk, req, false);
-
- /* generate the initial tweak */
- AES_encrypt(walk.iv, walk.iv, &ctx->twkey);
-
- while (walk.nbytes) {
- kernel_neon_begin();
- bsaes_xts_decrypt(walk.src.virt.addr, walk.dst.virt.addr,
- walk.nbytes, &ctx->dec, walk.iv);
- kernel_neon_end();
- err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
- }
- return err;
-}
-
-static struct skcipher_alg aesbs_algs[] = { {
- .base = {
- .cra_name = "__cbc(aes)",
- .cra_driver_name = "__cbc-aes-neonbs",
- .cra_priority = 300,
- .cra_flags = CRYPTO_ALG_INTERNAL,
- .cra_blocksize = AES_BLOCK_SIZE,
- .cra_ctxsize = sizeof(struct aesbs_cbc_ctx),
- .cra_alignmask = 7,
- .cra_module = THIS_MODULE,
- },
- .min_keysize = AES_MIN_KEY_SIZE,
- .max_keysize = AES_MAX_KEY_SIZE,
- .ivsize = AES_BLOCK_SIZE,
- .setkey = aesbs_cbc_set_key,
- .encrypt = aesbs_cbc_encrypt,
- .decrypt = aesbs_cbc_decrypt,
-}, {
- .base = {
- .cra_name = "__ctr(aes)",
- .cra_driver_name = "__ctr-aes-neonbs",
- .cra_priority = 300,
- .cra_flags = CRYPTO_ALG_INTERNAL,
- .cra_blocksize = 1,
- .cra_ctxsize = sizeof(struct aesbs_ctr_ctx),
- .cra_alignmask = 7,
- .cra_module = THIS_MODULE,
- },
- .min_keysize = AES_MIN_KEY_SIZE,
- .max_keysize = AES_MAX_KEY_SIZE,
- .ivsize = AES_BLOCK_SIZE,
- .chunksize = AES_BLOCK_SIZE,
- .setkey = aesbs_ctr_set_key,
- .encrypt = aesbs_ctr_encrypt,
- .decrypt = aesbs_ctr_encrypt,
-}, {
- .base = {
- .cra_name = "__xts(aes)",
- .cra_driver_name = "__xts-aes-neonbs",
- .cra_priority = 300,
- .cra_flags = CRYPTO_ALG_INTERNAL,
- .cra_blocksize = AES_BLOCK_SIZE,
- .cra_ctxsize = sizeof(struct aesbs_xts_ctx),
- .cra_alignmask = 7,
- .cra_module = THIS_MODULE,
- },
- .min_keysize = 2 * AES_MIN_KEY_SIZE,
- .max_keysize = 2 * AES_MAX_KEY_SIZE,
- .ivsize = AES_BLOCK_SIZE,
- .setkey = aesbs_xts_set_key,
- .encrypt = aesbs_xts_encrypt,
- .decrypt = aesbs_xts_decrypt,
-} };
-
-struct simd_skcipher_alg *aesbs_simd_algs[ARRAY_SIZE(aesbs_algs)];
-
-static void aesbs_mod_exit(void)
-{
- int i;
-
- for (i = 0; i < ARRAY_SIZE(aesbs_simd_algs) && aesbs_simd_algs[i]; i++)
- simd_skcipher_free(aesbs_simd_algs[i]);
-
- crypto_unregister_skciphers(aesbs_algs, ARRAY_SIZE(aesbs_algs));
-}
-
-static int __init aesbs_mod_init(void)
-{
- struct simd_skcipher_alg *simd;
- const char *basename;
- const char *algname;
- const char *drvname;
- int err;
- int i;
-
- if (!cpu_has_neon())
- return -ENODEV;
-
- err = crypto_register_skciphers(aesbs_algs, ARRAY_SIZE(aesbs_algs));
- if (err)
- return err;
-
- for (i = 0; i < ARRAY_SIZE(aesbs_algs); i++) {
- algname = aesbs_algs[i].base.cra_name + 2;
- drvname = aesbs_algs[i].base.cra_driver_name + 2;
- basename = aesbs_algs[i].base.cra_driver_name;
- simd = simd_skcipher_create_compat(algname, drvname, basename);
- err = PTR_ERR(simd);
- if (IS_ERR(simd))
- goto unregister_simds;
-
- aesbs_simd_algs[i] = simd;
- }
-
- return 0;
-
-unregister_simds:
- aesbs_mod_exit();
- return err;
-}
-
-module_init(aesbs_mod_init);
-module_exit(aesbs_mod_exit);
-
-MODULE_DESCRIPTION("Bit sliced AES in CBC/CTR/XTS modes using NEON");
-MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
-MODULE_LICENSE("GPL");
diff --git a/arch/arm/crypto/bsaes-armv7.pl b/arch/arm/crypto/bsaes-armv7.pl
deleted file mode 100644
index a4d3856e7d24..000000000000
--- a/arch/arm/crypto/bsaes-armv7.pl
+++ /dev/null
@@ -1,2471 +0,0 @@
-#!/usr/bin/env perl
-
-# ====================================================================
-# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
-# project. The module is, however, dual licensed under OpenSSL and
-# CRYPTOGAMS licenses depending on where you obtain it. For further
-# details see http://www.openssl.org/~appro/cryptogams/.
-#
-# Specific modes and adaptation for Linux kernel by Ard Biesheuvel
-# <ard.biesheuvel@linaro.org>. Permission to use under GPL terms is
-# granted.
-# ====================================================================
-
-# Bit-sliced AES for ARM NEON
-#
-# February 2012.
-#
-# This implementation is direct adaptation of bsaes-x86_64 module for
-# ARM NEON. Except that this module is endian-neutral [in sense that
-# it can be compiled for either endianness] by courtesy of vld1.8's
-# neutrality. Initial version doesn't implement interface to OpenSSL,
-# only low-level primitives and unsupported entry points, just enough
-# to collect performance results, which for Cortex-A8 core are:
-#
-# encrypt 19.5 cycles per byte processed with 128-bit key
-# decrypt 22.1 cycles per byte processed with 128-bit key
-# key conv. 440 cycles per 128-bit key/0.18 of 8x block
-#
-# Snapdragon S4 encrypts byte in 17.6 cycles and decrypts in 19.7,
-# which is [much] worse than anticipated (for further details see
-# http://www.openssl.org/~appro/Snapdragon-S4.html).
-#
-# Cortex-A15 manages in 14.2/16.1 cycles [when integer-only code
-# manages in 20.0 cycles].
-#
-# When comparing to x86_64 results keep in mind that NEON unit is
-# [mostly] single-issue and thus can't [fully] benefit from
-# instruction-level parallelism. And when comparing to aes-armv4
-# results keep in mind key schedule conversion overhead (see
-# bsaes-x86_64.pl for further details)...
-#
-# <appro@openssl.org>
-
-# April-August 2013
-#
-# Add CBC, CTR and XTS subroutines, adapt for kernel use.
-#
-# <ard.biesheuvel@linaro.org>
-
-while (($output=shift) && ($output!~/^\w[\w\-]*\.\w+$/)) {}
-open STDOUT,">$output";
-
-my ($inp,$out,$len,$key)=("r0","r1","r2","r3");
-my @XMM=map("q$_",(0..15));
-
-{
-my ($key,$rounds,$const)=("r4","r5","r6");
-
-sub Dlo() { shift=~m|q([1]?[0-9])|?"d".($1*2):""; }
-sub Dhi() { shift=~m|q([1]?[0-9])|?"d".($1*2+1):""; }
-
-sub Sbox {
-# input in lsb > [b0, b1, b2, b3, b4, b5, b6, b7] < msb
-# output in lsb > [b0, b1, b4, b6, b3, b7, b2, b5] < msb
-my @b=@_[0..7];
-my @t=@_[8..11];
-my @s=@_[12..15];
- &InBasisChange (@b);
- &Inv_GF256 (@b[6,5,0,3,7,1,4,2],@t,@s);
- &OutBasisChange (@b[7,1,4,2,6,5,0,3]);
-}
-
-sub InBasisChange {
-# input in lsb > [b0, b1, b2, b3, b4, b5, b6, b7] < msb
-# output in lsb > [b6, b5, b0, b3, b7, b1, b4, b2] < msb
-my @b=@_[0..7];
-$code.=<<___;
- veor @b[2], @b[2], @b[1]
- veor @b[5], @b[5], @b[6]
- veor @b[3], @b[3], @b[0]
- veor @b[6], @b[6], @b[2]
- veor @b[5], @b[5], @b[0]
-
- veor @b[6], @b[6], @b[3]
- veor @b[3], @b[3], @b[7]
- veor @b[7], @b[7], @b[5]
- veor @b[3], @b[3], @b[4]
- veor @b[4], @b[4], @b[5]
-
- veor @b[2], @b[2], @b[7]
- veor @b[3], @b[3], @b[1]
- veor @b[1], @b[1], @b[5]
-___
-}
-
-sub OutBasisChange {
-# input in lsb > [b0, b1, b2, b3, b4, b5, b6, b7] < msb
-# output in lsb > [b6, b1, b2, b4, b7, b0, b3, b5] < msb
-my @b=@_[0..7];
-$code.=<<___;
- veor @b[0], @b[0], @b[6]
- veor @b[1], @b[1], @b[4]
- veor @b[4], @b[4], @b[6]
- veor @b[2], @b[2], @b[0]
- veor @b[6], @b[6], @b[1]
-
- veor @b[1], @b[1], @b[5]
- veor @b[5], @b[5], @b[3]
- veor @b[3], @b[3], @b[7]
- veor @b[7], @b[7], @b[5]
- veor @b[2], @b[2], @b[5]
-
- veor @b[4], @b[4], @b[7]
-___
-}
-
-sub InvSbox {
-# input in lsb > [b0, b1, b2, b3, b4, b5, b6, b7] < msb
-# output in lsb > [b0, b1, b6, b4, b2, b7, b3, b5] < msb
-my @b=@_[0..7];
-my @t=@_[8..11];
-my @s=@_[12..15];
- &InvInBasisChange (@b);
- &Inv_GF256 (@b[5,1,2,6,3,7,0,4],@t,@s);
- &InvOutBasisChange (@b[3,7,0,4,5,1,2,6]);
-}
-
-sub InvInBasisChange { # OutBasisChange in reverse (with twist)
-my @b=@_[5,1,2,6,3,7,0,4];
-$code.=<<___
- veor @b[1], @b[1], @b[7]
- veor @b[4], @b[4], @b[7]
-
- veor @b[7], @b[7], @b[5]
- veor @b[1], @b[1], @b[3]
- veor @b[2], @b[2], @b[5]
- veor @b[3], @b[3], @b[7]
-
- veor @b[6], @b[6], @b[1]
- veor @b[2], @b[2], @b[0]
- veor @b[5], @b[5], @b[3]
- veor @b[4], @b[4], @b[6]
- veor @b[0], @b[0], @b[6]
- veor @b[1], @b[1], @b[4]
-___
-}
-
-sub InvOutBasisChange { # InBasisChange in reverse
-my @b=@_[2,5,7,3,6,1,0,4];
-$code.=<<___;
- veor @b[1], @b[1], @b[5]
- veor @b[2], @b[2], @b[7]
-
- veor @b[3], @b[3], @b[1]
- veor @b[4], @b[4], @b[5]
- veor @b[7], @b[7], @b[5]
- veor @b[3], @b[3], @b[4]
- veor @b[5], @b[5], @b[0]
- veor @b[3], @b[3], @b[7]
- veor @b[6], @b[6], @b[2]
- veor @b[2], @b[2], @b[1]
- veor @b[6], @b[6], @b[3]
-
- veor @b[3], @b[3], @b[0]
- veor @b[5], @b[5], @b[6]
-___
-}
-
-sub Mul_GF4 {
-#;*************************************************************
-#;* Mul_GF4: Input x0-x1,y0-y1 Output x0-x1 Temp t0 (8) *
-#;*************************************************************
-my ($x0,$x1,$y0,$y1,$t0,$t1)=@_;
-$code.=<<___;
- veor $t0, $y0, $y1
- vand $t0, $t0, $x0
- veor $x0, $x0, $x1
- vand $t1, $x1, $y0
- vand $x0, $x0, $y1
- veor $x1, $t1, $t0
- veor $x0, $x0, $t1
-___
-}
-
-sub Mul_GF4_N { # not used, see next subroutine
-# multiply and scale by N
-my ($x0,$x1,$y0,$y1,$t0)=@_;
-$code.=<<___;
- veor $t0, $y0, $y1
- vand $t0, $t0, $x0
- veor $x0, $x0, $x1
- vand $x1, $x1, $y0
- vand $x0, $x0, $y1
- veor $x1, $x1, $x0
- veor $x0, $x0, $t0
-___
-}
-
-sub Mul_GF4_N_GF4 {
-# interleaved Mul_GF4_N and Mul_GF4
-my ($x0,$x1,$y0,$y1,$t0,
- $x2,$x3,$y2,$y3,$t1)=@_;
-$code.=<<___;
- veor $t0, $y0, $y1
- veor $t1, $y2, $y3
- vand $t0, $t0, $x0
- vand $t1, $t1, $x2
- veor $x0, $x0, $x1
- veor $x2, $x2, $x3
- vand $x1, $x1, $y0
- vand $x3, $x3, $y2
- vand $x0, $x0, $y1
- vand $x2, $x2, $y3
- veor $x1, $x1, $x0
- veor $x2, $x2, $x3
- veor $x0, $x0, $t0
- veor $x3, $x3, $t1
-___
-}
-sub Mul_GF16_2 {
-my @x=@_[0..7];
-my @y=@_[8..11];
-my @t=@_[12..15];
-$code.=<<___;
- veor @t[0], @x[0], @x[2]
- veor @t[1], @x[1], @x[3]
-___
- &Mul_GF4 (@x[0], @x[1], @y[0], @y[1], @t[2..3]);
-$code.=<<___;
- veor @y[0], @y[0], @y[2]
- veor @y[1], @y[1], @y[3]
-___
- Mul_GF4_N_GF4 (@t[0], @t[1], @y[0], @y[1], @t[3],
- @x[2], @x[3], @y[2], @y[3], @t[2]);
-$code.=<<___;
- veor @x[0], @x[0], @t[0]
- veor @x[2], @x[2], @t[0]
- veor @x[1], @x[1], @t[1]
- veor @x[3], @x[3], @t[1]
-
- veor @t[0], @x[4], @x[6]
- veor @t[1], @x[5], @x[7]
-___
- &Mul_GF4_N_GF4 (@t[0], @t[1], @y[0], @y[1], @t[3],
- @x[6], @x[7], @y[2], @y[3], @t[2]);
-$code.=<<___;
- veor @y[0], @y[0], @y[2]
- veor @y[1], @y[1], @y[3]
-___
- &Mul_GF4 (@x[4], @x[5], @y[0], @y[1], @t[2..3]);
-$code.=<<___;
- veor @x[4], @x[4], @t[0]
- veor @x[6], @x[6], @t[0]
- veor @x[5], @x[5], @t[1]
- veor @x[7], @x[7], @t[1]
-___
-}
-sub Inv_GF256 {
-#;********************************************************************
-#;* Inv_GF256: Input x0-x7 Output x0-x7 Temp t0-t3,s0-s3 (144) *
-#;********************************************************************
-my @x=@_[0..7];
-my @t=@_[8..11];
-my @s=@_[12..15];
-# direct optimizations from hardware
-$code.=<<___;
- veor @t[3], @x[4], @x[6]
- veor @t[2], @x[5], @x[7]
- veor @t[1], @x[1], @x[3]
- veor @s[1], @x[7], @x[6]
- vmov @t[0], @t[2]
- veor @s[0], @x[0], @x[2]
-
- vorr @t[2], @t[2], @t[1]
- veor @s[3], @t[3], @t[0]
- vand @s[2], @t[3], @s[0]
- vorr @t[3], @t[3], @s[0]
- veor @s[0], @s[0], @t[1]
- vand @t[0], @t[0], @t[1]
- veor @t[1], @x[3], @x[2]
- vand @s[3], @s[3], @s[0]
- vand @s[1], @s[1], @t[1]
- veor @t[1], @x[4], @x[5]
- veor @s[0], @x[1], @x[0]
- veor @t[3], @t[3], @s[1]
- veor @t[2], @t[2], @s[1]
- vand @s[1], @t[1], @s[0]
- vorr @t[1], @t[1], @s[0]
- veor @t[3], @t[3], @s[3]
- veor @t[0], @t[0], @s[1]
- veor @t[2], @t[2], @s[2]
- veor @t[1], @t[1], @s[3]
- veor @t[0], @t[0], @s[2]
- vand @s[0], @x[7], @x[3]
- veor @t[1], @t[1], @s[2]
- vand @s[1], @x[6], @x[2]
- vand @s[2], @x[5], @x[1]
- vorr @s[3], @x[4], @x[0]
- veor @t[3], @t[3], @s[0]
- veor @t[1], @t[1], @s[2]
- veor @t[0], @t[0], @s[3]
- veor @t[2], @t[2], @s[1]
-
- @ Inv_GF16 \t0, \t1, \t2, \t3, \s0, \s1, \s2, \s3
-
- @ new smaller inversion
-
- vand @s[2], @t[3], @t[1]
- vmov @s[0], @t[0]
-
- veor @s[1], @t[2], @s[2]
- veor @s[3], @t[0], @s[2]
- veor @s[2], @t[0], @s[2] @ @s[2]=@s[3]
-
- vbsl @s[1], @t[1], @t[0]
- vbsl @s[3], @t[3], @t[2]
- veor @t[3], @t[3], @t[2]
-
- vbsl @s[0], @s[1], @s[2]
- vbsl @t[0], @s[2], @s[1]
-
- vand @s[2], @s[0], @s[3]
- veor @t[1], @t[1], @t[0]
-
- veor @s[2], @s[2], @t[3]
-___
-# output in s3, s2, s1, t1
-
-# Mul_GF16_2 \x0, \x1, \x2, \x3, \x4, \x5, \x6, \x7, \t2, \t3, \t0, \t1, \s0, \s1, \s2, \s3
-
-# Mul_GF16_2 \x0, \x1, \x2, \x3, \x4, \x5, \x6, \x7, \s3, \s2, \s1, \t1, \s0, \t0, \t2, \t3
- &Mul_GF16_2(@x,@s[3,2,1],@t[1],@s[0],@t[0,2,3]);
-
-### output msb > [x3,x2,x1,x0,x7,x6,x5,x4] < lsb
-}
-
-# AES linear components
-
-sub ShiftRows {
-my @x=@_[0..7];
-my @t=@_[8..11];
-my $mask=pop;
-$code.=<<___;
- vldmia $key!, {@t[0]-@t[3]}
- veor @t[0], @t[0], @x[0]
- veor @t[1], @t[1], @x[1]
- vtbl.8 `&Dlo(@x[0])`, {@t[0]}, `&Dlo($mask)`
- vtbl.8 `&Dhi(@x[0])`, {@t[0]}, `&Dhi($mask)`
- vldmia $key!, {@t[0]}
- veor @t[2], @t[2], @x[2]
- vtbl.8 `&Dlo(@x[1])`, {@t[1]}, `&Dlo($mask)`
- vtbl.8 `&Dhi(@x[1])`, {@t[1]}, `&Dhi($mask)`
- vldmia $key!, {@t[1]}
- veor @t[3], @t[3], @x[3]
- vtbl.8 `&Dlo(@x[2])`, {@t[2]}, `&Dlo($mask)`
- vtbl.8 `&Dhi(@x[2])`, {@t[2]}, `&Dhi($mask)`
- vldmia $key!, {@t[2]}
- vtbl.8 `&Dlo(@x[3])`, {@t[3]}, `&Dlo($mask)`
- vtbl.8 `&Dhi(@x[3])`, {@t[3]}, `&Dhi($mask)`
- vldmia $key!, {@t[3]}
- veor @t[0], @t[0], @x[4]
- veor @t[1], @t[1], @x[5]
- vtbl.8 `&Dlo(@x[4])`, {@t[0]}, `&Dlo($mask)`
- vtbl.8 `&Dhi(@x[4])`, {@t[0]}, `&Dhi($mask)`
- veor @t[2], @t[2], @x[6]
- vtbl.8 `&Dlo(@x[5])`, {@t[1]}, `&Dlo($mask)`
- vtbl.8 `&Dhi(@x[5])`, {@t[1]}, `&Dhi($mask)`
- veor @t[3], @t[3], @x[7]
- vtbl.8 `&Dlo(@x[6])`, {@t[2]}, `&Dlo($mask)`
- vtbl.8 `&Dhi(@x[6])`, {@t[2]}, `&Dhi($mask)`
- vtbl.8 `&Dlo(@x[7])`, {@t[3]}, `&Dlo($mask)`
- vtbl.8 `&Dhi(@x[7])`, {@t[3]}, `&Dhi($mask)`
-___
-}
-
-sub MixColumns {
-# modified to emit output in order suitable for feeding back to aesenc[last]
-my @x=@_[0..7];
-my @t=@_[8..15];
-my $inv=@_[16]; # optional
-$code.=<<___;
- vext.8 @t[0], @x[0], @x[0], #12 @ x0 <<< 32
- vext.8 @t[1], @x[1], @x[1], #12
- veor @x[0], @x[0], @t[0] @ x0 ^ (x0 <<< 32)
- vext.8 @t[2], @x[2], @x[2], #12
- veor @x[1], @x[1], @t[1]
- vext.8 @t[3], @x[3], @x[3], #12
- veor @x[2], @x[2], @t[2]
- vext.8 @t[4], @x[4], @x[4], #12
- veor @x[3], @x[3], @t[3]
- vext.8 @t[5], @x[5], @x[5], #12
- veor @x[4], @x[4], @t[4]
- vext.8 @t[6], @x[6], @x[6], #12
- veor @x[5], @x[5], @t[5]
- vext.8 @t[7], @x[7], @x[7], #12
- veor @x[6], @x[6], @t[6]
-
- veor @t[1], @t[1], @x[0]
- veor @x[7], @x[7], @t[7]
- vext.8 @x[0], @x[0], @x[0], #8 @ (x0 ^ (x0 <<< 32)) <<< 64)
- veor @t[2], @t[2], @x[1]
- veor @t[0], @t[0], @x[7]
- veor @t[1], @t[1], @x[7]
- vext.8 @x[1], @x[1], @x[1], #8
- veor @t[5], @t[5], @x[4]
- veor @x[0], @x[0], @t[0]
- veor @t[6], @t[6], @x[5]
- veor @x[1], @x[1], @t[1]
- vext.8 @t[0], @x[4], @x[4], #8
- veor @t[4], @t[4], @x[3]
- vext.8 @t[1], @x[5], @x[5], #8
- veor @t[7], @t[7], @x[6]
- vext.8 @x[4], @x[3], @x[3], #8
- veor @t[3], @t[3], @x[2]
- vext.8 @x[5], @x[7], @x[7], #8
- veor @t[4], @t[4], @x[7]
- vext.8 @x[3], @x[6], @x[6], #8
- veor @t[3], @t[3], @x[7]
- vext.8 @x[6], @x[2], @x[2], #8
- veor @x[7], @t[1], @t[5]
-___
-$code.=<<___ if (!$inv);
- veor @x[2], @t[0], @t[4]
- veor @x[4], @x[4], @t[3]
- veor @x[5], @x[5], @t[7]
- veor @x[3], @x[3], @t[6]
- @ vmov @x[2], @t[0]
- veor @x[6], @x[6], @t[2]
- @ vmov @x[7], @t[1]
-___
-$code.=<<___ if ($inv);
- veor @t[3], @t[3], @x[4]
- veor @x[5], @x[5], @t[7]
- veor @x[2], @x[3], @t[6]
- veor @x[3], @t[0], @t[4]
- veor @x[4], @x[6], @t[2]
- vmov @x[6], @t[3]
- @ vmov @x[7], @t[1]
-___
-}
-
-sub InvMixColumns_orig {
-my @x=@_[0..7];
-my @t=@_[8..15];
-
-$code.=<<___;
- @ multiplication by 0x0e
- vext.8 @t[7], @x[7], @x[7], #12
- vmov @t[2], @x[2]
- veor @x[2], @x[2], @x[5] @ 2 5
- veor @x[7], @x[7], @x[5] @ 7 5
- vext.8 @t[0], @x[0], @x[0], #12
- vmov @t[5], @x[5]
- veor @x[5], @x[5], @x[0] @ 5 0 [1]
- veor @x[0], @x[0], @x[1] @ 0 1
- vext.8 @t[1], @x[1], @x[1], #12
- veor @x[1], @x[1], @x[2] @ 1 25
- veor @x[0], @x[0], @x[6] @ 01 6 [2]
- vext.8 @t[3], @x[3], @x[3], #12
- veor @x[1], @x[1], @x[3] @ 125 3 [4]
- veor @x[2], @x[2], @x[0] @ 25 016 [3]
- veor @x[3], @x[3], @x[7] @ 3 75
- veor @x[7], @x[7], @x[6] @ 75 6 [0]
- vext.8 @t[6], @x[6], @x[6], #12
- vmov @t[4], @x[4]
- veor @x[6], @x[6], @x[4] @ 6 4
- veor @x[4], @x[4], @x[3] @ 4 375 [6]
- veor @x[3], @x[3], @x[7] @ 375 756=36
- veor @x[6], @x[6], @t[5] @ 64 5 [7]
- veor @x[3], @x[3], @t[2] @ 36 2
- vext.8 @t[5], @t[5], @t[5], #12
- veor @x[3], @x[3], @t[4] @ 362 4 [5]
-___
- my @y = @x[7,5,0,2,1,3,4,6];
-$code.=<<___;
- @ multiplication by 0x0b
- veor @y[1], @y[1], @y[0]
- veor @y[0], @y[0], @t[0]
- vext.8 @t[2], @t[2], @t[2], #12
- veor @y[1], @y[1], @t[1]
- veor @y[0], @y[0], @t[5]
- vext.8 @t[4], @t[4], @t[4], #12
- veor @y[1], @y[1], @t[6]
- veor @y[0], @y[0], @t[7]
- veor @t[7], @t[7], @t[6] @ clobber t[7]
-
- veor @y[3], @y[3], @t[0]
- veor @y[1], @y[1], @y[0]
- vext.8 @t[0], @t[0], @t[0], #12
- veor @y[2], @y[2], @t[1]
- veor @y[4], @y[4], @t[1]
- vext.8 @t[1], @t[1], @t[1], #12
- veor @y[2], @y[2], @t[2]
- veor @y[3], @y[3], @t[2]
- veor @y[5], @y[5], @t[2]
- veor @y[2], @y[2], @t[7]
- vext.8 @t[2], @t[2], @t[2], #12
- veor @y[3], @y[3], @t[3]
- veor @y[6], @y[6], @t[3]
- veor @y[4], @y[4], @t[3]
- veor @y[7], @y[7], @t[4]
- vext.8 @t[3], @t[3], @t[3], #12
- veor @y[5], @y[5], @t[4]
- veor @y[7], @y[7], @t[7]
- veor @t[7], @t[7], @t[5] @ clobber t[7] even more
- veor @y[3], @y[3], @t[5]
- veor @y[4], @y[4], @t[4]
-
- veor @y[5], @y[5], @t[7]
- vext.8 @t[4], @t[4], @t[4], #12
- veor @y[6], @y[6], @t[7]
- veor @y[4], @y[4], @t[7]
-
- veor @t[7], @t[7], @t[5]
- vext.8 @t[5], @t[5], @t[5], #12
-
- @ multiplication by 0x0d
- veor @y[4], @y[4], @y[7]
- veor @t[7], @t[7], @t[6] @ restore t[7]
- veor @y[7], @y[7], @t[4]
- vext.8 @t[6], @t[6], @t[6], #12
- veor @y[2], @y[2], @t[0]
- veor @y[7], @y[7], @t[5]
- vext.8 @t[7], @t[7], @t[7], #12
- veor @y[2], @y[2], @t[2]
-
- veor @y[3], @y[3], @y[1]
- veor @y[1], @y[1], @t[1]
- veor @y[0], @y[0], @t[0]
- veor @y[3], @y[3], @t[0]
- veor @y[1], @y[1], @t[5]
- veor @y[0], @y[0], @t[5]
- vext.8 @t[0], @t[0], @t[0], #12
- veor @y[1], @y[1], @t[7]
- veor @y[0], @y[0], @t[6]
- veor @y[3], @y[3], @y[1]
- veor @y[4], @y[4], @t[1]
- vext.8 @t[1], @t[1], @t[1], #12
-
- veor @y[7], @y[7], @t[7]
- veor @y[4], @y[4], @t[2]
- veor @y[5], @y[5], @t[2]
- veor @y[2], @y[2], @t[6]
- veor @t[6], @t[6], @t[3] @ clobber t[6]
- vext.8 @t[2], @t[2], @t[2], #12
- veor @y[4], @y[4], @y[7]
- veor @y[3], @y[3], @t[6]
-
- veor @y[6], @y[6], @t[6]
- veor @y[5], @y[5], @t[5]
- vext.8 @t[5], @t[5], @t[5], #12
- veor @y[6], @y[6], @t[4]
- vext.8 @t[4], @t[4], @t[4], #12
- veor @y[5], @y[5], @t[6]
- veor @y[6], @y[6], @t[7]
- vext.8 @t[7], @t[7], @t[7], #12
- veor @t[6], @t[6], @t[3] @ restore t[6]
- vext.8 @t[3], @t[3], @t[3], #12
-
- @ multiplication by 0x09
- veor @y[4], @y[4], @y[1]
- veor @t[1], @t[1], @y[1] @ t[1]=y[1]
- veor @t[0], @t[0], @t[5] @ clobber t[0]
- vext.8 @t[6], @t[6], @t[6], #12
- veor @t[1], @t[1], @t[5]
- veor @y[3], @y[3], @t[0]
- veor @t[0], @t[0], @y[0] @ t[0]=y[0]
- veor @t[1], @t[1], @t[6]
- veor @t[6], @t[6], @t[7] @ clobber t[6]
- veor @y[4], @y[4], @t[1]
- veor @y[7], @y[7], @t[4]
- veor @y[6], @y[6], @t[3]
- veor @y[5], @y[5], @t[2]
- veor @t[4], @t[4], @y[4] @ t[4]=y[4]
- veor @t[3], @t[3], @y[3] @ t[3]=y[3]
- veor @t[5], @t[5], @y[5] @ t[5]=y[5]
- veor @t[2], @t[2], @y[2] @ t[2]=y[2]
- veor @t[3], @t[3], @t[7]
- veor @XMM[5], @t[5], @t[6]
- veor @XMM[6], @t[6], @y[6] @ t[6]=y[6]
- veor @XMM[2], @t[2], @t[6]
- veor @XMM[7], @t[7], @y[7] @ t[7]=y[7]
-
- vmov @XMM[0], @t[0]
- vmov @XMM[1], @t[1]
- @ vmov @XMM[2], @t[2]
- vmov @XMM[3], @t[3]
- vmov @XMM[4], @t[4]
- @ vmov @XMM[5], @t[5]
- @ vmov @XMM[6], @t[6]
- @ vmov @XMM[7], @t[7]
-___
-}
-
-sub InvMixColumns {
-my @x=@_[0..7];
-my @t=@_[8..15];
-
-# Thanks to Jussi Kivilinna for providing pointer to
-#
-# | 0e 0b 0d 09 | | 02 03 01 01 | | 05 00 04 00 |
-# | 09 0e 0b 0d | = | 01 02 03 01 | x | 00 05 00 04 |
-# | 0d 09 0e 0b | | 01 01 02 03 | | 04 00 05 00 |
-# | 0b 0d 09 0e | | 03 01 01 02 | | 00 04 00 05 |
-
-$code.=<<___;
- @ multiplication by 0x05-0x00-0x04-0x00
- vext.8 @t[0], @x[0], @x[0], #8
- vext.8 @t[6], @x[6], @x[6], #8
- vext.8 @t[7], @x[7], @x[7], #8
- veor @t[0], @t[0], @x[0]
- vext.8 @t[1], @x[1], @x[1], #8
- veor @t[6], @t[6], @x[6]
- vext.8 @t[2], @x[2], @x[2], #8
- veor @t[7], @t[7], @x[7]
- vext.8 @t[3], @x[3], @x[3], #8
- veor @t[1], @t[1], @x[1]
- vext.8 @t[4], @x[4], @x[4], #8
- veor @t[2], @t[2], @x[2]
- vext.8 @t[5], @x[5], @x[5], #8
- veor @t[3], @t[3], @x[3]
- veor @t[4], @t[4], @x[4]
- veor @t[5], @t[5], @x[5]
-
- veor @x[0], @x[0], @t[6]
- veor @x[1], @x[1], @t[6]
- veor @x[2], @x[2], @t[0]
- veor @x[4], @x[4], @t[2]
- veor @x[3], @x[3], @t[1]
- veor @x[1], @x[1], @t[7]
- veor @x[2], @x[2], @t[7]
- veor @x[4], @x[4], @t[6]
- veor @x[5], @x[5], @t[3]
- veor @x[3], @x[3], @t[6]
- veor @x[6], @x[6], @t[4]
- veor @x[4], @x[4], @t[7]
- veor @x[5], @x[5], @t[7]
- veor @x[7], @x[7], @t[5]
-___
- &MixColumns (@x,@t,1); # flipped 2<->3 and 4<->6
-}
-
-sub swapmove {
-my ($a,$b,$n,$mask,$t)=@_;
-$code.=<<___;
- vshr.u64 $t, $b, #$n
- veor $t, $t, $a
- vand $t, $t, $mask
- veor $a, $a, $t
- vshl.u64 $t, $t, #$n
- veor $b, $b, $t
-___
-}
-sub swapmove2x {
-my ($a0,$b0,$a1,$b1,$n,$mask,$t0,$t1)=@_;
-$code.=<<___;
- vshr.u64 $t0, $b0, #$n
- vshr.u64 $t1, $b1, #$n
- veor $t0, $t0, $a0
- veor $t1, $t1, $a1
- vand $t0, $t0, $mask
- vand $t1, $t1, $mask
- veor $a0, $a0, $t0
- vshl.u64 $t0, $t0, #$n
- veor $a1, $a1, $t1
- vshl.u64 $t1, $t1, #$n
- veor $b0, $b0, $t0
- veor $b1, $b1, $t1
-___
-}
-
-sub bitslice {
-my @x=reverse(@_[0..7]);
-my ($t0,$t1,$t2,$t3)=@_[8..11];
-$code.=<<___;
- vmov.i8 $t0,#0x55 @ compose .LBS0
- vmov.i8 $t1,#0x33 @ compose .LBS1
-___
- &swapmove2x(@x[0,1,2,3],1,$t0,$t2,$t3);
- &swapmove2x(@x[4,5,6,7],1,$t0,$t2,$t3);
-$code.=<<___;
- vmov.i8 $t0,#0x0f @ compose .LBS2
-___
- &swapmove2x(@x[0,2,1,3],2,$t1,$t2,$t3);
- &swapmove2x(@x[4,6,5,7],2,$t1,$t2,$t3);
-
- &swapmove2x(@x[0,4,1,5],4,$t0,$t2,$t3);
- &swapmove2x(@x[2,6,3,7],4,$t0,$t2,$t3);
-}
-
-$code.=<<___;
-#ifndef __KERNEL__
-# include "arm_arch.h"
-
-# define VFP_ABI_PUSH vstmdb sp!,{d8-d15}
-# define VFP_ABI_POP vldmia sp!,{d8-d15}
-# define VFP_ABI_FRAME 0x40
-#else
-# define VFP_ABI_PUSH
-# define VFP_ABI_POP
-# define VFP_ABI_FRAME 0
-# define BSAES_ASM_EXTENDED_KEY
-# define XTS_CHAIN_TWEAK
-# define __ARM_ARCH__ __LINUX_ARM_ARCH__
-# define __ARM_MAX_ARCH__ 7
-#endif
-
-#ifdef __thumb__
-# define adrl adr
-#endif
-
-#if __ARM_MAX_ARCH__>=7
-.arch armv7-a
-.fpu neon
-
-.text
-.syntax unified @ ARMv7-capable assembler is expected to handle this
-#ifdef __thumb2__
-.thumb
-#else
-.code 32
-#endif
-
-.type _bsaes_decrypt8,%function
-.align 4
-_bsaes_decrypt8:
- adr $const,_bsaes_decrypt8
- vldmia $key!, {@XMM[9]} @ round 0 key
- add $const,$const,#.LM0ISR-_bsaes_decrypt8
-
- vldmia $const!, {@XMM[8]} @ .LM0ISR
- veor @XMM[10], @XMM[0], @XMM[9] @ xor with round0 key
- veor @XMM[11], @XMM[1], @XMM[9]
- vtbl.8 `&Dlo(@XMM[0])`, {@XMM[10]}, `&Dlo(@XMM[8])`
- vtbl.8 `&Dhi(@XMM[0])`, {@XMM[10]}, `&Dhi(@XMM[8])`
- veor @XMM[12], @XMM[2], @XMM[9]
- vtbl.8 `&Dlo(@XMM[1])`, {@XMM[11]}, `&Dlo(@XMM[8])`
- vtbl.8 `&Dhi(@XMM[1])`, {@XMM[11]}, `&Dhi(@XMM[8])`
- veor @XMM[13], @XMM[3], @XMM[9]
- vtbl.8 `&Dlo(@XMM[2])`, {@XMM[12]}, `&Dlo(@XMM[8])`
- vtbl.8 `&Dhi(@XMM[2])`, {@XMM[12]}, `&Dhi(@XMM[8])`
- veor @XMM[14], @XMM[4], @XMM[9]
- vtbl.8 `&Dlo(@XMM[3])`, {@XMM[13]}, `&Dlo(@XMM[8])`
- vtbl.8 `&Dhi(@XMM[3])`, {@XMM[13]}, `&Dhi(@XMM[8])`
- veor @XMM[15], @XMM[5], @XMM[9]
- vtbl.8 `&Dlo(@XMM[4])`, {@XMM[14]}, `&Dlo(@XMM[8])`
- vtbl.8 `&Dhi(@XMM[4])`, {@XMM[14]}, `&Dhi(@XMM[8])`
- veor @XMM[10], @XMM[6], @XMM[9]
- vtbl.8 `&Dlo(@XMM[5])`, {@XMM[15]}, `&Dlo(@XMM[8])`
- vtbl.8 `&Dhi(@XMM[5])`, {@XMM[15]}, `&Dhi(@XMM[8])`
- veor @XMM[11], @XMM[7], @XMM[9]
- vtbl.8 `&Dlo(@XMM[6])`, {@XMM[10]}, `&Dlo(@XMM[8])`
- vtbl.8 `&Dhi(@XMM[6])`, {@XMM[10]}, `&Dhi(@XMM[8])`
- vtbl.8 `&Dlo(@XMM[7])`, {@XMM[11]}, `&Dlo(@XMM[8])`
- vtbl.8 `&Dhi(@XMM[7])`, {@XMM[11]}, `&Dhi(@XMM[8])`
-___
- &bitslice (@XMM[0..7, 8..11]);
-$code.=<<___;
- sub $rounds,$rounds,#1
- b .Ldec_sbox
-.align 4
-.Ldec_loop:
-___
- &ShiftRows (@XMM[0..7, 8..12]);
-$code.=".Ldec_sbox:\n";
- &InvSbox (@XMM[0..7, 8..15]);
-$code.=<<___;
- subs $rounds,$rounds,#1
- bcc .Ldec_done
-___
- &InvMixColumns (@XMM[0,1,6,4,2,7,3,5, 8..15]);
-$code.=<<___;
- vldmia $const, {@XMM[12]} @ .LISR
- ite eq @ Thumb2 thing, sanity check in ARM
- addeq $const,$const,#0x10
- bne .Ldec_loop
- vldmia $const, {@XMM[12]} @ .LISRM0
- b .Ldec_loop
-.align 4
-.Ldec_done:
-___
- &bitslice (@XMM[0,1,6,4,2,7,3,5, 8..11]);
-$code.=<<___;
- vldmia $key, {@XMM[8]} @ last round key
- veor @XMM[6], @XMM[6], @XMM[8]
- veor @XMM[4], @XMM[4], @XMM[8]
- veor @XMM[2], @XMM[2], @XMM[8]
- veor @XMM[7], @XMM[7], @XMM[8]
- veor @XMM[3], @XMM[3], @XMM[8]
- veor @XMM[5], @XMM[5], @XMM[8]
- veor @XMM[0], @XMM[0], @XMM[8]
- veor @XMM[1], @XMM[1], @XMM[8]
- bx lr
-.size _bsaes_decrypt8,.-_bsaes_decrypt8
-
-.type _bsaes_const,%object
-.align 6
-_bsaes_const:
-.LM0ISR: @ InvShiftRows constants
- .quad 0x0a0e0206070b0f03, 0x0004080c0d010509
-.LISR:
- .quad 0x0504070602010003, 0x0f0e0d0c080b0a09
-.LISRM0:
- .quad 0x01040b0e0205080f, 0x0306090c00070a0d
-.LM0SR: @ ShiftRows constants
- .quad 0x0a0e02060f03070b, 0x0004080c05090d01
-.LSR:
- .quad 0x0504070600030201, 0x0f0e0d0c0a09080b
-.LSRM0:
- .quad 0x0304090e00050a0f, 0x01060b0c0207080d
-.LM0:
- .quad 0x02060a0e03070b0f, 0x0004080c0105090d
-.LREVM0SR:
- .quad 0x090d01050c000408, 0x03070b0f060a0e02
-.asciz "Bit-sliced AES for NEON, CRYPTOGAMS by <appro\@openssl.org>"
-.align 6
-.size _bsaes_const,.-_bsaes_const
-
-.type _bsaes_encrypt8,%function
-.align 4
-_bsaes_encrypt8:
- adr $const,_bsaes_encrypt8
- vldmia $key!, {@XMM[9]} @ round 0 key
- sub $const,$const,#_bsaes_encrypt8-.LM0SR
-
- vldmia $const!, {@XMM[8]} @ .LM0SR
-_bsaes_encrypt8_alt:
- veor @XMM[10], @XMM[0], @XMM[9] @ xor with round0 key
- veor @XMM[11], @XMM[1], @XMM[9]
- vtbl.8 `&Dlo(@XMM[0])`, {@XMM[10]}, `&Dlo(@XMM[8])`
- vtbl.8 `&Dhi(@XMM[0])`, {@XMM[10]}, `&Dhi(@XMM[8])`
- veor @XMM[12], @XMM[2], @XMM[9]
- vtbl.8 `&Dlo(@XMM[1])`, {@XMM[11]}, `&Dlo(@XMM[8])`
- vtbl.8 `&Dhi(@XMM[1])`, {@XMM[11]}, `&Dhi(@XMM[8])`
- veor @XMM[13], @XMM[3], @XMM[9]
- vtbl.8 `&Dlo(@XMM[2])`, {@XMM[12]}, `&Dlo(@XMM[8])`
- vtbl.8 `&Dhi(@XMM[2])`, {@XMM[12]}, `&Dhi(@XMM[8])`
- veor @XMM[14], @XMM[4], @XMM[9]
- vtbl.8 `&Dlo(@XMM[3])`, {@XMM[13]}, `&Dlo(@XMM[8])`
- vtbl.8 `&Dhi(@XMM[3])`, {@XMM[13]}, `&Dhi(@XMM[8])`
- veor @XMM[15], @XMM[5], @XMM[9]
- vtbl.8 `&Dlo(@XMM[4])`, {@XMM[14]}, `&Dlo(@XMM[8])`
- vtbl.8 `&Dhi(@XMM[4])`, {@XMM[14]}, `&Dhi(@XMM[8])`
- veor @XMM[10], @XMM[6], @XMM[9]
- vtbl.8 `&Dlo(@XMM[5])`, {@XMM[15]}, `&Dlo(@XMM[8])`
- vtbl.8 `&Dhi(@XMM[5])`, {@XMM[15]}, `&Dhi(@XMM[8])`
- veor @XMM[11], @XMM[7], @XMM[9]
- vtbl.8 `&Dlo(@XMM[6])`, {@XMM[10]}, `&Dlo(@XMM[8])`
- vtbl.8 `&Dhi(@XMM[6])`, {@XMM[10]}, `&Dhi(@XMM[8])`
- vtbl.8 `&Dlo(@XMM[7])`, {@XMM[11]}, `&Dlo(@XMM[8])`
- vtbl.8 `&Dhi(@XMM[7])`, {@XMM[11]}, `&Dhi(@XMM[8])`
-_bsaes_encrypt8_bitslice:
-___
- &bitslice (@XMM[0..7, 8..11]);
-$code.=<<___;
- sub $rounds,$rounds,#1
- b .Lenc_sbox
-.align 4
-.Lenc_loop:
-___
- &ShiftRows (@XMM[0..7, 8..12]);
-$code.=".Lenc_sbox:\n";
- &Sbox (@XMM[0..7, 8..15]);
-$code.=<<___;
- subs $rounds,$rounds,#1
- bcc .Lenc_done
-___
- &MixColumns (@XMM[0,1,4,6,3,7,2,5, 8..15]);
-$code.=<<___;
- vldmia $const, {@XMM[12]} @ .LSR
- ite eq @ Thumb2 thing, samity check in ARM
- addeq $const,$const,#0x10
- bne .Lenc_loop
- vldmia $const, {@XMM[12]} @ .LSRM0
- b .Lenc_loop
-.align 4
-.Lenc_done:
-___
- # output in lsb > [t0, t1, t4, t6, t3, t7, t2, t5] < msb
- &bitslice (@XMM[0,1,4,6,3,7,2,5, 8..11]);
-$code.=<<___;
- vldmia $key, {@XMM[8]} @ last round key
- veor @XMM[4], @XMM[4], @XMM[8]
- veor @XMM[6], @XMM[6], @XMM[8]
- veor @XMM[3], @XMM[3], @XMM[8]
- veor @XMM[7], @XMM[7], @XMM[8]
- veor @XMM[2], @XMM[2], @XMM[8]
- veor @XMM[5], @XMM[5], @XMM[8]
- veor @XMM[0], @XMM[0], @XMM[8]
- veor @XMM[1], @XMM[1], @XMM[8]
- bx lr
-.size _bsaes_encrypt8,.-_bsaes_encrypt8
-___
-}
-{
-my ($out,$inp,$rounds,$const)=("r12","r4","r5","r6");
-
-sub bitslice_key {
-my @x=reverse(@_[0..7]);
-my ($bs0,$bs1,$bs2,$t2,$t3)=@_[8..12];
-
- &swapmove (@x[0,1],1,$bs0,$t2,$t3);
-$code.=<<___;
- @ &swapmove(@x[2,3],1,$t0,$t2,$t3);
- vmov @x[2], @x[0]
- vmov @x[3], @x[1]
-___
- #&swapmove2x(@x[4,5,6,7],1,$t0,$t2,$t3);
-
- &swapmove2x (@x[0,2,1,3],2,$bs1,$t2,$t3);
-$code.=<<___;
- @ &swapmove2x(@x[4,6,5,7],2,$t1,$t2,$t3);
- vmov @x[4], @x[0]
- vmov @x[6], @x[2]
- vmov @x[5], @x[1]
- vmov @x[7], @x[3]
-___
- &swapmove2x (@x[0,4,1,5],4,$bs2,$t2,$t3);
- &swapmove2x (@x[2,6,3,7],4,$bs2,$t2,$t3);
-}
-
-$code.=<<___;
-.type _bsaes_key_convert,%function
-.align 4
-_bsaes_key_convert:
- adr $const,_bsaes_key_convert
- vld1.8 {@XMM[7]}, [$inp]! @ load round 0 key
- sub $const,$const,#_bsaes_key_convert-.LM0
- vld1.8 {@XMM[15]}, [$inp]! @ load round 1 key
-
- vmov.i8 @XMM[8], #0x01 @ bit masks
- vmov.i8 @XMM[9], #0x02
- vmov.i8 @XMM[10], #0x04
- vmov.i8 @XMM[11], #0x08
- vmov.i8 @XMM[12], #0x10
- vmov.i8 @XMM[13], #0x20
- vldmia $const, {@XMM[14]} @ .LM0
-
-#ifdef __ARMEL__
- vrev32.8 @XMM[7], @XMM[7]
- vrev32.8 @XMM[15], @XMM[15]
-#endif
- sub $rounds,$rounds,#1
- vstmia $out!, {@XMM[7]} @ save round 0 key
- b .Lkey_loop
-
-.align 4
-.Lkey_loop:
- vtbl.8 `&Dlo(@XMM[7])`,{@XMM[15]},`&Dlo(@XMM[14])`
- vtbl.8 `&Dhi(@XMM[7])`,{@XMM[15]},`&Dhi(@XMM[14])`
- vmov.i8 @XMM[6], #0x40
- vmov.i8 @XMM[15], #0x80
-
- vtst.8 @XMM[0], @XMM[7], @XMM[8]
- vtst.8 @XMM[1], @XMM[7], @XMM[9]
- vtst.8 @XMM[2], @XMM[7], @XMM[10]
- vtst.8 @XMM[3], @XMM[7], @XMM[11]
- vtst.8 @XMM[4], @XMM[7], @XMM[12]
- vtst.8 @XMM[5], @XMM[7], @XMM[13]
- vtst.8 @XMM[6], @XMM[7], @XMM[6]
- vtst.8 @XMM[7], @XMM[7], @XMM[15]
- vld1.8 {@XMM[15]}, [$inp]! @ load next round key
- vmvn @XMM[0], @XMM[0] @ "pnot"
- vmvn @XMM[1], @XMM[1]
- vmvn @XMM[5], @XMM[5]
- vmvn @XMM[6], @XMM[6]
-#ifdef __ARMEL__
- vrev32.8 @XMM[15], @XMM[15]
-#endif
- subs $rounds,$rounds,#1
- vstmia $out!,{@XMM[0]-@XMM[7]} @ write bit-sliced round key
- bne .Lkey_loop
-
- vmov.i8 @XMM[7],#0x63 @ compose .L63
- @ don't save last round key
- bx lr
-.size _bsaes_key_convert,.-_bsaes_key_convert
-___
-}
-
-if (0) { # following four functions are unsupported interface
- # used for benchmarking...
-$code.=<<___;
-.globl bsaes_enc_key_convert
-.type bsaes_enc_key_convert,%function
-.align 4
-bsaes_enc_key_convert:
- stmdb sp!,{r4-r6,lr}
- vstmdb sp!,{d8-d15} @ ABI specification says so
-
- ldr r5,[$inp,#240] @ pass rounds
- mov r4,$inp @ pass key
- mov r12,$out @ pass key schedule
- bl _bsaes_key_convert
- veor @XMM[7],@XMM[7],@XMM[15] @ fix up last round key
- vstmia r12, {@XMM[7]} @ save last round key
-
- vldmia sp!,{d8-d15}
- ldmia sp!,{r4-r6,pc}
-.size bsaes_enc_key_convert,.-bsaes_enc_key_convert
-
-.globl bsaes_encrypt_128
-.type bsaes_encrypt_128,%function
-.align 4
-bsaes_encrypt_128:
- stmdb sp!,{r4-r6,lr}
- vstmdb sp!,{d8-d15} @ ABI specification says so
-.Lenc128_loop:
- vld1.8 {@XMM[0]-@XMM[1]}, [$inp]! @ load input
- vld1.8 {@XMM[2]-@XMM[3]}, [$inp]!
- mov r4,$key @ pass the key
- vld1.8 {@XMM[4]-@XMM[5]}, [$inp]!
- mov r5,#10 @ pass rounds
- vld1.8 {@XMM[6]-@XMM[7]}, [$inp]!
-
- bl _bsaes_encrypt8
-
- vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output
- vst1.8 {@XMM[4]}, [$out]!
- vst1.8 {@XMM[6]}, [$out]!
- vst1.8 {@XMM[3]}, [$out]!
- vst1.8 {@XMM[7]}, [$out]!
- vst1.8 {@XMM[2]}, [$out]!
- subs $len,$len,#0x80
- vst1.8 {@XMM[5]}, [$out]!
- bhi .Lenc128_loop
-
- vldmia sp!,{d8-d15}
- ldmia sp!,{r4-r6,pc}
-.size bsaes_encrypt_128,.-bsaes_encrypt_128
-
-.globl bsaes_dec_key_convert
-.type bsaes_dec_key_convert,%function
-.align 4
-bsaes_dec_key_convert:
- stmdb sp!,{r4-r6,lr}
- vstmdb sp!,{d8-d15} @ ABI specification says so
-
- ldr r5,[$inp,#240] @ pass rounds
- mov r4,$inp @ pass key
- mov r12,$out @ pass key schedule
- bl _bsaes_key_convert
- vldmia $out, {@XMM[6]}
- vstmia r12, {@XMM[15]} @ save last round key
- veor @XMM[7], @XMM[7], @XMM[6] @ fix up round 0 key
- vstmia $out, {@XMM[7]}
-
- vldmia sp!,{d8-d15}
- ldmia sp!,{r4-r6,pc}
-.size bsaes_dec_key_convert,.-bsaes_dec_key_convert
-
-.globl bsaes_decrypt_128
-.type bsaes_decrypt_128,%function
-.align 4
-bsaes_decrypt_128:
- stmdb sp!,{r4-r6,lr}
- vstmdb sp!,{d8-d15} @ ABI specification says so
-.Ldec128_loop:
- vld1.8 {@XMM[0]-@XMM[1]}, [$inp]! @ load input
- vld1.8 {@XMM[2]-@XMM[3]}, [$inp]!
- mov r4,$key @ pass the key
- vld1.8 {@XMM[4]-@XMM[5]}, [$inp]!
- mov r5,#10 @ pass rounds
- vld1.8 {@XMM[6]-@XMM[7]}, [$inp]!
-
- bl _bsaes_decrypt8
-
- vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output
- vst1.8 {@XMM[6]}, [$out]!
- vst1.8 {@XMM[4]}, [$out]!
- vst1.8 {@XMM[2]}, [$out]!
- vst1.8 {@XMM[7]}, [$out]!
- vst1.8 {@XMM[3]}, [$out]!
- subs $len,$len,#0x80
- vst1.8 {@XMM[5]}, [$out]!
- bhi .Ldec128_loop
-
- vldmia sp!,{d8-d15}
- ldmia sp!,{r4-r6,pc}
-.size bsaes_decrypt_128,.-bsaes_decrypt_128
-___
-}
-{
-my ($inp,$out,$len,$key, $ivp,$fp,$rounds)=map("r$_",(0..3,8..10));
-my ($keysched)=("sp");
-
-$code.=<<___;
-.extern AES_cbc_encrypt
-.extern AES_decrypt
-
-.global bsaes_cbc_encrypt
-.type bsaes_cbc_encrypt,%function
-.align 5
-bsaes_cbc_encrypt:
-#ifndef __KERNEL__
- cmp $len, #128
-#ifndef __thumb__
- blo AES_cbc_encrypt
-#else
- bhs 1f
- b AES_cbc_encrypt
-1:
-#endif
-#endif
-
- @ it is up to the caller to make sure we are called with enc == 0
-
- mov ip, sp
- stmdb sp!, {r4-r10, lr}
- VFP_ABI_PUSH
- ldr $ivp, [ip] @ IV is 1st arg on the stack
- mov $len, $len, lsr#4 @ len in 16 byte blocks
- sub sp, #0x10 @ scratch space to carry over the IV
- mov $fp, sp @ save sp
-
- ldr $rounds, [$key, #240] @ get # of rounds
-#ifndef BSAES_ASM_EXTENDED_KEY
- @ allocate the key schedule on the stack
- sub r12, sp, $rounds, lsl#7 @ 128 bytes per inner round key
- add r12, #`128-32` @ sifze of bit-slices key schedule
-
- @ populate the key schedule
- mov r4, $key @ pass key
- mov r5, $rounds @ pass # of rounds
- mov sp, r12 @ sp is $keysched
- bl _bsaes_key_convert
- vldmia $keysched, {@XMM[6]}
- vstmia r12, {@XMM[15]} @ save last round key
- veor @XMM[7], @XMM[7], @XMM[6] @ fix up round 0 key
- vstmia $keysched, {@XMM[7]}
-#else
- ldr r12, [$key, #244]
- eors r12, #1
- beq 0f
-
- @ populate the key schedule
- str r12, [$key, #244]
- mov r4, $key @ pass key
- mov r5, $rounds @ pass # of rounds
- add r12, $key, #248 @ pass key schedule
- bl _bsaes_key_convert
- add r4, $key, #248
- vldmia r4, {@XMM[6]}
- vstmia r12, {@XMM[15]} @ save last round key
- veor @XMM[7], @XMM[7], @XMM[6] @ fix up round 0 key
- vstmia r4, {@XMM[7]}
-
-.align 2
-0:
-#endif
-
- vld1.8 {@XMM[15]}, [$ivp] @ load IV
- b .Lcbc_dec_loop
-
-.align 4
-.Lcbc_dec_loop:
- subs $len, $len, #0x8
- bmi .Lcbc_dec_loop_finish
-
- vld1.8 {@XMM[0]-@XMM[1]}, [$inp]! @ load input
- vld1.8 {@XMM[2]-@XMM[3]}, [$inp]!
-#ifndef BSAES_ASM_EXTENDED_KEY
- mov r4, $keysched @ pass the key
-#else
- add r4, $key, #248
-#endif
- vld1.8 {@XMM[4]-@XMM[5]}, [$inp]!
- mov r5, $rounds
- vld1.8 {@XMM[6]-@XMM[7]}, [$inp]
- sub $inp, $inp, #0x60
- vstmia $fp, {@XMM[15]} @ put aside IV
-
- bl _bsaes_decrypt8
-
- vldmia $fp, {@XMM[14]} @ reload IV
- vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ reload input
- veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV
- vld1.8 {@XMM[10]-@XMM[11]}, [$inp]!
- veor @XMM[1], @XMM[1], @XMM[8]
- veor @XMM[6], @XMM[6], @XMM[9]
- vld1.8 {@XMM[12]-@XMM[13]}, [$inp]!
- veor @XMM[4], @XMM[4], @XMM[10]
- veor @XMM[2], @XMM[2], @XMM[11]
- vld1.8 {@XMM[14]-@XMM[15]}, [$inp]!
- veor @XMM[7], @XMM[7], @XMM[12]
- vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output
- veor @XMM[3], @XMM[3], @XMM[13]
- vst1.8 {@XMM[6]}, [$out]!
- veor @XMM[5], @XMM[5], @XMM[14]
- vst1.8 {@XMM[4]}, [$out]!
- vst1.8 {@XMM[2]}, [$out]!
- vst1.8 {@XMM[7]}, [$out]!
- vst1.8 {@XMM[3]}, [$out]!
- vst1.8 {@XMM[5]}, [$out]!
-
- b .Lcbc_dec_loop
-
-.Lcbc_dec_loop_finish:
- adds $len, $len, #8
- beq .Lcbc_dec_done
-
- vld1.8 {@XMM[0]}, [$inp]! @ load input
- cmp $len, #2
- blo .Lcbc_dec_one
- vld1.8 {@XMM[1]}, [$inp]!
-#ifndef BSAES_ASM_EXTENDED_KEY
- mov r4, $keysched @ pass the key
-#else
- add r4, $key, #248
-#endif
- mov r5, $rounds
- vstmia $fp, {@XMM[15]} @ put aside IV
- beq .Lcbc_dec_two
- vld1.8 {@XMM[2]}, [$inp]!
- cmp $len, #4
- blo .Lcbc_dec_three
- vld1.8 {@XMM[3]}, [$inp]!
- beq .Lcbc_dec_four
- vld1.8 {@XMM[4]}, [$inp]!
- cmp $len, #6
- blo .Lcbc_dec_five
- vld1.8 {@XMM[5]}, [$inp]!
- beq .Lcbc_dec_six
- vld1.8 {@XMM[6]}, [$inp]!
- sub $inp, $inp, #0x70
-
- bl _bsaes_decrypt8
-
- vldmia $fp, {@XMM[14]} @ reload IV
- vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ reload input
- veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV
- vld1.8 {@XMM[10]-@XMM[11]}, [$inp]!
- veor @XMM[1], @XMM[1], @XMM[8]
- veor @XMM[6], @XMM[6], @XMM[9]
- vld1.8 {@XMM[12]-@XMM[13]}, [$inp]!
- veor @XMM[4], @XMM[4], @XMM[10]
- veor @XMM[2], @XMM[2], @XMM[11]
- vld1.8 {@XMM[15]}, [$inp]!
- veor @XMM[7], @XMM[7], @XMM[12]
- vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output
- veor @XMM[3], @XMM[3], @XMM[13]
- vst1.8 {@XMM[6]}, [$out]!
- vst1.8 {@XMM[4]}, [$out]!
- vst1.8 {@XMM[2]}, [$out]!
- vst1.8 {@XMM[7]}, [$out]!
- vst1.8 {@XMM[3]}, [$out]!
- b .Lcbc_dec_done
-.align 4
-.Lcbc_dec_six:
- sub $inp, $inp, #0x60
- bl _bsaes_decrypt8
- vldmia $fp,{@XMM[14]} @ reload IV
- vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ reload input
- veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV
- vld1.8 {@XMM[10]-@XMM[11]}, [$inp]!
- veor @XMM[1], @XMM[1], @XMM[8]
- veor @XMM[6], @XMM[6], @XMM[9]
- vld1.8 {@XMM[12]}, [$inp]!
- veor @XMM[4], @XMM[4], @XMM[10]
- veor @XMM[2], @XMM[2], @XMM[11]
- vld1.8 {@XMM[15]}, [$inp]!
- veor @XMM[7], @XMM[7], @XMM[12]
- vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output
- vst1.8 {@XMM[6]}, [$out]!
- vst1.8 {@XMM[4]}, [$out]!
- vst1.8 {@XMM[2]}, [$out]!
- vst1.8 {@XMM[7]}, [$out]!
- b .Lcbc_dec_done
-.align 4
-.Lcbc_dec_five:
- sub $inp, $inp, #0x50
- bl _bsaes_decrypt8
- vldmia $fp, {@XMM[14]} @ reload IV
- vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ reload input
- veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV
- vld1.8 {@XMM[10]-@XMM[11]}, [$inp]!
- veor @XMM[1], @XMM[1], @XMM[8]
- veor @XMM[6], @XMM[6], @XMM[9]
- vld1.8 {@XMM[15]}, [$inp]!
- veor @XMM[4], @XMM[4], @XMM[10]
- vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output
- veor @XMM[2], @XMM[2], @XMM[11]
- vst1.8 {@XMM[6]}, [$out]!
- vst1.8 {@XMM[4]}, [$out]!
- vst1.8 {@XMM[2]}, [$out]!
- b .Lcbc_dec_done
-.align 4
-.Lcbc_dec_four:
- sub $inp, $inp, #0x40
- bl _bsaes_decrypt8
- vldmia $fp, {@XMM[14]} @ reload IV
- vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ reload input
- veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV
- vld1.8 {@XMM[10]}, [$inp]!
- veor @XMM[1], @XMM[1], @XMM[8]
- veor @XMM[6], @XMM[6], @XMM[9]
- vld1.8 {@XMM[15]}, [$inp]!
- veor @XMM[4], @XMM[4], @XMM[10]
- vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output
- vst1.8 {@XMM[6]}, [$out]!
- vst1.8 {@XMM[4]}, [$out]!
- b .Lcbc_dec_done
-.align 4
-.Lcbc_dec_three:
- sub $inp, $inp, #0x30
- bl _bsaes_decrypt8
- vldmia $fp, {@XMM[14]} @ reload IV
- vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ reload input
- veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV
- vld1.8 {@XMM[15]}, [$inp]!
- veor @XMM[1], @XMM[1], @XMM[8]
- veor @XMM[6], @XMM[6], @XMM[9]
- vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output
- vst1.8 {@XMM[6]}, [$out]!
- b .Lcbc_dec_done
-.align 4
-.Lcbc_dec_two:
- sub $inp, $inp, #0x20
- bl _bsaes_decrypt8
- vldmia $fp, {@XMM[14]} @ reload IV
- vld1.8 {@XMM[8]}, [$inp]! @ reload input
- veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV
- vld1.8 {@XMM[15]}, [$inp]! @ reload input
- veor @XMM[1], @XMM[1], @XMM[8]
- vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output
- b .Lcbc_dec_done
-.align 4
-.Lcbc_dec_one:
- sub $inp, $inp, #0x10
- mov $rounds, $out @ save original out pointer
- mov $out, $fp @ use the iv scratch space as out buffer
- mov r2, $key
- vmov @XMM[4],@XMM[15] @ just in case ensure that IV
- vmov @XMM[5],@XMM[0] @ and input are preserved
- bl AES_decrypt
- vld1.8 {@XMM[0]}, [$fp,:64] @ load result
- veor @XMM[0], @XMM[0], @XMM[4] @ ^= IV
- vmov @XMM[15], @XMM[5] @ @XMM[5] holds input
- vst1.8 {@XMM[0]}, [$rounds] @ write output
-
-.Lcbc_dec_done:
-#ifndef BSAES_ASM_EXTENDED_KEY
- vmov.i32 q0, #0
- vmov.i32 q1, #0
-.Lcbc_dec_bzero: @ wipe key schedule [if any]
- vstmia $keysched!, {q0-q1}
- cmp $keysched, $fp
- bne .Lcbc_dec_bzero
-#endif
-
- mov sp, $fp
- add sp, #0x10 @ add sp,$fp,#0x10 is no good for thumb
- vst1.8 {@XMM[15]}, [$ivp] @ return IV
- VFP_ABI_POP
- ldmia sp!, {r4-r10, pc}
-.size bsaes_cbc_encrypt,.-bsaes_cbc_encrypt
-___
-}
-{
-my ($inp,$out,$len,$key, $ctr,$fp,$rounds)=(map("r$_",(0..3,8..10)));
-my $const = "r6"; # shared with _bsaes_encrypt8_alt
-my $keysched = "sp";
-
-$code.=<<___;
-.extern AES_encrypt
-.global bsaes_ctr32_encrypt_blocks
-.type bsaes_ctr32_encrypt_blocks,%function
-.align 5
-bsaes_ctr32_encrypt_blocks:
- cmp $len, #8 @ use plain AES for
- blo .Lctr_enc_short @ small sizes
-
- mov ip, sp
- stmdb sp!, {r4-r10, lr}
- VFP_ABI_PUSH
- ldr $ctr, [ip] @ ctr is 1st arg on the stack
- sub sp, sp, #0x10 @ scratch space to carry over the ctr
- mov $fp, sp @ save sp
-
- ldr $rounds, [$key, #240] @ get # of rounds
-#ifndef BSAES_ASM_EXTENDED_KEY
- @ allocate the key schedule on the stack
- sub r12, sp, $rounds, lsl#7 @ 128 bytes per inner round key
- add r12, #`128-32` @ size of bit-sliced key schedule
-
- @ populate the key schedule
- mov r4, $key @ pass key
- mov r5, $rounds @ pass # of rounds
- mov sp, r12 @ sp is $keysched
- bl _bsaes_key_convert
- veor @XMM[7],@XMM[7],@XMM[15] @ fix up last round key
- vstmia r12, {@XMM[7]} @ save last round key
-
- vld1.8 {@XMM[0]}, [$ctr] @ load counter
- add $ctr, $const, #.LREVM0SR-.LM0 @ borrow $ctr
- vldmia $keysched, {@XMM[4]} @ load round0 key
-#else
- ldr r12, [$key, #244]
- eors r12, #1
- beq 0f
-
- @ populate the key schedule
- str r12, [$key, #244]
- mov r4, $key @ pass key
- mov r5, $rounds @ pass # of rounds
- add r12, $key, #248 @ pass key schedule
- bl _bsaes_key_convert
- veor @XMM[7],@XMM[7],@XMM[15] @ fix up last round key
- vstmia r12, {@XMM[7]} @ save last round key
-
-.align 2
-0: add r12, $key, #248
- vld1.8 {@XMM[0]}, [$ctr] @ load counter
- adrl $ctr, .LREVM0SR @ borrow $ctr
- vldmia r12, {@XMM[4]} @ load round0 key
- sub sp, #0x10 @ place for adjusted round0 key
-#endif
-
- vmov.i32 @XMM[8],#1 @ compose 1<<96
- veor @XMM[9],@XMM[9],@XMM[9]
- vrev32.8 @XMM[0],@XMM[0]
- vext.8 @XMM[8],@XMM[9],@XMM[8],#4
- vrev32.8 @XMM[4],@XMM[4]
- vadd.u32 @XMM[9],@XMM[8],@XMM[8] @ compose 2<<96
- vstmia $keysched, {@XMM[4]} @ save adjusted round0 key
- b .Lctr_enc_loop
-
-.align 4
-.Lctr_enc_loop:
- vadd.u32 @XMM[10], @XMM[8], @XMM[9] @ compose 3<<96
- vadd.u32 @XMM[1], @XMM[0], @XMM[8] @ +1
- vadd.u32 @XMM[2], @XMM[0], @XMM[9] @ +2
- vadd.u32 @XMM[3], @XMM[0], @XMM[10] @ +3
- vadd.u32 @XMM[4], @XMM[1], @XMM[10]
- vadd.u32 @XMM[5], @XMM[2], @XMM[10]
- vadd.u32 @XMM[6], @XMM[3], @XMM[10]
- vadd.u32 @XMM[7], @XMM[4], @XMM[10]
- vadd.u32 @XMM[10], @XMM[5], @XMM[10] @ next counter
-
- @ Borrow prologue from _bsaes_encrypt8 to use the opportunity
- @ to flip byte order in 32-bit counter
-
- vldmia $keysched, {@XMM[9]} @ load round0 key
-#ifndef BSAES_ASM_EXTENDED_KEY
- add r4, $keysched, #0x10 @ pass next round key
-#else
- add r4, $key, #`248+16`
-#endif
- vldmia $ctr, {@XMM[8]} @ .LREVM0SR
- mov r5, $rounds @ pass rounds
- vstmia $fp, {@XMM[10]} @ save next counter
- sub $const, $ctr, #.LREVM0SR-.LSR @ pass constants
-
- bl _bsaes_encrypt8_alt
-
- subs $len, $len, #8
- blo .Lctr_enc_loop_done
-
- vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ load input
- vld1.8 {@XMM[10]-@XMM[11]}, [$inp]!
- veor @XMM[0], @XMM[8]
- veor @XMM[1], @XMM[9]
- vld1.8 {@XMM[12]-@XMM[13]}, [$inp]!
- veor @XMM[4], @XMM[10]
- veor @XMM[6], @XMM[11]
- vld1.8 {@XMM[14]-@XMM[15]}, [$inp]!
- veor @XMM[3], @XMM[12]
- vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output
- veor @XMM[7], @XMM[13]
- veor @XMM[2], @XMM[14]
- vst1.8 {@XMM[4]}, [$out]!
- veor @XMM[5], @XMM[15]
- vst1.8 {@XMM[6]}, [$out]!
- vmov.i32 @XMM[8], #1 @ compose 1<<96
- vst1.8 {@XMM[3]}, [$out]!
- veor @XMM[9], @XMM[9], @XMM[9]
- vst1.8 {@XMM[7]}, [$out]!
- vext.8 @XMM[8], @XMM[9], @XMM[8], #4
- vst1.8 {@XMM[2]}, [$out]!
- vadd.u32 @XMM[9],@XMM[8],@XMM[8] @ compose 2<<96
- vst1.8 {@XMM[5]}, [$out]!
- vldmia $fp, {@XMM[0]} @ load counter
-
- bne .Lctr_enc_loop
- b .Lctr_enc_done
-
-.align 4
-.Lctr_enc_loop_done:
- add $len, $len, #8
- vld1.8 {@XMM[8]}, [$inp]! @ load input
- veor @XMM[0], @XMM[8]
- vst1.8 {@XMM[0]}, [$out]! @ write output
- cmp $len, #2
- blo .Lctr_enc_done
- vld1.8 {@XMM[9]}, [$inp]!
- veor @XMM[1], @XMM[9]
- vst1.8 {@XMM[1]}, [$out]!
- beq .Lctr_enc_done
- vld1.8 {@XMM[10]}, [$inp]!
- veor @XMM[4], @XMM[10]
- vst1.8 {@XMM[4]}, [$out]!
- cmp $len, #4
- blo .Lctr_enc_done
- vld1.8 {@XMM[11]}, [$inp]!
- veor @XMM[6], @XMM[11]
- vst1.8 {@XMM[6]}, [$out]!
- beq .Lctr_enc_done
- vld1.8 {@XMM[12]}, [$inp]!
- veor @XMM[3], @XMM[12]
- vst1.8 {@XMM[3]}, [$out]!
- cmp $len, #6
- blo .Lctr_enc_done
- vld1.8 {@XMM[13]}, [$inp]!
- veor @XMM[7], @XMM[13]
- vst1.8 {@XMM[7]}, [$out]!
- beq .Lctr_enc_done
- vld1.8 {@XMM[14]}, [$inp]
- veor @XMM[2], @XMM[14]
- vst1.8 {@XMM[2]}, [$out]!
-
-.Lctr_enc_done:
- vmov.i32 q0, #0
- vmov.i32 q1, #0
-#ifndef BSAES_ASM_EXTENDED_KEY
-.Lctr_enc_bzero: @ wipe key schedule [if any]
- vstmia $keysched!, {q0-q1}
- cmp $keysched, $fp
- bne .Lctr_enc_bzero
-#else
- vstmia $keysched, {q0-q1}
-#endif
-
- mov sp, $fp
- add sp, #0x10 @ add sp,$fp,#0x10 is no good for thumb
- VFP_ABI_POP
- ldmia sp!, {r4-r10, pc} @ return
-
-.align 4
-.Lctr_enc_short:
- ldr ip, [sp] @ ctr pointer is passed on stack
- stmdb sp!, {r4-r8, lr}
-
- mov r4, $inp @ copy arguments
- mov r5, $out
- mov r6, $len
- mov r7, $key
- ldr r8, [ip, #12] @ load counter LSW
- vld1.8 {@XMM[1]}, [ip] @ load whole counter value
-#ifdef __ARMEL__
- rev r8, r8
-#endif
- sub sp, sp, #0x10
- vst1.8 {@XMM[1]}, [sp,:64] @ copy counter value
- sub sp, sp, #0x10
-
-.Lctr_enc_short_loop:
- add r0, sp, #0x10 @ input counter value
- mov r1, sp @ output on the stack
- mov r2, r7 @ key
-
- bl AES_encrypt
-
- vld1.8 {@XMM[0]}, [r4]! @ load input
- vld1.8 {@XMM[1]}, [sp,:64] @ load encrypted counter
- add r8, r8, #1
-#ifdef __ARMEL__
- rev r0, r8
- str r0, [sp, #0x1c] @ next counter value
-#else
- str r8, [sp, #0x1c] @ next counter value
-#endif
- veor @XMM[0],@XMM[0],@XMM[1]
- vst1.8 {@XMM[0]}, [r5]! @ store output
- subs r6, r6, #1
- bne .Lctr_enc_short_loop
-
- vmov.i32 q0, #0
- vmov.i32 q1, #0
- vstmia sp!, {q0-q1}
-
- ldmia sp!, {r4-r8, pc}
-.size bsaes_ctr32_encrypt_blocks,.-bsaes_ctr32_encrypt_blocks
-___
-}
-{
-######################################################################
-# void bsaes_xts_[en|de]crypt(const char *inp,char *out,size_t len,
-# const AES_KEY *key1, const AES_KEY *key2,
-# const unsigned char iv[16]);
-#
-my ($inp,$out,$len,$key,$rounds,$magic,$fp)=(map("r$_",(7..10,1..3)));
-my $const="r6"; # returned by _bsaes_key_convert
-my $twmask=@XMM[5];
-my @T=@XMM[6..7];
-
-$code.=<<___;
-.globl bsaes_xts_encrypt
-.type bsaes_xts_encrypt,%function
-.align 4
-bsaes_xts_encrypt:
- mov ip, sp
- stmdb sp!, {r4-r10, lr} @ 0x20
- VFP_ABI_PUSH
- mov r6, sp @ future $fp
-
- mov $inp, r0
- mov $out, r1
- mov $len, r2
- mov $key, r3
-
- sub r0, sp, #0x10 @ 0x10
- bic r0, #0xf @ align at 16 bytes
- mov sp, r0
-
-#ifdef XTS_CHAIN_TWEAK
- ldr r0, [ip] @ pointer to input tweak
-#else
- @ generate initial tweak
- ldr r0, [ip, #4] @ iv[]
- mov r1, sp
- ldr r2, [ip, #0] @ key2
- bl AES_encrypt
- mov r0,sp @ pointer to initial tweak
-#endif
-
- ldr $rounds, [$key, #240] @ get # of rounds
- mov $fp, r6
-#ifndef BSAES_ASM_EXTENDED_KEY
- @ allocate the key schedule on the stack
- sub r12, sp, $rounds, lsl#7 @ 128 bytes per inner round key
- @ add r12, #`128-32` @ size of bit-sliced key schedule
- sub r12, #`32+16` @ place for tweak[9]
-
- @ populate the key schedule
- mov r4, $key @ pass key
- mov r5, $rounds @ pass # of rounds
- mov sp, r12
- add r12, #0x90 @ pass key schedule
- bl _bsaes_key_convert
- veor @XMM[7], @XMM[7], @XMM[15] @ fix up last round key
- vstmia r12, {@XMM[7]} @ save last round key
-#else
- ldr r12, [$key, #244]
- eors r12, #1
- beq 0f
-
- str r12, [$key, #244]
- mov r4, $key @ pass key
- mov r5, $rounds @ pass # of rounds
- add r12, $key, #248 @ pass key schedule
- bl _bsaes_key_convert
- veor @XMM[7], @XMM[7], @XMM[15] @ fix up last round key
- vstmia r12, {@XMM[7]}
-
-.align 2
-0: sub sp, #0x90 @ place for tweak[9]
-#endif
-
- vld1.8 {@XMM[8]}, [r0] @ initial tweak
- adr $magic, .Lxts_magic
-
- subs $len, #0x80
- blo .Lxts_enc_short
- b .Lxts_enc_loop
-
-.align 4
-.Lxts_enc_loop:
- vldmia $magic, {$twmask} @ load XTS magic
- vshr.s64 @T[0], @XMM[8], #63
- mov r0, sp
- vand @T[0], @T[0], $twmask
-___
-for($i=9;$i<16;$i++) {
-$code.=<<___;
- vadd.u64 @XMM[$i], @XMM[$i-1], @XMM[$i-1]
- vst1.64 {@XMM[$i-1]}, [r0,:128]!
- vswp `&Dhi("@T[0]")`,`&Dlo("@T[0]")`
- vshr.s64 @T[1], @XMM[$i], #63
- veor @XMM[$i], @XMM[$i], @T[0]
- vand @T[1], @T[1], $twmask
-___
- @T=reverse(@T);
-
-$code.=<<___ if ($i>=10);
- vld1.8 {@XMM[$i-10]}, [$inp]!
-___
-$code.=<<___ if ($i>=11);
- veor @XMM[$i-11], @XMM[$i-11], @XMM[$i-3]
-___
-}
-$code.=<<___;
- vadd.u64 @XMM[8], @XMM[15], @XMM[15]
- vst1.64 {@XMM[15]}, [r0,:128]!
- vswp `&Dhi("@T[0]")`,`&Dlo("@T[0]")`
- veor @XMM[8], @XMM[8], @T[0]
- vst1.64 {@XMM[8]}, [r0,:128] @ next round tweak
-
- vld1.8 {@XMM[6]-@XMM[7]}, [$inp]!
- veor @XMM[5], @XMM[5], @XMM[13]
-#ifndef BSAES_ASM_EXTENDED_KEY
- add r4, sp, #0x90 @ pass key schedule
-#else
- add r4, $key, #248 @ pass key schedule
-#endif
- veor @XMM[6], @XMM[6], @XMM[14]
- mov r5, $rounds @ pass rounds
- veor @XMM[7], @XMM[7], @XMM[15]
- mov r0, sp
-
- bl _bsaes_encrypt8
-
- vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]!
- vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]!
- veor @XMM[0], @XMM[0], @XMM[ 8]
- vld1.64 {@XMM[12]-@XMM[13]}, [r0,:128]!
- veor @XMM[1], @XMM[1], @XMM[ 9]
- veor @XMM[8], @XMM[4], @XMM[10]
- vst1.8 {@XMM[0]-@XMM[1]}, [$out]!
- veor @XMM[9], @XMM[6], @XMM[11]
- vld1.64 {@XMM[14]-@XMM[15]}, [r0,:128]!
- veor @XMM[10], @XMM[3], @XMM[12]
- vst1.8 {@XMM[8]-@XMM[9]}, [$out]!
- veor @XMM[11], @XMM[7], @XMM[13]
- veor @XMM[12], @XMM[2], @XMM[14]
- vst1.8 {@XMM[10]-@XMM[11]}, [$out]!
- veor @XMM[13], @XMM[5], @XMM[15]
- vst1.8 {@XMM[12]-@XMM[13]}, [$out]!
-
- vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak
-
- subs $len, #0x80
- bpl .Lxts_enc_loop
-
-.Lxts_enc_short:
- adds $len, #0x70
- bmi .Lxts_enc_done
-
- vldmia $magic, {$twmask} @ load XTS magic
- vshr.s64 @T[0], @XMM[8], #63
- mov r0, sp
- vand @T[0], @T[0], $twmask
-___
-for($i=9;$i<16;$i++) {
-$code.=<<___;
- vadd.u64 @XMM[$i], @XMM[$i-1], @XMM[$i-1]
- vst1.64 {@XMM[$i-1]}, [r0,:128]!
- vswp `&Dhi("@T[0]")`,`&Dlo("@T[0]")`
- vshr.s64 @T[1], @XMM[$i], #63
- veor @XMM[$i], @XMM[$i], @T[0]
- vand @T[1], @T[1], $twmask
-___
- @T=reverse(@T);
-
-$code.=<<___ if ($i>=10);
- vld1.8 {@XMM[$i-10]}, [$inp]!
- subs $len, #0x10
- bmi .Lxts_enc_`$i-9`
-___
-$code.=<<___ if ($i>=11);
- veor @XMM[$i-11], @XMM[$i-11], @XMM[$i-3]
-___
-}
-$code.=<<___;
- sub $len, #0x10
- vst1.64 {@XMM[15]}, [r0,:128] @ next round tweak
-
- vld1.8 {@XMM[6]}, [$inp]!
- veor @XMM[5], @XMM[5], @XMM[13]
-#ifndef BSAES_ASM_EXTENDED_KEY
- add r4, sp, #0x90 @ pass key schedule
-#else
- add r4, $key, #248 @ pass key schedule
-#endif
- veor @XMM[6], @XMM[6], @XMM[14]
- mov r5, $rounds @ pass rounds
- mov r0, sp
-
- bl _bsaes_encrypt8
-
- vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]!
- vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]!
- veor @XMM[0], @XMM[0], @XMM[ 8]
- vld1.64 {@XMM[12]-@XMM[13]}, [r0,:128]!
- veor @XMM[1], @XMM[1], @XMM[ 9]
- veor @XMM[8], @XMM[4], @XMM[10]
- vst1.8 {@XMM[0]-@XMM[1]}, [$out]!
- veor @XMM[9], @XMM[6], @XMM[11]
- vld1.64 {@XMM[14]}, [r0,:128]!
- veor @XMM[10], @XMM[3], @XMM[12]
- vst1.8 {@XMM[8]-@XMM[9]}, [$out]!
- veor @XMM[11], @XMM[7], @XMM[13]
- veor @XMM[12], @XMM[2], @XMM[14]
- vst1.8 {@XMM[10]-@XMM[11]}, [$out]!
- vst1.8 {@XMM[12]}, [$out]!
-
- vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak
- b .Lxts_enc_done
-.align 4
-.Lxts_enc_6:
- vst1.64 {@XMM[14]}, [r0,:128] @ next round tweak
-
- veor @XMM[4], @XMM[4], @XMM[12]
-#ifndef BSAES_ASM_EXTENDED_KEY
- add r4, sp, #0x90 @ pass key schedule
-#else
- add r4, $key, #248 @ pass key schedule
-#endif
- veor @XMM[5], @XMM[5], @XMM[13]
- mov r5, $rounds @ pass rounds
- mov r0, sp
-
- bl _bsaes_encrypt8
-
- vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]!
- vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]!
- veor @XMM[0], @XMM[0], @XMM[ 8]
- vld1.64 {@XMM[12]-@XMM[13]}, [r0,:128]!
- veor @XMM[1], @XMM[1], @XMM[ 9]
- veor @XMM[8], @XMM[4], @XMM[10]
- vst1.8 {@XMM[0]-@XMM[1]}, [$out]!
- veor @XMM[9], @XMM[6], @XMM[11]
- veor @XMM[10], @XMM[3], @XMM[12]
- vst1.8 {@XMM[8]-@XMM[9]}, [$out]!
- veor @XMM[11], @XMM[7], @XMM[13]
- vst1.8 {@XMM[10]-@XMM[11]}, [$out]!
-
- vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak
- b .Lxts_enc_done
-
-@ put this in range for both ARM and Thumb mode adr instructions
-.align 5
-.Lxts_magic:
- .quad 1, 0x87
-
-.align 5
-.Lxts_enc_5:
- vst1.64 {@XMM[13]}, [r0,:128] @ next round tweak
-
- veor @XMM[3], @XMM[3], @XMM[11]
-#ifndef BSAES_ASM_EXTENDED_KEY
- add r4, sp, #0x90 @ pass key schedule
-#else
- add r4, $key, #248 @ pass key schedule
-#endif
- veor @XMM[4], @XMM[4], @XMM[12]
- mov r5, $rounds @ pass rounds
- mov r0, sp
-
- bl _bsaes_encrypt8
-
- vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]!
- vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]!
- veor @XMM[0], @XMM[0], @XMM[ 8]
- vld1.64 {@XMM[12]}, [r0,:128]!
- veor @XMM[1], @XMM[1], @XMM[ 9]
- veor @XMM[8], @XMM[4], @XMM[10]
- vst1.8 {@XMM[0]-@XMM[1]}, [$out]!
- veor @XMM[9], @XMM[6], @XMM[11]
- veor @XMM[10], @XMM[3], @XMM[12]
- vst1.8 {@XMM[8]-@XMM[9]}, [$out]!
- vst1.8 {@XMM[10]}, [$out]!
-
- vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak
- b .Lxts_enc_done
-.align 4
-.Lxts_enc_4:
- vst1.64 {@XMM[12]}, [r0,:128] @ next round tweak
-
- veor @XMM[2], @XMM[2], @XMM[10]
-#ifndef BSAES_ASM_EXTENDED_KEY
- add r4, sp, #0x90 @ pass key schedule
-#else
- add r4, $key, #248 @ pass key schedule
-#endif
- veor @XMM[3], @XMM[3], @XMM[11]
- mov r5, $rounds @ pass rounds
- mov r0, sp
-
- bl _bsaes_encrypt8
-
- vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]!
- vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]!
- veor @XMM[0], @XMM[0], @XMM[ 8]
- veor @XMM[1], @XMM[1], @XMM[ 9]
- veor @XMM[8], @XMM[4], @XMM[10]
- vst1.8 {@XMM[0]-@XMM[1]}, [$out]!
- veor @XMM[9], @XMM[6], @XMM[11]
- vst1.8 {@XMM[8]-@XMM[9]}, [$out]!
-
- vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak
- b .Lxts_enc_done
-.align 4
-.Lxts_enc_3:
- vst1.64 {@XMM[11]}, [r0,:128] @ next round tweak
-
- veor @XMM[1], @XMM[1], @XMM[9]
-#ifndef BSAES_ASM_EXTENDED_KEY
- add r4, sp, #0x90 @ pass key schedule
-#else
- add r4, $key, #248 @ pass key schedule
-#endif
- veor @XMM[2], @XMM[2], @XMM[10]
- mov r5, $rounds @ pass rounds
- mov r0, sp
-
- bl _bsaes_encrypt8
-
- vld1.64 {@XMM[8]-@XMM[9]}, [r0,:128]!
- vld1.64 {@XMM[10]}, [r0,:128]!
- veor @XMM[0], @XMM[0], @XMM[ 8]
- veor @XMM[1], @XMM[1], @XMM[ 9]
- veor @XMM[8], @XMM[4], @XMM[10]
- vst1.8 {@XMM[0]-@XMM[1]}, [$out]!
- vst1.8 {@XMM[8]}, [$out]!
-
- vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak
- b .Lxts_enc_done
-.align 4
-.Lxts_enc_2:
- vst1.64 {@XMM[10]}, [r0,:128] @ next round tweak
-
- veor @XMM[0], @XMM[0], @XMM[8]
-#ifndef BSAES_ASM_EXTENDED_KEY
- add r4, sp, #0x90 @ pass key schedule
-#else
- add r4, $key, #248 @ pass key schedule
-#endif
- veor @XMM[1], @XMM[1], @XMM[9]
- mov r5, $rounds @ pass rounds
- mov r0, sp
-
- bl _bsaes_encrypt8
-
- vld1.64 {@XMM[8]-@XMM[9]}, [r0,:128]!
- veor @XMM[0], @XMM[0], @XMM[ 8]
- veor @XMM[1], @XMM[1], @XMM[ 9]
- vst1.8 {@XMM[0]-@XMM[1]}, [$out]!
-
- vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak
- b .Lxts_enc_done
-.align 4
-.Lxts_enc_1:
- mov r0, sp
- veor @XMM[0], @XMM[8]
- mov r1, sp
- vst1.8 {@XMM[0]}, [sp,:128]
- mov r2, $key
- mov r4, $fp @ preserve fp
-
- bl AES_encrypt
-
- vld1.8 {@XMM[0]}, [sp,:128]
- veor @XMM[0], @XMM[0], @XMM[8]
- vst1.8 {@XMM[0]}, [$out]!
- mov $fp, r4
-
- vmov @XMM[8], @XMM[9] @ next round tweak
-
-.Lxts_enc_done:
-#ifndef XTS_CHAIN_TWEAK
- adds $len, #0x10
- beq .Lxts_enc_ret
- sub r6, $out, #0x10
-
-.Lxts_enc_steal:
- ldrb r0, [$inp], #1
- ldrb r1, [$out, #-0x10]
- strb r0, [$out, #-0x10]
- strb r1, [$out], #1
-
- subs $len, #1
- bhi .Lxts_enc_steal
-
- vld1.8 {@XMM[0]}, [r6]
- mov r0, sp
- veor @XMM[0], @XMM[0], @XMM[8]
- mov r1, sp
- vst1.8 {@XMM[0]}, [sp,:128]
- mov r2, $key
- mov r4, $fp @ preserve fp
-
- bl AES_encrypt
-
- vld1.8 {@XMM[0]}, [sp,:128]
- veor @XMM[0], @XMM[0], @XMM[8]
- vst1.8 {@XMM[0]}, [r6]
- mov $fp, r4
-#endif
-
-.Lxts_enc_ret:
- bic r0, $fp, #0xf
- vmov.i32 q0, #0
- vmov.i32 q1, #0
-#ifdef XTS_CHAIN_TWEAK
- ldr r1, [$fp, #0x20+VFP_ABI_FRAME] @ chain tweak
-#endif
-.Lxts_enc_bzero: @ wipe key schedule [if any]
- vstmia sp!, {q0-q1}
- cmp sp, r0
- bne .Lxts_enc_bzero
-
- mov sp, $fp
-#ifdef XTS_CHAIN_TWEAK
- vst1.8 {@XMM[8]}, [r1]
-#endif
- VFP_ABI_POP
- ldmia sp!, {r4-r10, pc} @ return
-
-.size bsaes_xts_encrypt,.-bsaes_xts_encrypt
-
-.globl bsaes_xts_decrypt
-.type bsaes_xts_decrypt,%function
-.align 4
-bsaes_xts_decrypt:
- mov ip, sp
- stmdb sp!, {r4-r10, lr} @ 0x20
- VFP_ABI_PUSH
- mov r6, sp @ future $fp
-
- mov $inp, r0
- mov $out, r1
- mov $len, r2
- mov $key, r3
-
- sub r0, sp, #0x10 @ 0x10
- bic r0, #0xf @ align at 16 bytes
- mov sp, r0
-
-#ifdef XTS_CHAIN_TWEAK
- ldr r0, [ip] @ pointer to input tweak
-#else
- @ generate initial tweak
- ldr r0, [ip, #4] @ iv[]
- mov r1, sp
- ldr r2, [ip, #0] @ key2
- bl AES_encrypt
- mov r0, sp @ pointer to initial tweak
-#endif
-
- ldr $rounds, [$key, #240] @ get # of rounds
- mov $fp, r6
-#ifndef BSAES_ASM_EXTENDED_KEY
- @ allocate the key schedule on the stack
- sub r12, sp, $rounds, lsl#7 @ 128 bytes per inner round key
- @ add r12, #`128-32` @ size of bit-sliced key schedule
- sub r12, #`32+16` @ place for tweak[9]
-
- @ populate the key schedule
- mov r4, $key @ pass key
- mov r5, $rounds @ pass # of rounds
- mov sp, r12
- add r12, #0x90 @ pass key schedule
- bl _bsaes_key_convert
- add r4, sp, #0x90
- vldmia r4, {@XMM[6]}
- vstmia r12, {@XMM[15]} @ save last round key
- veor @XMM[7], @XMM[7], @XMM[6] @ fix up round 0 key
- vstmia r4, {@XMM[7]}
-#else
- ldr r12, [$key, #244]
- eors r12, #1
- beq 0f
-
- str r12, [$key, #244]
- mov r4, $key @ pass key
- mov r5, $rounds @ pass # of rounds
- add r12, $key, #248 @ pass key schedule
- bl _bsaes_key_convert
- add r4, $key, #248
- vldmia r4, {@XMM[6]}
- vstmia r12, {@XMM[15]} @ save last round key
- veor @XMM[7], @XMM[7], @XMM[6] @ fix up round 0 key
- vstmia r4, {@XMM[7]}
-
-.align 2
-0: sub sp, #0x90 @ place for tweak[9]
-#endif
- vld1.8 {@XMM[8]}, [r0] @ initial tweak
- adr $magic, .Lxts_magic
-
-#ifndef XTS_CHAIN_TWEAK
- tst $len, #0xf @ if not multiple of 16
- it ne @ Thumb2 thing, sanity check in ARM
- subne $len, #0x10 @ subtract another 16 bytes
-#endif
- subs $len, #0x80
-
- blo .Lxts_dec_short
- b .Lxts_dec_loop
-
-.align 4
-.Lxts_dec_loop:
- vldmia $magic, {$twmask} @ load XTS magic
- vshr.s64 @T[0], @XMM[8], #63
- mov r0, sp
- vand @T[0], @T[0], $twmask
-___
-for($i=9;$i<16;$i++) {
-$code.=<<___;
- vadd.u64 @XMM[$i], @XMM[$i-1], @XMM[$i-1]
- vst1.64 {@XMM[$i-1]}, [r0,:128]!
- vswp `&Dhi("@T[0]")`,`&Dlo("@T[0]")`
- vshr.s64 @T[1], @XMM[$i], #63
- veor @XMM[$i], @XMM[$i], @T[0]
- vand @T[1], @T[1], $twmask
-___
- @T=reverse(@T);
-
-$code.=<<___ if ($i>=10);
- vld1.8 {@XMM[$i-10]}, [$inp]!
-___
-$code.=<<___ if ($i>=11);
- veor @XMM[$i-11], @XMM[$i-11], @XMM[$i-3]
-___
-}
-$code.=<<___;
- vadd.u64 @XMM[8], @XMM[15], @XMM[15]
- vst1.64 {@XMM[15]}, [r0,:128]!
- vswp `&Dhi("@T[0]")`,`&Dlo("@T[0]")`
- veor @XMM[8], @XMM[8], @T[0]
- vst1.64 {@XMM[8]}, [r0,:128] @ next round tweak
-
- vld1.8 {@XMM[6]-@XMM[7]}, [$inp]!
- veor @XMM[5], @XMM[5], @XMM[13]
-#ifndef BSAES_ASM_EXTENDED_KEY
- add r4, sp, #0x90 @ pass key schedule
-#else
- add r4, $key, #248 @ pass key schedule
-#endif
- veor @XMM[6], @XMM[6], @XMM[14]
- mov r5, $rounds @ pass rounds
- veor @XMM[7], @XMM[7], @XMM[15]
- mov r0, sp
-
- bl _bsaes_decrypt8
-
- vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]!
- vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]!
- veor @XMM[0], @XMM[0], @XMM[ 8]
- vld1.64 {@XMM[12]-@XMM[13]}, [r0,:128]!
- veor @XMM[1], @XMM[1], @XMM[ 9]
- veor @XMM[8], @XMM[6], @XMM[10]
- vst1.8 {@XMM[0]-@XMM[1]}, [$out]!
- veor @XMM[9], @XMM[4], @XMM[11]
- vld1.64 {@XMM[14]-@XMM[15]}, [r0,:128]!
- veor @XMM[10], @XMM[2], @XMM[12]
- vst1.8 {@XMM[8]-@XMM[9]}, [$out]!
- veor @XMM[11], @XMM[7], @XMM[13]
- veor @XMM[12], @XMM[3], @XMM[14]
- vst1.8 {@XMM[10]-@XMM[11]}, [$out]!
- veor @XMM[13], @XMM[5], @XMM[15]
- vst1.8 {@XMM[12]-@XMM[13]}, [$out]!
-
- vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak
-
- subs $len, #0x80
- bpl .Lxts_dec_loop
-
-.Lxts_dec_short:
- adds $len, #0x70
- bmi .Lxts_dec_done
-
- vldmia $magic, {$twmask} @ load XTS magic
- vshr.s64 @T[0], @XMM[8], #63
- mov r0, sp
- vand @T[0], @T[0], $twmask
-___
-for($i=9;$i<16;$i++) {
-$code.=<<___;
- vadd.u64 @XMM[$i], @XMM[$i-1], @XMM[$i-1]
- vst1.64 {@XMM[$i-1]}, [r0,:128]!
- vswp `&Dhi("@T[0]")`,`&Dlo("@T[0]")`
- vshr.s64 @T[1], @XMM[$i], #63
- veor @XMM[$i], @XMM[$i], @T[0]
- vand @T[1], @T[1], $twmask
-___
- @T=reverse(@T);
-
-$code.=<<___ if ($i>=10);
- vld1.8 {@XMM[$i-10]}, [$inp]!
- subs $len, #0x10
- bmi .Lxts_dec_`$i-9`
-___
-$code.=<<___ if ($i>=11);
- veor @XMM[$i-11], @XMM[$i-11], @XMM[$i-3]
-___
-}
-$code.=<<___;
- sub $len, #0x10
- vst1.64 {@XMM[15]}, [r0,:128] @ next round tweak
-
- vld1.8 {@XMM[6]}, [$inp]!
- veor @XMM[5], @XMM[5], @XMM[13]
-#ifndef BSAES_ASM_EXTENDED_KEY
- add r4, sp, #0x90 @ pass key schedule
-#else
- add r4, $key, #248 @ pass key schedule
-#endif
- veor @XMM[6], @XMM[6], @XMM[14]
- mov r5, $rounds @ pass rounds
- mov r0, sp
-
- bl _bsaes_decrypt8
-
- vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]!
- vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]!
- veor @XMM[0], @XMM[0], @XMM[ 8]
- vld1.64 {@XMM[12]-@XMM[13]}, [r0,:128]!
- veor @XMM[1], @XMM[1], @XMM[ 9]
- veor @XMM[8], @XMM[6], @XMM[10]
- vst1.8 {@XMM[0]-@XMM[1]}, [$out]!
- veor @XMM[9], @XMM[4], @XMM[11]
- vld1.64 {@XMM[14]}, [r0,:128]!
- veor @XMM[10], @XMM[2], @XMM[12]
- vst1.8 {@XMM[8]-@XMM[9]}, [$out]!
- veor @XMM[11], @XMM[7], @XMM[13]
- veor @XMM[12], @XMM[3], @XMM[14]
- vst1.8 {@XMM[10]-@XMM[11]}, [$out]!
- vst1.8 {@XMM[12]}, [$out]!
-
- vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak
- b .Lxts_dec_done
-.align 4
-.Lxts_dec_6:
- vst1.64 {@XMM[14]}, [r0,:128] @ next round tweak
-
- veor @XMM[4], @XMM[4], @XMM[12]
-#ifndef BSAES_ASM_EXTENDED_KEY
- add r4, sp, #0x90 @ pass key schedule
-#else
- add r4, $key, #248 @ pass key schedule
-#endif
- veor @XMM[5], @XMM[5], @XMM[13]
- mov r5, $rounds @ pass rounds
- mov r0, sp
-
- bl _bsaes_decrypt8
-
- vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]!
- vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]!
- veor @XMM[0], @XMM[0], @XMM[ 8]
- vld1.64 {@XMM[12]-@XMM[13]}, [r0,:128]!
- veor @XMM[1], @XMM[1], @XMM[ 9]
- veor @XMM[8], @XMM[6], @XMM[10]
- vst1.8 {@XMM[0]-@XMM[1]}, [$out]!
- veor @XMM[9], @XMM[4], @XMM[11]
- veor @XMM[10], @XMM[2], @XMM[12]
- vst1.8 {@XMM[8]-@XMM[9]}, [$out]!
- veor @XMM[11], @XMM[7], @XMM[13]
- vst1.8 {@XMM[10]-@XMM[11]}, [$out]!
-
- vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak
- b .Lxts_dec_done
-.align 4
-.Lxts_dec_5:
- vst1.64 {@XMM[13]}, [r0,:128] @ next round tweak
-
- veor @XMM[3], @XMM[3], @XMM[11]
-#ifndef BSAES_ASM_EXTENDED_KEY
- add r4, sp, #0x90 @ pass key schedule
-#else
- add r4, $key, #248 @ pass key schedule
-#endif
- veor @XMM[4], @XMM[4], @XMM[12]
- mov r5, $rounds @ pass rounds
- mov r0, sp
-
- bl _bsaes_decrypt8
-
- vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]!
- vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]!
- veor @XMM[0], @XMM[0], @XMM[ 8]
- vld1.64 {@XMM[12]}, [r0,:128]!
- veor @XMM[1], @XMM[1], @XMM[ 9]
- veor @XMM[8], @XMM[6], @XMM[10]
- vst1.8 {@XMM[0]-@XMM[1]}, [$out]!
- veor @XMM[9], @XMM[4], @XMM[11]
- veor @XMM[10], @XMM[2], @XMM[12]
- vst1.8 {@XMM[8]-@XMM[9]}, [$out]!
- vst1.8 {@XMM[10]}, [$out]!
-
- vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak
- b .Lxts_dec_done
-.align 4
-.Lxts_dec_4:
- vst1.64 {@XMM[12]}, [r0,:128] @ next round tweak
-
- veor @XMM[2], @XMM[2], @XMM[10]
-#ifndef BSAES_ASM_EXTENDED_KEY
- add r4, sp, #0x90 @ pass key schedule
-#else
- add r4, $key, #248 @ pass key schedule
-#endif
- veor @XMM[3], @XMM[3], @XMM[11]
- mov r5, $rounds @ pass rounds
- mov r0, sp
-
- bl _bsaes_decrypt8
-
- vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]!
- vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]!
- veor @XMM[0], @XMM[0], @XMM[ 8]
- veor @XMM[1], @XMM[1], @XMM[ 9]
- veor @XMM[8], @XMM[6], @XMM[10]
- vst1.8 {@XMM[0]-@XMM[1]}, [$out]!
- veor @XMM[9], @XMM[4], @XMM[11]
- vst1.8 {@XMM[8]-@XMM[9]}, [$out]!
-
- vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak
- b .Lxts_dec_done
-.align 4
-.Lxts_dec_3:
- vst1.64 {@XMM[11]}, [r0,:128] @ next round tweak
-
- veor @XMM[1], @XMM[1], @XMM[9]
-#ifndef BSAES_ASM_EXTENDED_KEY
- add r4, sp, #0x90 @ pass key schedule
-#else
- add r4, $key, #248 @ pass key schedule
-#endif
- veor @XMM[2], @XMM[2], @XMM[10]
- mov r5, $rounds @ pass rounds
- mov r0, sp
-
- bl _bsaes_decrypt8
-
- vld1.64 {@XMM[8]-@XMM[9]}, [r0,:128]!
- vld1.64 {@XMM[10]}, [r0,:128]!
- veor @XMM[0], @XMM[0], @XMM[ 8]
- veor @XMM[1], @XMM[1], @XMM[ 9]
- veor @XMM[8], @XMM[6], @XMM[10]
- vst1.8 {@XMM[0]-@XMM[1]}, [$out]!
- vst1.8 {@XMM[8]}, [$out]!
-
- vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak
- b .Lxts_dec_done
-.align 4
-.Lxts_dec_2:
- vst1.64 {@XMM[10]}, [r0,:128] @ next round tweak
-
- veor @XMM[0], @XMM[0], @XMM[8]
-#ifndef BSAES_ASM_EXTENDED_KEY
- add r4, sp, #0x90 @ pass key schedule
-#else
- add r4, $key, #248 @ pass key schedule
-#endif
- veor @XMM[1], @XMM[1], @XMM[9]
- mov r5, $rounds @ pass rounds
- mov r0, sp
-
- bl _bsaes_decrypt8
-
- vld1.64 {@XMM[8]-@XMM[9]}, [r0,:128]!
- veor @XMM[0], @XMM[0], @XMM[ 8]
- veor @XMM[1], @XMM[1], @XMM[ 9]
- vst1.8 {@XMM[0]-@XMM[1]}, [$out]!
-
- vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak
- b .Lxts_dec_done
-.align 4
-.Lxts_dec_1:
- mov r0, sp
- veor @XMM[0], @XMM[8]
- mov r1, sp
- vst1.8 {@XMM[0]}, [sp,:128]
- mov r2, $key
- mov r4, $fp @ preserve fp
- mov r5, $magic @ preserve magic
-
- bl AES_decrypt
-
- vld1.8 {@XMM[0]}, [sp,:128]
- veor @XMM[0], @XMM[0], @XMM[8]
- vst1.8 {@XMM[0]}, [$out]!
- mov $fp, r4
- mov $magic, r5
-
- vmov @XMM[8], @XMM[9] @ next round tweak
-
-.Lxts_dec_done:
-#ifndef XTS_CHAIN_TWEAK
- adds $len, #0x10
- beq .Lxts_dec_ret
-
- @ calculate one round of extra tweak for the stolen ciphertext
- vldmia $magic, {$twmask}
- vshr.s64 @XMM[6], @XMM[8], #63
- vand @XMM[6], @XMM[6], $twmask
- vadd.u64 @XMM[9], @XMM[8], @XMM[8]
- vswp `&Dhi("@XMM[6]")`,`&Dlo("@XMM[6]")`
- veor @XMM[9], @XMM[9], @XMM[6]
-
- @ perform the final decryption with the last tweak value
- vld1.8 {@XMM[0]}, [$inp]!
- mov r0, sp
- veor @XMM[0], @XMM[0], @XMM[9]
- mov r1, sp
- vst1.8 {@XMM[0]}, [sp,:128]
- mov r2, $key
- mov r4, $fp @ preserve fp
-
- bl AES_decrypt
-
- vld1.8 {@XMM[0]}, [sp,:128]
- veor @XMM[0], @XMM[0], @XMM[9]
- vst1.8 {@XMM[0]}, [$out]
-
- mov r6, $out
-.Lxts_dec_steal:
- ldrb r1, [$out]
- ldrb r0, [$inp], #1
- strb r1, [$out, #0x10]
- strb r0, [$out], #1
-
- subs $len, #1
- bhi .Lxts_dec_steal
-
- vld1.8 {@XMM[0]}, [r6]
- mov r0, sp
- veor @XMM[0], @XMM[8]
- mov r1, sp
- vst1.8 {@XMM[0]}, [sp,:128]
- mov r2, $key
-
- bl AES_decrypt
-
- vld1.8 {@XMM[0]}, [sp,:128]
- veor @XMM[0], @XMM[0], @XMM[8]
- vst1.8 {@XMM[0]}, [r6]
- mov $fp, r4
-#endif
-
-.Lxts_dec_ret:
- bic r0, $fp, #0xf
- vmov.i32 q0, #0
- vmov.i32 q1, #0
-#ifdef XTS_CHAIN_TWEAK
- ldr r1, [$fp, #0x20+VFP_ABI_FRAME] @ chain tweak
-#endif
-.Lxts_dec_bzero: @ wipe key schedule [if any]
- vstmia sp!, {q0-q1}
- cmp sp, r0
- bne .Lxts_dec_bzero
-
- mov sp, $fp
-#ifdef XTS_CHAIN_TWEAK
- vst1.8 {@XMM[8]}, [r1]
-#endif
- VFP_ABI_POP
- ldmia sp!, {r4-r10, pc} @ return
-
-.size bsaes_xts_decrypt,.-bsaes_xts_decrypt
-___
-}
-$code.=<<___;
-#endif
-___
-
-$code =~ s/\`([^\`]*)\`/eval($1)/gem;
-
-open SELF,$0;
-while(<SELF>) {
- next if (/^#!/);
- last if (!s/^#/@/ and !/^$/);
- print;
-}
-close SELF;
-
-print $code;
-
-close STDOUT;
diff --git a/arch/arm/crypto/chacha20-neon-core.S b/arch/arm/crypto/chacha20-neon-core.S
new file mode 100644
index 000000000000..3fecb2124c35
--- /dev/null
+++ b/arch/arm/crypto/chacha20-neon-core.S
@@ -0,0 +1,523 @@
+/*
+ * ChaCha20 256-bit cipher algorithm, RFC7539, ARM NEON functions
+ *
+ * Copyright (C) 2016 Linaro, Ltd. <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * Based on:
+ * ChaCha20 256-bit cipher algorithm, RFC7539, x64 SSE3 functions
+ *
+ * Copyright (C) 2015 Martin Willi
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <linux/linkage.h>
+
+ .text
+ .fpu neon
+ .align 5
+
+ENTRY(chacha20_block_xor_neon)
+ // r0: Input state matrix, s
+ // r1: 1 data block output, o
+ // r2: 1 data block input, i
+
+ //
+ // This function encrypts one ChaCha20 block by loading the state matrix
+ // in four NEON registers. It performs matrix operation on four words in
+ // parallel, but requireds shuffling to rearrange the words after each
+ // round.
+ //
+
+ // x0..3 = s0..3
+ add ip, r0, #0x20
+ vld1.32 {q0-q1}, [r0]
+ vld1.32 {q2-q3}, [ip]
+
+ vmov q8, q0
+ vmov q9, q1
+ vmov q10, q2
+ vmov q11, q3
+
+ mov r3, #10
+
+.Ldoubleround:
+ // x0 += x1, x3 = rotl32(x3 ^ x0, 16)
+ vadd.i32 q0, q0, q1
+ veor q4, q3, q0
+ vshl.u32 q3, q4, #16
+ vsri.u32 q3, q4, #16
+
+ // x2 += x3, x1 = rotl32(x1 ^ x2, 12)
+ vadd.i32 q2, q2, q3
+ veor q4, q1, q2
+ vshl.u32 q1, q4, #12
+ vsri.u32 q1, q4, #20
+
+ // x0 += x1, x3 = rotl32(x3 ^ x0, 8)
+ vadd.i32 q0, q0, q1
+ veor q4, q3, q0
+ vshl.u32 q3, q4, #8
+ vsri.u32 q3, q4, #24
+
+ // x2 += x3, x1 = rotl32(x1 ^ x2, 7)
+ vadd.i32 q2, q2, q3
+ veor q4, q1, q2
+ vshl.u32 q1, q4, #7
+ vsri.u32 q1, q4, #25
+
+ // x1 = shuffle32(x1, MASK(0, 3, 2, 1))
+ vext.8 q1, q1, q1, #4
+ // x2 = shuffle32(x2, MASK(1, 0, 3, 2))
+ vext.8 q2, q2, q2, #8
+ // x3 = shuffle32(x3, MASK(2, 1, 0, 3))
+ vext.8 q3, q3, q3, #12
+
+ // x0 += x1, x3 = rotl32(x3 ^ x0, 16)
+ vadd.i32 q0, q0, q1
+ veor q4, q3, q0
+ vshl.u32 q3, q4, #16
+ vsri.u32 q3, q4, #16
+
+ // x2 += x3, x1 = rotl32(x1 ^ x2, 12)
+ vadd.i32 q2, q2, q3
+ veor q4, q1, q2
+ vshl.u32 q1, q4, #12
+ vsri.u32 q1, q4, #20
+
+ // x0 += x1, x3 = rotl32(x3 ^ x0, 8)
+ vadd.i32 q0, q0, q1
+ veor q4, q3, q0
+ vshl.u32 q3, q4, #8
+ vsri.u32 q3, q4, #24
+
+ // x2 += x3, x1 = rotl32(x1 ^ x2, 7)
+ vadd.i32 q2, q2, q3
+ veor q4, q1, q2
+ vshl.u32 q1, q4, #7
+ vsri.u32 q1, q4, #25
+
+ // x1 = shuffle32(x1, MASK(2, 1, 0, 3))
+ vext.8 q1, q1, q1, #12
+ // x2 = shuffle32(x2, MASK(1, 0, 3, 2))
+ vext.8 q2, q2, q2, #8
+ // x3 = shuffle32(x3, MASK(0, 3, 2, 1))
+ vext.8 q3, q3, q3, #4
+
+ subs r3, r3, #1
+ bne .Ldoubleround
+
+ add ip, r2, #0x20
+ vld1.8 {q4-q5}, [r2]
+ vld1.8 {q6-q7}, [ip]
+
+ // o0 = i0 ^ (x0 + s0)
+ vadd.i32 q0, q0, q8
+ veor q0, q0, q4
+
+ // o1 = i1 ^ (x1 + s1)
+ vadd.i32 q1, q1, q9
+ veor q1, q1, q5
+
+ // o2 = i2 ^ (x2 + s2)
+ vadd.i32 q2, q2, q10
+ veor q2, q2, q6
+
+ // o3 = i3 ^ (x3 + s3)
+ vadd.i32 q3, q3, q11
+ veor q3, q3, q7
+
+ add ip, r1, #0x20
+ vst1.8 {q0-q1}, [r1]
+ vst1.8 {q2-q3}, [ip]
+
+ bx lr
+ENDPROC(chacha20_block_xor_neon)
+
+ .align 5
+ENTRY(chacha20_4block_xor_neon)
+ push {r4-r6, lr}
+ mov ip, sp // preserve the stack pointer
+ sub r3, sp, #0x20 // allocate a 32 byte buffer
+ bic r3, r3, #0x1f // aligned to 32 bytes
+ mov sp, r3
+
+ // r0: Input state matrix, s
+ // r1: 4 data blocks output, o
+ // r2: 4 data blocks input, i
+
+ //
+ // This function encrypts four consecutive ChaCha20 blocks by loading
+ // the state matrix in NEON registers four times. The algorithm performs
+ // each operation on the corresponding word of each state matrix, hence
+ // requires no word shuffling. For final XORing step we transpose the
+ // matrix by interleaving 32- and then 64-bit words, which allows us to
+ // do XOR in NEON registers.
+ //
+
+ // x0..15[0-3] = s0..3[0..3]
+ add r3, r0, #0x20
+ vld1.32 {q0-q1}, [r0]
+ vld1.32 {q2-q3}, [r3]
+
+ adr r3, CTRINC
+ vdup.32 q15, d7[1]
+ vdup.32 q14, d7[0]
+ vld1.32 {q11}, [r3, :128]
+ vdup.32 q13, d6[1]
+ vdup.32 q12, d6[0]
+ vadd.i32 q12, q12, q11 // x12 += counter values 0-3
+ vdup.32 q11, d5[1]
+ vdup.32 q10, d5[0]
+ vdup.32 q9, d4[1]
+ vdup.32 q8, d4[0]
+ vdup.32 q7, d3[1]
+ vdup.32 q6, d3[0]
+ vdup.32 q5, d2[1]
+ vdup.32 q4, d2[0]
+ vdup.32 q3, d1[1]
+ vdup.32 q2, d1[0]
+ vdup.32 q1, d0[1]
+ vdup.32 q0, d0[0]
+
+ mov r3, #10
+
+.Ldoubleround4:
+ // x0 += x4, x12 = rotl32(x12 ^ x0, 16)
+ // x1 += x5, x13 = rotl32(x13 ^ x1, 16)
+ // x2 += x6, x14 = rotl32(x14 ^ x2, 16)
+ // x3 += x7, x15 = rotl32(x15 ^ x3, 16)
+ vadd.i32 q0, q0, q4
+ vadd.i32 q1, q1, q5
+ vadd.i32 q2, q2, q6
+ vadd.i32 q3, q3, q7
+
+ veor q12, q12, q0
+ veor q13, q13, q1
+ veor q14, q14, q2
+ veor q15, q15, q3
+
+ vrev32.16 q12, q12
+ vrev32.16 q13, q13
+ vrev32.16 q14, q14
+ vrev32.16 q15, q15
+
+ // x8 += x12, x4 = rotl32(x4 ^ x8, 12)
+ // x9 += x13, x5 = rotl32(x5 ^ x9, 12)
+ // x10 += x14, x6 = rotl32(x6 ^ x10, 12)
+ // x11 += x15, x7 = rotl32(x7 ^ x11, 12)
+ vadd.i32 q8, q8, q12
+ vadd.i32 q9, q9, q13
+ vadd.i32 q10, q10, q14
+ vadd.i32 q11, q11, q15
+
+ vst1.32 {q8-q9}, [sp, :256]
+
+ veor q8, q4, q8
+ veor q9, q5, q9
+ vshl.u32 q4, q8, #12
+ vshl.u32 q5, q9, #12
+ vsri.u32 q4, q8, #20
+ vsri.u32 q5, q9, #20
+
+ veor q8, q6, q10
+ veor q9, q7, q11
+ vshl.u32 q6, q8, #12
+ vshl.u32 q7, q9, #12
+ vsri.u32 q6, q8, #20
+ vsri.u32 q7, q9, #20
+
+ // x0 += x4, x12 = rotl32(x12 ^ x0, 8)
+ // x1 += x5, x13 = rotl32(x13 ^ x1, 8)
+ // x2 += x6, x14 = rotl32(x14 ^ x2, 8)
+ // x3 += x7, x15 = rotl32(x15 ^ x3, 8)
+ vadd.i32 q0, q0, q4
+ vadd.i32 q1, q1, q5
+ vadd.i32 q2, q2, q6
+ vadd.i32 q3, q3, q7
+
+ veor q8, q12, q0
+ veor q9, q13, q1
+ vshl.u32 q12, q8, #8
+ vshl.u32 q13, q9, #8
+ vsri.u32 q12, q8, #24
+ vsri.u32 q13, q9, #24
+
+ veor q8, q14, q2
+ veor q9, q15, q3
+ vshl.u32 q14, q8, #8
+ vshl.u32 q15, q9, #8
+ vsri.u32 q14, q8, #24
+ vsri.u32 q15, q9, #24
+
+ vld1.32 {q8-q9}, [sp, :256]
+
+ // x8 += x12, x4 = rotl32(x4 ^ x8, 7)
+ // x9 += x13, x5 = rotl32(x5 ^ x9, 7)
+ // x10 += x14, x6 = rotl32(x6 ^ x10, 7)
+ // x11 += x15, x7 = rotl32(x7 ^ x11, 7)
+ vadd.i32 q8, q8, q12
+ vadd.i32 q9, q9, q13
+ vadd.i32 q10, q10, q14
+ vadd.i32 q11, q11, q15
+
+ vst1.32 {q8-q9}, [sp, :256]
+
+ veor q8, q4, q8
+ veor q9, q5, q9
+ vshl.u32 q4, q8, #7
+ vshl.u32 q5, q9, #7
+ vsri.u32 q4, q8, #25
+ vsri.u32 q5, q9, #25
+
+ veor q8, q6, q10
+ veor q9, q7, q11
+ vshl.u32 q6, q8, #7
+ vshl.u32 q7, q9, #7
+ vsri.u32 q6, q8, #25
+ vsri.u32 q7, q9, #25
+
+ vld1.32 {q8-q9}, [sp, :256]
+
+ // x0 += x5, x15 = rotl32(x15 ^ x0, 16)
+ // x1 += x6, x12 = rotl32(x12 ^ x1, 16)
+ // x2 += x7, x13 = rotl32(x13 ^ x2, 16)
+ // x3 += x4, x14 = rotl32(x14 ^ x3, 16)
+ vadd.i32 q0, q0, q5
+ vadd.i32 q1, q1, q6
+ vadd.i32 q2, q2, q7
+ vadd.i32 q3, q3, q4
+
+ veor q15, q15, q0
+ veor q12, q12, q1
+ veor q13, q13, q2
+ veor q14, q14, q3
+
+ vrev32.16 q15, q15
+ vrev32.16 q12, q12
+ vrev32.16 q13, q13
+ vrev32.16 q14, q14
+
+ // x10 += x15, x5 = rotl32(x5 ^ x10, 12)
+ // x11 += x12, x6 = rotl32(x6 ^ x11, 12)
+ // x8 += x13, x7 = rotl32(x7 ^ x8, 12)
+ // x9 += x14, x4 = rotl32(x4 ^ x9, 12)
+ vadd.i32 q10, q10, q15
+ vadd.i32 q11, q11, q12
+ vadd.i32 q8, q8, q13
+ vadd.i32 q9, q9, q14
+
+ vst1.32 {q8-q9}, [sp, :256]
+
+ veor q8, q7, q8
+ veor q9, q4, q9
+ vshl.u32 q7, q8, #12
+ vshl.u32 q4, q9, #12
+ vsri.u32 q7, q8, #20
+ vsri.u32 q4, q9, #20
+
+ veor q8, q5, q10
+ veor q9, q6, q11
+ vshl.u32 q5, q8, #12
+ vshl.u32 q6, q9, #12
+ vsri.u32 q5, q8, #20
+ vsri.u32 q6, q9, #20
+
+ // x0 += x5, x15 = rotl32(x15 ^ x0, 8)
+ // x1 += x6, x12 = rotl32(x12 ^ x1, 8)
+ // x2 += x7, x13 = rotl32(x13 ^ x2, 8)
+ // x3 += x4, x14 = rotl32(x14 ^ x3, 8)
+ vadd.i32 q0, q0, q5
+ vadd.i32 q1, q1, q6
+ vadd.i32 q2, q2, q7
+ vadd.i32 q3, q3, q4
+
+ veor q8, q15, q0
+ veor q9, q12, q1
+ vshl.u32 q15, q8, #8
+ vshl.u32 q12, q9, #8
+ vsri.u32 q15, q8, #24
+ vsri.u32 q12, q9, #24
+
+ veor q8, q13, q2
+ veor q9, q14, q3
+ vshl.u32 q13, q8, #8
+ vshl.u32 q14, q9, #8
+ vsri.u32 q13, q8, #24
+ vsri.u32 q14, q9, #24
+
+ vld1.32 {q8-q9}, [sp, :256]
+
+ // x10 += x15, x5 = rotl32(x5 ^ x10, 7)
+ // x11 += x12, x6 = rotl32(x6 ^ x11, 7)
+ // x8 += x13, x7 = rotl32(x7 ^ x8, 7)
+ // x9 += x14, x4 = rotl32(x4 ^ x9, 7)
+ vadd.i32 q10, q10, q15
+ vadd.i32 q11, q11, q12
+ vadd.i32 q8, q8, q13
+ vadd.i32 q9, q9, q14
+
+ vst1.32 {q8-q9}, [sp, :256]
+
+ veor q8, q7, q8
+ veor q9, q4, q9
+ vshl.u32 q7, q8, #7
+ vshl.u32 q4, q9, #7
+ vsri.u32 q7, q8, #25
+ vsri.u32 q4, q9, #25
+
+ veor q8, q5, q10
+ veor q9, q6, q11
+ vshl.u32 q5, q8, #7
+ vshl.u32 q6, q9, #7
+ vsri.u32 q5, q8, #25
+ vsri.u32 q6, q9, #25
+
+ subs r3, r3, #1
+ beq 0f
+
+ vld1.32 {q8-q9}, [sp, :256]
+ b .Ldoubleround4
+
+ // x0[0-3] += s0[0]
+ // x1[0-3] += s0[1]
+ // x2[0-3] += s0[2]
+ // x3[0-3] += s0[3]
+0: ldmia r0!, {r3-r6}
+ vdup.32 q8, r3
+ vdup.32 q9, r4
+ vadd.i32 q0, q0, q8
+ vadd.i32 q1, q1, q9
+ vdup.32 q8, r5
+ vdup.32 q9, r6
+ vadd.i32 q2, q2, q8
+ vadd.i32 q3, q3, q9
+
+ // x4[0-3] += s1[0]
+ // x5[0-3] += s1[1]
+ // x6[0-3] += s1[2]
+ // x7[0-3] += s1[3]
+ ldmia r0!, {r3-r6}
+ vdup.32 q8, r3
+ vdup.32 q9, r4
+ vadd.i32 q4, q4, q8
+ vadd.i32 q5, q5, q9
+ vdup.32 q8, r5
+ vdup.32 q9, r6
+ vadd.i32 q6, q6, q8
+ vadd.i32 q7, q7, q9
+
+ // interleave 32-bit words in state n, n+1
+ vzip.32 q0, q1
+ vzip.32 q2, q3
+ vzip.32 q4, q5
+ vzip.32 q6, q7
+
+ // interleave 64-bit words in state n, n+2
+ vswp d1, d4
+ vswp d3, d6
+ vswp d9, d12
+ vswp d11, d14
+
+ // xor with corresponding input, write to output
+ vld1.8 {q8-q9}, [r2]!
+ veor q8, q8, q0
+ veor q9, q9, q4
+ vst1.8 {q8-q9}, [r1]!
+
+ vld1.32 {q8-q9}, [sp, :256]
+
+ // x8[0-3] += s2[0]
+ // x9[0-3] += s2[1]
+ // x10[0-3] += s2[2]
+ // x11[0-3] += s2[3]
+ ldmia r0!, {r3-r6}
+ vdup.32 q0, r3
+ vdup.32 q4, r4
+ vadd.i32 q8, q8, q0
+ vadd.i32 q9, q9, q4
+ vdup.32 q0, r5
+ vdup.32 q4, r6
+ vadd.i32 q10, q10, q0
+ vadd.i32 q11, q11, q4
+
+ // x12[0-3] += s3[0]
+ // x13[0-3] += s3[1]
+ // x14[0-3] += s3[2]
+ // x15[0-3] += s3[3]
+ ldmia r0!, {r3-r6}
+ vdup.32 q0, r3
+ vdup.32 q4, r4
+ adr r3, CTRINC
+ vadd.i32 q12, q12, q0
+ vld1.32 {q0}, [r3, :128]
+ vadd.i32 q13, q13, q4
+ vadd.i32 q12, q12, q0 // x12 += counter values 0-3
+
+ vdup.32 q0, r5
+ vdup.32 q4, r6
+ vadd.i32 q14, q14, q0
+ vadd.i32 q15, q15, q4
+
+ // interleave 32-bit words in state n, n+1
+ vzip.32 q8, q9
+ vzip.32 q10, q11
+ vzip.32 q12, q13
+ vzip.32 q14, q15
+
+ // interleave 64-bit words in state n, n+2
+ vswp d17, d20
+ vswp d19, d22
+ vswp d25, d28
+ vswp d27, d30
+
+ vmov q4, q1
+
+ vld1.8 {q0-q1}, [r2]!
+ veor q0, q0, q8
+ veor q1, q1, q12
+ vst1.8 {q0-q1}, [r1]!
+
+ vld1.8 {q0-q1}, [r2]!
+ veor q0, q0, q2
+ veor q1, q1, q6
+ vst1.8 {q0-q1}, [r1]!
+
+ vld1.8 {q0-q1}, [r2]!
+ veor q0, q0, q10
+ veor q1, q1, q14
+ vst1.8 {q0-q1}, [r1]!
+
+ vld1.8 {q0-q1}, [r2]!
+ veor q0, q0, q4
+ veor q1, q1, q5
+ vst1.8 {q0-q1}, [r1]!
+
+ vld1.8 {q0-q1}, [r2]!
+ veor q0, q0, q9
+ veor q1, q1, q13
+ vst1.8 {q0-q1}, [r1]!
+
+ vld1.8 {q0-q1}, [r2]!
+ veor q0, q0, q3
+ veor q1, q1, q7
+ vst1.8 {q0-q1}, [r1]!
+
+ vld1.8 {q0-q1}, [r2]
+ veor q0, q0, q11
+ veor q1, q1, q15
+ vst1.8 {q0-q1}, [r1]
+
+ mov sp, ip
+ pop {r4-r6, pc}
+ENDPROC(chacha20_4block_xor_neon)
+
+ .align 4
+CTRINC: .word 0, 1, 2, 3
diff --git a/arch/arm/crypto/chacha20-neon-glue.c b/arch/arm/crypto/chacha20-neon-glue.c
new file mode 100644
index 000000000000..59a7be08e80c
--- /dev/null
+++ b/arch/arm/crypto/chacha20-neon-glue.c
@@ -0,0 +1,127 @@
+/*
+ * ChaCha20 256-bit cipher algorithm, RFC7539, ARM NEON functions
+ *
+ * Copyright (C) 2016 Linaro, Ltd. <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * Based on:
+ * ChaCha20 256-bit cipher algorithm, RFC7539, SIMD glue code
+ *
+ * Copyright (C) 2015 Martin Willi
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <crypto/algapi.h>
+#include <crypto/chacha20.h>
+#include <crypto/internal/skcipher.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+
+#include <asm/hwcap.h>
+#include <asm/neon.h>
+#include <asm/simd.h>
+
+asmlinkage void chacha20_block_xor_neon(u32 *state, u8 *dst, const u8 *src);
+asmlinkage void chacha20_4block_xor_neon(u32 *state, u8 *dst, const u8 *src);
+
+static void chacha20_doneon(u32 *state, u8 *dst, const u8 *src,
+ unsigned int bytes)
+{
+ u8 buf[CHACHA20_BLOCK_SIZE];
+
+ while (bytes >= CHACHA20_BLOCK_SIZE * 4) {
+ chacha20_4block_xor_neon(state, dst, src);
+ bytes -= CHACHA20_BLOCK_SIZE * 4;
+ src += CHACHA20_BLOCK_SIZE * 4;
+ dst += CHACHA20_BLOCK_SIZE * 4;
+ state[12] += 4;
+ }
+ while (bytes >= CHACHA20_BLOCK_SIZE) {
+ chacha20_block_xor_neon(state, dst, src);
+ bytes -= CHACHA20_BLOCK_SIZE;
+ src += CHACHA20_BLOCK_SIZE;
+ dst += CHACHA20_BLOCK_SIZE;
+ state[12]++;
+ }
+ if (bytes) {
+ memcpy(buf, src, bytes);
+ chacha20_block_xor_neon(state, buf, buf);
+ memcpy(dst, buf, bytes);
+ }
+}
+
+static int chacha20_neon(struct skcipher_request *req)
+{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+ struct chacha20_ctx *ctx = crypto_skcipher_ctx(tfm);
+ struct skcipher_walk walk;
+ u32 state[16];
+ int err;
+
+ if (req->cryptlen <= CHACHA20_BLOCK_SIZE || !may_use_simd())
+ return crypto_chacha20_crypt(req);
+
+ err = skcipher_walk_virt(&walk, req, true);
+
+ crypto_chacha20_init(state, ctx, walk.iv);
+
+ kernel_neon_begin();
+ while (walk.nbytes > 0) {
+ unsigned int nbytes = walk.nbytes;
+
+ if (nbytes < walk.total)
+ nbytes = round_down(nbytes, walk.stride);
+
+ chacha20_doneon(state, walk.dst.virt.addr, walk.src.virt.addr,
+ nbytes);
+ err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
+ }
+ kernel_neon_end();
+
+ return err;
+}
+
+static struct skcipher_alg alg = {
+ .base.cra_name = "chacha20",
+ .base.cra_driver_name = "chacha20-neon",
+ .base.cra_priority = 300,
+ .base.cra_blocksize = 1,
+ .base.cra_ctxsize = sizeof(struct chacha20_ctx),
+ .base.cra_module = THIS_MODULE,
+
+ .min_keysize = CHACHA20_KEY_SIZE,
+ .max_keysize = CHACHA20_KEY_SIZE,
+ .ivsize = CHACHA20_IV_SIZE,
+ .chunksize = CHACHA20_BLOCK_SIZE,
+ .walksize = 4 * CHACHA20_BLOCK_SIZE,
+ .setkey = crypto_chacha20_setkey,
+ .encrypt = chacha20_neon,
+ .decrypt = chacha20_neon,
+};
+
+static int __init chacha20_simd_mod_init(void)
+{
+ if (!(elf_hwcap & HWCAP_NEON))
+ return -ENODEV;
+
+ return crypto_register_skcipher(&alg);
+}
+
+static void __exit chacha20_simd_mod_fini(void)
+{
+ crypto_unregister_skcipher(&alg);
+}
+
+module_init(chacha20_simd_mod_init);
+module_exit(chacha20_simd_mod_fini);
+
+MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS_CRYPTO("chacha20");
diff --git a/arch/arm/include/asm/Kbuild b/arch/arm/include/asm/Kbuild
index efb21757d41f..b14e8c7d71bd 100644
--- a/arch/arm/include/asm/Kbuild
+++ b/arch/arm/include/asm/Kbuild
@@ -2,7 +2,6 @@
generic-y += bitsperlong.h
generic-y += clkdev.h
-generic-y += cputime.h
generic-y += current.h
generic-y += early_ioremap.h
generic-y += emergency-restart.h
diff --git a/arch/arm/include/asm/cacheflush.h b/arch/arm/include/asm/cacheflush.h
index bdd283bc5842..02454fa15d2c 100644
--- a/arch/arm/include/asm/cacheflush.h
+++ b/arch/arm/include/asm/cacheflush.h
@@ -490,7 +490,7 @@ static inline int set_memory_x(unsigned long addr, int numpages) { return 0; }
static inline int set_memory_nx(unsigned long addr, int numpages) { return 0; }
#endif
-#ifdef CONFIG_DEBUG_RODATA
+#ifdef CONFIG_STRICT_KERNEL_RWX
void set_kernel_text_rw(void);
void set_kernel_text_ro(void);
#else
diff --git a/arch/arm/include/asm/cputype.h b/arch/arm/include/asm/cputype.h
index 522b5feb4eaa..b62eaeb147aa 100644
--- a/arch/arm/include/asm/cputype.h
+++ b/arch/arm/include/asm/cputype.h
@@ -94,6 +94,9 @@
#define ARM_CPU_XSCALE_ARCH_V2 0x4000
#define ARM_CPU_XSCALE_ARCH_V3 0x6000
+/* Qualcomm implemented cores */
+#define ARM_CPU_PART_SCORPION 0x510002d0
+
extern unsigned int processor_id;
#ifdef CONFIG_CPU_CP15
diff --git a/arch/arm/include/asm/efi.h b/arch/arm/include/asm/efi.h
index 0b06f5341b45..e4e6a9d6a825 100644
--- a/arch/arm/include/asm/efi.h
+++ b/arch/arm/include/asm/efi.h
@@ -55,6 +55,7 @@ void efi_virtmap_unload(void);
#define efi_call_early(f, ...) sys_table_arg->boottime->f(__VA_ARGS__)
#define __efi_call_early(f, ...) f(__VA_ARGS__)
+#define efi_call_runtime(f, ...) sys_table_arg->runtime->f(__VA_ARGS__)
#define efi_is_64bit() (false)
#define efi_call_proto(protocol, f, instance, ...) \
diff --git a/arch/arm/include/asm/ftrace.h b/arch/arm/include/asm/ftrace.h
index bfe2a2f5a644..22b73112b75f 100644
--- a/arch/arm/include/asm/ftrace.h
+++ b/arch/arm/include/asm/ftrace.h
@@ -54,6 +54,24 @@ static inline void *return_address(unsigned int level)
#define ftrace_return_address(n) return_address(n)
+#define ARCH_HAS_SYSCALL_MATCH_SYM_NAME
+
+static inline bool arch_syscall_match_sym_name(const char *sym,
+ const char *name)
+{
+ if (!strcmp(sym, "sys_mmap2"))
+ sym = "sys_mmap_pgoff";
+ else if (!strcmp(sym, "sys_statfs64_wrapper"))
+ sym = "sys_statfs64";
+ else if (!strcmp(sym, "sys_fstatfs64_wrapper"))
+ sym = "sys_fstatfs64";
+ else if (!strcmp(sym, "sys_arm_fadvise64_64"))
+ sym = "sys_fadvise64_64";
+
+ /* Ignore case since sym may start with "SyS" instead of "sys" */
+ return !strcasecmp(sym, name);
+}
+
#endif /* ifndef __ASSEMBLY__ */
#endif /* _ASM_ARM_FTRACE */
diff --git a/arch/arm/include/asm/kvm_host.h b/arch/arm/include/asm/kvm_host.h
index d5423ab15ed5..cc495d799c67 100644
--- a/arch/arm/include/asm/kvm_host.h
+++ b/arch/arm/include/asm/kvm_host.h
@@ -60,9 +60,6 @@ struct kvm_arch {
/* The last vcpu id that ran on each physical CPU */
int __percpu *last_vcpu_ran;
- /* Timer */
- struct arch_timer_kvm timer;
-
/*
* Anything that is not used directly from assembly code goes
* here.
diff --git a/arch/arm/include/asm/kvm_mmu.h b/arch/arm/include/asm/kvm_mmu.h
index 74a44727f8e1..95f38dcd611d 100644
--- a/arch/arm/include/asm/kvm_mmu.h
+++ b/arch/arm/include/asm/kvm_mmu.h
@@ -129,8 +129,7 @@ static inline bool vcpu_has_cache_enabled(struct kvm_vcpu *vcpu)
static inline void __coherent_cache_guest_page(struct kvm_vcpu *vcpu,
kvm_pfn_t pfn,
- unsigned long size,
- bool ipa_uncached)
+ unsigned long size)
{
/*
* If we are going to insert an instruction page and the icache is
@@ -150,18 +149,12 @@ static inline void __coherent_cache_guest_page(struct kvm_vcpu *vcpu,
* and iterate over the range.
*/
- bool need_flush = !vcpu_has_cache_enabled(vcpu) || ipa_uncached;
-
VM_BUG_ON(size & ~PAGE_MASK);
- if (!need_flush && !icache_is_pipt())
- goto vipt_cache;
-
while (size) {
void *va = kmap_atomic_pfn(pfn);
- if (need_flush)
- kvm_flush_dcache_to_poc(va, PAGE_SIZE);
+ kvm_flush_dcache_to_poc(va, PAGE_SIZE);
if (icache_is_pipt())
__cpuc_coherent_user_range((unsigned long)va,
@@ -173,7 +166,6 @@ static inline void __coherent_cache_guest_page(struct kvm_vcpu *vcpu,
kunmap_atomic(va);
}
-vipt_cache:
if (!icache_is_pipt() && !icache_is_vivt_asid_tagged()) {
/* any kind of VIPT cache */
__flush_icache_all();
diff --git a/arch/arm/include/asm/uaccess.h b/arch/arm/include/asm/uaccess.h
index 1f59ea051bab..b7e0125c0bbf 100644
--- a/arch/arm/include/asm/uaccess.h
+++ b/arch/arm/include/asm/uaccess.h
@@ -478,11 +478,10 @@ extern unsigned long __must_check
arm_copy_from_user(void *to, const void __user *from, unsigned long n);
static inline unsigned long __must_check
-__copy_from_user(void *to, const void __user *from, unsigned long n)
+__arch_copy_from_user(void *to, const void __user *from, unsigned long n)
{
unsigned int __ua_flags;
- check_object_size(to, n, false);
__ua_flags = uaccess_save_and_enable();
n = arm_copy_from_user(to, from, n);
uaccess_restore(__ua_flags);
@@ -495,18 +494,15 @@ extern unsigned long __must_check
__copy_to_user_std(void __user *to, const void *from, unsigned long n);
static inline unsigned long __must_check
-__copy_to_user(void __user *to, const void *from, unsigned long n)
+__arch_copy_to_user(void __user *to, const void *from, unsigned long n)
{
#ifndef CONFIG_UACCESS_WITH_MEMCPY
unsigned int __ua_flags;
-
- check_object_size(from, n, true);
__ua_flags = uaccess_save_and_enable();
n = arm_copy_to_user(to, from, n);
uaccess_restore(__ua_flags);
return n;
#else
- check_object_size(from, n, true);
return arm_copy_to_user(to, from, n);
#endif
}
@@ -526,25 +522,49 @@ __clear_user(void __user *addr, unsigned long n)
}
#else
-#define __copy_from_user(to, from, n) (memcpy(to, (void __force *)from, n), 0)
-#define __copy_to_user(to, from, n) (memcpy((void __force *)to, from, n), 0)
+#define __arch_copy_from_user(to, from, n) \
+ (memcpy(to, (void __force *)from, n), 0)
+#define __arch_copy_to_user(to, from, n) \
+ (memcpy((void __force *)to, from, n), 0)
#define __clear_user(addr, n) (memset((void __force *)addr, 0, n), 0)
#endif
-static inline unsigned long __must_check copy_from_user(void *to, const void __user *from, unsigned long n)
+static inline unsigned long __must_check
+__copy_from_user(void *to, const void __user *from, unsigned long n)
+{
+ check_object_size(to, n, false);
+ return __arch_copy_from_user(to, from, n);
+}
+
+static inline unsigned long __must_check
+copy_from_user(void *to, const void __user *from, unsigned long n)
{
unsigned long res = n;
+
+ check_object_size(to, n, false);
+
if (likely(access_ok(VERIFY_READ, from, n)))
- res = __copy_from_user(to, from, n);
+ res = __arch_copy_from_user(to, from, n);
if (unlikely(res))
memset(to + (n - res), 0, res);
return res;
}
-static inline unsigned long __must_check copy_to_user(void __user *to, const void *from, unsigned long n)
+static inline unsigned long __must_check
+__copy_to_user(void __user *to, const void *from, unsigned long n)
{
+ check_object_size(from, n, true);
+
+ return __arch_copy_to_user(to, from, n);
+}
+
+static inline unsigned long __must_check
+copy_to_user(void __user *to, const void *from, unsigned long n)
+{
+ check_object_size(from, n, true);
+
if (access_ok(VERIFY_WRITE, to, n))
- n = __copy_to_user(to, from, n);
+ n = __arch_copy_to_user(to, from, n);
return n;
}
diff --git a/arch/arm/include/asm/virt.h b/arch/arm/include/asm/virt.h
index a2e75b84e2ae..6dae1956c74d 100644
--- a/arch/arm/include/asm/virt.h
+++ b/arch/arm/include/asm/virt.h
@@ -80,6 +80,11 @@ static inline bool is_kernel_in_hyp_mode(void)
return false;
}
+static inline bool has_vhe(void)
+{
+ return false;
+}
+
/* The section containing the hypervisor idmap text */
extern char __hyp_idmap_text_start[];
extern char __hyp_idmap_text_end[];
diff --git a/arch/arm/include/uapi/asm/kvm.h b/arch/arm/include/uapi/asm/kvm.h
index af05f8e0903e..6ebd3e6a1fd1 100644
--- a/arch/arm/include/uapi/asm/kvm.h
+++ b/arch/arm/include/uapi/asm/kvm.h
@@ -181,10 +181,23 @@ struct kvm_arch_memory_slot {
#define KVM_DEV_ARM_VGIC_GRP_CPU_REGS 2
#define KVM_DEV_ARM_VGIC_CPUID_SHIFT 32
#define KVM_DEV_ARM_VGIC_CPUID_MASK (0xffULL << KVM_DEV_ARM_VGIC_CPUID_SHIFT)
+#define KVM_DEV_ARM_VGIC_V3_MPIDR_SHIFT 32
+#define KVM_DEV_ARM_VGIC_V3_MPIDR_MASK \
+ (0xffffffffULL << KVM_DEV_ARM_VGIC_V3_MPIDR_SHIFT)
#define KVM_DEV_ARM_VGIC_OFFSET_SHIFT 0
#define KVM_DEV_ARM_VGIC_OFFSET_MASK (0xffffffffULL << KVM_DEV_ARM_VGIC_OFFSET_SHIFT)
+#define KVM_DEV_ARM_VGIC_SYSREG_INSTR_MASK (0xffff)
#define KVM_DEV_ARM_VGIC_GRP_NR_IRQS 3
#define KVM_DEV_ARM_VGIC_GRP_CTRL 4
+#define KVM_DEV_ARM_VGIC_GRP_REDIST_REGS 5
+#define KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS 6
+#define KVM_DEV_ARM_VGIC_GRP_LEVEL_INFO 7
+#define KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_SHIFT 10
+#define KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_MASK \
+ (0x3fffffULL << KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_SHIFT)
+#define KVM_DEV_ARM_VGIC_LINE_LEVEL_INTID_MASK 0x3ff
+#define VGIC_LEVEL_INFO_LINE_LEVEL 0
+
#define KVM_DEV_ARM_VGIC_CTRL_INIT 0
/* KVM_IRQ_LINE irq field index values */
diff --git a/arch/arm/include/asm/types.h b/arch/arm/include/uapi/asm/types.h
index a53cdb8f068c..9435a42f575e 100644
--- a/arch/arm/include/asm/types.h
+++ b/arch/arm/include/uapi/asm/types.h
@@ -1,5 +1,5 @@
-#ifndef _ASM_TYPES_H
-#define _ASM_TYPES_H
+#ifndef _UAPI_ASM_TYPES_H
+#define _UAPI_ASM_TYPES_H
#include <asm-generic/int-ll64.h>
@@ -37,4 +37,4 @@
#define __UINTPTR_TYPE__ unsigned long
#endif
-#endif /* _ASM_TYPES_H */
+#endif /* _UAPI_ASM_TYPES_H */
diff --git a/arch/arm/kernel/armksyms.c b/arch/arm/kernel/armksyms.c
index 7e45f69a0ddc..8e8d20cdbce7 100644
--- a/arch/arm/kernel/armksyms.c
+++ b/arch/arm/kernel/armksyms.c
@@ -178,6 +178,6 @@ EXPORT_SYMBOL(__pv_offset);
#endif
#ifdef CONFIG_HAVE_ARM_SMCCC
-EXPORT_SYMBOL(arm_smccc_smc);
-EXPORT_SYMBOL(arm_smccc_hvc);
+EXPORT_SYMBOL(__arm_smccc_smc);
+EXPORT_SYMBOL(__arm_smccc_hvc);
#endif
diff --git a/arch/arm/kernel/hw_breakpoint.c b/arch/arm/kernel/hw_breakpoint.c
index 188180b5523d..be3b3fbd382f 100644
--- a/arch/arm/kernel/hw_breakpoint.c
+++ b/arch/arm/kernel/hw_breakpoint.c
@@ -1063,6 +1063,22 @@ static int __init arch_hw_breakpoint_init(void)
return 0;
}
+ /*
+ * Scorpion CPUs (at least those in APQ8060) seem to set DBGPRSR.SPD
+ * whenever a WFI is issued, even if the core is not powered down, in
+ * violation of the architecture. When DBGPRSR.SPD is set, accesses to
+ * breakpoint and watchpoint registers are treated as undefined, so
+ * this results in boot time and runtime failures when these are
+ * accessed and we unexpectedly take a trap.
+ *
+ * It's not clear if/how this can be worked around, so we blacklist
+ * Scorpion CPUs to avoid these issues.
+ */
+ if (read_cpuid_part() == ARM_CPU_PART_SCORPION) {
+ pr_info("Scorpion CPU detected. Hardware breakpoints and watchpoints disabled\n");
+ return 0;
+ }
+
has_ossr = core_has_os_save_restore();
/* Determine how many BRPs/WRPs are available. */
diff --git a/arch/arm/kernel/patch.c b/arch/arm/kernel/patch.c
index 69bda1a5707e..020560b2dcb7 100644
--- a/arch/arm/kernel/patch.c
+++ b/arch/arm/kernel/patch.c
@@ -24,9 +24,9 @@ static void __kprobes *patch_map(void *addr, int fixmap, unsigned long *flags)
bool module = !core_kernel_text(uintaddr);
struct page *page;
- if (module && IS_ENABLED(CONFIG_DEBUG_SET_MODULE_RONX))
+ if (module && IS_ENABLED(CONFIG_STRICT_MODULE_RWX))
page = vmalloc_to_page(addr);
- else if (!module && IS_ENABLED(CONFIG_DEBUG_RODATA))
+ else if (!module && IS_ENABLED(CONFIG_STRICT_KERNEL_RWX))
page = virt_to_page(addr);
else
return addr;
diff --git a/arch/arm/kernel/perf_event_v6.c b/arch/arm/kernel/perf_event_v6.c
index 09413e7b49aa..96b7a477a8db 100644
--- a/arch/arm/kernel/perf_event_v6.c
+++ b/arch/arm/kernel/perf_event_v6.c
@@ -581,9 +581,5 @@ static struct platform_driver armv6_pmu_driver = {
.probe = armv6_pmu_device_probe,
};
-static int __init register_armv6_pmu_driver(void)
-{
- return platform_driver_register(&armv6_pmu_driver);
-}
-device_initcall(register_armv6_pmu_driver);
+builtin_platform_driver(armv6_pmu_driver);
#endif /* CONFIG_CPU_V6 || CONFIG_CPU_V6K */
diff --git a/arch/arm/kernel/perf_event_v7.c b/arch/arm/kernel/perf_event_v7.c
index b9423491b9d7..ab6522b43659 100644
--- a/arch/arm/kernel/perf_event_v7.c
+++ b/arch/arm/kernel/perf_event_v7.c
@@ -2034,9 +2034,5 @@ static struct platform_driver armv7_pmu_driver = {
.probe = armv7_pmu_device_probe,
};
-static int __init register_armv7_pmu_driver(void)
-{
- return platform_driver_register(&armv7_pmu_driver);
-}
-device_initcall(register_armv7_pmu_driver);
+builtin_platform_driver(armv7_pmu_driver);
#endif /* CONFIG_CPU_V7 */
diff --git a/arch/arm/kernel/perf_event_xscale.c b/arch/arm/kernel/perf_event_xscale.c
index aa0499e2eef7..0e51f5e4f879 100644
--- a/arch/arm/kernel/perf_event_xscale.c
+++ b/arch/arm/kernel/perf_event_xscale.c
@@ -767,9 +767,5 @@ static struct platform_driver xscale_pmu_driver = {
.probe = xscale_pmu_device_probe,
};
-static int __init register_xscale_pmu_driver(void)
-{
- return platform_driver_register(&xscale_pmu_driver);
-}
-device_initcall(register_xscale_pmu_driver);
+builtin_platform_driver(xscale_pmu_driver);
#endif /* CONFIG_CPU_XSCALE */
diff --git a/arch/arm/kernel/ptrace.c b/arch/arm/kernel/ptrace.c
index ce131ed5939d..ae738a6319f6 100644
--- a/arch/arm/kernel/ptrace.c
+++ b/arch/arm/kernel/ptrace.c
@@ -600,7 +600,7 @@ static int gpr_set(struct task_struct *target,
const void *kbuf, const void __user *ubuf)
{
int ret;
- struct pt_regs newregs;
+ struct pt_regs newregs = *task_pt_regs(target);
ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
&newregs,
diff --git a/arch/arm/kernel/smccc-call.S b/arch/arm/kernel/smccc-call.S
index 2e48b674aab1..e5d43066b889 100644
--- a/arch/arm/kernel/smccc-call.S
+++ b/arch/arm/kernel/smccc-call.S
@@ -46,17 +46,19 @@ UNWIND( .fnend)
/*
* void smccc_smc(unsigned long a0, unsigned long a1, unsigned long a2,
* unsigned long a3, unsigned long a4, unsigned long a5,
- * unsigned long a6, unsigned long a7, struct arm_smccc_res *res)
+ * unsigned long a6, unsigned long a7, struct arm_smccc_res *res,
+ * struct arm_smccc_quirk *quirk)
*/
-ENTRY(arm_smccc_smc)
+ENTRY(__arm_smccc_smc)
SMCCC SMCCC_SMC
-ENDPROC(arm_smccc_smc)
+ENDPROC(__arm_smccc_smc)
/*
* void smccc_hvc(unsigned long a0, unsigned long a1, unsigned long a2,
* unsigned long a3, unsigned long a4, unsigned long a5,
- * unsigned long a6, unsigned long a7, struct arm_smccc_res *res)
+ * unsigned long a6, unsigned long a7, struct arm_smccc_res *res,
+ * struct arm_smccc_quirk *quirk)
*/
-ENTRY(arm_smccc_hvc)
+ENTRY(__arm_smccc_hvc)
SMCCC SMCCC_HVC
-ENDPROC(arm_smccc_hvc)
+ENDPROC(__arm_smccc_hvc)
diff --git a/arch/arm/kernel/smp_tlb.c b/arch/arm/kernel/smp_tlb.c
index 22313cb53362..9af0701f7094 100644
--- a/arch/arm/kernel/smp_tlb.c
+++ b/arch/arm/kernel/smp_tlb.c
@@ -9,6 +9,7 @@
*/
#include <linux/preempt.h>
#include <linux/smp.h>
+#include <linux/uaccess.h>
#include <asm/smp_plat.h>
#include <asm/tlbflush.h>
@@ -40,8 +41,11 @@ static inline void ipi_flush_tlb_mm(void *arg)
static inline void ipi_flush_tlb_page(void *arg)
{
struct tlb_args *ta = (struct tlb_args *)arg;
+ unsigned int __ua_flags = uaccess_save_and_enable();
local_flush_tlb_page(ta->ta_vma, ta->ta_start);
+
+ uaccess_restore(__ua_flags);
}
static inline void ipi_flush_tlb_kernel_page(void *arg)
@@ -54,8 +58,11 @@ static inline void ipi_flush_tlb_kernel_page(void *arg)
static inline void ipi_flush_tlb_range(void *arg)
{
struct tlb_args *ta = (struct tlb_args *)arg;
+ unsigned int __ua_flags = uaccess_save_and_enable();
local_flush_tlb_range(ta->ta_vma, ta->ta_start, ta->ta_end);
+
+ uaccess_restore(__ua_flags);
}
static inline void ipi_flush_tlb_kernel_range(void *arg)
diff --git a/arch/arm/kernel/vmlinux.lds.S b/arch/arm/kernel/vmlinux.lds.S
index f7f55df0bf7b..ce18007f9e4e 100644
--- a/arch/arm/kernel/vmlinux.lds.S
+++ b/arch/arm/kernel/vmlinux.lds.S
@@ -97,7 +97,7 @@ SECTIONS
HEAD_TEXT
}
-#ifdef CONFIG_DEBUG_RODATA
+#ifdef CONFIG_STRICT_KERNEL_RWX
. = ALIGN(1<<SECTION_SHIFT);
#endif
@@ -158,7 +158,7 @@ SECTIONS
NOTES
-#ifdef CONFIG_DEBUG_RODATA
+#ifdef CONFIG_STRICT_KERNEL_RWX
. = ALIGN(1<<SECTION_SHIFT);
#else
. = ALIGN(PAGE_SIZE);
@@ -230,7 +230,7 @@ SECTIONS
PERCPU_SECTION(L1_CACHE_BYTES)
#endif
-#ifdef CONFIG_DEBUG_RODATA
+#ifdef CONFIG_STRICT_KERNEL_RWX
. = ALIGN(1<<SECTION_SHIFT);
#else
. = ALIGN(THREAD_SIZE);
@@ -325,7 +325,7 @@ SECTIONS
STABS_DEBUG
}
-#ifdef CONFIG_DEBUG_RODATA
+#ifdef CONFIG_STRICT_KERNEL_RWX
/*
* Without CONFIG_DEBUG_ALIGN_RODATA, __start_rodata_section_aligned will
* be the first section-aligned location after __start_rodata. Otherwise,
diff --git a/arch/arm/kvm/Makefile b/arch/arm/kvm/Makefile
index d571243ab4d1..7b3670c2ae7b 100644
--- a/arch/arm/kvm/Makefile
+++ b/arch/arm/kvm/Makefile
@@ -7,7 +7,7 @@ ifeq ($(plus_virt),+virt)
plus_virt_def := -DREQUIRES_VIRT=1
endif
-ccflags-y += -Iarch/arm/kvm
+ccflags-y += -Iarch/arm/kvm -Ivirt/kvm/arm/vgic
CFLAGS_arm.o := -I. $(plus_virt_def)
CFLAGS_mmu.o := -I.
@@ -20,7 +20,7 @@ kvm-arm-y = $(KVM)/kvm_main.o $(KVM)/coalesced_mmio.o $(KVM)/eventfd.o $(KVM)/vf
obj-$(CONFIG_KVM_ARM_HOST) += hyp/
obj-y += kvm-arm.o init.o interrupts.o
obj-y += arm.o handle_exit.o guest.o mmu.o emulate.o reset.o
-obj-y += coproc.o coproc_a15.o coproc_a7.o mmio.o psci.o perf.o
+obj-y += coproc.o coproc_a15.o coproc_a7.o mmio.o psci.o perf.o vgic-v3-coproc.o
obj-y += $(KVM)/arm/aarch32.o
obj-y += $(KVM)/arm/vgic/vgic.o
@@ -33,5 +33,6 @@ obj-y += $(KVM)/arm/vgic/vgic-mmio-v2.o
obj-y += $(KVM)/arm/vgic/vgic-mmio-v3.o
obj-y += $(KVM)/arm/vgic/vgic-kvm-device.o
obj-y += $(KVM)/arm/vgic/vgic-its.o
+obj-y += $(KVM)/arm/vgic/vgic-debug.o
obj-y += $(KVM)/irqchip.o
obj-y += $(KVM)/arm/arch_timer.o
diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
index 11676787ad49..c9a2103faeb9 100644
--- a/arch/arm/kvm/arm.c
+++ b/arch/arm/kvm/arm.c
@@ -135,7 +135,6 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
goto out_free_stage2_pgd;
kvm_vgic_early_init(kvm);
- kvm_timer_init(kvm);
/* Mark the initial VMID generation invalid */
kvm->arch.vmid_gen = 0;
@@ -207,6 +206,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
case KVM_CAP_ARM_PSCI_0_2:
case KVM_CAP_READONLY_MEM:
case KVM_CAP_MP_STATE:
+ case KVM_CAP_IMMEDIATE_EXIT:
r = 1;
break;
case KVM_CAP_COALESCED_MMIO:
@@ -301,7 +301,8 @@ void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
{
- return kvm_timer_should_fire(vcpu);
+ return kvm_timer_should_fire(vcpu_vtimer(vcpu)) ||
+ kvm_timer_should_fire(vcpu_ptimer(vcpu));
}
void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu)
@@ -604,6 +605,9 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
return ret;
}
+ if (run->immediate_exit)
+ return -EINTR;
+
if (vcpu->sigset_active)
sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
@@ -1099,6 +1103,9 @@ static void cpu_init_hyp_mode(void *dummy)
__cpu_init_hyp_mode(pgd_ptr, hyp_stack_ptr, vector_ptr);
__cpu_init_stage2();
+ if (is_kernel_in_hyp_mode())
+ kvm_timer_init_vhe();
+
kvm_arm_init_debug();
}
diff --git a/arch/arm/kvm/mmu.c b/arch/arm/kvm/mmu.c
index a5265edbeeab..962616fd4ddd 100644
--- a/arch/arm/kvm/mmu.c
+++ b/arch/arm/kvm/mmu.c
@@ -1232,9 +1232,9 @@ void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm,
}
static void coherent_cache_guest_page(struct kvm_vcpu *vcpu, kvm_pfn_t pfn,
- unsigned long size, bool uncached)
+ unsigned long size)
{
- __coherent_cache_guest_page(vcpu, pfn, size, uncached);
+ __coherent_cache_guest_page(vcpu, pfn, size);
}
static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
@@ -1250,7 +1250,6 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
struct vm_area_struct *vma;
kvm_pfn_t pfn;
pgprot_t mem_type = PAGE_S2;
- bool fault_ipa_uncached;
bool logging_active = memslot_is_logging(memslot);
unsigned long flags = 0;
@@ -1337,8 +1336,6 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
if (!hugetlb && !force_pte)
hugetlb = transparent_hugepage_adjust(&pfn, &fault_ipa);
- fault_ipa_uncached = memslot->flags & KVM_MEMSLOT_INCOHERENT;
-
if (hugetlb) {
pmd_t new_pmd = pfn_pmd(pfn, mem_type);
new_pmd = pmd_mkhuge(new_pmd);
@@ -1346,7 +1343,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
new_pmd = kvm_s2pmd_mkwrite(new_pmd);
kvm_set_pfn_dirty(pfn);
}
- coherent_cache_guest_page(vcpu, pfn, PMD_SIZE, fault_ipa_uncached);
+ coherent_cache_guest_page(vcpu, pfn, PMD_SIZE);
ret = stage2_set_pmd_huge(kvm, memcache, fault_ipa, &new_pmd);
} else {
pte_t new_pte = pfn_pte(pfn, mem_type);
@@ -1356,7 +1353,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
kvm_set_pfn_dirty(pfn);
mark_page_dirty(kvm, gfn);
}
- coherent_cache_guest_page(vcpu, pfn, PAGE_SIZE, fault_ipa_uncached);
+ coherent_cache_guest_page(vcpu, pfn, PAGE_SIZE);
ret = stage2_set_pte(kvm, memcache, fault_ipa, &new_pte, flags);
}
@@ -1879,15 +1876,6 @@ void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free,
int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
unsigned long npages)
{
- /*
- * Readonly memslots are not incoherent with the caches by definition,
- * but in practice, they are used mostly to emulate ROMs or NOR flashes
- * that the guest may consider devices and hence map as uncached.
- * To prevent incoherency issues in these cases, tag all readonly
- * regions as incoherent.
- */
- if (slot->flags & KVM_MEM_READONLY)
- slot->flags |= KVM_MEMSLOT_INCOHERENT;
return 0;
}
diff --git a/arch/arm/kvm/reset.c b/arch/arm/kvm/reset.c
index 4b5e802e57d1..1da8b2d14550 100644
--- a/arch/arm/kvm/reset.c
+++ b/arch/arm/kvm/reset.c
@@ -37,6 +37,11 @@ static struct kvm_regs cortexa_regs_reset = {
.usr_regs.ARM_cpsr = SVC_MODE | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT,
};
+static const struct kvm_irq_level cortexa_ptimer_irq = {
+ { .irq = 30 },
+ .level = 1,
+};
+
static const struct kvm_irq_level cortexa_vtimer_irq = {
{ .irq = 27 },
.level = 1,
@@ -58,6 +63,7 @@ int kvm_reset_vcpu(struct kvm_vcpu *vcpu)
{
struct kvm_regs *reset_regs;
const struct kvm_irq_level *cpu_vtimer_irq;
+ const struct kvm_irq_level *cpu_ptimer_irq;
switch (vcpu->arch.target) {
case KVM_ARM_TARGET_CORTEX_A7:
@@ -65,6 +71,7 @@ int kvm_reset_vcpu(struct kvm_vcpu *vcpu)
reset_regs = &cortexa_regs_reset;
vcpu->arch.midr = read_cpuid_id();
cpu_vtimer_irq = &cortexa_vtimer_irq;
+ cpu_ptimer_irq = &cortexa_ptimer_irq;
break;
default:
return -ENODEV;
@@ -77,5 +84,5 @@ int kvm_reset_vcpu(struct kvm_vcpu *vcpu)
kvm_reset_coprocs(vcpu);
/* Reset arch_timer context */
- return kvm_timer_vcpu_reset(vcpu, cpu_vtimer_irq);
+ return kvm_timer_vcpu_reset(vcpu, cpu_vtimer_irq, cpu_ptimer_irq);
}
diff --git a/arch/arm/kvm/vgic-v3-coproc.c b/arch/arm/kvm/vgic-v3-coproc.c
new file mode 100644
index 000000000000..f41abf76366f
--- /dev/null
+++ b/arch/arm/kvm/vgic-v3-coproc.c
@@ -0,0 +1,35 @@
+/*
+ * VGIC system registers handling functions for AArch32 mode
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/kvm.h>
+#include <linux/kvm_host.h>
+#include <asm/kvm_emulate.h>
+#include "vgic.h"
+
+int vgic_v3_has_cpu_sysregs_attr(struct kvm_vcpu *vcpu, bool is_write, u64 id,
+ u64 *reg)
+{
+ /*
+ * TODO: Implement for AArch32
+ */
+ return -ENXIO;
+}
+
+int vgic_v3_cpu_sysregs_uaccess(struct kvm_vcpu *vcpu, bool is_write, u64 id,
+ u64 *reg)
+{
+ /*
+ * TODO: Implement for AArch32
+ */
+ return -ENXIO;
+}
diff --git a/arch/arm/lib/getuser.S b/arch/arm/lib/getuser.S
index 8ecfd15c3a02..df73914e81c8 100644
--- a/arch/arm/lib/getuser.S
+++ b/arch/arm/lib/getuser.S
@@ -67,7 +67,7 @@ ENTRY(__get_user_4)
ENDPROC(__get_user_4)
ENTRY(__get_user_8)
- check_uaccess r0, 8, r1, r2, __get_user_bad
+ check_uaccess r0, 8, r1, r2, __get_user_bad8
#ifdef CONFIG_THUMB2_KERNEL
5: TUSER(ldr) r2, [r0]
6: TUSER(ldr) r3, [r0, #4]
diff --git a/arch/arm/mach-davinci/board-da850-evm.c b/arch/arm/mach-davinci/board-da850-evm.c
index aac3ab1a044f..df3ca38778af 100644
--- a/arch/arm/mach-davinci/board-da850-evm.c
+++ b/arch/arm/mach-davinci/board-da850-evm.c
@@ -18,6 +18,7 @@
#include <linux/gpio/machine.h>
#include <linux/init.h>
#include <linux/kernel.h>
+#include <linux/leds.h>
#include <linux/i2c.h>
#include <linux/platform_data/at24.h>
#include <linux/platform_data/pca953x.h>
diff --git a/arch/arm/mach-davinci/board-dm644x-evm.c b/arch/arm/mach-davinci/board-dm644x-evm.c
index 521e40977265..023480b75244 100644
--- a/arch/arm/mach-davinci/board-dm644x-evm.c
+++ b/arch/arm/mach-davinci/board-dm644x-evm.c
@@ -25,6 +25,7 @@
#include <linux/videodev2.h>
#include <linux/v4l2-dv-timings.h>
#include <linux/export.h>
+#include <linux/leds.h>
#include <media/i2c/tvp514x.h>
diff --git a/arch/arm/mach-davinci/board-neuros-osd2.c b/arch/arm/mach-davinci/board-neuros-osd2.c
index ad10017203c1..0a7838852649 100644
--- a/arch/arm/mach-davinci/board-neuros-osd2.c
+++ b/arch/arm/mach-davinci/board-neuros-osd2.c
@@ -25,6 +25,7 @@
*/
#include <linux/platform_device.h>
#include <linux/gpio.h>
+#include <linux/leds.h>
#include <linux/mtd/partitions.h>
#include <linux/platform_data/gpio-davinci.h>
#include <linux/platform_data/i2c-davinci.h>
diff --git a/arch/arm/mach-davinci/board-omapl138-hawk.c b/arch/arm/mach-davinci/board-omapl138-hawk.c
index 41d5500996b2..a3e78074be70 100644
--- a/arch/arm/mach-davinci/board-omapl138-hawk.c
+++ b/arch/arm/mach-davinci/board-omapl138-hawk.c
@@ -12,6 +12,7 @@
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/console.h>
+#include <linux/interrupt.h>
#include <linux/gpio.h>
#include <linux/gpio/machine.h>
#include <linux/platform_data/gpio-davinci.h>
diff --git a/arch/arm/mach-davinci/da850.c b/arch/arm/mach-davinci/da850.c
index d09765ba246b..ccad2f99dfc9 100644
--- a/arch/arm/mach-davinci/da850.c
+++ b/arch/arm/mach-davinci/da850.c
@@ -557,15 +557,7 @@ static struct clk_lookup da850_clks[] = {
CLK("da830-mmc.0", NULL, &mmcsd0_clk),
CLK("da830-mmc.1", NULL, &mmcsd1_clk),
CLK("ti-aemif", NULL, &aemif_clk),
- /*
- * The only user of this clock is davinci_nand and it get's it through
- * con_id. The nand node itself is created from within the aemif
- * driver to guarantee that it's probed after the aemif timing
- * parameters are configured. of_dev_auxdata is not accessible from
- * the aemif driver and can't be passed to of_platform_populate(). For
- * that reason we're leaving the dev_id here as NULL.
- */
- CLK(NULL, "aemif", &aemif_nand_clk),
+ CLK("davinci-nand.0", "aemif", &aemif_nand_clk),
CLK("ohci-da8xx", "usb11", &usb11_clk),
CLK("musb-da8xx", "usb20", &usb20_clk),
CLK("spi_davinci.0", NULL, &spi0_clk),
diff --git a/arch/arm/mach-davinci/da8xx-dt.c b/arch/arm/mach-davinci/da8xx-dt.c
index 071933b16a0c..e3cef503cd8f 100644
--- a/arch/arm/mach-davinci/da8xx-dt.c
+++ b/arch/arm/mach-davinci/da8xx-dt.c
@@ -11,6 +11,7 @@
#include <linux/of_irq.h>
#include <linux/of_platform.h>
#include <linux/irqdomain.h>
+#include <linux/platform_data/ti-aemif.h>
#include <asm/mach/arch.h>
@@ -18,6 +19,15 @@
#include "cp_intc.h"
#include <mach/da8xx.h>
+static struct of_dev_auxdata da850_aemif_auxdata_lookup[] = {
+ OF_DEV_AUXDATA("ti,davinci-nand", 0x62000000, "davinci-nand.0", NULL),
+ {}
+};
+
+static struct aemif_platform_data aemif_data = {
+ .dev_lookup = da850_aemif_auxdata_lookup,
+};
+
static struct of_dev_auxdata da850_auxdata_lookup[] __initdata = {
OF_DEV_AUXDATA("ti,davinci-i2c", 0x01c22000, "i2c_davinci.1", NULL),
OF_DEV_AUXDATA("ti,davinci-i2c", 0x01e28000, "i2c_davinci.2", NULL),
@@ -37,7 +47,7 @@ static struct of_dev_auxdata da850_auxdata_lookup[] __initdata = {
OF_DEV_AUXDATA("ti,davinci-dm6467-emac", 0x01e20000, "davinci_emac.1",
NULL),
OF_DEV_AUXDATA("ti,da830-mcasp-audio", 0x01d00000, "davinci-mcasp.0", NULL),
- OF_DEV_AUXDATA("ti,da850-aemif", 0x68000000, "ti-aemif", NULL),
+ OF_DEV_AUXDATA("ti,da850-aemif", 0x68000000, "ti-aemif", &aemif_data),
OF_DEV_AUXDATA("ti,da850-tilcdc", 0x01e13000, "da8xx_lcdc.0", NULL),
OF_DEV_AUXDATA("ti,da830-ohci", 0x01e25000, "ohci-da8xx", NULL),
OF_DEV_AUXDATA("ti,da830-musb", 0x01e00000, "musb-da8xx", NULL),
diff --git a/arch/arm/mach-ep93xx/edb93xx.c b/arch/arm/mach-ep93xx/edb93xx.c
index ad92d9f7e4df..0ac176386789 100644
--- a/arch/arm/mach-ep93xx/edb93xx.c
+++ b/arch/arm/mach-ep93xx/edb93xx.c
@@ -27,7 +27,6 @@
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/platform_device.h>
-#include <linux/gpio.h>
#include <linux/i2c.h>
#include <linux/i2c-gpio.h>
#include <linux/spi/spi.h>
@@ -106,33 +105,10 @@ static struct cs4271_platform_data edb93xx_cs4271_data = {
.gpio_nreset = -EINVAL, /* filled in later */
};
-static int edb93xx_cs4271_hw_setup(struct spi_device *spi)
-{
- return gpio_request_one(EP93XX_GPIO_LINE_EGPIO6,
- GPIOF_OUT_INIT_HIGH, spi->modalias);
-}
-
-static void edb93xx_cs4271_hw_cleanup(struct spi_device *spi)
-{
- gpio_free(EP93XX_GPIO_LINE_EGPIO6);
-}
-
-static void edb93xx_cs4271_hw_cs_control(struct spi_device *spi, int value)
-{
- gpio_set_value(EP93XX_GPIO_LINE_EGPIO6, value);
-}
-
-static struct ep93xx_spi_chip_ops edb93xx_cs4271_hw = {
- .setup = edb93xx_cs4271_hw_setup,
- .cleanup = edb93xx_cs4271_hw_cleanup,
- .cs_control = edb93xx_cs4271_hw_cs_control,
-};
-
static struct spi_board_info edb93xx_spi_board_info[] __initdata = {
{
.modalias = "cs4271",
.platform_data = &edb93xx_cs4271_data,
- .controller_data = &edb93xx_cs4271_hw,
.max_speed_hz = 6000000,
.bus_num = 0,
.chip_select = 0,
@@ -140,8 +116,13 @@ static struct spi_board_info edb93xx_spi_board_info[] __initdata = {
},
};
+static int edb93xx_spi_chipselects[] __initdata = {
+ EP93XX_GPIO_LINE_EGPIO6,
+};
+
static struct ep93xx_spi_info edb93xx_spi_info __initdata = {
- .num_chipselect = ARRAY_SIZE(edb93xx_spi_board_info),
+ .chipselect = edb93xx_spi_chipselects,
+ .num_chipselect = ARRAY_SIZE(edb93xx_spi_chipselects),
};
static void __init edb93xx_register_spi(void)
diff --git a/arch/arm/mach-ep93xx/simone.c b/arch/arm/mach-ep93xx/simone.c
index 7bb540c421ee..c7a40f245892 100644
--- a/arch/arm/mach-ep93xx/simone.c
+++ b/arch/arm/mach-ep93xx/simone.c
@@ -49,56 +49,6 @@ static struct ep93xxfb_mach_info __initdata simone_fb_info = {
#define MMC_CARD_DETECT_GPIO EP93XX_GPIO_LINE_EGPIO0
/*
- * Up to v1.3, the Sim.One used SFRMOUT as SD card chip select, but this goes
- * low between multi-message command blocks. From v1.4, it uses a GPIO instead.
- * v1.3 parts will still work, since the signal on SFRMOUT is automatic.
- */
-#define MMC_CHIP_SELECT_GPIO EP93XX_GPIO_LINE_EGPIO1
-
-/*
- * MMC SPI chip select GPIO handling. If you are using SFRMOUT (SFRM1) signal,
- * you can leave these empty and pass NULL as .controller_data.
- */
-
-static int simone_mmc_spi_setup(struct spi_device *spi)
-{
- unsigned int gpio = MMC_CHIP_SELECT_GPIO;
- int err;
-
- err = gpio_request(gpio, spi->modalias);
- if (err)
- return err;
-
- err = gpio_direction_output(gpio, 1);
- if (err) {
- gpio_free(gpio);
- return err;
- }
-
- return 0;
-}
-
-static void simone_mmc_spi_cleanup(struct spi_device *spi)
-{
- unsigned int gpio = MMC_CHIP_SELECT_GPIO;
-
- gpio_set_value(gpio, 1);
- gpio_direction_input(gpio);
- gpio_free(gpio);
-}
-
-static void simone_mmc_spi_cs_control(struct spi_device *spi, int value)
-{
- gpio_set_value(MMC_CHIP_SELECT_GPIO, value);
-}
-
-static struct ep93xx_spi_chip_ops simone_mmc_spi_ops = {
- .setup = simone_mmc_spi_setup,
- .cleanup = simone_mmc_spi_cleanup,
- .cs_control = simone_mmc_spi_cs_control,
-};
-
-/*
* MMC card detection GPIO setup.
*/
@@ -152,7 +102,6 @@ static struct mmc_spi_platform_data simone_mmc_spi_data = {
static struct spi_board_info simone_spi_devices[] __initdata = {
{
.modalias = "mmc_spi",
- .controller_data = &simone_mmc_spi_ops,
.platform_data = &simone_mmc_spi_data,
/*
* We use 10 MHz even though the maximum is 3.7 MHz. The driver
@@ -165,8 +114,18 @@ static struct spi_board_info simone_spi_devices[] __initdata = {
},
};
+/*
+ * Up to v1.3, the Sim.One used SFRMOUT as SD card chip select, but this goes
+ * low between multi-message command blocks. From v1.4, it uses a GPIO instead.
+ * v1.3 parts will still work, since the signal on SFRMOUT is automatic.
+ */
+static int simone_spi_chipselects[] __initdata = {
+ EP93XX_GPIO_LINE_EGPIO1,
+};
+
static struct ep93xx_spi_info simone_spi_info __initdata = {
- .num_chipselect = ARRAY_SIZE(simone_spi_devices),
+ .chipselect = simone_spi_chipselects,
+ .num_chipselect = ARRAY_SIZE(simone_spi_chipselects),
.use_dma = 1,
};
diff --git a/arch/arm/mach-ep93xx/vision_ep9307.c b/arch/arm/mach-ep93xx/vision_ep9307.c
index 5cced5988498..1daf9441058c 100644
--- a/arch/arm/mach-ep93xx/vision_ep9307.c
+++ b/arch/arm/mach-ep93xx/vision_ep9307.c
@@ -175,33 +175,9 @@ static struct cs4271_platform_data vision_cs4271_data = {
.gpio_nreset = EP93XX_GPIO_LINE_H(2),
};
-static int vision_cs4271_hw_setup(struct spi_device *spi)
-{
- return gpio_request_one(EP93XX_GPIO_LINE_EGPIO6,
- GPIOF_OUT_INIT_HIGH, spi->modalias);
-}
-
-static void vision_cs4271_hw_cleanup(struct spi_device *spi)
-{
- gpio_free(EP93XX_GPIO_LINE_EGPIO6);
-}
-
-static void vision_cs4271_hw_cs_control(struct spi_device *spi, int value)
-{
- gpio_set_value(EP93XX_GPIO_LINE_EGPIO6, value);
-}
-
-static struct ep93xx_spi_chip_ops vision_cs4271_hw = {
- .setup = vision_cs4271_hw_setup,
- .cleanup = vision_cs4271_hw_cleanup,
- .cs_control = vision_cs4271_hw_cs_control,
-};
-
/*************************************************************************
* SPI Flash
*************************************************************************/
-#define VISION_SPI_FLASH_CS EP93XX_GPIO_LINE_EGPIO7
-
static struct mtd_partition vision_spi_flash_partitions[] = {
{
.name = "SPI bootstrap",
@@ -224,68 +200,20 @@ static struct flash_platform_data vision_spi_flash_data = {
.nr_parts = ARRAY_SIZE(vision_spi_flash_partitions),
};
-static int vision_spi_flash_hw_setup(struct spi_device *spi)
-{
- return gpio_request_one(VISION_SPI_FLASH_CS, GPIOF_INIT_HIGH,
- spi->modalias);
-}
-
-static void vision_spi_flash_hw_cleanup(struct spi_device *spi)
-{
- gpio_free(VISION_SPI_FLASH_CS);
-}
-
-static void vision_spi_flash_hw_cs_control(struct spi_device *spi, int value)
-{
- gpio_set_value(VISION_SPI_FLASH_CS, value);
-}
-
-static struct ep93xx_spi_chip_ops vision_spi_flash_hw = {
- .setup = vision_spi_flash_hw_setup,
- .cleanup = vision_spi_flash_hw_cleanup,
- .cs_control = vision_spi_flash_hw_cs_control,
-};
-
/*************************************************************************
* SPI SD/MMC host
*************************************************************************/
-#define VISION_SPI_MMC_CS EP93XX_GPIO_LINE_G(2)
-#define VISION_SPI_MMC_WP EP93XX_GPIO_LINE_F(0)
-#define VISION_SPI_MMC_CD EP93XX_GPIO_LINE_EGPIO15
-
static struct mmc_spi_platform_data vision_spi_mmc_data = {
.detect_delay = 100,
.powerup_msecs = 100,
.ocr_mask = MMC_VDD_32_33 | MMC_VDD_33_34,
.flags = MMC_SPI_USE_CD_GPIO | MMC_SPI_USE_RO_GPIO,
- .cd_gpio = VISION_SPI_MMC_CD,
+ .cd_gpio = EP93XX_GPIO_LINE_EGPIO15,
.cd_debounce = 1,
- .ro_gpio = VISION_SPI_MMC_WP,
+ .ro_gpio = EP93XX_GPIO_LINE_F(0),
.caps2 = MMC_CAP2_RO_ACTIVE_HIGH,
};
-static int vision_spi_mmc_hw_setup(struct spi_device *spi)
-{
- return gpio_request_one(VISION_SPI_MMC_CS, GPIOF_INIT_HIGH,
- spi->modalias);
-}
-
-static void vision_spi_mmc_hw_cleanup(struct spi_device *spi)
-{
- gpio_free(VISION_SPI_MMC_CS);
-}
-
-static void vision_spi_mmc_hw_cs_control(struct spi_device *spi, int value)
-{
- gpio_set_value(VISION_SPI_MMC_CS, value);
-}
-
-static struct ep93xx_spi_chip_ops vision_spi_mmc_hw = {
- .setup = vision_spi_mmc_hw_setup,
- .cleanup = vision_spi_mmc_hw_cleanup,
- .cs_control = vision_spi_mmc_hw_cs_control,
-};
-
/*************************************************************************
* SPI Bus
*************************************************************************/
@@ -293,7 +221,6 @@ static struct spi_board_info vision_spi_board_info[] __initdata = {
{
.modalias = "cs4271",
.platform_data = &vision_cs4271_data,
- .controller_data = &vision_cs4271_hw,
.max_speed_hz = 6000000,
.bus_num = 0,
.chip_select = 0,
@@ -301,7 +228,6 @@ static struct spi_board_info vision_spi_board_info[] __initdata = {
}, {
.modalias = "sst25l",
.platform_data = &vision_spi_flash_data,
- .controller_data = &vision_spi_flash_hw,
.max_speed_hz = 20000000,
.bus_num = 0,
.chip_select = 1,
@@ -309,7 +235,6 @@ static struct spi_board_info vision_spi_board_info[] __initdata = {
}, {
.modalias = "mmc_spi",
.platform_data = &vision_spi_mmc_data,
- .controller_data = &vision_spi_mmc_hw,
.max_speed_hz = 20000000,
.bus_num = 0,
.chip_select = 2,
@@ -317,8 +242,15 @@ static struct spi_board_info vision_spi_board_info[] __initdata = {
},
};
+static int vision_spi_chipselects[] __initdata = {
+ EP93XX_GPIO_LINE_EGPIO6,
+ EP93XX_GPIO_LINE_EGPIO7,
+ EP93XX_GPIO_LINE_G(2),
+};
+
static struct ep93xx_spi_info vision_spi_master __initdata = {
- .num_chipselect = ARRAY_SIZE(vision_spi_board_info),
+ .chipselect = vision_spi_chipselects,
+ .num_chipselect = ARRAY_SIZE(vision_spi_chipselects),
.use_dma = 1,
};
diff --git a/arch/arm/mach-exynos/suspend.c b/arch/arm/mach-exynos/suspend.c
index 25e7c5326259..312d3a886e92 100644
--- a/arch/arm/mach-exynos/suspend.c
+++ b/arch/arm/mach-exynos/suspend.c
@@ -57,7 +57,6 @@ struct exynos_wkup_irq {
struct exynos_pm_data {
const struct exynos_wkup_irq *wkup_irq;
unsigned int wake_disable_mask;
- const unsigned int *release_ret_regs;
void (*pm_prepare)(void);
void (*pm_resume_prepare)(void);
@@ -95,47 +94,6 @@ static const struct exynos_wkup_irq exynos5250_wkup_irq[] = {
{ /* sentinel */ },
};
-static const unsigned int exynos_release_ret_regs[] = {
- S5P_PAD_RET_MAUDIO_OPTION,
- S5P_PAD_RET_GPIO_OPTION,
- S5P_PAD_RET_UART_OPTION,
- S5P_PAD_RET_MMCA_OPTION,
- S5P_PAD_RET_MMCB_OPTION,
- S5P_PAD_RET_EBIA_OPTION,
- S5P_PAD_RET_EBIB_OPTION,
- REG_TABLE_END,
-};
-
-static const unsigned int exynos3250_release_ret_regs[] = {
- S5P_PAD_RET_MAUDIO_OPTION,
- S5P_PAD_RET_GPIO_OPTION,
- S5P_PAD_RET_UART_OPTION,
- S5P_PAD_RET_MMCA_OPTION,
- S5P_PAD_RET_MMCB_OPTION,
- S5P_PAD_RET_EBIA_OPTION,
- S5P_PAD_RET_EBIB_OPTION,
- S5P_PAD_RET_MMC2_OPTION,
- S5P_PAD_RET_SPI_OPTION,
- REG_TABLE_END,
-};
-
-static const unsigned int exynos5420_release_ret_regs[] = {
- EXYNOS_PAD_RET_DRAM_OPTION,
- EXYNOS_PAD_RET_MAUDIO_OPTION,
- EXYNOS_PAD_RET_JTAG_OPTION,
- EXYNOS5420_PAD_RET_GPIO_OPTION,
- EXYNOS5420_PAD_RET_UART_OPTION,
- EXYNOS5420_PAD_RET_MMCA_OPTION,
- EXYNOS5420_PAD_RET_MMCB_OPTION,
- EXYNOS5420_PAD_RET_MMCC_OPTION,
- EXYNOS5420_PAD_RET_HSI_OPTION,
- EXYNOS_PAD_RET_EBIA_OPTION,
- EXYNOS_PAD_RET_EBIB_OPTION,
- EXYNOS5420_PAD_RET_SPI_OPTION,
- EXYNOS5420_PAD_RET_DRAM_COREBLK_OPTION,
- REG_TABLE_END,
-};
-
static int exynos_irq_set_wake(struct irq_data *data, unsigned int state)
{
const struct exynos_wkup_irq *wkup_irq;
@@ -441,15 +399,6 @@ static int exynos5420_pm_suspend(void)
return 0;
}
-static void exynos_pm_release_retention(void)
-{
- unsigned int i;
-
- for (i = 0; (pm_data->release_ret_regs[i] != REG_TABLE_END); i++)
- pmu_raw_writel(EXYNOS_WAKEUP_FROM_LOWPWR,
- pm_data->release_ret_regs[i]);
-}
-
static void exynos_pm_resume(void)
{
u32 cpuid = read_cpuid_part();
@@ -457,9 +406,6 @@ static void exynos_pm_resume(void)
if (exynos_pm_central_resume())
goto early_wakeup;
- /* For release retention */
- exynos_pm_release_retention();
-
if (cpuid == ARM_CPU_PART_CORTEX_A9)
scu_enable(S5P_VA_SCU);
@@ -481,9 +427,6 @@ static void exynos3250_pm_resume(void)
if (exynos_pm_central_resume())
goto early_wakeup;
- /* For release retention */
- exynos_pm_release_retention();
-
pmu_raw_writel(S5P_USE_STANDBY_WFI_ALL, S5P_CENTRAL_SEQ_OPTION);
if (call_firmware_op(resume) == -ENOSYS
@@ -521,9 +464,6 @@ static void exynos5420_pm_resume(void)
if (exynos_pm_central_resume())
goto early_wakeup;
- /* For release retention */
- exynos_pm_release_retention();
-
pmu_raw_writel(exynos_pmu_spare3, S5P_PMU_SPARE3);
early_wakeup:
@@ -636,7 +576,6 @@ static const struct platform_suspend_ops exynos_suspend_ops = {
static const struct exynos_pm_data exynos3250_pm_data = {
.wkup_irq = exynos3250_wkup_irq,
.wake_disable_mask = ((0xFF << 8) | (0x1F << 1)),
- .release_ret_regs = exynos3250_release_ret_regs,
.pm_suspend = exynos_pm_suspend,
.pm_resume = exynos3250_pm_resume,
.pm_prepare = exynos3250_pm_prepare,
@@ -646,7 +585,6 @@ static const struct exynos_pm_data exynos3250_pm_data = {
static const struct exynos_pm_data exynos4_pm_data = {
.wkup_irq = exynos4_wkup_irq,
.wake_disable_mask = ((0xFF << 8) | (0x1F << 1)),
- .release_ret_regs = exynos_release_ret_regs,
.pm_suspend = exynos_pm_suspend,
.pm_resume = exynos_pm_resume,
.pm_prepare = exynos_pm_prepare,
@@ -656,7 +594,6 @@ static const struct exynos_pm_data exynos4_pm_data = {
static const struct exynos_pm_data exynos5250_pm_data = {
.wkup_irq = exynos5250_wkup_irq,
.wake_disable_mask = ((0xFF << 8) | (0x1F << 1)),
- .release_ret_regs = exynos_release_ret_regs,
.pm_suspend = exynos_pm_suspend,
.pm_resume = exynos_pm_resume,
.pm_prepare = exynos_pm_prepare,
@@ -666,7 +603,6 @@ static const struct exynos_pm_data exynos5250_pm_data = {
static const struct exynos_pm_data exynos5420_pm_data = {
.wkup_irq = exynos5250_wkup_irq,
.wake_disable_mask = (0x7F << 7) | (0x1F << 1),
- .release_ret_regs = exynos5420_release_ret_regs,
.pm_resume_prepare = exynos5420_prepare_pm_resume,
.pm_resume = exynos5420_pm_resume,
.pm_suspend = exynos5420_pm_suspend,
diff --git a/arch/arm/mach-imx/mmdc.c b/arch/arm/mach-imx/mmdc.c
index 699157759120..c03bf28d8bbc 100644
--- a/arch/arm/mach-imx/mmdc.c
+++ b/arch/arm/mach-imx/mmdc.c
@@ -60,7 +60,6 @@
#define to_mmdc_pmu(p) container_of(p, struct mmdc_pmu, pmu)
-static enum cpuhp_state cpuhp_mmdc_state;
static int ddr_type;
struct fsl_mmdc_devtype_data {
@@ -82,6 +81,7 @@ static const struct of_device_id imx_mmdc_dt_ids[] = {
#ifdef CONFIG_PERF_EVENTS
+static enum cpuhp_state cpuhp_mmdc_state;
static DEFINE_IDA(mmdc_ida);
PMU_EVENT_ATTR_STRING(total-cycles, mmdc_pmu_total_cycles, "event=0x00")
diff --git a/arch/arm/mach-omap1/dma.c b/arch/arm/mach-omap1/dma.c
index f6ba589cd312..c821c1d5610e 100644
--- a/arch/arm/mach-omap1/dma.c
+++ b/arch/arm/mach-omap1/dma.c
@@ -32,7 +32,6 @@
#include "soc.h"
#define OMAP1_DMA_BASE (0xfffed800)
-#define OMAP1_LOGICAL_DMA_CH_COUNT 17
static u32 enable_1510_mode;
@@ -348,8 +347,6 @@ static int __init omap1_system_dma_init(void)
goto exit_iounmap;
}
- d->lch_count = OMAP1_LOGICAL_DMA_CH_COUNT;
-
/* Valid attributes for omap1 plus processors */
if (cpu_is_omap15xx())
d->dev_caps = ENABLE_1510_MODE;
@@ -366,13 +363,14 @@ static int __init omap1_system_dma_init(void)
d->dev_caps |= CLEAR_CSR_ON_READ;
d->dev_caps |= IS_WORD_16;
- if (cpu_is_omap15xx())
- d->chan_count = 9;
- else if (cpu_is_omap16xx() || cpu_is_omap7xx()) {
- if (!(d->dev_caps & ENABLE_1510_MODE))
- d->chan_count = 16;
+ /* available logical channels */
+ if (cpu_is_omap15xx()) {
+ d->lch_count = 9;
+ } else {
+ if (d->dev_caps & ENABLE_1510_MODE)
+ d->lch_count = 9;
else
- d->chan_count = 9;
+ d->lch_count = 16;
}
p = dma_plat_info;
diff --git a/arch/arm/mach-omap2/id.c b/arch/arm/mach-omap2/id.c
index cc6d9fa60924..e2274a162b74 100644
--- a/arch/arm/mach-omap2/id.c
+++ b/arch/arm/mach-omap2/id.c
@@ -223,7 +223,15 @@ static void __init omap3_cpuinfo(void)
* and CPU class bits.
*/
if (soc_is_omap3630()) {
- cpu_name = "OMAP3630";
+ if (omap3_has_iva() && omap3_has_sgx()) {
+ cpu_name = (omap3_has_isp()) ? "OMAP3630/DM3730" : "OMAP3621";
+ } else if (omap3_has_iva()) {
+ cpu_name = "DM3725";
+ } else if (omap3_has_sgx()) {
+ cpu_name = "OMAP3615/AM3715";
+ } else {
+ cpu_name = (omap3_has_isp()) ? "AM3703" : "OMAP3611";
+ }
} else if (soc_is_am35xx()) {
cpu_name = (omap3_has_sgx()) ? "AM3517" : "AM3505";
} else if (soc_is_ti816x()) {
diff --git a/arch/arm/mach-omap2/omap_hwmod_54xx_data.c b/arch/arm/mach-omap2/omap_hwmod_54xx_data.c
index 8cdfd9b7ab4f..a2d763a4cc57 100644
--- a/arch/arm/mach-omap2/omap_hwmod_54xx_data.c
+++ b/arch/arm/mach-omap2/omap_hwmod_54xx_data.c
@@ -1748,6 +1748,7 @@ static struct omap_hwmod omap54xx_uart1_hwmod = {
.name = "uart1",
.class = &omap54xx_uart_hwmod_class,
.clkdm_name = "l4per_clkdm",
+ .flags = HWMOD_SWSUP_SIDLE_ACT,
.main_clk = "func_48m_fclk",
.prcm = {
.omap4 = {
@@ -1763,6 +1764,7 @@ static struct omap_hwmod omap54xx_uart2_hwmod = {
.name = "uart2",
.class = &omap54xx_uart_hwmod_class,
.clkdm_name = "l4per_clkdm",
+ .flags = HWMOD_SWSUP_SIDLE_ACT,
.main_clk = "func_48m_fclk",
.prcm = {
.omap4 = {
@@ -1778,7 +1780,7 @@ static struct omap_hwmod omap54xx_uart3_hwmod = {
.name = "uart3",
.class = &omap54xx_uart_hwmod_class,
.clkdm_name = "l4per_clkdm",
- .flags = DEBUG_OMAP4UART3_FLAGS,
+ .flags = DEBUG_OMAP4UART3_FLAGS | HWMOD_SWSUP_SIDLE_ACT,
.main_clk = "func_48m_fclk",
.prcm = {
.omap4 = {
@@ -1794,7 +1796,7 @@ static struct omap_hwmod omap54xx_uart4_hwmod = {
.name = "uart4",
.class = &omap54xx_uart_hwmod_class,
.clkdm_name = "l4per_clkdm",
- .flags = DEBUG_OMAP4UART4_FLAGS,
+ .flags = DEBUG_OMAP4UART4_FLAGS | HWMOD_SWSUP_SIDLE_ACT,
.main_clk = "func_48m_fclk",
.prcm = {
.omap4 = {
@@ -1810,6 +1812,7 @@ static struct omap_hwmod omap54xx_uart5_hwmod = {
.name = "uart5",
.class = &omap54xx_uart_hwmod_class,
.clkdm_name = "l4per_clkdm",
+ .flags = HWMOD_SWSUP_SIDLE_ACT,
.main_clk = "func_48m_fclk",
.prcm = {
.omap4 = {
@@ -1825,6 +1828,7 @@ static struct omap_hwmod omap54xx_uart6_hwmod = {
.name = "uart6",
.class = &omap54xx_uart_hwmod_class,
.clkdm_name = "l4per_clkdm",
+ .flags = HWMOD_SWSUP_SIDLE_ACT,
.main_clk = "func_48m_fclk",
.prcm = {
.omap4 = {
diff --git a/arch/arm/mach-omap2/pdata-quirks.c b/arch/arm/mach-omap2/pdata-quirks.c
index 477910a48448..9700a8ef0f16 100644
--- a/arch/arm/mach-omap2/pdata-quirks.c
+++ b/arch/arm/mach-omap2/pdata-quirks.c
@@ -161,7 +161,7 @@ static struct ti_st_plat_data wilink7_pdata = {
.nshutdown_gpio = 162,
.dev_name = "/dev/ttyO1",
.flow_cntrl = 1,
- .baud_rate = 300000,
+ .baud_rate = 3000000,
};
static struct platform_device wl128x_device = {
@@ -484,15 +484,15 @@ static struct pwm_omap_dmtimer_pdata pwm_dmtimer_pdata = {
};
#endif
-static struct lirc_rx51_platform_data __maybe_unused rx51_lirc_data = {
+static struct ir_rx51_platform_data __maybe_unused rx51_ir_data = {
.set_max_mpu_wakeup_lat = omap_pm_set_max_mpu_wakeup_lat,
};
-static struct platform_device __maybe_unused rx51_lirc_device = {
- .name = "lirc_rx51",
+static struct platform_device __maybe_unused rx51_ir_device = {
+ .name = "ir_rx51",
.id = -1,
.dev = {
- .platform_data = &rx51_lirc_data,
+ .platform_data = &rx51_ir_data,
},
};
@@ -533,7 +533,7 @@ static struct of_dev_auxdata omap_auxdata_lookup[] __initdata = {
&omap3_iommu_pdata),
OF_DEV_AUXDATA("ti,omap3-hsmmc", 0x4809c000, "4809c000.mmc", &mmc_pdata[0]),
OF_DEV_AUXDATA("ti,omap3-hsmmc", 0x480b4000, "480b4000.mmc", &mmc_pdata[1]),
- OF_DEV_AUXDATA("nokia,n900-ir", 0, "n900-ir", &rx51_lirc_data),
+ OF_DEV_AUXDATA("nokia,n900-ir", 0, "n900-ir", &rx51_ir_data),
/* Only on am3517 */
OF_DEV_AUXDATA("ti,davinci_mdio", 0x5c030000, "davinci_mdio.0", NULL),
OF_DEV_AUXDATA("ti,am3517-emac", 0x5c000000, "davinci_emac.0",
@@ -599,7 +599,6 @@ static void pdata_quirks_check(struct pdata_init *quirks)
if (of_machine_is_compatible(quirks->compatible)) {
if (quirks->fn)
quirks->fn();
- break;
}
quirks++;
}
diff --git a/arch/arm/mach-omap2/pm.c b/arch/arm/mach-omap2/pm.c
index 76b0454ddc49..0598630c1778 100644
--- a/arch/arm/mach-omap2/pm.c
+++ b/arch/arm/mach-omap2/pm.c
@@ -130,17 +130,16 @@ static int __init omap2_set_init_voltage(char *vdd_name, char *clk_name,
freq = clk_get_rate(clk);
clk_put(clk);
- rcu_read_lock();
opp = dev_pm_opp_find_freq_ceil(dev, &freq);
if (IS_ERR(opp)) {
- rcu_read_unlock();
pr_err("%s: unable to find boot up OPP for vdd_%s\n",
__func__, vdd_name);
goto exit;
}
bootup_volt = dev_pm_opp_get_voltage(opp);
- rcu_read_unlock();
+ dev_pm_opp_put(opp);
+
if (!bootup_volt) {
pr_err("%s: unable to find voltage corresponding to the bootup OPP for vdd_%s\n",
__func__, vdd_name);
diff --git a/arch/arm/mach-orion5x/common.c b/arch/arm/mach-orion5x/common.c
index 04910764c385..83a7ec4c16d0 100644
--- a/arch/arm/mach-orion5x/common.c
+++ b/arch/arm/mach-orion5x/common.c
@@ -105,7 +105,7 @@ void __init orion5x_eth_init(struct mv643xx_eth_platform_data *eth_data)
/*****************************************************************************
* Ethernet switch
****************************************************************************/
-void __init orion5x_eth_switch_init(struct dsa_platform_data *d)
+void __init orion5x_eth_switch_init(struct dsa_chip_data *d)
{
orion_ge00_switch_init(d);
}
diff --git a/arch/arm/mach-orion5x/common.h b/arch/arm/mach-orion5x/common.h
index 8a4115bd441d..efeffc6b4ebb 100644
--- a/arch/arm/mach-orion5x/common.h
+++ b/arch/arm/mach-orion5x/common.h
@@ -3,7 +3,7 @@
#include <linux/reboot.h>
-struct dsa_platform_data;
+struct dsa_chip_data;
struct mv643xx_eth_platform_data;
struct mv_sata_platform_data;
@@ -41,7 +41,7 @@ void orion5x_setup_wins(void);
void orion5x_ehci0_init(void);
void orion5x_ehci1_init(void);
void orion5x_eth_init(struct mv643xx_eth_platform_data *eth_data);
-void orion5x_eth_switch_init(struct dsa_platform_data *d);
+void orion5x_eth_switch_init(struct dsa_chip_data *d);
void orion5x_i2c_init(void);
void orion5x_sata_init(struct mv_sata_platform_data *sata_data);
void orion5x_spi_init(void);
diff --git a/arch/arm/mach-orion5x/rd88f5181l-fxo-setup.c b/arch/arm/mach-orion5x/rd88f5181l-fxo-setup.c
index dccadf68ea2b..a3c1336d30c9 100644
--- a/arch/arm/mach-orion5x/rd88f5181l-fxo-setup.c
+++ b/arch/arm/mach-orion5x/rd88f5181l-fxo-setup.c
@@ -101,11 +101,6 @@ static struct dsa_chip_data rd88f5181l_fxo_switch_chip_data = {
.port_names[7] = "lan3",
};
-static struct dsa_platform_data __initdata rd88f5181l_fxo_switch_plat_data = {
- .nr_chips = 1,
- .chip = &rd88f5181l_fxo_switch_chip_data,
-};
-
static void __init rd88f5181l_fxo_init(void)
{
/*
@@ -120,7 +115,7 @@ static void __init rd88f5181l_fxo_init(void)
*/
orion5x_ehci0_init();
orion5x_eth_init(&rd88f5181l_fxo_eth_data);
- orion5x_eth_switch_init(&rd88f5181l_fxo_switch_plat_data);
+ orion5x_eth_switch_init(&rd88f5181l_fxo_switch_chip_data);
orion5x_uart0_init();
mvebu_mbus_add_window_by_id(ORION_MBUS_DEVBUS_BOOT_TARGET,
diff --git a/arch/arm/mach-orion5x/rd88f5181l-ge-setup.c b/arch/arm/mach-orion5x/rd88f5181l-ge-setup.c
index affe5ec825de..252efe29bd1a 100644
--- a/arch/arm/mach-orion5x/rd88f5181l-ge-setup.c
+++ b/arch/arm/mach-orion5x/rd88f5181l-ge-setup.c
@@ -102,11 +102,6 @@ static struct dsa_chip_data rd88f5181l_ge_switch_chip_data = {
.port_names[7] = "lan3",
};
-static struct dsa_platform_data __initdata rd88f5181l_ge_switch_plat_data = {
- .nr_chips = 1,
- .chip = &rd88f5181l_ge_switch_chip_data,
-};
-
static struct i2c_board_info __initdata rd88f5181l_ge_i2c_rtc = {
I2C_BOARD_INFO("ds1338", 0x68),
};
@@ -125,7 +120,7 @@ static void __init rd88f5181l_ge_init(void)
*/
orion5x_ehci0_init();
orion5x_eth_init(&rd88f5181l_ge_eth_data);
- orion5x_eth_switch_init(&rd88f5181l_ge_switch_plat_data);
+ orion5x_eth_switch_init(&rd88f5181l_ge_switch_chip_data);
orion5x_i2c_init();
orion5x_uart0_init();
diff --git a/arch/arm/mach-orion5x/rd88f6183ap-ge-setup.c b/arch/arm/mach-orion5x/rd88f6183ap-ge-setup.c
index 67ee8571b03c..f4f1dbe1d91d 100644
--- a/arch/arm/mach-orion5x/rd88f6183ap-ge-setup.c
+++ b/arch/arm/mach-orion5x/rd88f6183ap-ge-setup.c
@@ -40,11 +40,6 @@ static struct dsa_chip_data rd88f6183ap_ge_switch_chip_data = {
.port_names[5] = "cpu",
};
-static struct dsa_platform_data __initdata rd88f6183ap_ge_switch_plat_data = {
- .nr_chips = 1,
- .chip = &rd88f6183ap_ge_switch_chip_data,
-};
-
static struct mtd_partition rd88f6183ap_ge_partitions[] = {
{
.name = "kernel",
@@ -89,7 +84,7 @@ static void __init rd88f6183ap_ge_init(void)
*/
orion5x_ehci0_init();
orion5x_eth_init(&rd88f6183ap_ge_eth_data);
- orion5x_eth_switch_init(&rd88f6183ap_ge_switch_plat_data);
+ orion5x_eth_switch_init(&rd88f6183ap_ge_switch_chip_data);
spi_register_board_info(rd88f6183ap_ge_spi_slave_info,
ARRAY_SIZE(rd88f6183ap_ge_spi_slave_info));
orion5x_spi_init();
diff --git a/arch/arm/mach-orion5x/wnr854t-setup.c b/arch/arm/mach-orion5x/wnr854t-setup.c
index 4dbcdbe1de7c..d162d4c7f85d 100644
--- a/arch/arm/mach-orion5x/wnr854t-setup.c
+++ b/arch/arm/mach-orion5x/wnr854t-setup.c
@@ -106,11 +106,6 @@ static struct dsa_chip_data wnr854t_switch_chip_data = {
.port_names[7] = "lan2",
};
-static struct dsa_platform_data __initdata wnr854t_switch_plat_data = {
- .nr_chips = 1,
- .chip = &wnr854t_switch_chip_data,
-};
-
static void __init wnr854t_init(void)
{
/*
@@ -124,7 +119,7 @@ static void __init wnr854t_init(void)
* Configure peripherals.
*/
orion5x_eth_init(&wnr854t_eth_data);
- orion5x_eth_switch_init(&wnr854t_switch_plat_data);
+ orion5x_eth_switch_init(&wnr854t_switch_chip_data);
orion5x_uart0_init();
mvebu_mbus_add_window_by_id(ORION_MBUS_DEVBUS_BOOT_TARGET,
diff --git a/arch/arm/mach-orion5x/wrt350n-v2-setup.c b/arch/arm/mach-orion5x/wrt350n-v2-setup.c
index a6a8c4648d74..9250bb2e429c 100644
--- a/arch/arm/mach-orion5x/wrt350n-v2-setup.c
+++ b/arch/arm/mach-orion5x/wrt350n-v2-setup.c
@@ -191,11 +191,6 @@ static struct dsa_chip_data wrt350n_v2_switch_chip_data = {
.port_names[7] = "lan4",
};
-static struct dsa_platform_data __initdata wrt350n_v2_switch_plat_data = {
- .nr_chips = 1,
- .chip = &wrt350n_v2_switch_chip_data,
-};
-
static void __init wrt350n_v2_init(void)
{
/*
@@ -210,7 +205,7 @@ static void __init wrt350n_v2_init(void)
*/
orion5x_ehci0_init();
orion5x_eth_init(&wrt350n_v2_eth_data);
- orion5x_eth_switch_init(&wrt350n_v2_switch_plat_data);
+ orion5x_eth_switch_init(&wrt350n_v2_switch_chip_data);
orion5x_uart0_init();
mvebu_mbus_add_window_by_id(ORION_MBUS_DEVBUS_BOOT_TARGET,
diff --git a/arch/arm/mach-pxa/balloon3.c b/arch/arm/mach-pxa/balloon3.c
index 8a3c409294bf..d452a49c0396 100644
--- a/arch/arm/mach-pxa/balloon3.c
+++ b/arch/arm/mach-pxa/balloon3.c
@@ -17,6 +17,7 @@
#include <linux/init.h>
#include <linux/platform_device.h>
#include <linux/interrupt.h>
+#include <linux/leds.h>
#include <linux/sched.h>
#include <linux/bitops.h>
#include <linux/fb.h>
diff --git a/arch/arm/mach-pxa/colibri-pxa270-income.c b/arch/arm/mach-pxa/colibri-pxa270-income.c
index 8cff770e6a00..d7cf47d03618 100644
--- a/arch/arm/mach-pxa/colibri-pxa270-income.c
+++ b/arch/arm/mach-pxa/colibri-pxa270-income.c
@@ -17,6 +17,7 @@
#include <linux/gpio.h>
#include <linux/init.h>
#include <linux/interrupt.h>
+#include <linux/leds.h>
#include <linux/ioport.h>
#include <linux/kernel.h>
#include <linux/platform_device.h>
diff --git a/arch/arm/mach-pxa/corgi.c b/arch/arm/mach-pxa/corgi.c
index 183cd3446f25..7270f0db3432 100644
--- a/arch/arm/mach-pxa/corgi.c
+++ b/arch/arm/mach-pxa/corgi.c
@@ -19,6 +19,7 @@
#include <linux/major.h>
#include <linux/fs.h>
#include <linux/interrupt.h>
+#include <linux/leds.h>
#include <linux/mmc/host.h>
#include <linux/mtd/physmap.h>
#include <linux/pm.h>
diff --git a/arch/arm/mach-pxa/trizeps4.c b/arch/arm/mach-pxa/trizeps4.c
index ea78bc5c4198..3dd13b44c311 100644
--- a/arch/arm/mach-pxa/trizeps4.c
+++ b/arch/arm/mach-pxa/trizeps4.c
@@ -16,6 +16,7 @@
#include <linux/kernel.h>
#include <linux/platform_device.h>
#include <linux/interrupt.h>
+#include <linux/leds.h>
#include <linux/export.h>
#include <linux/sched.h>
#include <linux/bitops.h>
diff --git a/arch/arm/mach-pxa/vpac270.c b/arch/arm/mach-pxa/vpac270.c
index c006ee902a8f..70ab3ad28237 100644
--- a/arch/arm/mach-pxa/vpac270.c
+++ b/arch/arm/mach-pxa/vpac270.c
@@ -15,6 +15,7 @@
#include <linux/irq.h>
#include <linux/gpio_keys.h>
#include <linux/input.h>
+#include <linux/leds.h>
#include <linux/gpio.h>
#include <linux/usb/gpio_vbus.h>
#include <linux/mtd/mtd.h>
diff --git a/arch/arm/mach-pxa/zeus.c b/arch/arm/mach-pxa/zeus.c
index 3b94ecfb9426..ecbcaee5a2d5 100644
--- a/arch/arm/mach-pxa/zeus.c
+++ b/arch/arm/mach-pxa/zeus.c
@@ -13,6 +13,7 @@
#include <linux/cpufreq.h>
#include <linux/interrupt.h>
+#include <linux/leds.h>
#include <linux/irq.h>
#include <linux/pm.h>
#include <linux/gpio.h>
diff --git a/arch/arm/mach-pxa/zylonite.c b/arch/arm/mach-pxa/zylonite.c
index 3642389b301a..4268552d600d 100644
--- a/arch/arm/mach-pxa/zylonite.c
+++ b/arch/arm/mach-pxa/zylonite.c
@@ -16,6 +16,7 @@
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/interrupt.h>
+#include <linux/leds.h>
#include <linux/init.h>
#include <linux/platform_device.h>
#include <linux/gpio.h>
diff --git a/arch/arm/mach-s3c64xx/dev-audio.c b/arch/arm/mach-s3c64xx/dev-audio.c
index a3f39dd114f9..cb953e238b2a 100644
--- a/arch/arm/mach-s3c64xx/dev-audio.c
+++ b/arch/arm/mach-s3c64xx/dev-audio.c
@@ -97,9 +97,7 @@ static struct resource s3c64xx_iisv4_resource[] = {
static struct s3c_audio_pdata i2sv4_pdata = {
.cfg_gpio = s3c64xx_i2s_cfg_gpio,
.type = {
- .i2s = {
- .quirks = QUIRK_PRI_6CHAN,
- },
+ .quirks = QUIRK_PRI_6CHAN,
},
};
diff --git a/arch/arm/mach-s5pv210/pm.c b/arch/arm/mach-s5pv210/pm.c
index 21b4b13c5ab7..7d69666de5ba 100644
--- a/arch/arm/mach-s5pv210/pm.c
+++ b/arch/arm/mach-s5pv210/pm.c
@@ -155,13 +155,6 @@ static const struct platform_suspend_ops s5pv210_suspend_ops = {
*/
static void s5pv210_pm_resume(void)
{
- u32 tmp;
-
- tmp = __raw_readl(S5P_OTHERS);
- tmp |= (S5P_OTHERS_RET_IO | S5P_OTHERS_RET_CF |\
- S5P_OTHERS_RET_MMC | S5P_OTHERS_RET_UART);
- __raw_writel(tmp , S5P_OTHERS);
-
s3c_pm_do_restore_core(s5pv210_core_save, ARRAY_SIZE(s5pv210_core_save));
}
diff --git a/arch/arm/mach-s5pv210/regs-clock.h b/arch/arm/mach-s5pv210/regs-clock.h
index 4640f0f03c12..fb3eb77412db 100644
--- a/arch/arm/mach-s5pv210/regs-clock.h
+++ b/arch/arm/mach-s5pv210/regs-clock.h
@@ -188,10 +188,6 @@
#define S5P_SLEEP_CFG_USBOSC_EN (1 << 1)
/* OTHERS Resgister */
-#define S5P_OTHERS_RET_IO (1 << 31)
-#define S5P_OTHERS_RET_CF (1 << 30)
-#define S5P_OTHERS_RET_MMC (1 << 29)
-#define S5P_OTHERS_RET_UART (1 << 28)
#define S5P_OTHERS_USB_SIG_MASK (1 << 16)
/* S5P_DAC_CONTROL */
diff --git a/arch/arm/mach-shmobile/Kconfig b/arch/arm/mach-shmobile/Kconfig
index 2bb4b09f079e..ad7d604ff001 100644
--- a/arch/arm/mach-shmobile/Kconfig
+++ b/arch/arm/mach-shmobile/Kconfig
@@ -57,6 +57,7 @@ config ARCH_R7S72100
select PM
select PM_GENERIC_DOMAINS
select SYS_SUPPORTS_SH_MTU2
+ select RENESAS_OSTM
config ARCH_R8A73A4
bool "R-Mobile APE6 (R8A73A40)"
diff --git a/arch/arm/mach-ux500/pm.c b/arch/arm/mach-ux500/pm.c
index 8538910db202..a970e7fcba9e 100644
--- a/arch/arm/mach-ux500/pm.c
+++ b/arch/arm/mach-ux500/pm.c
@@ -134,8 +134,8 @@ bool prcmu_pending_irq(void)
*/
bool prcmu_is_cpu_in_wfi(int cpu)
{
- return readl(PRCM_ARM_WFI_STANDBY) & cpu ? PRCM_ARM_WFI_STANDBY_WFI1 :
- PRCM_ARM_WFI_STANDBY_WFI0;
+ return readl(PRCM_ARM_WFI_STANDBY) &
+ (cpu ? PRCM_ARM_WFI_STANDBY_WFI1 : PRCM_ARM_WFI_STANDBY_WFI0);
}
/*
diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig
index f68e8ec29447..35e3a56e5d86 100644
--- a/arch/arm/mm/Kconfig
+++ b/arch/arm/mm/Kconfig
@@ -1051,21 +1051,9 @@ config ARCH_SUPPORTS_BIG_ENDIAN
This option specifies the architecture can support big endian
operation.
-config DEBUG_RODATA
- bool "Make kernel text and rodata read-only"
- depends on MMU && !XIP_KERNEL
- default y if CPU_V7
- help
- If this is set, kernel text and rodata memory will be made
- read-only, and non-text kernel memory will be made non-executable.
- The tradeoff is that each region is padded to section-size (1MiB)
- boundaries (because their permissions are different and splitting
- the 1M pages into 4K ones causes TLB performance problems), which
- can waste memory.
-
config DEBUG_ALIGN_RODATA
bool "Make rodata strictly non-executable"
- depends on DEBUG_RODATA
+ depends on STRICT_KERNEL_RWX
default y
help
If this is set, rodata will be made explicitly non-executable. This
diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c
index ab7710002ba6..82d3e79ec82b 100644
--- a/arch/arm/mm/dma-mapping.c
+++ b/arch/arm/mm/dma-mapping.c
@@ -1171,6 +1171,25 @@ core_initcall(dma_debug_do_init);
#ifdef CONFIG_ARM_DMA_USE_IOMMU
+static int __dma_info_to_prot(enum dma_data_direction dir, unsigned long attrs)
+{
+ int prot = 0;
+
+ if (attrs & DMA_ATTR_PRIVILEGED)
+ prot |= IOMMU_PRIV;
+
+ switch (dir) {
+ case DMA_BIDIRECTIONAL:
+ return prot | IOMMU_READ | IOMMU_WRITE;
+ case DMA_TO_DEVICE:
+ return prot | IOMMU_READ;
+ case DMA_FROM_DEVICE:
+ return prot | IOMMU_WRITE;
+ default:
+ return prot;
+ }
+}
+
/* IOMMU */
static int extend_iommu_mapping(struct dma_iommu_mapping *mapping);
@@ -1394,7 +1413,8 @@ __iommu_alloc_remap(struct page **pages, size_t size, gfp_t gfp, pgprot_t prot,
* Create a mapping in device IO address space for specified pages
*/
static dma_addr_t
-__iommu_create_mapping(struct device *dev, struct page **pages, size_t size)
+__iommu_create_mapping(struct device *dev, struct page **pages, size_t size,
+ unsigned long attrs)
{
struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
@@ -1419,7 +1439,7 @@ __iommu_create_mapping(struct device *dev, struct page **pages, size_t size)
len = (j - i) << PAGE_SHIFT;
ret = iommu_map(mapping->domain, iova, phys, len,
- IOMMU_READ|IOMMU_WRITE);
+ __dma_info_to_prot(DMA_BIDIRECTIONAL, attrs));
if (ret < 0)
goto fail;
iova += len;
@@ -1476,7 +1496,8 @@ static struct page **__iommu_get_pages(void *cpu_addr, unsigned long attrs)
}
static void *__iommu_alloc_simple(struct device *dev, size_t size, gfp_t gfp,
- dma_addr_t *handle, int coherent_flag)
+ dma_addr_t *handle, int coherent_flag,
+ unsigned long attrs)
{
struct page *page;
void *addr;
@@ -1488,7 +1509,7 @@ static void *__iommu_alloc_simple(struct device *dev, size_t size, gfp_t gfp,
if (!addr)
return NULL;
- *handle = __iommu_create_mapping(dev, &page, size);
+ *handle = __iommu_create_mapping(dev, &page, size, attrs);
if (*handle == DMA_ERROR_CODE)
goto err_mapping;
@@ -1522,7 +1543,7 @@ static void *__arm_iommu_alloc_attrs(struct device *dev, size_t size,
if (coherent_flag == COHERENT || !gfpflags_allow_blocking(gfp))
return __iommu_alloc_simple(dev, size, gfp, handle,
- coherent_flag);
+ coherent_flag, attrs);
/*
* Following is a work-around (a.k.a. hack) to prevent pages
@@ -1537,7 +1558,7 @@ static void *__arm_iommu_alloc_attrs(struct device *dev, size_t size,
if (!pages)
return NULL;
- *handle = __iommu_create_mapping(dev, pages, size);
+ *handle = __iommu_create_mapping(dev, pages, size, attrs);
if (*handle == DMA_ERROR_CODE)
goto err_buffer;
@@ -1672,27 +1693,6 @@ static int arm_iommu_get_sgtable(struct device *dev, struct sg_table *sgt,
GFP_KERNEL);
}
-static int __dma_direction_to_prot(enum dma_data_direction dir)
-{
- int prot;
-
- switch (dir) {
- case DMA_BIDIRECTIONAL:
- prot = IOMMU_READ | IOMMU_WRITE;
- break;
- case DMA_TO_DEVICE:
- prot = IOMMU_READ;
- break;
- case DMA_FROM_DEVICE:
- prot = IOMMU_WRITE;
- break;
- default:
- prot = 0;
- }
-
- return prot;
-}
-
/*
* Map a part of the scatter-gather list into contiguous io address space
*/
@@ -1722,7 +1722,7 @@ static int __map_sg_chunk(struct device *dev, struct scatterlist *sg,
if (!is_coherent && (attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)
__dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir);
- prot = __dma_direction_to_prot(dir);
+ prot = __dma_info_to_prot(dir, attrs);
ret = iommu_map(mapping->domain, iova, phys, len, prot);
if (ret < 0)
@@ -1930,7 +1930,7 @@ static dma_addr_t arm_coherent_iommu_map_page(struct device *dev, struct page *p
if (dma_addr == DMA_ERROR_CODE)
return dma_addr;
- prot = __dma_direction_to_prot(dir);
+ prot = __dma_info_to_prot(dir, attrs);
ret = iommu_map(mapping->domain, dma_addr, page_to_phys(page), len, prot);
if (ret < 0)
@@ -2036,7 +2036,7 @@ static dma_addr_t arm_iommu_map_resource(struct device *dev,
if (dma_addr == DMA_ERROR_CODE)
return dma_addr;
- prot = __dma_direction_to_prot(dir) | IOMMU_MMIO;
+ prot = __dma_info_to_prot(dir, attrs) | IOMMU_MMIO;
ret = iommu_map(mapping->domain, dma_addr, addr, len, prot);
if (ret < 0)
diff --git a/arch/arm/mm/extable.c b/arch/arm/mm/extable.c
index 312e15e6d00b..f436f7439e46 100644
--- a/arch/arm/mm/extable.c
+++ b/arch/arm/mm/extable.c
@@ -1,7 +1,7 @@
/*
* linux/arch/arm/mm/extable.c
*/
-#include <linux/module.h>
+#include <linux/extable.h>
#include <linux/uaccess.h>
int fixup_exception(struct pt_regs *regs)
diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c
index 3a2e678b8d30..c2b5b9892fd1 100644
--- a/arch/arm/mm/fault.c
+++ b/arch/arm/mm/fault.c
@@ -8,7 +8,7 @@
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*/
-#include <linux/module.h>
+#include <linux/extable.h>
#include <linux/signal.h>
#include <linux/mm.h>
#include <linux/hardirq.h>
@@ -610,9 +610,9 @@ static int __init early_abort_handler(unsigned long addr, unsigned int fsr,
void __init early_abt_enable(void)
{
- fsr_info[22].fn = early_abort_handler;
+ fsr_info[FSR_FS_AEA].fn = early_abort_handler;
local_abt_enable();
- fsr_info[22].fn = do_bad;
+ fsr_info[FSR_FS_AEA].fn = do_bad;
}
#ifndef CONFIG_ARM_LPAE
diff --git a/arch/arm/mm/fault.h b/arch/arm/mm/fault.h
index 67532f242271..afc1f84e763b 100644
--- a/arch/arm/mm/fault.h
+++ b/arch/arm/mm/fault.h
@@ -11,11 +11,15 @@
#define FSR_FS5_0 (0x3f)
#ifdef CONFIG_ARM_LPAE
+#define FSR_FS_AEA 17
+
static inline int fsr_fs(unsigned int fsr)
{
return fsr & FSR_FS5_0;
}
#else
+#define FSR_FS_AEA 22
+
static inline int fsr_fs(unsigned int fsr)
{
return (fsr & FSR_FS3_0) | (fsr & FSR_FS4) >> 6;
diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c
index 370581aeb871..4be0bee4c357 100644
--- a/arch/arm/mm/init.c
+++ b/arch/arm/mm/init.c
@@ -572,7 +572,7 @@ void __init mem_init(void)
}
}
-#ifdef CONFIG_DEBUG_RODATA
+#ifdef CONFIG_STRICT_KERNEL_RWX
struct section_perm {
const char *name;
unsigned long start;
@@ -741,7 +741,7 @@ void set_kernel_text_ro(void)
#else
static inline void fix_kernmem_perms(void) { }
-#endif /* CONFIG_DEBUG_RODATA */
+#endif /* CONFIG_STRICT_KERNEL_RWX */
void free_tcmmem(void)
{
diff --git a/arch/arm/plat-orion/common.c b/arch/arm/plat-orion/common.c
index 272f49b2c68f..9255b6d67ba5 100644
--- a/arch/arm/plat-orion/common.c
+++ b/arch/arm/plat-orion/common.c
@@ -22,6 +22,7 @@
#include <linux/platform_data/dma-mv_xor.h>
#include <linux/platform_data/usb-ehci-orion.h>
#include <plat/common.h>
+#include <linux/phy.h>
/* Create a clkdev entry for a given device/clk */
void __init orion_clkdev_add(const char *con_id, const char *dev_id,
@@ -470,15 +471,27 @@ void __init orion_ge11_init(struct mv643xx_eth_platform_data *eth_data,
/*****************************************************************************
* Ethernet switch
****************************************************************************/
-void __init orion_ge00_switch_init(struct dsa_platform_data *d)
+static __initconst const char *orion_ge00_mvmdio_bus_name = "orion-mii";
+static __initdata struct mdio_board_info
+ orion_ge00_switch_board_info;
+
+void __init orion_ge00_switch_init(struct dsa_chip_data *d)
{
- int i;
+ struct mdio_board_info *bd;
+ unsigned int i;
+
+ for (i = 0; i < ARRAY_SIZE(d->port_names); i++)
+ if (!strcmp(d->port_names[i], "cpu"))
+ break;
- d->netdev = &orion_ge00.dev;
- for (i = 0; i < d->nr_chips; i++)
- d->chip[i].host_dev = &orion_ge_mvmdio.dev;
+ bd = &orion_ge00_switch_board_info;
+ bd->bus_id = orion_ge00_mvmdio_bus_name;
+ bd->mdio_addr = d->sw_addr;
+ d->netdev[i] = &orion_ge00.dev;
+ strcpy(bd->modalias, "mv88e6085");
+ bd->platform_data = d;
- platform_device_register_data(NULL, "dsa", 0, d, sizeof(d));
+ mdiobus_register_board_info(&orion_ge00_switch_board_info, 1);
}
/*****************************************************************************
diff --git a/arch/arm/plat-orion/include/plat/common.h b/arch/arm/plat-orion/include/plat/common.h
index 9347f3c58a6d..3647d3b33c20 100644
--- a/arch/arm/plat-orion/include/plat/common.h
+++ b/arch/arm/plat-orion/include/plat/common.h
@@ -12,7 +12,7 @@
#include <linux/mv643xx_eth.h>
#include <linux/platform_data/usb-ehci-orion.h>
-struct dsa_platform_data;
+struct dsa_chip_data;
struct mv_sata_platform_data;
void __init orion_uart0_init(void __iomem *membase,
@@ -57,7 +57,7 @@ void __init orion_ge11_init(struct mv643xx_eth_platform_data *eth_data,
unsigned long mapbase,
unsigned long irq);
-void __init orion_ge00_switch_init(struct dsa_platform_data *d);
+void __init orion_ge00_switch_init(struct dsa_chip_data *d);
void __init orion_i2c_init(unsigned long mapbase,
unsigned long irq,
diff --git a/arch/arm/xen/enlighten.c b/arch/arm/xen/enlighten.c
index 11d9f2898b16..81e3217b12d3 100644
--- a/arch/arm/xen/enlighten.c
+++ b/arch/arm/xen/enlighten.c
@@ -457,4 +457,5 @@ EXPORT_SYMBOL_GPL(HYPERVISOR_tmem_op);
EXPORT_SYMBOL_GPL(HYPERVISOR_platform_op);
EXPORT_SYMBOL_GPL(HYPERVISOR_multicall);
EXPORT_SYMBOL_GPL(HYPERVISOR_vm_assist);
+EXPORT_SYMBOL_GPL(HYPERVISOR_dm_op);
EXPORT_SYMBOL_GPL(privcmd_call);
diff --git a/arch/arm/xen/hypercall.S b/arch/arm/xen/hypercall.S
index a648dfc3be30..b0b80c0f09f3 100644
--- a/arch/arm/xen/hypercall.S
+++ b/arch/arm/xen/hypercall.S
@@ -92,6 +92,7 @@ HYPERCALL1(tmem_op);
HYPERCALL1(platform_op_raw);
HYPERCALL2(multicall);
HYPERCALL2(vm_assist);
+HYPERCALL3(dm_op);
ENTRY(privcmd_call)
stmdb sp!, {r4}