diff options
Diffstat (limited to 'arch/x86/crypto/twofish-avx2-asm_64.S')
| -rw-r--r-- | arch/x86/crypto/twofish-avx2-asm_64.S | 600 |
1 files changed, 600 insertions, 0 deletions
diff --git a/arch/x86/crypto/twofish-avx2-asm_64.S b/arch/x86/crypto/twofish-avx2-asm_64.S new file mode 100644 index 000000000000..e1a83b9cd389 --- /dev/null +++ b/arch/x86/crypto/twofish-avx2-asm_64.S @@ -0,0 +1,600 @@ +/* + * x86_64/AVX2 assembler optimized version of Twofish + * + * Copyright © 2012-2013 Jussi Kivilinna <jussi.kivilinna@iki.fi> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + */ + +#include <linux/linkage.h> +#include "glue_helper-asm-avx2.S" + +.file "twofish-avx2-asm_64.S" + +.data +.align 16 + +.Lvpshufb_mask0: +.long 0x80808000 +.long 0x80808004 +.long 0x80808008 +.long 0x8080800c + +.Lbswap128_mask: + .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 +.Lxts_gf128mul_and_shl1_mask_0: + .byte 0x87, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0 +.Lxts_gf128mul_and_shl1_mask_1: + .byte 0x0e, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0 + +.text + +/* structure of crypto context */ +#define s0 0 +#define s1 1024 +#define s2 2048 +#define s3 3072 +#define w 4096 +#define k 4128 + +/* register macros */ +#define CTX %rdi + +#define RS0 CTX +#define RS1 %r8 +#define RS2 %r9 +#define RS3 %r10 +#define RK %r11 +#define RW %rax +#define RROUND %r12 +#define RROUNDd %r12d + +#define RA0 %ymm8 +#define RB0 %ymm9 +#define RC0 %ymm10 +#define RD0 %ymm11 +#define RA1 %ymm12 +#define RB1 %ymm13 +#define RC1 %ymm14 +#define RD1 %ymm15 + +/* temp regs */ +#define RX0 %ymm0 +#define RY0 %ymm1 +#define RX1 %ymm2 +#define RY1 %ymm3 +#define RT0 %ymm4 +#define RIDX %ymm5 + +#define RX0x %xmm0 +#define RY0x %xmm1 +#define RX1x %xmm2 +#define RY1x %xmm3 +#define RT0x %xmm4 + +/* vpgatherdd mask and '-1' */ +#define RNOT %ymm6 + +/* byte mask, (-1 >> 24) */ +#define RBYTE %ymm7 + +/********************************************************************** + 16-way AVX2 twofish + **********************************************************************/ +#define init_round_constants() \ + vpcmpeqd RNOT, RNOT, RNOT; \ + vpsrld $24, RNOT, RBYTE; \ + leaq k(CTX), RK; \ + leaq w(CTX), RW; \ + leaq s1(CTX), RS1; \ + leaq s2(CTX), RS2; \ + leaq s3(CTX), RS3; \ + +#define g16(ab, rs0, rs1, rs2, rs3, xy) \ + vpand RBYTE, ab ## 0, RIDX; \ + vpgatherdd RNOT, (rs0, RIDX, 4), xy ## 0; \ + vpcmpeqd RNOT, RNOT, RNOT; \ + \ + vpand RBYTE, ab ## 1, RIDX; \ + vpgatherdd RNOT, (rs0, RIDX, 4), xy ## 1; \ + vpcmpeqd RNOT, RNOT, RNOT; \ + \ + vpsrld $8, ab ## 0, RIDX; \ + vpand RBYTE, RIDX, RIDX; \ + vpgatherdd RNOT, (rs1, RIDX, 4), RT0; \ + vpcmpeqd RNOT, RNOT, RNOT; \ + vpxor RT0, xy ## 0, xy ## 0; \ + \ + vpsrld $8, ab ## 1, RIDX; \ + vpand RBYTE, RIDX, RIDX; \ + vpgatherdd RNOT, (rs1, RIDX, 4), RT0; \ + vpcmpeqd RNOT, RNOT, RNOT; \ + vpxor RT0, xy ## 1, xy ## 1; \ + \ + vpsrld $16, ab ## 0, RIDX; \ + vpand RBYTE, RIDX, RIDX; \ + vpgatherdd RNOT, (rs2, RIDX, 4), RT0; \ + vpcmpeqd RNOT, RNOT, RNOT; \ + vpxor RT0, xy ## 0, xy ## 0; \ + \ + vpsrld $16, ab ## 1, RIDX; \ + vpand RBYTE, RIDX, RIDX; \ + vpgatherdd RNOT, (rs2, RIDX, 4), RT0; \ + vpcmpeqd RNOT, RNOT, RNOT; \ + vpxor RT0, xy ## 1, xy ## 1; \ + \ + vpsrld $24, ab ## 0, RIDX; \ + vpgatherdd RNOT, (rs3, RIDX, 4), RT0; \ + vpcmpeqd RNOT, RNOT, RNOT; \ + vpxor RT0, xy ## 0, xy ## 0; \ + \ + vpsrld $24, ab ## 1, RIDX; \ + vpgatherdd RNOT, (rs3, RIDX, 4), RT0; \ + vpcmpeqd RNOT, RNOT, RNOT; \ + vpxor RT0, xy ## 1, xy ## 1; + +#define g1_16(a, x) \ + g16(a, RS0, RS1, RS2, RS3, x); + +#define g2_16(b, y) \ + g16(b, RS1, RS2, RS3, RS0, y); + +#define encrypt_round_end16(a, b, c, d, nk) \ + vpaddd RY0, RX0, RX0; \ + vpaddd RX0, RY0, RY0; \ + vpbroadcastd nk(RK,RROUND,8), RT0; \ + vpaddd RT0, RX0, RX0; \ + vpbroadcastd 4+nk(RK,RROUND,8), RT0; \ + vpaddd RT0, RY0, RY0; \ + \ + vpxor RY0, d ## 0, d ## 0; \ + \ + vpxor RX0, c ## 0, c ## 0; \ + vpsrld $1, c ## 0, RT0; \ + vpslld $31, c ## 0, c ## 0; \ + vpor RT0, c ## 0, c ## 0; \ + \ + vpaddd RY1, RX1, RX1; \ + vpaddd RX1, RY1, RY1; \ + vpbroadcastd nk(RK,RROUND,8), RT0; \ + vpaddd RT0, RX1, RX1; \ + vpbroadcastd 4+nk(RK,RROUND,8), RT0; \ + vpaddd RT0, RY1, RY1; \ + \ + vpxor RY1, d ## 1, d ## 1; \ + \ + vpxor RX1, c ## 1, c ## 1; \ + vpsrld $1, c ## 1, RT0; \ + vpslld $31, c ## 1, c ## 1; \ + vpor RT0, c ## 1, c ## 1; \ + +#define encrypt_round16(a, b, c, d, nk) \ + g2_16(b, RY); \ + \ + vpslld $1, b ## 0, RT0; \ + vpsrld $31, b ## 0, b ## 0; \ + vpor RT0, b ## 0, b ## 0; \ + \ + vpslld $1, b ## 1, RT0; \ + vpsrld $31, b ## 1, b ## 1; \ + vpor RT0, b ## 1, b ## 1; \ + \ + g1_16(a, RX); \ + \ + encrypt_round_end16(a, b, c, d, nk); + +#define encrypt_round_first16(a, b, c, d, nk) \ + vpslld $1, d ## 0, RT0; \ + vpsrld $31, d ## 0, d ## 0; \ + vpor RT0, d ## 0, d ## 0; \ + \ + vpslld $1, d ## 1, RT0; \ + vpsrld $31, d ## 1, d ## 1; \ + vpor RT0, d ## 1, d ## 1; \ + \ + encrypt_round16(a, b, c, d, nk); + +#define encrypt_round_last16(a, b, c, d, nk) \ + g2_16(b, RY); \ + \ + g1_16(a, RX); \ + \ + encrypt_round_end16(a, b, c, d, nk); + +#define decrypt_round_end16(a, b, c, d, nk) \ + vpaddd RY0, RX0, RX0; \ + vpaddd RX0, RY0, RY0; \ + vpbroadcastd nk(RK,RROUND,8), RT0; \ + vpaddd RT0, RX0, RX0; \ + vpbroadcastd 4+nk(RK,RROUND,8), RT0; \ + vpaddd RT0, RY0, RY0; \ + \ + vpxor RX0, c ## 0, c ## 0; \ + \ + vpxor RY0, d ## 0, d ## 0; \ + vpsrld $1, d ## 0, RT0; \ + vpslld $31, d ## 0, d ## 0; \ + vpor RT0, d ## 0, d ## 0; \ + \ + vpaddd RY1, RX1, RX1; \ + vpaddd RX1, RY1, RY1; \ + vpbroadcastd nk(RK,RROUND,8), RT0; \ + vpaddd RT0, RX1, RX1; \ + vpbroadcastd 4+nk(RK,RROUND,8), RT0; \ + vpaddd RT0, RY1, RY1; \ + \ + vpxor RX1, c ## 1, c ## 1; \ + \ + vpxor RY1, d ## 1, d ## 1; \ + vpsrld $1, d ## 1, RT0; \ + vpslld $31, d ## 1, d ## 1; \ + vpor RT0, d ## 1, d ## 1; + +#define decrypt_round16(a, b, c, d, nk) \ + g1_16(a, RX); \ + \ + vpslld $1, a ## 0, RT0; \ + vpsrld $31, a ## 0, a ## 0; \ + vpor RT0, a ## 0, a ## 0; \ + \ + vpslld $1, a ## 1, RT0; \ + vpsrld $31, a ## 1, a ## 1; \ + vpor RT0, a ## 1, a ## 1; \ + \ + g2_16(b, RY); \ + \ + decrypt_round_end16(a, b, c, d, nk); + +#define decrypt_round_first16(a, b, c, d, nk) \ + vpslld $1, c ## 0, RT0; \ + vpsrld $31, c ## 0, c ## 0; \ + vpor RT0, c ## 0, c ## 0; \ + \ + vpslld $1, c ## 1, RT0; \ + vpsrld $31, c ## 1, c ## 1; \ + vpor RT0, c ## 1, c ## 1; \ + \ + decrypt_round16(a, b, c, d, nk) + +#define decrypt_round_last16(a, b, c, d, nk) \ + g1_16(a, RX); \ + \ + g2_16(b, RY); \ + \ + decrypt_round_end16(a, b, c, d, nk); + +#define encrypt_cycle16() \ + encrypt_round16(RA, RB, RC, RD, 0); \ + encrypt_round16(RC, RD, RA, RB, 8); + +#define encrypt_cycle_first16() \ + encrypt_round_first16(RA, RB, RC, RD, 0); \ + encrypt_round16(RC, RD, RA, RB, 8); + +#define encrypt_cycle_last16() \ + encrypt_round16(RA, RB, RC, RD, 0); \ + encrypt_round_last16(RC, RD, RA, RB, 8); + +#define decrypt_cycle16(n) \ + decrypt_round16(RC, RD, RA, RB, 8); \ + decrypt_round16(RA, RB, RC, RD, 0); + +#define decrypt_cycle_first16(n) \ + decrypt_round_first16(RC, RD, RA, RB, 8); \ + decrypt_round16(RA, RB, RC, RD, 0); + +#define decrypt_cycle_last16(n) \ + decrypt_round16(RC, RD, RA, RB, 8); \ + decrypt_round_last16(RA, RB, RC, RD, 0); + +#define transpose_4x4(x0,x1,x2,x3,t1,t2) \ + vpunpckhdq x1, x0, t2; \ + vpunpckldq x1, x0, x0; \ + \ + vpunpckldq x3, x2, t1; \ + vpunpckhdq x3, x2, x2; \ + \ + vpunpckhqdq t1, x0, x1; \ + vpunpcklqdq t1, x0, x0; \ + \ + vpunpckhqdq x2, t2, x3; \ + vpunpcklqdq x2, t2, x2; + +#define read_blocks8(offs,a,b,c,d) \ + transpose_4x4(a, b, c, d, RX0, RY0); + +#define write_blocks8(offs,a,b,c,d) \ + transpose_4x4(a, b, c, d, RX0, RY0); + +#define inpack_enc8(a,b,c,d) \ + vpbroadcastd 4*0(RW), RT0; \ + vpxor RT0, a, a; \ + \ + vpbroadcastd 4*1(RW), RT0; \ + vpxor RT0, b, b; \ + \ + vpbroadcastd 4*2(RW), RT0; \ + vpxor RT0, c, c; \ + \ + vpbroadcastd 4*3(RW), RT0; \ + vpxor RT0, d, d; + +#define outunpack_enc8(a,b,c,d) \ + vpbroadcastd 4*4(RW), RX0; \ + vpbroadcastd 4*5(RW), RY0; \ + vpxor RX0, c, RX0; \ + vpxor RY0, d, RY0; \ + \ + vpbroadcastd 4*6(RW), RT0; \ + vpxor RT0, a, c; \ + vpbroadcastd 4*7(RW), RT0; \ + vpxor RT0, b, d; \ + \ + vmovdqa RX0, a; \ + vmovdqa RY0, b; + +#define inpack_dec8(a,b,c,d) \ + vpbroadcastd 4*4(RW), RX0; \ + vpbroadcastd 4*5(RW), RY0; \ + vpxor RX0, a, RX0; \ + vpxor RY0, b, RY0; \ + \ + vpbroadcastd 4*6(RW), RT0; \ + vpxor RT0, c, a; \ + vpbroadcastd 4*7(RW), RT0; \ + vpxor RT0, d, b; \ + \ + vmovdqa RX0, c; \ + vmovdqa RY0, d; + +#define outunpack_dec8(a,b,c,d) \ + vpbroadcastd 4*0(RW), RT0; \ + vpxor RT0, a, a; \ + \ + vpbroadcastd 4*1(RW), RT0; \ + vpxor RT0, b, b; \ + \ + vpbroadcastd 4*2(RW), RT0; \ + vpxor RT0, c, c; \ + \ + vpbroadcastd 4*3(RW), RT0; \ + vpxor RT0, d, d; + +#define read_blocks16(a,b,c,d) \ + read_blocks8(0, a ## 0, b ## 0, c ## 0, d ## 0); \ + read_blocks8(8, a ## 1, b ## 1, c ## 1, d ## 1); + +#define write_blocks16(a,b,c,d) \ + write_blocks8(0, a ## 0, b ## 0, c ## 0, d ## 0); \ + write_blocks8(8, a ## 1, b ## 1, c ## 1, d ## 1); + +#define xor_blocks16(a,b,c,d) \ + xor_blocks8(0, a ## 0, b ## 0, c ## 0, d ## 0); \ + xor_blocks8(8, a ## 1, b ## 1, c ## 1, d ## 1); + +#define inpack_enc16(a,b,c,d) \ + inpack_enc8(a ## 0, b ## 0, c ## 0, d ## 0); \ + inpack_enc8(a ## 1, b ## 1, c ## 1, d ## 1); + +#define outunpack_enc16(a,b,c,d) \ + outunpack_enc8(a ## 0, b ## 0, c ## 0, d ## 0); \ + outunpack_enc8(a ## 1, b ## 1, c ## 1, d ## 1); + +#define inpack_dec16(a,b,c,d) \ + inpack_dec8(a ## 0, b ## 0, c ## 0, d ## 0); \ + inpack_dec8(a ## 1, b ## 1, c ## 1, d ## 1); + +#define outunpack_dec16(a,b,c,d) \ + outunpack_dec8(a ## 0, b ## 0, c ## 0, d ## 0); \ + outunpack_dec8(a ## 1, b ## 1, c ## 1, d ## 1); + +.align 8 +__twofish_enc_blk16: + /* input: + * %rdi: ctx, CTX + * RA0, RB0, RC0, RD0, RA1, RB1, RC1, RD1: plaintext + * output: + * RA0, RB0, RC0, RD0, RA1, RB1, RC1, RD1: ciphertext + */ + init_round_constants(); + + read_blocks16(RA, RB, RC, RD); + inpack_enc16(RA, RB, RC, RD); + + xorl RROUNDd, RROUNDd; + encrypt_cycle_first16(); + movl $2, RROUNDd; + +.align 4 +.L__enc_loop: + encrypt_cycle16(); + + addl $2, RROUNDd; + cmpl $14, RROUNDd; + jne .L__enc_loop; + + encrypt_cycle_last16(); + + outunpack_enc16(RA, RB, RC, RD); + write_blocks16(RA, RB, RC, RD); + + ret; +ENDPROC(__twofish_enc_blk16) + +.align 8 +__twofish_dec_blk16: + /* input: + * %rdi: ctx, CTX + * RA0, RB0, RC0, RD0, RA1, RB1, RC1, RD1: ciphertext + * output: + * RA0, RB0, RC0, RD0, RA1, RB1, RC1, RD1: plaintext + */ + init_round_constants(); + + read_blocks16(RA, RB, RC, RD); + inpack_dec16(RA, RB, RC, RD); + + movl $14, RROUNDd; + decrypt_cycle_first16(); + movl $12, RROUNDd; + +.align 4 +.L__dec_loop: + decrypt_cycle16(); + + addl $-2, RROUNDd; + jnz .L__dec_loop; + + decrypt_cycle_last16(); + + outunpack_dec16(RA, RB, RC, RD); + write_blocks16(RA, RB, RC, RD); + + ret; +ENDPROC(__twofish_dec_blk16) + +ENTRY(twofish_ecb_enc_16way) + /* input: + * %rdi: ctx, CTX + * %rsi: dst + * %rdx: src + */ + + vzeroupper; + pushq %r12; + + load_16way(%rdx, RA0, RB0, RC0, RD0, RA1, RB1, RC1, RD1); + + call __twofish_enc_blk16; + + store_16way(%rsi, RA0, RB0, RC0, RD0, RA1, RB1, RC1, RD1); + + popq %r12; + vzeroupper; + + ret; +ENDPROC(twofish_ecb_enc_16way) + +ENTRY(twofish_ecb_dec_16way) + /* input: + * %rdi: ctx, CTX + * %rsi: dst + * %rdx: src + */ + + vzeroupper; + pushq %r12; + + load_16way(%rdx, RA0, RB0, RC0, RD0, RA1, RB1, RC1, RD1); + + call __twofish_dec_blk16; + + store_16way(%rsi, RA0, RB0, RC0, RD0, RA1, RB1, RC1, RD1); + + popq %r12; + vzeroupper; + + ret; +ENDPROC(twofish_ecb_dec_16way) + +ENTRY(twofish_cbc_dec_16way) + /* input: + * %rdi: ctx, CTX + * %rsi: dst + * %rdx: src + */ + + vzeroupper; + pushq %r12; + + load_16way(%rdx, RA0, RB0, RC0, RD0, RA1, RB1, RC1, RD1); + + call __twofish_dec_blk16; + + store_cbc_16way(%rdx, %rsi, RA0, RB0, RC0, RD0, RA1, RB1, RC1, RD1, + RX0); + + popq %r12; + vzeroupper; + + ret; +ENDPROC(twofish_cbc_dec_16way) + +ENTRY(twofish_ctr_16way) + /* input: + * %rdi: ctx, CTX + * %rsi: dst (16 blocks) + * %rdx: src (16 blocks) + * %rcx: iv (little endian, 128bit) + */ + + vzeroupper; + pushq %r12; + + load_ctr_16way(%rcx, .Lbswap128_mask, RA0, RB0, RC0, RD0, RA1, RB1, RC1, + RD1, RX0, RX0x, RX1, RX1x, RY0, RY0x, RY1, RY1x, RNOT, + RBYTE); + + call __twofish_enc_blk16; + + store_ctr_16way(%rdx, %rsi, RA0, RB0, RC0, RD0, RA1, RB1, RC1, RD1); + + popq %r12; + vzeroupper; + + ret; +ENDPROC(twofish_ctr_16way) + +.align 8 +twofish_xts_crypt_16way: + /* input: + * %rdi: ctx, CTX + * %rsi: dst (16 blocks) + * %rdx: src (16 blocks) + * %rcx: iv (t ⊕ αⁿ ∈ GF(2¹²⁸)) + * %r8: pointer to __twofish_enc_blk16 or __twofish_dec_blk16 + */ + + vzeroupper; + pushq %r12; + + load_xts_16way(%rcx, %rdx, %rsi, RA0, RB0, RC0, RD0, RA1, RB1, RC1, + RD1, RX0, RX0x, RX1, RX1x, RY0, RY0x, RY1, RY1x, RNOT, + .Lxts_gf128mul_and_shl1_mask_0, + .Lxts_gf128mul_and_shl1_mask_1); + + call *%r8; + + store_xts_16way(%rsi, RA0, RB0, RC0, RD0, RA1, RB1, RC1, RD1); + + popq %r12; + vzeroupper; + + ret; +ENDPROC(twofish_xts_crypt_16way) + +ENTRY(twofish_xts_enc_16way) + /* input: + * %rdi: ctx, CTX + * %rsi: dst (16 blocks) + * %rdx: src (16 blocks) + * %rcx: iv (t ⊕ αⁿ ∈ GF(2¹²⁸)) + */ + leaq __twofish_enc_blk16, %r8; + jmp twofish_xts_crypt_16way; +ENDPROC(twofish_xts_enc_16way) + +ENTRY(twofish_xts_dec_16way) + /* input: + * %rdi: ctx, CTX + * %rsi: dst (16 blocks) + * %rdx: src (16 blocks) + * %rcx: iv (t ⊕ αⁿ ∈ GF(2¹²⁸)) + */ + leaq __twofish_dec_blk16, %r8; + jmp twofish_xts_crypt_16way; +ENDPROC(twofish_xts_dec_16way) |