diff options
229 files changed, 9465 insertions, 3102 deletions
diff --git a/Documentation/crypto/userspace-if.rst b/Documentation/crypto/userspace-if.rst index 52019e905900..b45dabbf69d6 100644 --- a/Documentation/crypto/userspace-if.rst +++ b/Documentation/crypto/userspace-if.rst @@ -296,15 +296,16 @@ follows: struct sockaddr_alg sa = { .salg_family = AF_ALG, - .salg_type = "rng", /* this selects the symmetric cipher */ - .salg_name = "drbg_nopr_sha256" /* this is the cipher name */ + .salg_type = "rng", /* this selects the random number generator */ + .salg_name = "drbg_nopr_sha256" /* this is the RNG name */ }; Depending on the RNG type, the RNG must be seeded. The seed is provided using the setsockopt interface to set the key. For example, the ansi_cprng requires a seed. The DRBGs do not require a seed, but may be -seeded. +seeded. The seed is also known as a *Personalization String* in NIST SP 800-90A +standard. Using the read()/recvmsg() system calls, random numbers can be obtained. The kernel generates at most 128 bytes in one call. If user space @@ -314,6 +315,16 @@ WARNING: The user space caller may invoke the initially mentioned accept system call multiple times. In this case, the returned file descriptors have the same state. +Following CAVP testing interfaces are enabled when kernel is built with +CRYPTO_USER_API_RNG_CAVP option: + +- the concatenation of *Entropy* and *Nonce* can be provided to the RNG via + ALG_SET_DRBG_ENTROPY setsockopt interface. Setting the entropy requires + CAP_SYS_ADMIN permission. + +- *Additional Data* can be provided using the send()/sendmsg() system calls, + but only after the entropy has been set. + Zero-Copy Interface ------------------- @@ -377,6 +388,9 @@ mentioned optname: provided ciphertext is assumed to contain an authentication tag of the given size (see section about AEAD memory layout below). +- ALG_SET_DRBG_ENTROPY -- Setting the entropy of the random number generator. + This option is applicable to RNG cipher type only. + User space API example ---------------------- diff --git a/Documentation/devicetree/bindings/rng/ingenic,trng.yaml b/Documentation/devicetree/bindings/rng/ingenic,trng.yaml new file mode 100644 index 000000000000..808f247c8421 --- /dev/null +++ b/Documentation/devicetree/bindings/rng/ingenic,trng.yaml @@ -0,0 +1,43 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/rng/ingenic,trng.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Bindings for DTRNG in Ingenic SoCs + +maintainers: + - 周琰杰 (Zhou Yanjie) <zhouyanjie@wanyeetech.com> + +description: + The True Random Number Generator in Ingenic SoCs. + +properties: + compatible: + enum: + - ingenic,x1830-dtrng + + reg: + maxItems: 1 + + clocks: + maxItems: 1 + +required: + - compatible + - reg + - clocks + +additionalProperties: false + +examples: + - | + #include <dt-bindings/clock/x1830-cgu.h> + + dtrng: trng@10072000 { + compatible = "ingenic,x1830-dtrng"; + reg = <0x10072000 0xc>; + + clocks = <&cgu X1830_CLK_DTRNG>; + }; +... diff --git a/Documentation/devicetree/bindings/rng/xiphera,xip8001b-trng.yaml b/Documentation/devicetree/bindings/rng/xiphera,xip8001b-trng.yaml new file mode 100644 index 000000000000..1e17e55762f1 --- /dev/null +++ b/Documentation/devicetree/bindings/rng/xiphera,xip8001b-trng.yaml @@ -0,0 +1,33 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/rng/xiphera,xip8001b-trng.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Xiphera XIP8001B-trng bindings + +maintainers: + - Atte Tommiska <atte.tommiska@xiphera.com> + +description: | + Xiphera FPGA-based true random number generator intellectual property core. + +properties: + compatible: + const: xiphera,xip8001b-trng + + reg: + maxItems: 1 + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + rng@43c00000 { + compatible = "xiphera,xip8001b-trng"; + reg = <0x43c00000 0x10000>; + }; diff --git a/Documentation/devicetree/bindings/vendor-prefixes.yaml b/Documentation/devicetree/bindings/vendor-prefixes.yaml index 63996ab03521..7d58834c5aab 100644 --- a/Documentation/devicetree/bindings/vendor-prefixes.yaml +++ b/Documentation/devicetree/bindings/vendor-prefixes.yaml @@ -1174,6 +1174,8 @@ patternProperties: description: Shenzhen Xingbangda Display Technology Co., Ltd "^xinpeng,.*": description: Shenzhen Xinpeng Technology Co., Ltd + "^xiphera,.*": + description: Xiphera Ltd. "^xlnx,.*": description: Xilinx "^xnano,.*": diff --git a/MAINTAINERS b/MAINTAINERS index 6594f0966716..b81a3ed71a5f 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -13068,7 +13068,9 @@ F: lib/packing.c PADATA PARALLEL EXECUTION MECHANISM M: Steffen Klassert <steffen.klassert@secunet.com> +M: Daniel Jordan <daniel.m.jordan@oracle.com> L: linux-crypto@vger.kernel.org +L: linux-kernel@vger.kernel.org S: Maintained F: Documentation/core-api/padata.rst F: include/linux/padata.h diff --git a/arch/arm/crypto/aes-neonbs-core.S b/arch/arm/crypto/aes-neonbs-core.S index cfaed4e67535..7d0cc7f226a5 100644 --- a/arch/arm/crypto/aes-neonbs-core.S +++ b/arch/arm/crypto/aes-neonbs-core.S @@ -77,11 +77,6 @@ vldr \out\()h, \sym + 8 .endm - .macro __adr, reg, lbl - adr \reg, \lbl -THUMB( orr \reg, \reg, #1 ) - .endm - .macro in_bs_ch, b0, b1, b2, b3, b4, b5, b6, b7 veor \b2, \b2, \b1 veor \b5, \b5, \b6 @@ -629,11 +624,11 @@ ENDPROC(aesbs_decrypt8) push {r4-r6, lr} ldr r5, [sp, #16] // number of blocks -99: __adr ip, 0f +99: adr ip, 0f and lr, r5, #7 cmp r5, #8 sub ip, ip, lr, lsl #2 - bxlt ip // computed goto if blocks < 8 + movlt pc, ip // computed goto if blocks < 8 vld1.8 {q0}, [r1]! vld1.8 {q1}, [r1]! @@ -648,11 +643,11 @@ ENDPROC(aesbs_decrypt8) mov rounds, r3 bl \do8 - __adr ip, 1f + adr ip, 1f and lr, r5, #7 cmp r5, #8 sub ip, ip, lr, lsl #2 - bxlt ip // computed goto if blocks < 8 + movlt pc, ip // computed goto if blocks < 8 vst1.8 {\o0}, [r0]! vst1.8 {\o1}, [r0]! @@ -689,12 +684,12 @@ ENTRY(aesbs_cbc_decrypt) push {r4-r6, lr} ldm ip, {r5-r6} // load args 4-5 -99: __adr ip, 0f +99: adr ip, 0f and lr, r5, #7 cmp r5, #8 sub ip, ip, lr, lsl #2 mov lr, r1 - bxlt ip // computed goto if blocks < 8 + movlt pc, ip // computed goto if blocks < 8 vld1.8 {q0}, [lr]! vld1.8 {q1}, [lr]! @@ -718,11 +713,11 @@ ENTRY(aesbs_cbc_decrypt) vmov q14, q8 vmov q15, q8 - __adr ip, 1f + adr ip, 1f and lr, r5, #7 cmp r5, #8 sub ip, ip, lr, lsl #2 - bxlt ip // computed goto if blocks < 8 + movlt pc, ip // computed goto if blocks < 8 vld1.8 {q9}, [r1]! vld1.8 {q10}, [r1]! @@ -733,9 +728,9 @@ ENTRY(aesbs_cbc_decrypt) vld1.8 {q15}, [r1]! W(nop) -1: __adr ip, 2f +1: adr ip, 2f sub ip, ip, lr, lsl #3 - bxlt ip // computed goto if blocks < 8 + movlt pc, ip // computed goto if blocks < 8 veor q0, q0, q8 vst1.8 {q0}, [r0]! @@ -804,13 +799,13 @@ ENTRY(aesbs_ctr_encrypt) vmov q6, q0 vmov q7, q0 - __adr ip, 0f + adr ip, 0f sub lr, r5, #1 and lr, lr, #7 cmp r5, #8 sub ip, ip, lr, lsl #5 sub ip, ip, lr, lsl #2 - bxlt ip // computed goto if blocks < 8 + movlt pc, ip // computed goto if blocks < 8 next_ctr q1 next_ctr q2 @@ -824,13 +819,13 @@ ENTRY(aesbs_ctr_encrypt) mov rounds, r3 bl aesbs_encrypt8 - __adr ip, 1f + adr ip, 1f and lr, r5, #7 cmp r5, #8 movgt r4, #0 ldrle r4, [sp, #40] // load final in the last round sub ip, ip, lr, lsl #2 - bxlt ip // computed goto if blocks < 8 + movlt pc, ip // computed goto if blocks < 8 vld1.8 {q8}, [r1]! vld1.8 {q9}, [r1]! @@ -843,10 +838,10 @@ ENTRY(aesbs_ctr_encrypt) 1: bne 2f vld1.8 {q15}, [r1]! -2: __adr ip, 3f +2: adr ip, 3f cmp r5, #8 sub ip, ip, lr, lsl #3 - bxlt ip // computed goto if blocks < 8 + movlt pc, ip // computed goto if blocks < 8 veor q0, q0, q8 vst1.8 {q0}, [r0]! @@ -900,12 +895,12 @@ __xts_prepare8: vshr.u64 d30, d31, #7 vmov q12, q14 - __adr ip, 0f + adr ip, 0f and r4, r6, #7 cmp r6, #8 sub ip, ip, r4, lsl #5 mov r4, sp - bxlt ip // computed goto if blocks < 8 + movlt pc, ip // computed goto if blocks < 8 vld1.8 {q0}, [r1]! next_tweak q12, q14, q15, q13 @@ -961,8 +956,7 @@ ENDPROC(__xts_prepare8) push {r4-r8, lr} mov r5, sp // preserve sp ldrd r6, r7, [sp, #24] // get blocks and iv args - ldr r8, [sp, #32] // reorder final tweak? - rsb r8, r8, #1 + rsb r8, ip, #1 sub ip, sp, #128 // make room for 8x tweak bic ip, ip, #0xf // align sp to 16 bytes mov sp, ip @@ -973,12 +967,12 @@ ENDPROC(__xts_prepare8) mov rounds, r3 bl \do8 - __adr ip, 0f + adr ip, 0f and lr, r6, #7 cmp r6, #8 sub ip, ip, lr, lsl #2 mov r4, sp - bxlt ip // computed goto if blocks < 8 + movlt pc, ip // computed goto if blocks < 8 vld1.8 {q8}, [r4, :128]! vld1.8 {q9}, [r4, :128]! @@ -989,9 +983,9 @@ ENDPROC(__xts_prepare8) vld1.8 {q14}, [r4, :128]! vld1.8 {q15}, [r4, :128] -0: __adr ip, 1f +0: adr ip, 1f sub ip, ip, lr, lsl #3 - bxlt ip // computed goto if blocks < 8 + movlt pc, ip // computed goto if blocks < 8 veor \o0, \o0, q8 vst1.8 {\o0}, [r0]! @@ -1018,9 +1012,11 @@ ENDPROC(__xts_prepare8) .endm ENTRY(aesbs_xts_encrypt) + mov ip, #0 // never reorder final tweak __xts_crypt aesbs_encrypt8, q0, q1, q4, q6, q3, q7, q2, q5 ENDPROC(aesbs_xts_encrypt) ENTRY(aesbs_xts_decrypt) + ldr ip, [sp, #8] // reorder final tweak? __xts_crypt aesbs_decrypt8, q0, q1, q6, q4, q2, q7, q3, q5 ENDPROC(aesbs_xts_decrypt) diff --git a/arch/arm/crypto/aes-neonbs-glue.c b/arch/arm/crypto/aes-neonbs-glue.c index e6fd32919c81..bda8bf17631e 100644 --- a/arch/arm/crypto/aes-neonbs-glue.c +++ b/arch/arm/crypto/aes-neonbs-glue.c @@ -8,7 +8,6 @@ #include <asm/neon.h> #include <asm/simd.h> #include <crypto/aes.h> -#include <crypto/cbc.h> #include <crypto/ctr.h> #include <crypto/internal/simd.h> #include <crypto/internal/skcipher.h> @@ -49,7 +48,7 @@ struct aesbs_ctx { struct aesbs_cbc_ctx { struct aesbs_ctx key; - struct crypto_cipher *enc_tfm; + struct crypto_skcipher *enc_tfm; }; struct aesbs_xts_ctx { @@ -140,19 +139,23 @@ static int aesbs_cbc_setkey(struct crypto_skcipher *tfm, const u8 *in_key, kernel_neon_end(); memzero_explicit(&rk, sizeof(rk)); - return crypto_cipher_setkey(ctx->enc_tfm, in_key, key_len); + return crypto_skcipher_setkey(ctx->enc_tfm, in_key, key_len); } -static void cbc_encrypt_one(struct crypto_skcipher *tfm, const u8 *src, u8 *dst) +static int cbc_encrypt(struct skcipher_request *req) { + struct skcipher_request *subreq = skcipher_request_ctx(req); + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); - crypto_cipher_encrypt_one(ctx->enc_tfm, dst, src); -} + skcipher_request_set_tfm(subreq, ctx->enc_tfm); + skcipher_request_set_callback(subreq, + skcipher_request_flags(req), + NULL, NULL); + skcipher_request_set_crypt(subreq, req->src, req->dst, + req->cryptlen, req->iv); -static int cbc_encrypt(struct skcipher_request *req) -{ - return crypto_cbc_encrypt_walk(req, cbc_encrypt_one); + return crypto_skcipher_encrypt(subreq); } static int cbc_decrypt(struct skcipher_request *req) @@ -183,20 +186,27 @@ static int cbc_decrypt(struct skcipher_request *req) return err; } -static int cbc_init(struct crypto_tfm *tfm) +static int cbc_init(struct crypto_skcipher *tfm) { - struct aesbs_cbc_ctx *ctx = crypto_tfm_ctx(tfm); + struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); + unsigned int reqsize; - ctx->enc_tfm = crypto_alloc_cipher("aes", 0, 0); + ctx->enc_tfm = crypto_alloc_skcipher("cbc(aes)", 0, CRYPTO_ALG_ASYNC); + if (IS_ERR(ctx->enc_tfm)) + return PTR_ERR(ctx->enc_tfm); - return PTR_ERR_OR_ZERO(ctx->enc_tfm); + reqsize = sizeof(struct skcipher_request); + reqsize += crypto_skcipher_reqsize(ctx->enc_tfm); + crypto_skcipher_set_reqsize(tfm, reqsize); + + return 0; } -static void cbc_exit(struct crypto_tfm *tfm) +static void cbc_exit(struct crypto_skcipher *tfm) { - struct aesbs_cbc_ctx *ctx = crypto_tfm_ctx(tfm); + struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); - crypto_free_cipher(ctx->enc_tfm); + crypto_free_skcipher(ctx->enc_tfm); } static int aesbs_ctr_setkey_sync(struct crypto_skcipher *tfm, const u8 *in_key, @@ -304,9 +314,9 @@ static int aesbs_xts_setkey(struct crypto_skcipher *tfm, const u8 *in_key, return aesbs_setkey(tfm, in_key, key_len); } -static int xts_init(struct crypto_tfm *tfm) +static int xts_init(struct crypto_skcipher *tfm) { - struct aesbs_xts_ctx *ctx = crypto_tfm_ctx(tfm); + struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm); ctx->cts_tfm = crypto_alloc_cipher("aes", 0, 0); if (IS_ERR(ctx->cts_tfm)) @@ -319,9 +329,9 @@ static int xts_init(struct crypto_tfm *tfm) return PTR_ERR_OR_ZERO(ctx->tweak_tfm); } -static void xts_exit(struct crypto_tfm *tfm) +static void xts_exit(struct crypto_skcipher *tfm) { - struct aesbs_xts_ctx *ctx = crypto_tfm_ctx(tfm); + struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm); crypto_free_cipher(ctx->tweak_tfm); crypto_free_cipher(ctx->cts_tfm); @@ -432,8 +442,6 @@ static struct skcipher_alg aes_algs[] = { { .base.cra_ctxsize = sizeof(struct aesbs_cbc_ctx), .base.cra_module = THIS_MODULE, .base.cra_flags = CRYPTO_ALG_INTERNAL, - .base.cra_init = cbc_init, - .base.cra_exit = cbc_exit, .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, @@ -442,6 +450,8 @@ static struct skcipher_alg aes_algs[] = { { .setkey = aesbs_cbc_setkey, .encrypt = cbc_encrypt, .decrypt = cbc_decrypt, + .init = cbc_init, + .exit = cbc_exit, }, { .base.cra_name = "__ctr(aes)", .base.cra_driver_name = "__ctr-aes-neonbs", @@ -483,8 +493,6 @@ static struct skcipher_alg aes_algs[] = { { .base.cra_ctxsize = sizeof(struct aesbs_xts_ctx), .base.cra_module = THIS_MODULE, .base.cra_flags = CRYPTO_ALG_INTERNAL, - .base.cra_init = xts_init, - .base.cra_exit = xts_exit, .min_keysize = 2 * AES_MIN_KEY_SIZE, .max_keysize = 2 * AES_MAX_KEY_SIZE, @@ -493,6 +501,8 @@ static struct skcipher_alg aes_algs[] = { { .setkey = aesbs_xts_setkey, .encrypt = xts_encrypt, .decrypt = xts_decrypt, + .init = xts_init, + .exit = xts_exit, } }; static struct simd_skcipher_alg *aes_simd_algs[ARRAY_SIZE(aes_algs)]; diff --git a/arch/arm/crypto/curve25519-glue.c b/arch/arm/crypto/curve25519-glue.c index 776ae07e0469..31eb75b6002f 100644 --- a/arch/arm/crypto/curve25519-glue.c +++ b/arch/arm/crypto/curve25519-glue.c @@ -16,6 +16,7 @@ #include <linux/module.h> #include <linux/init.h> #include <linux/jump_label.h> +#include <linux/scatterlist.h> #include <crypto/curve25519.h> asmlinkage void curve25519_neon(u8 mypublic[CURVE25519_KEY_SIZE], diff --git a/arch/arm/crypto/poly1305-glue.c b/arch/arm/crypto/poly1305-glue.c index 13cfef4ae22e..3023c1acfa19 100644 --- a/arch/arm/crypto/poly1305-glue.c +++ b/arch/arm/crypto/poly1305-glue.c @@ -20,6 +20,7 @@ void poly1305_init_arm(void *state, const u8 *key); void poly1305_blocks_arm(void *state, const u8 *src, u32 len, u32 hibit); +void poly1305_blocks_neon(void *state, const u8 *src, u32 len, u32 hibit); void poly1305_emit_arm(void *state, u8 *digest, const u32 *nonce); void __weak poly1305_blocks_neon(void *state, const u8 *src, u32 len, u32 hibit) diff --git a/arch/arm/crypto/sha256-armv4.pl b/arch/arm/crypto/sha256-armv4.pl index 9f96ff48e4a8..f3a2b54efd4e 100644 --- a/arch/arm/crypto/sha256-armv4.pl +++ b/arch/arm/crypto/sha256-armv4.pl @@ -175,7 +175,6 @@ $code=<<___; #else .syntax unified # ifdef __thumb2__ -# define adrl adr .thumb # else .code 32 @@ -471,7 +470,8 @@ sha256_block_data_order_neon: stmdb sp!,{r4-r12,lr} sub $H,sp,#16*4+16 - adrl $Ktbl,K256 + adr $Ktbl,.Lsha256_block_data_order + sub $Ktbl,$Ktbl,#.Lsha256_block_data_order-K256 bic $H,$H,#15 @ align for 128-bit stores mov $t2,sp mov sp,$H @ alloca diff --git a/arch/arm/crypto/sha256-core.S_shipped b/arch/arm/crypto/sha256-core.S_shipped index ea04b2ab0c33..6363014a50d7 100644 --- a/arch/arm/crypto/sha256-core.S_shipped +++ b/arch/arm/crypto/sha256-core.S_shipped @@ -56,7 +56,6 @@ #else .syntax unified # ifdef __thumb2__ -# define adrl adr .thumb # else .code 32 @@ -1885,7 +1884,8 @@ sha256_block_data_order_neon: stmdb sp!,{r4-r12,lr} sub r11,sp,#16*4+16 - adrl r14,K256 + adr r14,.Lsha256_block_data_order + sub r14,r14,#.Lsha256_block_data_order-K256 bic r11,r11,#15 @ align for 128-bit stores mov r12,sp mov sp,r11 @ alloca diff --git a/arch/arm/crypto/sha512-armv4.pl b/arch/arm/crypto/sha512-armv4.pl index 69df68981acd..2fc3516912fa 100644 --- a/arch/arm/crypto/sha512-armv4.pl +++ b/arch/arm/crypto/sha512-armv4.pl @@ -212,7 +212,6 @@ $code=<<___; #else .syntax unified # ifdef __thumb2__ -# define adrl adr .thumb # else .code 32 @@ -602,7 +601,8 @@ sha512_block_data_order_neon: dmb @ errata #451034 on early Cortex A8 add $len,$inp,$len,lsl#7 @ len to point at the end of inp VFP_ABI_PUSH - adrl $Ktbl,K512 + adr $Ktbl,.Lsha512_block_data_order + sub $Ktbl,$Ktbl,.Lsha512_block_data_order-K512 vldmia $ctx,{$A-$H} @ load context .Loop_neon: ___ diff --git a/arch/arm/crypto/sha512-core.S_shipped b/arch/arm/crypto/sha512-core.S_shipped index cb147db5cbfe..03014624f2ab 100644 --- a/arch/arm/crypto/sha512-core.S_shipped +++ b/arch/arm/crypto/sha512-core.S_shipped @@ -79,7 +79,6 @@ #else .syntax unified # ifdef __thumb2__ -# define adrl adr .thumb # else .code 32 @@ -543,7 +542,8 @@ sha512_block_data_order_neon: dmb @ errata #451034 on early Cortex A8 add r2,r1,r2,lsl#7 @ len to point at the end of inp VFP_ABI_PUSH - adrl r3,K512 + adr r3,.Lsha512_block_data_order + sub r3,r3,.Lsha512_block_data_order-K512 vldmia r0,{d16-d23} @ load context .Loop_neon: vshr.u64 d24,d20,#14 @ 0 diff --git a/arch/arm64/crypto/ghash-ce-glue.c b/arch/arm64/crypto/ghash-ce-glue.c index da1034867aaa..8536008e3e35 100644 --- a/arch/arm64/crypto/ghash-ce-glue.c +++ b/arch/arm64/crypto/ghash-ce-glue.c @@ -347,7 +347,7 @@ static int gcm_encrypt(struct aead_request *req) u8 buf[AES_BLOCK_SIZE]; u8 iv[AES_BLOCK_SIZE]; u64 dg[2] = {}; - u128 lengths; + be128 lengths; u8 *tag; int err; @@ -461,7 +461,7 @@ static int gcm_decrypt(struct aead_request *req) u8 buf[AES_BLOCK_SIZE]; u8 iv[AES_BLOCK_SIZE]; u64 dg[2] = {}; - u128 lengths; + be128 lengths; u8 *tag; int err; diff --git a/arch/arm64/crypto/sha1-ce-glue.c b/arch/arm64/crypto/sha1-ce-glue.c index 565ef604ca04..c63b99211db3 100644 --- a/arch/arm64/crypto/sha1-ce-glue.c +++ b/arch/arm64/crypto/sha1-ce-glue.c @@ -25,6 +25,9 @@ struct sha1_ce_state { u32 finalize; }; +extern const u32 sha1_ce_offsetof_count; +extern const u32 sha1_ce_offsetof_finalize; + asmlinkage void sha1_ce_transform(struct sha1_ce_state *sst, u8 const *src, int blocks); diff --git a/arch/arm64/crypto/sha2-ce-glue.c b/arch/arm64/crypto/sha2-ce-glue.c index 9450d19b9e6e..5e956d7582a5 100644 --- a/arch/arm64/crypto/sha2-ce-glue.c +++ b/arch/arm64/crypto/sha2-ce-glue.c @@ -25,6 +25,9 @@ struct sha256_ce_state { u32 finalize; }; +extern const u32 sha256_ce_offsetof_count; +extern const u32 sha256_ce_offsetof_finalize; + asmlinkage void sha2_ce_transform(struct sha256_ce_state *sst, u8 const *src, int blocks); diff --git a/arch/powerpc/crypto/crc-vpmsum_test.c b/arch/powerpc/crypto/crc-vpmsum_test.c index dce86e75f1a8..c1c1ef9457fb 100644 --- a/arch/powerpc/crypto/crc-vpmsum_test.c +++ b/arch/powerpc/crypto/crc-vpmsum_test.c @@ -9,6 +9,7 @@ #include <crypto/internal/hash.h> #include <linux/init.h> #include <linux/module.h> +#include <linux/random.h> #include <linux/string.h> #include <linux/kernel.h> #include <linux/cpufeature.h> @@ -22,10 +23,11 @@ static unsigned long iterations = 10000; static int __init crc_test_init(void) { u16 crc16 = 0, verify16 = 0; - u32 crc32 = 0, verify32 = 0; __le32 verify32le = 0; unsigned char *data; + u32 verify32 = 0; unsigned long i; + __le32 crc32; int ret; struct crypto_shash *crct10dif_tfm; @@ -98,7 +100,7 @@ static int __init crc_test_init(void) crypto_shash_final(crc32c_shash, (u8 *)(&crc32)); verify32 = le32_to_cpu(verify32le); verify32le = ~cpu_to_le32(__crc32c_le(~verify32, data+offset, len)); - if (crc32 != (u32)verify32le) { + if (crc32 != verify32le) { pr_err("FAILURE in CRC32: got 0x%08x expected 0x%08x (len %lu)\n", crc32, verify32, len); break; diff --git a/arch/x86/crypto/blake2s-glue.c b/arch/x86/crypto/blake2s-glue.c index 6737bcea1fa1..c025a01cf708 100644 --- a/arch/x86/crypto/blake2s-glue.c +++ b/arch/x86/crypto/blake2s-glue.c @@ -11,6 +11,7 @@ #include <linux/jump_label.h> #include <linux/kernel.h> #include <linux/module.h> +#include <linux/sizes.h> #include <asm/cpufeature.h> #include <asm/fpu/api.h> diff --git a/arch/x86/crypto/chacha_glue.c b/arch/x86/crypto/chacha_glue.c index e67a59130025..7b3a1cf0984b 100644 --- a/arch/x86/crypto/chacha_glue.c +++ b/arch/x86/crypto/chacha_glue.c @@ -12,6 +12,7 @@ #include <crypto/internal/skcipher.h> #include <linux/kernel.h> #include <linux/module.h> +#include <linux/sizes.h> #include <asm/simd.h> asmlinkage void chacha_block_xor_ssse3(u32 *state, u8 *dst, const u8 *src, diff --git a/arch/x86/crypto/crc32c-intel_glue.c b/arch/x86/crypto/crc32c-intel_glue.c index d2d069bd459b..feccb5254c7e 100644 --- a/arch/x86/crypto/crc32c-intel_glue.c +++ b/arch/x86/crypto/crc32c-intel_glue.c @@ -28,9 +28,9 @@ #define SCALE_F sizeof(unsigned long) #ifdef CONFIG_X86_64 -#define REX_PRE "0x48, " +#define CRC32_INST "crc32q %1, %q0" #else -#define REX_PRE +#define CRC32_INST "crc32l %1, %0" #endif #ifdef CONFIG_X86_64 @@ -48,11 +48,8 @@ asmlinkage unsigned int crc_pcl(const u8 *buffer, int len, static u32 crc32c_intel_le_hw_byte(u32 crc, unsigned char const *data, size_t length) { while (length--) { - __asm__ __volatile__( - ".byte 0xf2, 0xf, 0x38, 0xf0, 0xf1" - :"=S"(crc) - :"0"(crc), "c"(*data) - ); + asm("crc32b %1, %0" + : "+r" (crc) : "rm" (*data)); data++; } @@ -66,11 +63,8 @@ static u32 __pure crc32c_intel_le_hw(u32 crc, unsigned char const *p, size_t len unsigned long *ptmp = (unsigned long *)p; while (iquotient--) { - __asm__ __volatile__( - ".byte 0xf2, " REX_PRE "0xf, 0x38, 0xf1, 0xf1;" - :"=S"(crc) - :"0"(crc), "c"(*ptmp) - ); + asm(CRC32_INST + : "+r" (crc) : "rm" (*ptmp)); ptmp++; } diff --git a/arch/x86/crypto/curve25519-x86_64.c b/arch/x86/crypto/curve25519-x86_64.c index 8acbb6584a37..5af8021b98ce 100644 --- a/arch/x86/crypto/curve25519-x86_64.c +++ b/arch/x86/crypto/curve25519-x86_64.c @@ -11,6 +11,7 @@ #include <linux/jump_label.h> #include <linux/kernel.h> #include <linux/module.h> +#include <linux/scatterlist.h> #include <asm/cpufeature.h> #include <asm/processor.h> @@ -45,11 +46,11 @@ static inline u64 add_scalar(u64 *out, const u64 *f1, u64 f2) asm volatile( /* Clear registers to propagate the carry bit */ - " xor %%r8, %%r8;" - " xor %%r9, %%r9;" - " xor %%r10, %%r10;" - " xor %%r11, %%r11;" - " xor %1, %1;" + " xor %%r8d, %%r8d;" + " xor %%r9d, %%r9d;" + " xor %%r10d, %%r10d;" + " xor %%r11d, %%r11d;" + " xor %k1, %k1;" /* Begin addition chain */ " addq 0(%3), %0;" @@ -93,7 +94,7 @@ static inline void fadd(u64 *out, const u64 *f1, const u64 *f2) " cmovc %0, %%rax;" /* Step 2: Add carry*38 to the original sum */ - " xor %%rcx, %%rcx;" + " xor %%ecx, %%ecx;" " add %%rax, %%r8;" " adcx %%rcx, %%r9;" " movq %%r9, 8(%1);" @@ -165,28 +166,28 @@ static inline void fmul(u64 *out, const u64 *f1, const u64 *f2, u64 *tmp) /* Compute src1[0] * src2 */ " movq 0(%1), %%rdx;" - " mulxq 0(%3), %%r8, %%r9;" " xor %%r10, %%r10;" " movq %%r8, 0(%0);" + " mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " movq %%r8, 0(%0);" " mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " movq %%r10, 8(%0);" " mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " mov $0, %%rax;" " adox %%rdx, %%rax;" /* Compute src1[1] * src2 */ " movq 8(%1), %%rdx;" - " mulxq 0(%3), %%r8, %%r9;" " xor %%r10, %%r10;" " adcxq 8(%0), %%r8;" " movq %%r8, 8(%0);" + " mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 8(%0), %%r8;" " movq %%r8, 8(%0);" " mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 16(%0);" " mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;" " mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;" " adox %%rdx, %%rax;" " adcx %%r8, %%rax;" /* Compute src1[2] * src2 */ " movq 16(%1), %%rdx;" - " mulxq 0(%3), %%r8, %%r9;" " xor %%r10, %%r10;" " adcxq 16(%0), %%r8;" " movq %%r8, 16(%0);" + " mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 16(%0), %%r8;" " movq %%r8, 16(%0);" " mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 24(%0);" " mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;" " mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;" " adox %%rdx, %%rax;" " adcx %%r8, %%rax;" /* Compute src1[3] * src2 */ " movq 24(%1), %%rdx;" - " mulxq 0(%3), %%r8, %%r9;" " xor %%r10, %%r10;" " adcxq 24(%0), %%r8;" " movq %%r8, 24(%0);" + " mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 24(%0), %%r8;" " movq %%r8, 24(%0);" " mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 32(%0);" " mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " movq %%rbx, 40(%0);" " mov $0, %%r8;" " mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " movq %%r14, 48(%0);" " mov $0, %%rax;" @@ -200,7 +201,7 @@ static inline void fmul(u64 *out, const u64 *f1, const u64 *f2, u64 *tmp) /* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */ " mov $38, %%rdx;" " mulxq 32(%1), %%r8, %%r13;" - " xor %3, %3;" + " xor %k3, %k3;" " adoxq 0(%1), %%r8;" " mulxq 40(%1), %%r9, %%rbx;" " adcx %%r13, %%r9;" @@ -246,28 +247,28 @@ static inline void fmul2(u64 *out, const u64 *f1, const u64 *f2, u64 *tmp) /* Compute src1[0] * src2 */ " movq 0(%1), %%rdx;" - " mulxq 0(%3), %%r8, %%r9;" " xor %%r10, %%r10;" " movq %%r8, 0(%0);" + " mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " movq %%r8, 0(%0);" " mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " movq %%r10, 8(%0);" " mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " mov $0, %%rax;" " adox %%rdx, %%rax;" /* Compute src1[1] * src2 */ " movq 8(%1), %%rdx;" - " mulxq 0(%3), %%r8, %%r9;" " xor %%r10, %%r10;" " adcxq 8(%0), %%r8;" " movq %%r8, 8(%0);" + " mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 8(%0), %%r8;" " movq %%r8, 8(%0);" " mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 16(%0);" " mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;" " mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;" " adox %%rdx, %%rax;" " adcx %%r8, %%rax;" /* Compute src1[2] * src2 */ " movq 16(%1), %%rdx;" - " mulxq 0(%3), %%r8, %%r9;" " xor %%r10, %%r10;" " adcxq 16(%0), %%r8;" " movq %%r8, 16(%0);" + " mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 16(%0), %%r8;" " movq %%r8, 16(%0);" " mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 24(%0);" " mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;" " mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;" " adox %%rdx, %%rax;" " adcx %%r8, %%rax;" /* Compute src1[3] * src2 */ " movq 24(%1), %%rdx;" - " mulxq 0(%3), %%r8, %%r9;" " xor %%r10, %%r10;" " adcxq 24(%0), %%r8;" " movq %%r8, 24(%0);" + " mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 24(%0), %%r8;" " movq %%r8, 24(%0);" " mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 32(%0);" " mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " movq %%rbx, 40(%0);" " mov $0, %%r8;" " mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " movq %%r14, 48(%0);" " mov $0, %%rax;" @@ -277,29 +278,29 @@ static inline void fmul2(u64 *out, const u64 *f1, const u64 *f2, u64 *tmp) /* Compute src1[0] * src2 */ " movq 32(%1), %%rdx;" - " mulxq 32(%3), %%r8, %%r9;" " xor %%r10, %%r10;" " movq %%r8, 64(%0);" - " mulxq 40(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " movq %%r10, 72(%0);" + " mulxq 32(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " movq %%r8, 64(%0);" + " mulxq 40(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " movq %%r10, 72(%0);" " mulxq 48(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " mulxq 56(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " mov $0, %%rax;" " adox %%rdx, %%rax;" /* Compute src1[1] * src2 */ " movq 40(%1), %%rdx;" - " mulxq 32(%3), %%r8, %%r9;" " xor %%r10, %%r10;" " adcxq 72(%0), %%r8;" " movq %%r8, 72(%0);" - " mulxq 40(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 80(%0);" + " mulxq 32(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 72(%0), %%r8;" " movq %%r8, 72(%0);" + " mulxq 40(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 80(%0);" " mulxq 48(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;" " mulxq 56(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;" " adox %%rdx, %%rax;" " adcx %%r8, %%rax;" /* Compute src1[2] * src2 */ " movq 48(%1), %%rdx;" - " mulxq 32(%3), %%r8, %%r9;" " xor %%r10, %%r10;" " adcxq 80(%0), %%r8;" " movq %%r8, 80(%0);" - " mulxq 40(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 88(%0);" + " mulxq 32(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 80(%0), %%r8;" " movq %%r8, 80(%0);" + " mulxq 40(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 88(%0);" " mulxq 48(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;" " mulxq 56(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;" " adox %%rdx, %%rax;" " adcx %%r8, %%rax;" /* Compute src1[3] * src2 */ " movq 56(%1), %%rdx;" - " mulxq 32(%3), %%r8, %%r9;" " xor %%r10, %%r10;" " adcxq 88(%0), %%r8;" " movq %%r8, 88(%0);" - " mulxq 40(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 96(%0);" + " mulxq 32(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 88(%0), %%r8;" " movq %%r8, 88(%0);" + " mulxq 40(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 96(%0);" " mulxq 48(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " movq %%rbx, 104(%0);" " mov $0, %%r8;" " mulxq 56(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " movq %%r14, 112(%0);" " mov $0, %%rax;" " adox %%rdx, %%rax;" " adcx %%r8, %%rax;" " movq %%rax, 120(%0);" @@ -312,7 +313,7 @@ static inline void fmul2(u64 *out, const u64 *f1, const u64 *f2, u64 *tmp) /* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */ " mov $38, %%rdx;" " mulxq 32(%1), %%r8, %%r13;" - " xor %3, %3;" + " xor %k3, %k3;" " adoxq 0(%1), %%r8;" " mulxq 40(%1), %%r9, %%rbx;" " adcx %%r13, %%r9;" @@ -345,7 +346,7 @@ static inline void fmul2(u64 *out, const u64 *f1, const u64 *f2, u64 *tmp) /* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */ " mov $38, %%rdx;" " mulxq 96(%1), %%r8, %%r13;" - " xor %3, %3;" + " xor %k3, %k3;" " adoxq 64(%1), %%r8;" " mulxq 104(%1), %%r9, %%rbx;" " adcx %%r13, %%r9;" @@ -516,7 +517,7 @@ static inline void fsqr(u64 *out, const u64 *f, u64 *tmp) /* Step 1: Compute all partial products */ " movq 0(%1), %%rdx;" /* f[0] */ - " mulxq 8(%1), %%r8, %%r14;" " xor %%r15, %%r15;" /* f[1]*f[0] */ + " mulxq 8(%1), %%r8, %%r14;" " xor %%r15d, %%r15d;" /* f[1]*f[0] */ " mulxq 16(%1), %%r9, %%r10;" " adcx %%r14, %%r9;" /* f[2]*f[0] */ " mulxq 24(%1), %%rax, %%rcx;" " adcx %%rax, %%r10;" /* f[3]*f[0] */ " movq 24(%1), %%rdx;" /* f[3] */ @@ -526,7 +527,7 @@ static inline void fsqr(u64 *out, const u64 *f, u64 *tmp) " mulxq 16(%1), %%rax, %%rcx;" " mov $0, %%r14;" /* f[2]*f[1] */ /* Step 2: Compute two parallel carry chains */ - " xor %%r15, %%r15;" + " xor %%r15d, %%r15d;" " adox %%rax, %%r10;" " adcx %%r8, %%r8;" " adox %%rcx, %%r11;" @@ -563,7 +564,7 @@ static inline void fsqr(u64 *out, const u64 *f, u64 *tmp) /* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */ " mov $38, %%rdx;" " mulxq 32(%1), %%r8, %%r13;" - " xor %%rcx, %%rcx;" + " xor %%ecx, %%ecx;" " adoxq 0(%1), %%r8;" " mulxq 40(%1), %%r9, %%rbx;" " adcx %%r13, %%r9;" @@ -607,7 +608,7 @@ static inline void fsqr2(u64 *out, const u64 *f, u64 *tmp) asm volatile( /* Step 1: Compute all partial products */ " movq 0(%1), %%rdx;" /* f[0] */ - " mulxq 8(%1), %%r8, %%r14;" " xor %%r15, %%r15;" /* f[1]*f[0] */ + " mulxq 8(%1), %%r8, %%r14;" " xor %%r15d, %%r15d;" /* f[1]*f[0] */ " mulxq 16(%1), %%r9, %%r10;" " adcx %%r14, %%r9;" /* f[2]*f[0] */ " mulxq 24(%1), %%rax, %%rcx;" " adcx %%rax, %%r10;" /* f[3]*f[0] */ " movq 24(%1), %%rdx;" /* f[3] */ @@ -617,7 +618,7 @@ static inline void fsqr2(u64 *out, const u64 *f, u64 *tmp) " mulxq 16(%1), %%rax, %%rcx;" " mov $0, %%r14;" /* f[2]*f[1] */ /* Step 2: Compute two parallel carry chains */ - " xor %%r15, %%r15;" + " xor %%r15d, %%r15d;" " adox %%rax, %%r10;" " adcx %%r8, %%r8;" " adox %%rcx, %%r11;" @@ -647,7 +648,7 @@ static inline void fsqr2(u64 *out, const u64 *f, u64 *tmp) /* Step 1: Compute all partial products */ " movq 32(%1), %%rdx;" /* f[0] */ - " mulxq 40(%1), %%r8, %%r14;" " xor %%r15, %%r15;" /* f[1]*f[0] */ + " mulxq 40(%1), %%r8, %%r14;" " xor %%r15d, %%r15d;" /* f[1]*f[0] */ " mulxq 48(%1), %%r9, %%r10;" " adcx %%r14, %%r9;" /* f[2]*f[0] */ " mulxq 56(%1), %%rax, %%rcx;" " adcx %%rax, %%r10;" /* f[3]*f[0] */ " movq 56(%1), %%rdx;" /* f[3] */ @@ -657,7 +658,7 @@ static inline void fsqr2(u64 *out, const u64 *f, u64 *tmp) " mulxq 48(%1), %%rax, %%rcx;" " mov $0, %%r14;" /* f[2]*f[1] */ /* Step 2: Compute two parallel carry chains */ - " xor %%r15, %%r15;" + " xor %%r15d, %%r15d;" " adox %%rax, %%r10;" " adcx %%r8, %%r8;" " adox %%rcx, %%r11;" @@ -692,7 +693,7 @@ static inline void fsqr2(u64 *out, const u64 *f, u64 *tmp) /* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */ " mov $38, %%rdx;" " mulxq 32(%1), %%r8, %%r13;" - " xor %%rcx, %%rcx;" + " xor %%ecx, %%ecx;" " adoxq 0(%1), %%r8;" " mulxq 40(%1), %%r9, %%rbx;" " adcx %%r13, %%r9;" @@ -725,7 +726,7 @@ static inline void fsqr2(u64 *out, const u64 *f, u64 *tmp) /* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */ " mov $38, %%rdx;" " mulxq 96(%1), %%r8, %%r13;" - " xor %%rcx, %%rcx;" + " xor %%ecx, %%ecx;" " adoxq 64(%1), %%r8;" " mulxq 104(%1), %%r9, %%rbx;" " adcx %%r13, %%r9;" diff --git a/arch/x86/crypto/nhpoly1305-avx2-glue.c b/arch/x86/crypto/nhpoly1305-avx2-glue.c index 80fcb85736e1..8ea5ab0f1ca7 100644 --- a/arch/x86/crypto/nhpoly1305-avx2-glue.c +++ b/arch/x86/crypto/nhpoly1305-avx2-glue.c @@ -10,6 +10,7 @@ #include <crypto/internal/simd.h> #include <crypto/nhpoly1305.h> #include <linux/module.h> +#include <linux/sizes.h> #include <asm/simd.h> asmlinkage void nh_avx2(const u32 *key, const u8 *message, size_t message_len, diff --git a/arch/x86/crypto/nhpoly1305-sse2-glue.c b/arch/x86/crypto/nhpoly1305-sse2-glue.c index cc6b7c1a2705..2b353d42ed13 100644 --- a/arch/x86/crypto/nhpoly1305-sse2-glue.c +++ b/arch/x86/crypto/nhpoly1305-sse2-glue.c @@ -10,6 +10,7 @@ #include <crypto/internal/simd.h> #include <crypto/nhpoly1305.h> #include <linux/module.h> +#include <linux/sizes.h> #include <asm/simd.h> asmlinkage void nh_sse2(const u32 *key, const u8 *message, size_t message_len, diff --git a/arch/x86/crypto/poly1305-x86_64-cryptogams.pl b/arch/x86/crypto/poly1305-x86_64-cryptogams.pl index 137edcf038cb..7d568012cc15 100644 --- a/arch/x86/crypto/poly1305-x86_64-cryptogams.pl +++ b/arch/x86/crypto/poly1305-x86_64-cryptogams.pl @@ -246,7 +246,7 @@ $code.=<<___ if (!$kernel); ___ &declare_function("poly1305_init_x86_64", 32, 3); $code.=<<___; - xor %rax,%rax + xor %eax,%eax mov %rax,0($ctx) # initialize hash value mov %rax,8($ctx) mov %rax,16($ctx) @@ -2853,7 +2853,7 @@ $code.=<<___; .type poly1305_init_base2_44,\@function,3 .align 32 poly1305_init_base2_44: - xor %rax,%rax + xor %eax,%eax mov %rax,0($ctx) # initialize hash value mov %rax,8($ctx) mov %rax,16($ctx) @@ -3947,7 +3947,7 @@ xor128_decrypt_n_pad: mov \$16,$len sub %r10,$len xor %eax,%eax - xor %r11,%r11 + xor %r11d,%r11d .Loop_dec_byte: mov ($inp,$otp),%r11b mov ($otp),%al @@ -4085,7 +4085,7 @@ avx_handler: .long 0xa548f3fc # cld; rep movsq mov $disp,%rsi - xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER + xor %ecx,%ecx # arg1, UNW_FLAG_NHANDLER mov 8(%rsi),%rdx # arg2, disp->ImageBase mov 0(%rsi),%r8 # arg3, disp->ControlPc mov 16(%rsi),%r9 # arg4, disp->FunctionEntry diff --git a/arch/x86/crypto/poly1305_glue.c b/arch/x86/crypto/poly1305_glue.c index dfe921efa9b2..e508dbd91813 100644 --- a/arch/x86/crypto/poly1305_glue.c +++ b/arch/x86/crypto/poly1305_glue.c @@ -11,6 +11,7 @@ #include <linux/jump_label.h> #include <linux/kernel.h> #include <linux/module.h> +#include <linux/sizes.h> #include <asm/intel-family.h> #include <asm/simd.h> @@ -157,9 +158,6 @@ static unsigned int crypto_poly1305_setdctxkey(struct poly1305_desc_ctx *dctx, dctx->s[1] = get_unaligned_le32(&inp[4]); dctx->s[2] = get_unaligned_le32(&inp[8]); dctx->s[3] = get_unaligned_le32(&inp[12]); - inp += POLY1305_BLOCK_SIZE; - len -= POLY1305_BLOCK_SIZE; - acc += POLY1305_BLOCK_SIZE; dctx->sset = true; } } diff --git a/crypto/Kconfig b/crypto/Kconfig index 1b57419fa2e7..094ef56ab7b4 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -260,6 +260,23 @@ config CRYPTO_ECRDSA standard algorithms (called GOST algorithms). Only signature verification is implemented. +config CRYPTO_SM2 + tristate "SM2 algorithm" + select CRYPTO_SM3 + select CRYPTO_AKCIPHER + select CRYPTO_MANAGER + select MPILIB + select ASN1 + help + Generic implementation of the SM2 public key algorithm. It was + published by State Encryption Management Bureau, China. + as specified by OSCCA GM/T 0003.1-2012 -- 0003.5-2012. + + References: + https://tools.ietf.org/html/draft-shen-sm2-ecdsa-02 + http://www.oscca.gov.cn/sca/xxgk/2010-12/17/content_1002386.shtml + http://www.gmbz.org.cn/main/bzlb.html + config CRYPTO_CURVE25519 tristate "Curve25519 algorithm" select CRYPTO_KPP @@ -1185,6 +1202,7 @@ config CRYPTO_AES_PPC_SPE config CRYPTO_ANUBIS tristate "Anubis cipher algorithm" + depends on CRYPTO_USER_API_ENABLE_OBSOLETE select CRYPTO_ALGAPI help Anubis cipher algorithm. @@ -1199,6 +1217,7 @@ config CRYPTO_ANUBIS config CRYPTO_ARC4 tristate "ARC4 cipher algorithm" + depends on CRYPTO_USER_API_ENABLE_OBSOLETE select CRYPTO_SKCIPHER select CRYPTO_LIB_ARC4 help @@ -1423,6 +1442,7 @@ config CRYPTO_FCRYPT config CRYPTO_KHAZAD tristate "Khazad cipher algorithm" + depends on CRYPTO_USER_API_ENABLE_OBSOLETE select CRYPTO_ALGAPI help Khazad cipher algorithm. @@ -1486,6 +1506,7 @@ config CRYPTO_CHACHA_MIPS config CRYPTO_SEED tristate "SEED cipher algorithm" + depends on CRYPTO_USER_API_ENABLE_OBSOLETE select CRYPTO_ALGAPI help SEED cipher algorithm (RFC4269). @@ -1612,6 +1633,7 @@ config CRYPTO_SM4 config CRYPTO_TEA tristate "TEA, XTEA and XETA cipher algorithms" + depends on CRYPTO_USER_API_ENABLE_OBSOLETE select CRYPTO_ALGAPI help TEA cipher algorithm. @@ -1870,6 +1892,15 @@ config CRYPTO_USER_API_RNG This option enables the user-spaces interface for random number generator algorithms. +config CRYPTO_USER_API_RNG_CAVP + bool "Enable CAVP testing of DRBG" + depends on CRYPTO_USER_API_RNG && CRYPTO_DRBG + help + This option enables extra API for CAVP testing via the user-space + interface: resetting of DRBG entropy, and providing Additional Data. + This should only be enabled for CAVP testing. You should say + no unless you know what this is. + config CRYPTO_USER_API_AEAD tristate "User-space interface for AEAD cipher algorithms" depends on NET @@ -1881,6 +1912,15 @@ config CRYPTO_USER_API_AEAD This option enables the user-spaces interface for AEAD cipher algorithms. +config CRYPTO_USER_API_ENABLE_OBSOLETE + bool "Enable obsolete cryptographic algorithms for userspace" + depends on CRYPTO_USER_API + default y + help + Allow obsolete cryptographic algorithms to be selected that have + already been phased out from internal use by the kernel, and are + only useful for userspace clients that still rely on them. + config CRYPTO_STATS bool "Crypto usage statistics for User-space" depends on CRYPTO_USER diff --git a/crypto/Makefile b/crypto/Makefile index 4ca12b6044f7..b279483fba50 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -42,6 +42,14 @@ rsa_generic-y += rsa_helper.o rsa_generic-y += rsa-pkcs1pad.o obj-$(CONFIG_CRYPTO_RSA) += rsa_generic.o +$(obj)/sm2signature.asn1.o: $(obj)/sm2signature.asn1.c $(obj)/sm2signature.asn1.h +$(obj)/sm2.o: $(obj)/sm2signature.asn1.h + +sm2_generic-y += sm2signature.asn1.o +sm2_generic-y += sm2.o + +obj-$(CONFIG_CRYPTO_SM2) += sm2_generic.o + crypto_acompress-y := acompress.o crypto_acompress-y += scompress.o obj-$(CONFIG_CRYPTO_ACOMP2) += crypto_acompress.o diff --git a/crypto/af_alg.c b/crypto/af_alg.c index 8be8bec07cdd..d11db80d24cd 100644 --- a/crypto/af_alg.c +++ b/crypto/af_alg.c @@ -254,6 +254,14 @@ static int alg_setsockopt(struct socket *sock, int level, int optname, if (!type->setauthsize) goto unlock; err = type->setauthsize(ask->private, optlen); + break; + case ALG_SET_DRBG_ENTROPY: + if (sock->state == SS_CONNECTED) + goto unlock; + if (!type->setentropy) + goto unlock; + + err = type->setentropy(ask->private, optval, optlen); } unlock: @@ -286,6 +294,11 @@ int af_alg_accept(struct sock *sk, struct socket *newsock, bool kern) security_sock_graft(sk2, newsock); security_sk_clone(sk, sk2); + /* + * newsock->ops assigned here to allow type->accept call to override + * them when required. + */ + newsock->ops = type->ops; err = type->accept(ask->private, sk2); nokey = err == -ENOKEY; @@ -304,7 +317,6 @@ int af_alg_accept(struct sock *sk, struct socket *newsock, bool kern) alg_sk(sk2)->parent = sk; alg_sk(sk2)->type = type; - newsock->ops = type->ops; newsock->state = SS_CONNECTED; if (nokey) diff --git a/crypto/ahash.c b/crypto/ahash.c index d9d65d1cc669..c2ca631a111f 100644 --- a/crypto/ahash.c +++ b/crypto/ahash.c @@ -10,7 +10,6 @@ #include <crypto/internal/hash.h> #include <crypto/scatterwalk.h> -#include <linux/bug.h> #include <linux/err.h> #include <linux/kernel.h> #include <linux/module.h> @@ -46,10 +45,7 @@ static int hash_walk_next(struct crypto_hash_walk *walk) unsigned int nbytes = min(walk->entrylen, ((unsigned int)(PAGE_SIZE)) - offset); - if (walk->flags & CRYPTO_ALG_ASYNC) - walk->data = kmap(walk->pg); - else - walk->data = kmap_atomic(walk->pg); + walk->data = kmap_atomic(walk->pg); walk->data += offset; if (offset & alignmask) { @@ -99,16 +95,8 @@ int crypto_hash_walk_done(struct crypto_hash_walk *walk, int err) } } - if (walk->flags & CRYPTO_ALG_ASYNC) - kunmap(walk->pg); - else { - kunmap_atomic(walk->data); - /* - * The may sleep test only makes sense for sync users. - * Async users don't need to sleep here anyway. - */ - crypto_yield(walk->flags); - } + kunmap_atomic(walk->data); + crypto_yield(walk->flags); if (err) return err; @@ -140,33 +128,12 @@ int crypto_hash_walk_first(struct ahash_request *req, walk->alignmask = crypto_ahash_alignmask(crypto_ahash_reqtfm(req)); walk->sg = req->src; - walk->flags = req->base.flags & CRYPTO_TFM_REQ_MASK; + walk->flags = req->base.flags; return hash_walk_new_entry(walk); } EXPORT_SYMBOL_GPL(crypto_hash_walk_first); -int crypto_ahash_walk_first(struct ahash_request *req, - struct crypto_hash_walk *walk) -{ - walk->total = req->nbytes; - - if (!walk->total) { - walk->entrylen = 0; - return 0; - } - - walk->alignmask = crypto_ahash_alignmask(crypto_ahash_reqtfm(req)); - walk->sg = req->src; - walk->flags = req->base.flags & CRYPTO_TFM_REQ_MASK; - walk->flags |= CRYPTO_ALG_ASYNC; - - BUILD_BUG_ON(CRYPTO_TFM_REQ_MASK & CRYPTO_ALG_ASYNC); - - return hash_walk_new_entry(walk); -} -EXPORT_SYMBOL_GPL(crypto_ahash_walk_first); - static int ahash_setkey_unaligned(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen) { @@ -477,6 +444,14 @@ static int ahash_def_finup(struct ahash_request *req) return ahash_def_finup_finish1(req, err); } +static void crypto_ahash_exit_tfm(struct crypto_tfm *tfm) +{ + struct crypto_ahash *hash = __crypto_ahash_cast(tfm); + struct ahash_alg *alg = crypto_ahash_alg(hash); + + alg->exit_tfm(hash); +} + static int crypto_ahash_init_tfm(struct crypto_tfm *tfm) { struct crypto_ahash *hash = __crypto_ahash_cast(tfm); @@ -500,7 +475,10 @@ static int crypto_ahash_init_tfm(struct crypto_tfm *tfm) ahash_set_needkey(hash); } - return 0; + if (alg->exit_tfm) + tfm->exit = crypto_ahash_exit_tfm; + + return alg->init_tfm ? alg->init_tfm(hash) : 0; } static unsigned int crypto_ahash_extsize(struct crypto_alg *alg) diff --git a/crypto/algif_aead.c b/crypto/algif_aead.c index 21efa786f09c..42493b4d8ce4 100644 --- a/crypto/algif_aead.c +++ b/crypto/algif_aead.c @@ -78,7 +78,7 @@ static int crypto_aead_copy_sgl(struct crypto_sync_skcipher *null_tfm, SYNC_SKCIPHER_REQUEST_ON_STACK(skreq, null_tfm); skcipher_request_set_sync_tfm(skreq, null_tfm); - skcipher_request_set_callback(skreq, CRYPTO_TFM_REQ_MAY_BACKLOG, + skcipher_request_set_callback(skreq, CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); skcipher_request_set_crypt(skreq, src, dst, len, NULL); @@ -120,7 +120,7 @@ static int _aead_recvmsg(struct socket *sock, struct msghdr *msg, /* * Make sure sufficient data is present -- note, the same check is - * is also present in sendmsg/sendpage. The checks in sendpage/sendmsg + * also present in sendmsg/sendpage. The checks in sendpage/sendmsg * shall provide an information to the data sender that something is * wrong, but they are irrelevant to maintain the kernel integrity. * We need this check here too in case user space decides to not honor @@ -291,19 +291,20 @@ static int _aead_recvmsg(struct socket *sock, struct msghdr *msg, areq->outlen = outlen; aead_request_set_callback(&areq->cra_u.aead_req, - CRYPTO_TFM_REQ_MAY_BACKLOG, + CRYPTO_TFM_REQ_MAY_SLEEP, af_alg_async_cb, areq); err = ctx->enc ? crypto_aead_encrypt(&areq->cra_u.aead_req) : crypto_aead_decrypt(&areq->cra_u.aead_req); /* AIO operation in progress */ - if (err == -EINPROGRESS || err == -EBUSY) + if (err == -EINPROGRESS) return -EIOCBQUEUED; sock_put(sk); } else { /* Synchronous operation */ aead_request_set_callback(&areq->cra_u.aead_req, + CRYPTO_TFM_REQ_MAY_SLEEP | CRYPTO_TFM_REQ_MAY_BACKLOG, crypto_req_done, &ctx->wait); err = crypto_wait_req(ctx->enc ? diff --git a/crypto/algif_rng.c b/crypto/algif_rng.c index 6300e0566dc5..407408c43730 100644 --- a/crypto/algif_rng.c +++ b/crypto/algif_rng.c @@ -38,6 +38,7 @@ * DAMAGE. */ +#include <linux/capability.h> #include <linux/module.h> #include <crypto/rng.h> #include <linux/random.h> @@ -53,15 +54,26 @@ struct rng_ctx { #define MAXSIZE 128 unsigned int len; struct crypto_rng *drng; + u8 *addtl; + size_t addtl_len; }; -static int rng_recvmsg(struct socket *sock, struct msghdr *msg, size_t len, - int flags) +struct rng_parent_ctx { + struct crypto_rng *drng; + u8 *entropy; +}; + +static void rng_reset_addtl(struct rng_ctx *ctx) { - struct sock *sk = sock->sk; - struct alg_sock *ask = alg_sk(sk); - struct rng_ctx *ctx = ask->private; - int err; + kfree_sensitive(ctx->addtl); + ctx->addtl = NULL; + ctx->addtl_len = 0; +} + +static int _rng_recvmsg(struct crypto_rng *drng, struct msghdr *msg, size_t len, + u8 *addtl, size_t addtl_len) +{ + int err = 0; int genlen = 0; u8 result[MAXSIZE]; @@ -82,7 +94,7 @@ static int rng_recvmsg(struct socket *sock, struct msghdr *msg, size_t len, * seeding as they automatically seed. The X9.31 DRNG will return * an error if it was not seeded properly. */ - genlen = crypto_rng_get_bytes(ctx->drng, result, len); + genlen = crypto_rng_generate(drng, addtl, addtl_len, result, len); if (genlen < 0) return genlen; @@ -92,6 +104,63 @@ static int rng_recvmsg(struct socket *sock, struct msghdr *msg, size_t len, return err ? err : len; } +static int rng_recvmsg(struct socket *sock, struct msghdr *msg, size_t len, + int flags) +{ + struct sock *sk = sock->sk; + struct alg_sock *ask = alg_sk(sk); + struct rng_ctx *ctx = ask->private; + + return _rng_recvmsg(ctx->drng, msg, len, NULL, 0); +} + +static int rng_test_recvmsg(struct socket *sock, struct msghdr *msg, size_t len, + int flags) +{ + struct sock *sk = sock->sk; + struct alg_sock *ask = alg_sk(sk); + struct rng_ctx *ctx = ask->private; + int ret; + + lock_sock(sock->sk); + ret = _rng_recvmsg(ctx->drng, msg, len, ctx->addtl, ctx->addtl_len); + rng_reset_addtl(ctx); + release_sock(sock->sk); + + return ret; +} + +static int rng_test_sendmsg(struct socket *sock, struct msghdr *msg, size_t len) +{ + int err; + struct alg_sock *ask = alg_sk(sock->sk); + struct rng_ctx *ctx = ask->private; + + lock_sock(sock->sk); + if (len > MAXSIZE) { + err = -EMSGSIZE; + goto unlock; + } + + rng_reset_addtl(ctx); + ctx->addtl = kmalloc(len, GFP_KERNEL); + if (!ctx->addtl) { + err = -ENOMEM; + goto unlock; + } + + err = memcpy_from_msg(ctx->addtl, msg, len); + if (err) { + rng_reset_addtl(ctx); + goto unlock; + } + ctx->addtl_len = len; + +unlock: + release_sock(sock->sk); + return err ? err : len; +} + static struct proto_ops algif_rng_ops = { .family = PF_ALG, @@ -111,14 +180,53 @@ static struct proto_ops algif_rng_ops = { .recvmsg = rng_recvmsg, }; +static struct proto_ops __maybe_unused algif_rng_test_ops = { + .family = PF_ALG, + + .connect = sock_no_connect, + .socketpair = sock_no_socketpair, + .getname = sock_no_getname, + .ioctl = sock_no_ioctl, + .listen = sock_no_listen, + .shutdown = sock_no_shutdown, + .mmap = sock_no_mmap, + .bind = sock_no_bind, + .accept = sock_no_accept, + .sendpage = sock_no_sendpage, + + .release = af_alg_release, + .recvmsg = rng_test_recvmsg, + .sendmsg = rng_test_sendmsg, +}; + static void *rng_bind(const char *name, u32 type, u32 mask) { - return crypto_alloc_rng(name, type, mask); + struct rng_parent_ctx *pctx; + struct crypto_rng *rng; + + pctx = kzalloc(sizeof(*pctx), GFP_KERNEL); + if (!pctx) + return ERR_PTR(-ENOMEM); + + rng = crypto_alloc_rng(name, type, mask); + if (IS_ERR(rng)) { + kfree(pctx); + return ERR_CAST(rng); + } + + pctx->drng = rng; + return pctx; } static void rng_release(void *private) { - crypto_free_rng(private); + struct rng_parent_ctx *pctx = private; + + if (unlikely(!pctx)) + return; + crypto_free_rng(pctx->drng); + kfree_sensitive(pctx->entropy); + kfree_sensitive(pctx); } static void rng_sock_destruct(struct sock *sk) @@ -126,6 +234,7 @@ static void rng_sock_destruct(struct sock *sk) struct alg_sock *ask = alg_sk(sk); struct rng_ctx *ctx = ask->private; + rng_reset_addtl(ctx); sock_kfree_s(sk, ctx, ctx->len); af_alg_release_parent(sk); } @@ -133,6 +242,7 @@ static void rng_sock_destruct(struct sock *sk) static int rng_accept_parent(void *private, struct sock *sk) { struct rng_ctx *ctx; + struct rng_parent_ctx *pctx = private; struct alg_sock *ask = alg_sk(sk); unsigned int len = sizeof(*ctx); @@ -141,6 +251,8 @@ static int rng_accept_parent(void *private, struct sock *sk) return -ENOMEM; ctx->len = len; + ctx->addtl = NULL; + ctx->addtl_len = 0; /* * No seeding done at that point -- if multiple accepts are @@ -148,20 +260,58 @@ static int rng_accept_parent(void *private, struct sock *sk) * state of the RNG. */ - ctx->drng = private; + ctx->drng = pctx->drng; ask->private = ctx; sk->sk_destruct = rng_sock_destruct; + /* + * Non NULL pctx->entropy means that CAVP test has been initiated on + * this socket, replace proto_ops algif_rng_ops with algif_rng_test_ops. + */ + if (IS_ENABLED(CONFIG_CRYPTO_USER_API_RNG_CAVP) && pctx->entropy) + sk->sk_socket->ops = &algif_rng_test_ops; + return 0; } static int rng_setkey(void *private, const u8 *seed, unsigned int seedlen) { + struct rng_parent_ctx *pctx = private; /* * Check whether seedlen is of sufficient size is done in RNG * implementations. */ - return crypto_rng_reset(private, seed, seedlen); + return crypto_rng_reset(pctx->drng, seed, seedlen); +} + +static int __maybe_unused rng_setentropy(void *private, sockptr_t entropy, + unsigned int len) +{ + struct rng_parent_ctx *pctx = private; + u8 *kentropy = NULL; + + if (!capable(CAP_SYS_ADMIN)) + return -EACCES; + + if (pctx->entropy) + return -EINVAL; + + if (len > MAXSIZE) + return -EMSGSIZE; + + if (len) { + kentropy = memdup_sockptr(entropy, len); + if (IS_ERR(kentropy)) + return PTR_ERR(kentropy); + } + + crypto_rng_alg(pctx->drng)->set_ent(pctx->drng, kentropy, len); + /* + * Since rng doesn't perform any memory management for the entropy + * buffer, save kentropy pointer to pctx now to free it after use. + */ + pctx->entropy = kentropy; + return 0; } static const struct af_alg_type algif_type_rng = { @@ -169,6 +319,9 @@ static const struct af_alg_type algif_type_rng = { .release = rng_release, .accept = rng_accept_parent, .setkey = rng_setkey, +#ifdef CONFIG_CRYPTO_USER_API_RNG_CAVP + .setentropy = rng_setentropy, +#endif .ops = &algif_rng_ops, .name = "rng", .owner = THIS_MODULE diff --git a/crypto/algif_skcipher.c b/crypto/algif_skcipher.c index 478f3b8f5bd5..ee8890ee8f33 100644 --- a/crypto/algif_skcipher.c +++ b/crypto/algif_skcipher.c @@ -123,7 +123,7 @@ static int _skcipher_recvmsg(struct socket *sock, struct msghdr *msg, crypto_skcipher_decrypt(&areq->cra_u.skcipher_req); /* AIO operation in progress */ - if (err == -EINPROGRESS || err == -EBUSY) + if (err == -EINPROGRESS) return -EIOCBQUEUED; sock_put(sk); diff --git a/crypto/arc4.c b/crypto/arc4.c index aa79571dbd49..3254dcc34368 100644 --- a/crypto/arc4.c +++ b/crypto/arc4.c @@ -11,7 +11,9 @@ #include <crypto/arc4.h> #include <crypto/internal/skcipher.h> #include <linux/init.h> +#include <linux/kernel.h> #include <linux/module.h> +#include <linux/sched.h> static int crypto_arc4_setkey(struct crypto_skcipher *tfm, const u8 *in_key, unsigned int key_len) @@ -39,6 +41,14 @@ static int crypto_arc4_crypt(struct skcipher_request *req) return err; } +static int crypto_arc4_init(struct crypto_skcipher *tfm) +{ + pr_warn_ratelimited("\"%s\" (%ld) uses obsolete ecb(arc4) skcipher\n", + current->comm, (unsigned long)current->pid); + + return 0; +} + static struct skcipher_alg arc4_alg = { /* * For legacy reasons, this is named "ecb(arc4)", not "arc4". @@ -55,6 +65,7 @@ static struct skcipher_alg arc4_alg = { .setkey = crypto_arc4_setkey, .encrypt = crypto_arc4_crypt, .decrypt = crypto_arc4_crypt, + .init = crypto_arc4_init, }; static int __init arc4_init(void) diff --git a/crypto/asymmetric_keys/public_key.c b/crypto/asymmetric_keys/public_key.c index d8410ffd7f12..8892908ad58c 100644 --- a/crypto/asymmetric_keys/public_key.c +++ b/crypto/asymmetric_keys/public_key.c @@ -17,6 +17,8 @@ #include <keys/asymmetric-subtype.h> #include <crypto/public_key.h> #include <crypto/akcipher.h> +#include <crypto/sm2.h> +#include <crypto/sm3_base.h> MODULE_DESCRIPTION("In-software asymmetric public-key subtype"); MODULE_AUTHOR("Red Hat, Inc."); @@ -246,6 +248,61 @@ error_free_tfm: return ret; } +#if IS_REACHABLE(CONFIG_CRYPTO_SM2) +static int cert_sig_digest_update(const struct public_key_signature *sig, + struct crypto_akcipher *tfm_pkey) +{ + struct crypto_shash *tfm; + struct shash_desc *desc; + size_t desc_size; + unsigned char dgst[SM3_DIGEST_SIZE]; + int ret; + + BUG_ON(!sig->data); + + ret = sm2_compute_z_digest(tfm_pkey, SM2_DEFAULT_USERID, + SM2_DEFAULT_USERID_LEN, dgst); + if (ret) + return ret; + + tfm = crypto_alloc_shash(sig->hash_algo, 0, 0); + if (IS_ERR(tfm)) + return PTR_ERR(tfm); + + desc_size = crypto_shash_descsize(tfm) + sizeof(*desc); + desc = kzalloc(desc_size, GFP_KERNEL); + if (!desc) { + ret = -ENOMEM; + goto error_free_tfm; + } + + desc->tfm = tfm; + + ret = crypto_shash_init(desc); + if (ret < 0) + goto error_free_desc; + + ret = crypto_shash_update(desc, dgst, SM3_DIGEST_SIZE); + if (ret < 0) + goto error_free_desc; + + ret = crypto_shash_finup(desc, sig->data, sig->data_size, sig->digest); + +error_free_desc: + kfree(desc); +error_free_tfm: + crypto_free_shash(tfm); + return ret; +} +#else +static inline int cert_sig_digest_update( + const struct public_key_signature *sig, + struct crypto_akcipher *tfm_pkey) +{ + return -ENOTSUPP; +} +#endif /* ! IS_REACHABLE(CONFIG_CRYPTO_SM2) */ + /* * Verify a signature using a public key. */ @@ -299,6 +356,12 @@ int public_key_verify_signature(const struct public_key *pkey, if (ret) goto error_free_key; + if (strcmp(sig->pkey_algo, "sm2") == 0 && sig->data_size) { + ret = cert_sig_digest_update(sig, tfm); + if (ret) + goto error_free_key; + } + sg_init_table(src_sg, 2); sg_set_buf(&src_sg[0], sig->s, sig->s_size); sg_set_buf(&src_sg[1], sig->digest, sig->digest_size); diff --git a/crypto/asymmetric_keys/x509_cert_parser.c b/crypto/asymmetric_keys/x509_cert_parser.c index 26ec20ef4899..52c9b455fc7d 100644 --- a/crypto/asymmetric_keys/x509_cert_parser.c +++ b/crypto/asymmetric_keys/x509_cert_parser.c @@ -234,6 +234,10 @@ int x509_note_pkey_algo(void *context, size_t hdrlen, case OID_gost2012Signature512: ctx->cert->sig->hash_algo = "streebog512"; goto ecrdsa; + + case OID_SM2_with_SM3: + ctx->cert->sig->hash_algo = "sm3"; + goto sm2; } rsa_pkcs1: @@ -246,6 +250,11 @@ ecrdsa: ctx->cert->sig->encoding = "raw"; ctx->algo_oid = ctx->last_oid; return 0; +sm2: + ctx->cert->sig->pkey_algo = "sm2"; + ctx->cert->sig->encoding = "raw"; + ctx->algo_oid = ctx->last_oid; + return 0; } /* @@ -266,7 +275,8 @@ int x509_note_signature(void *context, size_t hdrlen, } if (strcmp(ctx->cert->sig->pkey_algo, "rsa") == 0 || - strcmp(ctx->cert->sig->pkey_algo, "ecrdsa") == 0) { + strcmp(ctx->cert->sig->pkey_algo, "ecrdsa") == 0 || + strcmp(ctx->cert->sig->pkey_algo, "sm2") == 0) { /* Discard the BIT STRING metadata */ if (vlen < 1 || *(const u8 *)value != 0) return -EBADMSG; @@ -451,13 +461,20 @@ int x509_extract_key_data(void *context, size_t hdrlen, struct x509_parse_context *ctx = context; ctx->key_algo = ctx->last_oid; - if (ctx->last_oid == OID_rsaEncryption) + switch (ctx->last_oid) { + case OID_rsaEncryption: ctx->cert->pub->pkey_algo = "rsa"; - else if (ctx->last_oid == OID_gost2012PKey256 || - ctx->last_oid == OID_gost2012PKey512) + break; + case OID_gost2012PKey256: + case OID_gost2012PKey512: ctx->cert->pub->pkey_algo = "ecrdsa"; - else + break; + case OID_id_ecPublicKey: + ctx->cert->pub->pkey_algo = "sm2"; + break; + default: return -ENOPKG; + } /* Discard the BIT STRING metadata */ if (vlen < 1 || *(const u8 *)value != 0) diff --git a/crypto/asymmetric_keys/x509_public_key.c b/crypto/asymmetric_keys/x509_public_key.c index d964cc82b69c..ae450eb8be14 100644 --- a/crypto/asymmetric_keys/x509_public_key.c +++ b/crypto/asymmetric_keys/x509_public_key.c @@ -30,6 +30,9 @@ int x509_get_sig_params(struct x509_certificate *cert) pr_devel("==>%s()\n", __func__); + sig->data = cert->tbs; + sig->data_size = cert->tbs_size; + if (!cert->pub->pkey_algo) cert->unsupported_key = true; diff --git a/crypto/cbc.c b/crypto/cbc.c index e6f6273a7d39..0d9509dff891 100644 --- a/crypto/cbc.c +++ b/crypto/cbc.c @@ -6,7 +6,6 @@ */ #include <crypto/algapi.h> -#include <crypto/cbc.h> #include <crypto/internal/skcipher.h> #include <linux/err.h> #include <linux/init.h> @@ -14,34 +13,157 @@ #include <linux/log2.h> #include <linux/module.h> -static inline void crypto_cbc_encrypt_one(struct crypto_skcipher *tfm, - const u8 *src, u8 *dst) +static int crypto_cbc_encrypt_segment(struct skcipher_walk *walk, + struct crypto_skcipher *skcipher) { - crypto_cipher_encrypt_one(skcipher_cipher_simple(tfm), dst, src); + unsigned int bsize = crypto_skcipher_blocksize(skcipher); + void (*fn)(struct crypto_tfm *, u8 *, const u8 *); + unsigned int nbytes = walk->nbytes; + u8 *src = walk->src.virt.addr; + u8 *dst = walk->dst.virt.addr; + struct crypto_cipher *cipher; + struct crypto_tfm *tfm; + u8 *iv = walk->iv; + + cipher = skcipher_cipher_simple(skcipher); + tfm = crypto_cipher_tfm(cipher); + fn = crypto_cipher_alg(cipher)->cia_encrypt; + + do { + crypto_xor(iv, src, bsize); + fn(tfm, dst, iv); + memcpy(iv, dst, bsize); + + src += bsize; + dst += bsize; + } while ((nbytes -= bsize) >= bsize); + + return nbytes; +} + +static int crypto_cbc_encrypt_inplace(struct skcipher_walk *walk, + struct crypto_skcipher *skcipher) +{ + unsigned int bsize = crypto_skcipher_blocksize(skcipher); + void (*fn)(struct crypto_tfm *, u8 *, const u8 *); + unsigned int nbytes = walk->nbytes; + u8 *src = walk->src.virt.addr; + struct crypto_cipher *cipher; + struct crypto_tfm *tfm; + u8 *iv = walk->iv; + + cipher = skcipher_cipher_simple(skcipher); + tfm = crypto_cipher_tfm(cipher); + fn = crypto_cipher_alg(cipher)->cia_encrypt; + + do { + crypto_xor(src, iv, bsize); + fn(tfm, src, src); + iv = src; + + src += bsize; + } while ((nbytes -= bsize) >= bsize); + + memcpy(walk->iv, iv, bsize); + + return nbytes; } static int crypto_cbc_encrypt(struct skcipher_request *req) { - return crypto_cbc_encrypt_walk(req, crypto_cbc_encrypt_one); + struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); + struct skcipher_walk walk; + int err; + + err = skcipher_walk_virt(&walk, req, false); + + while (walk.nbytes) { + if (walk.src.virt.addr == walk.dst.virt.addr) + err = crypto_cbc_encrypt_inplace(&walk, skcipher); + else + err = crypto_cbc_encrypt_segment(&walk, skcipher); + err = skcipher_walk_done(&walk, err); + } + + return err; +} + +static int crypto_cbc_decrypt_segment(struct skcipher_walk *walk, + struct crypto_skcipher *skcipher) +{ + unsigned int bsize = crypto_skcipher_blocksize(skcipher); + void (*fn)(struct crypto_tfm *, u8 *, const u8 *); + unsigned int nbytes = walk->nbytes; + u8 *src = walk->src.virt.addr; + u8 *dst = walk->dst.virt.addr; + struct crypto_cipher *cipher; + struct crypto_tfm *tfm; + u8 *iv = walk->iv; + + cipher = skcipher_cipher_simple(skcipher); + tfm = crypto_cipher_tfm(cipher); + fn = crypto_cipher_alg(cipher)->cia_decrypt; + + do { + fn(tfm, dst, src); + crypto_xor(dst, iv, bsize); + iv = src; + + src += bsize; + dst += bsize; + } while ((nbytes -= bsize) >= bsize); + + memcpy(walk->iv, iv, bsize); + + return nbytes; } -static inline void crypto_cbc_decrypt_one(struct crypto_skcipher *tfm, - const u8 *src, u8 *dst) +static int crypto_cbc_decrypt_inplace(struct skcipher_walk *walk, + struct crypto_skcipher *skcipher) { - crypto_cipher_decrypt_one(skcipher_cipher_simple(tfm), dst, src); + unsigned int bsize = crypto_skcipher_blocksize(skcipher); + void (*fn)(struct crypto_tfm *, u8 *, const u8 *); + unsigned int nbytes = walk->nbytes; + u8 *src = walk->src.virt.addr; + u8 last_iv[MAX_CIPHER_BLOCKSIZE]; + struct crypto_cipher *cipher; + struct crypto_tfm *tfm; + + cipher = skcipher_cipher_simple(skcipher); + tfm = crypto_cipher_tfm(cipher); + fn = crypto_cipher_alg(cipher)->cia_decrypt; + + /* Start of the last block. */ + src += nbytes - (nbytes & (bsize - 1)) - bsize; + memcpy(last_iv, src, bsize); + + for (;;) { + fn(tfm, src, src); + if ((nbytes -= bsize) < bsize) + break; + crypto_xor(src, src - bsize, bsize); + src -= bsize; + } + + crypto_xor(src, walk->iv, bsize); + memcpy(walk->iv, last_iv, bsize); + + return nbytes; } static int crypto_cbc_decrypt(struct skcipher_request *req) { - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); struct skcipher_walk walk; int err; err = skcipher_walk_virt(&walk, req, false); while (walk.nbytes) { - err = crypto_cbc_decrypt_blocks(&walk, tfm, - crypto_cbc_decrypt_one); + if (walk.src.virt.addr == walk.dst.virt.addr) + err = crypto_cbc_decrypt_inplace(&walk, skcipher); + else + err = crypto_cbc_decrypt_segment(&walk, skcipher); err = skcipher_walk_done(&walk, err); } diff --git a/crypto/crc32c_generic.c b/crypto/crc32c_generic.c index 7fa9b0788685..768614738541 100644 --- a/crypto/crc32c_generic.c +++ b/crypto/crc32c_generic.c @@ -15,7 +15,7 @@ * pages = {}, * month = {June}, *} - * Used by the iSCSI driver, possibly others, and derived from the + * Used by the iSCSI driver, possibly others, and derived from * the iscsi-crc.c module of the linux-iscsi driver at * http://linux-iscsi.sourceforge.net. * @@ -50,7 +50,7 @@ struct chksum_desc_ctx { }; /* - * Steps through buffer one byte at at time, calculates reflected + * Steps through buffer one byte at a time, calculates reflected * crc using table. */ diff --git a/crypto/crct10dif_generic.c b/crypto/crct10dif_generic.c index d90c0070710e..e843982073bb 100644 --- a/crypto/crct10dif_generic.c +++ b/crypto/crct10dif_generic.c @@ -35,7 +35,7 @@ struct chksum_desc_ctx { }; /* - * Steps through buffer one byte at at time, calculates reflected + * Steps through buffer one byte at a time, calculates reflected * crc using table. */ diff --git a/crypto/crypto_engine.c b/crypto/crypto_engine.c index 198a8eb1cd56..cff21f4e03e3 100644 --- a/crypto/crypto_engine.c +++ b/crypto/crypto_engine.c @@ -9,6 +9,7 @@ #include <linux/err.h> #include <linux/delay.h> +#include <linux/device.h> #include <crypto/engine.h> #include <uapi/linux/sched/types.h> #include "internal.h" @@ -465,7 +466,7 @@ EXPORT_SYMBOL_GPL(crypto_engine_stop); * crypto-engine queue. * @dev: the device attached with one hardware engine * @retry_support: whether hardware has support for retry mechanism - * @cbk_do_batch: pointer to a callback function to be invoked when executing a + * @cbk_do_batch: pointer to a callback function to be invoked when executing * a batch of requests. * This has the form: * callback(struct crypto_engine *engine) diff --git a/crypto/ecrdsa.c b/crypto/ecrdsa.c index 887ec21aee49..6a3fd09057d0 100644 --- a/crypto/ecrdsa.c +++ b/crypto/ecrdsa.c @@ -22,6 +22,7 @@ #include <crypto/internal/akcipher.h> #include <crypto/akcipher.h> #include <linux/oid_registry.h> +#include <linux/scatterlist.h> #include "ecrdsa_params.asn1.h" #include "ecrdsa_pub_key.asn1.h" #include "ecc.h" diff --git a/crypto/internal.h b/crypto/internal.h index 1b92a5a61852..976ec9dfc76d 100644 --- a/crypto/internal.h +++ b/crypto/internal.h @@ -10,16 +10,14 @@ #include <crypto/algapi.h> #include <linux/completion.h> -#include <linux/mm.h> -#include <linux/highmem.h> -#include <linux/interrupt.h> -#include <linux/init.h> #include <linux/list.h> #include <linux/module.h> -#include <linux/kernel.h> #include <linux/notifier.h> +#include <linux/numa.h> +#include <linux/refcount.h> #include <linux/rwsem.h> -#include <linux/slab.h> +#include <linux/sched.h> +#include <linux/types.h> struct crypto_instance; struct crypto_template; @@ -140,5 +138,11 @@ static inline void crypto_notify(unsigned long val, void *v) blocking_notifier_call_chain(&crypto_chain, val, v); } +static inline void crypto_yield(u32 flags) +{ + if (flags & CRYPTO_TFM_REQ_MAY_SLEEP) + cond_resched(); +} + #endif /* _CRYPTO_INTERNAL_H */ diff --git a/crypto/jitterentropy-kcapi.c b/crypto/jitterentropy-kcapi.c index eb7d1dd506bf..e8a4165a1874 100644 --- a/crypto/jitterentropy-kcapi.c +++ b/crypto/jitterentropy-kcapi.c @@ -37,11 +37,11 @@ * DAMAGE. */ +#include <linux/kernel.h> #include <linux/module.h> #include <linux/slab.h> #include <linux/fips.h> #include <linux/time.h> -#include <linux/crypto.h> #include <crypto/internal/rng.h> #include "jitterentropy.h" diff --git a/crypto/proc.c b/crypto/proc.c index 08d8c2bc7e62..12fccb9c5205 100644 --- a/crypto/proc.c +++ b/crypto/proc.c @@ -36,7 +36,7 @@ static void c_stop(struct seq_file *m, void *p) static int c_show(struct seq_file *m, void *p) { struct crypto_alg *alg = list_entry(p, struct crypto_alg, cra_list); - + seq_printf(m, "name : %s\n", alg->cra_name); seq_printf(m, "driver : %s\n", alg->cra_driver_name); seq_printf(m, "module : %s\n", module_name(alg->cra_module)); @@ -59,7 +59,7 @@ static int c_show(struct seq_file *m, void *p) alg->cra_type->show(m, alg); goto out; } - + switch (alg->cra_flags & CRYPTO_ALG_TYPE_MASK) { case CRYPTO_ALG_TYPE_CIPHER: seq_printf(m, "type : cipher\n"); diff --git a/crypto/rsa-pkcs1pad.c b/crypto/rsa-pkcs1pad.c index ddd3d10ffc15..8ac3e73e8ea6 100644 --- a/crypto/rsa-pkcs1pad.c +++ b/crypto/rsa-pkcs1pad.c @@ -14,6 +14,7 @@ #include <linux/kernel.h> #include <linux/module.h> #include <linux/random.h> +#include <linux/scatterlist.h> /* * Hash algorithm OIDs plus ASN.1 DER wrappings [RFC4880 sec 5.2.2]. diff --git a/crypto/sm2.c b/crypto/sm2.c new file mode 100644 index 000000000000..767e160333f6 --- /dev/null +++ b/crypto/sm2.c @@ -0,0 +1,481 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * SM2 asymmetric public-key algorithm + * as specified by OSCCA GM/T 0003.1-2012 -- 0003.5-2012 SM2 and + * described at https://tools.ietf.org/html/draft-shen-sm2-ecdsa-02 + * + * Copyright (c) 2020, Alibaba Group. + * Authors: Tianjia Zhang <tianjia.zhang@linux.alibaba.com> + */ + +#include <linux/module.h> +#include <linux/mpi.h> +#include <crypto/internal/akcipher.h> +#include <crypto/akcipher.h> +#include <crypto/hash.h> +#include <crypto/sm3_base.h> +#include <crypto/rng.h> +#include <crypto/sm2.h> +#include "sm2signature.asn1.h" + +#define MPI_NBYTES(m) ((mpi_get_nbits(m) + 7) / 8) + +struct ecc_domain_parms { + const char *desc; /* Description of the curve. */ + unsigned int nbits; /* Number of bits. */ + unsigned int fips:1; /* True if this is a FIPS140-2 approved curve */ + + /* The model describing this curve. This is mainly used to select + * the group equation. + */ + enum gcry_mpi_ec_models model; + + /* The actual ECC dialect used. This is used for curve specific + * optimizations and to select encodings etc. + */ + enum ecc_dialects dialect; + + const char *p; /* The prime defining the field. */ + const char *a, *b; /* The coefficients. For Twisted Edwards + * Curves b is used for d. For Montgomery + * Curves (a,b) has ((A-2)/4,B^-1). + */ + const char *n; /* The order of the base point. */ + const char *g_x, *g_y; /* Base point. */ + unsigned int h; /* Cofactor. */ +}; + +static const struct ecc_domain_parms sm2_ecp = { + .desc = "sm2p256v1", + .nbits = 256, + .fips = 0, + .model = MPI_EC_WEIERSTRASS, + .dialect = ECC_DIALECT_STANDARD, + .p = "0xfffffffeffffffffffffffffffffffffffffffff00000000ffffffffffffffff", + .a = "0xfffffffeffffffffffffffffffffffffffffffff00000000fffffffffffffffc", + .b = "0x28e9fa9e9d9f5e344d5a9e4bcf6509a7f39789f515ab8f92ddbcbd414d940e93", + .n = "0xfffffffeffffffffffffffffffffffff7203df6b21c6052b53bbf40939d54123", + .g_x = "0x32c4ae2c1f1981195f9904466a39c9948fe30bbff2660be1715a4589334c74c7", + .g_y = "0xbc3736a2f4f6779c59bdcee36b692153d0a9877cc62a474002df32e52139f0a0", + .h = 1 +}; + +static int sm2_ec_ctx_init(struct mpi_ec_ctx *ec) +{ + const struct ecc_domain_parms *ecp = &sm2_ecp; + MPI p, a, b; + MPI x, y; + int rc = -EINVAL; + + p = mpi_scanval(ecp->p); + a = mpi_scanval(ecp->a); + b = mpi_scanval(ecp->b); + if (!p || !a || !b) + goto free_p; + + x = mpi_scanval(ecp->g_x); + y = mpi_scanval(ecp->g_y); + if (!x || !y) + goto free; + + rc = -ENOMEM; + /* mpi_ec_setup_elliptic_curve */ + ec->G = mpi_point_new(0); + if (!ec->G) + goto free; + + mpi_set(ec->G->x, x); + mpi_set(ec->G->y, y); + mpi_set_ui(ec->G->z, 1); + + rc = -EINVAL; + ec->n = mpi_scanval(ecp->n); + if (!ec->n) { + mpi_point_release(ec->G); + goto free; + } + + ec->h = ecp->h; + ec->name = ecp->desc; + mpi_ec_init(ec, ecp->model, ecp->dialect, 0, p, a, b); + + rc = 0; + +free: + mpi_free(x); + mpi_free(y); +free_p: + mpi_free(p); + mpi_free(a); + mpi_free(b); + + return rc; +} + +static void sm2_ec_ctx_deinit(struct mpi_ec_ctx *ec) +{ + mpi_ec_deinit(ec); + + memset(ec, 0, sizeof(*ec)); +} + +static int sm2_ec_ctx_reset(struct mpi_ec_ctx *ec) +{ + sm2_ec_ctx_deinit(ec); + return sm2_ec_ctx_init(ec); +} + +/* RESULT must have been initialized and is set on success to the + * point given by VALUE. + */ +static int sm2_ecc_os2ec(MPI_POINT result, MPI value) +{ + int rc; + size_t n; + const unsigned char *buf; + unsigned char *buf_memory; + MPI x, y; + + n = (mpi_get_nbits(value)+7)/8; + buf_memory = kmalloc(n, GFP_KERNEL); + rc = mpi_print(GCRYMPI_FMT_USG, buf_memory, n, &n, value); + if (rc) { + kfree(buf_memory); + return rc; + } + buf = buf_memory; + + if (n < 1) { + kfree(buf_memory); + return -EINVAL; + } + if (*buf != 4) { + kfree(buf_memory); + return -EINVAL; /* No support for point compression. */ + } + if (((n-1)%2)) { + kfree(buf_memory); + return -EINVAL; + } + n = (n-1)/2; + x = mpi_read_raw_data(buf + 1, n); + if (!x) { + kfree(buf_memory); + return -ENOMEM; + } + y = mpi_read_raw_data(buf + 1 + n, n); + kfree(buf_memory); + if (!y) { + mpi_free(x); + return -ENOMEM; + } + + mpi_normalize(x); + mpi_normalize(y); + + mpi_set(result->x, x); + mpi_set(result->y, y); + mpi_set_ui(result->z, 1); + + mpi_free(x); + mpi_free(y); + + return 0; +} + +struct sm2_signature_ctx { + MPI sig_r; + MPI sig_s; +}; + +int sm2_get_signature_r(void *context, size_t hdrlen, unsigned char tag, + const void *value, size_t vlen) +{ + struct sm2_signature_ctx *sig = context; + + if (!value || !vlen) + return -EINVAL; + + sig->sig_r = mpi_read_raw_data(value, vlen); + if (!sig->sig_r) + return -ENOMEM; + + return 0; +} + +int sm2_get_signature_s(void *context, size_t hdrlen, unsigned char tag, + const void *value, size_t vlen) +{ + struct sm2_signature_ctx *sig = context; + + if (!value || !vlen) + return -EINVAL; + + sig->sig_s = mpi_read_raw_data(value, vlen); + if (!sig->sig_s) + return -ENOMEM; + + return 0; +} + +static int sm2_z_digest_update(struct shash_desc *desc, + MPI m, unsigned int pbytes) +{ + static const unsigned char zero[32]; + unsigned char *in; + unsigned int inlen; + + in = mpi_get_buffer(m, &inlen, NULL); + if (!in) + return -EINVAL; + + if (inlen < pbytes) { + /* padding with zero */ + crypto_sm3_update(desc, zero, pbytes - inlen); + crypto_sm3_update(desc, in, inlen); + } else if (inlen > pbytes) { + /* skip the starting zero */ + crypto_sm3_update(desc, in + inlen - pbytes, pbytes); + } else { + crypto_sm3_update(desc, in, inlen); + } + + kfree(in); + return 0; +} + +static int sm2_z_digest_update_point(struct shash_desc *desc, + MPI_POINT point, struct mpi_ec_ctx *ec, unsigned int pbytes) +{ + MPI x, y; + int ret = -EINVAL; + + x = mpi_new(0); + y = mpi_new(0); + + if (!mpi_ec_get_affine(x, y, point, ec) && + !sm2_z_digest_update(desc, x, pbytes) && + !sm2_z_digest_update(desc, y, pbytes)) + ret = 0; + + mpi_free(x); + mpi_free(y); + return ret; +} + +int sm2_compute_z_digest(struct crypto_akcipher *tfm, + const unsigned char *id, size_t id_len, + unsigned char dgst[SM3_DIGEST_SIZE]) +{ + struct mpi_ec_ctx *ec = akcipher_tfm_ctx(tfm); + uint16_t bits_len; + unsigned char entl[2]; + SHASH_DESC_ON_STACK(desc, NULL); + unsigned int pbytes; + + if (id_len > (USHRT_MAX / 8) || !ec->Q) + return -EINVAL; + + bits_len = (uint16_t)(id_len * 8); + entl[0] = bits_len >> 8; + entl[1] = bits_len & 0xff; + + pbytes = MPI_NBYTES(ec->p); + + /* ZA = H256(ENTLA | IDA | a | b | xG | yG | xA | yA) */ + sm3_base_init(desc); + crypto_sm3_update(desc, entl, 2); + crypto_sm3_update(desc, id, id_len); + + if (sm2_z_digest_update(desc, ec->a, pbytes) || + sm2_z_digest_update(desc, ec->b, pbytes) || + sm2_z_digest_update_point(desc, ec->G, ec, pbytes) || + sm2_z_digest_update_point(desc, ec->Q, ec, pbytes)) + return -EINVAL; + + crypto_sm3_final(desc, dgst); + return 0; +} +EXPORT_SYMBOL(sm2_compute_z_digest); + +static int _sm2_verify(struct mpi_ec_ctx *ec, MPI hash, MPI sig_r, MPI sig_s) +{ + int rc = -EINVAL; + struct gcry_mpi_point sG, tP; + MPI t = NULL; + MPI x1 = NULL, y1 = NULL; + + mpi_point_init(&sG); + mpi_point_init(&tP); + x1 = mpi_new(0); + y1 = mpi_new(0); + t = mpi_new(0); + + /* r, s in [1, n-1] */ + if (mpi_cmp_ui(sig_r, 1) < 0 || mpi_cmp(sig_r, ec->n) > 0 || + mpi_cmp_ui(sig_s, 1) < 0 || mpi_cmp(sig_s, ec->n) > 0) { + goto leave; + } + + /* t = (r + s) % n, t == 0 */ + mpi_addm(t, sig_r, sig_s, ec->n); + if (mpi_cmp_ui(t, 0) == 0) + goto leave; + + /* sG + tP = (x1, y1) */ + rc = -EBADMSG; + mpi_ec_mul_point(&sG, sig_s, ec->G, ec); + mpi_ec_mul_point(&tP, t, ec->Q, ec); + mpi_ec_add_points(&sG, &sG, &tP, ec); + if (mpi_ec_get_affine(x1, y1, &sG, ec)) + goto leave; + + /* R = (e + x1) % n */ + mpi_addm(t, hash, x1, ec->n); + + /* check R == r */ + rc = -EKEYREJECTED; + if (mpi_cmp(t, sig_r)) + goto leave; + + rc = 0; + +leave: + mpi_point_free_parts(&sG); + mpi_point_free_parts(&tP); + mpi_free(x1); + mpi_free(y1); + mpi_free(t); + + return rc; +} + +static int sm2_verify(struct akcipher_request *req) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct mpi_ec_ctx *ec = akcipher_tfm_ctx(tfm); + unsigned char *buffer; + struct sm2_signature_ctx sig; + MPI hash; + int ret; + + if (unlikely(!ec->Q)) + return -EINVAL; + + buffer = kmalloc(req->src_len + req->dst_len, GFP_KERNEL); + if (!buffer) + return -ENOMEM; + + sg_pcopy_to_buffer(req->src, + sg_nents_for_len(req->src, req->src_len + req->dst_len), + buffer, req->src_len + req->dst_len, 0); + + sig.sig_r = NULL; + sig.sig_s = NULL; + ret = asn1_ber_decoder(&sm2signature_decoder, &sig, + buffer, req->src_len); + if (ret) + goto error; + + ret = -ENOMEM; + hash = mpi_read_raw_data(buffer + req->src_len, req->dst_len); + if (!hash) + goto error; + + ret = _sm2_verify(ec, hash, sig.sig_r, sig.sig_s); + + mpi_free(hash); +error: + mpi_free(sig.sig_r); + mpi_free(sig.sig_s); + kfree(buffer); + return ret; +} + +static int sm2_set_pub_key(struct crypto_akcipher *tfm, + const void *key, unsigned int keylen) +{ + struct mpi_ec_ctx *ec = akcipher_tfm_ctx(tfm); + MPI a; + int rc; + + rc = sm2_ec_ctx_reset(ec); + if (rc) + return rc; + + ec->Q = mpi_point_new(0); + if (!ec->Q) + return -ENOMEM; + + /* include the uncompressed flag '0x04' */ + rc = -ENOMEM; + a = mpi_read_raw_data(key, keylen); + if (!a) + goto error; + + mpi_normalize(a); + rc = sm2_ecc_os2ec(ec->Q, a); + mpi_free(a); + if (rc) + goto error; + + return 0; + +error: + mpi_point_release(ec->Q); + ec->Q = NULL; + return rc; +} + +static unsigned int sm2_max_size(struct crypto_akcipher *tfm) +{ + /* Unlimited max size */ + return PAGE_SIZE; +} + +static int sm2_init_tfm(struct crypto_akcipher *tfm) +{ + struct mpi_ec_ctx *ec = akcipher_tfm_ctx(tfm); + + return sm2_ec_ctx_init(ec); +} + +static void sm2_exit_tfm(struct crypto_akcipher *tfm) +{ + struct mpi_ec_ctx *ec = akcipher_tfm_ctx(tfm); + + sm2_ec_ctx_deinit(ec); +} + +static struct akcipher_alg sm2 = { + .verify = sm2_verify, + .set_pub_key = sm2_set_pub_key, + .max_size = sm2_max_size, + .init = sm2_init_tfm, + .exit = sm2_exit_tfm, + .base = { + .cra_name = "sm2", + .cra_driver_name = "sm2-generic", + .cra_priority = 100, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct mpi_ec_ctx), + }, +}; + +static int sm2_init(void) +{ + return crypto_register_akcipher(&sm2); +} + +static void sm2_exit(void) +{ + crypto_unregister_akcipher(&sm2); +} + +subsys_initcall(sm2_init); +module_exit(sm2_exit); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Tianjia Zhang <tianjia.zhang@linux.alibaba.com>"); +MODULE_DESCRIPTION("SM2 generic algorithm"); +MODULE_ALIAS_CRYPTO("sm2-generic"); diff --git a/crypto/sm2signature.asn1 b/crypto/sm2signature.asn1 new file mode 100644 index 000000000000..ab8c0b754d21 --- /dev/null +++ b/crypto/sm2signature.asn1 @@ -0,0 +1,4 @@ +Sm2Signature ::= SEQUENCE { + sig_r INTEGER ({ sm2_get_signature_r }), + sig_s INTEGER ({ sm2_get_signature_s }) +} diff --git a/crypto/sm3_generic.c b/crypto/sm3_generic.c index 3468975215ca..193c4584bd00 100644 --- a/crypto/sm3_generic.c +++ b/crypto/sm3_generic.c @@ -149,17 +149,18 @@ int crypto_sm3_update(struct shash_desc *desc, const u8 *data, } EXPORT_SYMBOL(crypto_sm3_update); -static int sm3_final(struct shash_desc *desc, u8 *out) +int crypto_sm3_final(struct shash_desc *desc, u8 *out) { sm3_base_do_finalize(desc, sm3_generic_block_fn); return sm3_base_finish(desc, out); } +EXPORT_SYMBOL(crypto_sm3_final); int crypto_sm3_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *hash) { sm3_base_do_update(desc, data, len, sm3_generic_block_fn); - return sm3_final(desc, hash); + return crypto_sm3_final(desc, hash); } EXPORT_SYMBOL(crypto_sm3_finup); @@ -167,7 +168,7 @@ static struct shash_alg sm3_alg = { .digestsize = SM3_DIGEST_SIZE, .init = sm3_base_init, .update = crypto_sm3_update, - .final = sm3_final, + .final = crypto_sm3_final, .finup = crypto_sm3_finup, .descsize = sizeof(struct sm3_state), .base = { diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index 12e82a61b896..eea0f453cfb6 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -63,6 +63,7 @@ static u32 type; static u32 mask; static int mode; static u32 num_mb = 8; +static unsigned int klen; static char *tvmem[TVMEMSIZE]; static const char *check[] = { @@ -398,7 +399,7 @@ static void test_mb_aead_speed(const char *algo, int enc, int secs, ret = do_one_aead_op(cur->req, ret); if (ret) { - pr_err("calculating auth failed failed (%d)\n", + pr_err("calculating auth failed (%d)\n", ret); break; } @@ -648,7 +649,7 @@ static void test_aead_speed(const char *algo, int enc, unsigned int secs, crypto_aead_encrypt(req)); if (ret) { - pr_err("calculating auth failed failed (%d)\n", + pr_err("calculating auth failed (%d)\n", ret); break; } @@ -864,8 +865,8 @@ static void test_mb_ahash_speed(const char *algo, unsigned int secs, goto out; } - if (speed[i].klen) - crypto_ahash_setkey(tfm, tvmem[0], speed[i].klen); + if (klen) + crypto_ahash_setkey(tfm, tvmem[0], klen); for (k = 0; k < num_mb; k++) ahash_request_set_crypt(data[k].req, data[k].sg, @@ -1099,8 +1100,8 @@ static void test_ahash_speed_common(const char *algo, unsigned int secs, break; } - if (speed[i].klen) - crypto_ahash_setkey(tfm, tvmem[0], speed[i].klen); + if (klen) + crypto_ahash_setkey(tfm, tvmem[0], klen); pr_info("test%3u " "(%5u byte blocks,%5u bytes per update,%4u updates): ", @@ -2418,7 +2419,8 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) if (mode > 300 && mode < 400) break; fallthrough; case 318: - test_hash_speed("ghash-generic", sec, hash_speed_template_16); + klen = 16; + test_hash_speed("ghash", sec, generic_hash_speed_template); if (mode > 300 && mode < 400) break; fallthrough; case 319: @@ -3076,6 +3078,8 @@ MODULE_PARM_DESC(sec, "Length in seconds of speed tests " "(defaults to zero which uses CPU cycles instead)"); module_param(num_mb, uint, 0000); MODULE_PARM_DESC(num_mb, "Number of concurrent requests to be used in mb speed tests (defaults to 8)"); +module_param(klen, uint, 0); +MODULE_PARM_DESC(klen, "Key length (defaults to 0)"); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Quick & dirty crypto testing module"); diff --git a/crypto/tcrypt.h b/crypto/tcrypt.h index 7e5fea811670..9f654677172a 100644 --- a/crypto/tcrypt.h +++ b/crypto/tcrypt.h @@ -25,7 +25,6 @@ struct aead_speed_template { struct hash_speed { unsigned int blen; /* buffer length */ unsigned int plen; /* per-update length */ - unsigned int klen; /* key length */ }; /* @@ -97,34 +96,6 @@ static struct hash_speed generic_hash_speed_template[] = { { .blen = 0, .plen = 0, } }; -static struct hash_speed hash_speed_template_16[] = { - { .blen = 16, .plen = 16, .klen = 16, }, - { .blen = 64, .plen = 16, .klen = 16, }, - { .blen = 64, .plen = 64, .klen = 16, }, - { .blen = 256, .plen = 16, .klen = 16, }, - { .blen = 256, .plen = 64, .klen = 16, }, - { .blen = 256, .plen = 256, .klen = 16, }, - { .blen = 1024, .plen = 16, .klen = 16, }, - { .blen = 1024, .plen = 256, .klen = 16, }, - { .blen = 1024, .plen = 1024, .klen = 16, }, - { .blen = 2048, .plen = 16, .klen = 16, }, - { .blen = 2048, .plen = 256, .klen = 16, }, - { .blen = 2048, .plen = 1024, .klen = 16, }, - { .blen = 2048, .plen = 2048, .klen = 16, }, - { .blen = 4096, .plen = 16, .klen = 16, }, - { .blen = 4096, .plen = 256, .klen = 16, }, - { .blen = 4096, .plen = 1024, .klen = 16, }, - { .blen = 4096, .plen = 4096, .klen = 16, }, - { .blen = 8192, .plen = 16, .klen = 16, }, - { .blen = 8192, .plen = 256, .klen = 16, }, - { .blen = 8192, .plen = 1024, .klen = 16, }, - { .blen = 8192, .plen = 4096, .klen = 16, }, - { .blen = 8192, .plen = 8192, .klen = 16, }, - - /* End marker */ - { .blen = 0, .plen = 0, .klen = 0, } -}; - static struct hash_speed poly1305_speed_template[] = { { .blen = 96, .plen = 16, }, { .blen = 96, .plen = 32, }, diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 23c27fc96394..a64a639eddfa 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -27,6 +27,7 @@ #include <linux/scatterlist.h> #include <linux/slab.h> #include <linux/string.h> +#include <linux/uio.h> #include <crypto/rng.h> #include <crypto/drbg.h> #include <crypto/akcipher.h> @@ -3954,7 +3955,7 @@ static int test_akcipher_one(struct crypto_akcipher *tfm, key = kmalloc(vecs->key_len + sizeof(u32) * 2 + vecs->param_len, GFP_KERNEL); if (!key) - goto free_xbuf; + goto free_req; memcpy(key, vecs->key, vecs->key_len); ptr = key + vecs->key_len; ptr = test_pack_u32(ptr, vecs->algo); @@ -3966,7 +3967,7 @@ static int test_akcipher_one(struct crypto_akcipher *tfm, else err = crypto_akcipher_set_priv_key(tfm, key, vecs->key_len); if (err) - goto free_req; + goto free_key; /* * First run test which do not require a private key, such as @@ -3976,7 +3977,7 @@ static int test_akcipher_one(struct crypto_akcipher *tfm, out_len_max = crypto_akcipher_maxsize(tfm); outbuf_enc = kzalloc(out_len_max, GFP_KERNEL); if (!outbuf_enc) - goto free_req; + goto free_key; if (!vecs->siggen_sigver_test) { m = vecs->m; @@ -3995,6 +3996,7 @@ static int test_akcipher_one(struct crypto_akcipher *tfm, op = "verify"; } + err = -E2BIG; if (WARN_ON(m_size > PAGE_SIZE)) goto free_all; memcpy(xbuf[0], m, m_size); @@ -4025,7 +4027,7 @@ static int test_akcipher_one(struct crypto_akcipher *tfm, pr_err("alg: akcipher: %s test failed. err %d\n", op, err); goto free_all; } - if (!vecs->siggen_sigver_test) { + if (!vecs->siggen_sigver_test && c) { if (req->dst_len != c_size) { pr_err("alg: akcipher: %s test failed. Invalid output len\n", op); @@ -4056,6 +4058,12 @@ static int test_akcipher_one(struct crypto_akcipher *tfm, goto free_all; } + if (!vecs->siggen_sigver_test && !c) { + c = outbuf_enc; + c_size = req->dst_len; + } + + err = -E2BIG; op = vecs->siggen_sigver_test ? "sign" : "decrypt"; if (WARN_ON(c_size > PAGE_SIZE)) goto free_all; @@ -4092,9 +4100,10 @@ static int test_akcipher_one(struct crypto_akcipher *tfm, free_all: kfree(outbuf_dec); kfree(outbuf_enc); +free_key: + kfree(key); free_req: akcipher_request_free(req); - kfree(key); free_xbuf: testmgr_free_buf(xbuf); return err; @@ -5377,6 +5386,12 @@ static const struct alg_test_desc alg_test_descs[] = { .hash = __VECS(sha512_tv_template) } }, { + .alg = "sm2", + .test = alg_test_akcipher, + .suite = { + .akcipher = __VECS(sm2_tv_template) + } + }, { .alg = "sm3", .test = alg_test_hash, .suite = { diff --git a/crypto/testmgr.h b/crypto/testmgr.h index b9a2d73d9f8d..8c83811c0e35 100644 --- a/crypto/testmgr.h +++ b/crypto/testmgr.h @@ -3792,6 +3792,65 @@ static const struct hash_testvec hmac_streebog512_tv_template[] = { }, }; +/* + * SM2 test vectors. + */ +static const struct akcipher_testvec sm2_tv_template[] = { + { /* Generated from openssl */ + .key = + "\x04" + "\x8e\xa0\x33\x69\x91\x7e\x3d\xec\xad\x8e\xf0\x45\x5e\x13\x3e\x68" + "\x5b\x8c\xab\x5c\xc6\xc8\x50\xdf\x91\x00\xe0\x24\x73\x4d\x31\xf2" + "\x2e\xc0\xd5\x6b\xee\xda\x98\x93\xec\xd8\x36\xaa\xb9\xcf\x63\x82" + "\xef\xa7\x1a\x03\xed\x16\xba\x74\xb8\x8b\xf9\xe5\x70\x39\xa4\x70", + .key_len = 65, + .param_len = 0, + .c = + "\x30\x45" + "\x02\x20" + "\x70\xab\xb6\x7d\xd6\x54\x80\x64\x42\x7e\x2d\x05\x08\x36\xc9\x96" + "\x25\xc2\xbb\xff\x08\xe5\x43\x15\x5e\xf3\x06\xd9\x2b\x2f\x0a\x9f" + "\x02\x21" + "\x00" + "\xbf\x21\x5f\x7e\x5d\x3f\x1a\x4d\x8f\x84\xc2\xe9\xa6\x4c\xa4\x18" + "\xb2\xb8\x46\xf4\x32\x96\xfa\x57\xc6\x29\xd4\x89\xae\xcc\xda\xdb", + .c_size = 71, + .algo = OID_SM2_with_SM3, + .m = + "\x47\xa7\xbf\xd3\xda\xc4\x79\xee\xda\x8b\x4f\xe8\x40\x94\xd4\x32" + "\x8f\xf1\xcd\x68\x4d\xbd\x9b\x1d\xe0\xd8\x9a\x5d\xad\x85\x47\x5c", + .m_size = 32, + .public_key_vec = true, + .siggen_sigver_test = true, + }, + { /* From libgcrypt */ + .key = + "\x04" + "\x87\x59\x38\x9a\x34\xaa\xad\x07\xec\xf4\xe0\xc8\xc2\x65\x0a\x44" + "\x59\xc8\xd9\x26\xee\x23\x78\x32\x4e\x02\x61\xc5\x25\x38\xcb\x47" + "\x75\x28\x10\x6b\x1e\x0b\x7c\x8d\xd5\xff\x29\xa9\xc8\x6a\x89\x06" + "\x56\x56\xeb\x33\x15\x4b\xc0\x55\x60\x91\xef\x8a\xc9\xd1\x7d\x78", + .key_len = 65, + .param_len = 0, + .c = + "\x30\x44" + "\x02\x20" + "\xd9\xec\xef\xe8\x5f\xee\x3c\x59\x57\x8e\x5b\xab\xb3\x02\xe1\x42" + "\x4b\x67\x2c\x0b\x26\xb6\x51\x2c\x3e\xfc\xc6\x49\xec\xfe\x89\xe5" + "\x02\x20" + "\x43\x45\xd0\xa5\xff\xe5\x13\x27\x26\xd0\xec\x37\xad\x24\x1e\x9a" + "\x71\x9a\xa4\x89\xb0\x7e\x0f\xc4\xbb\x2d\x50\xd0\xe5\x7f\x7a\x68", + .c_size = 70, + .algo = OID_SM2_with_SM3, + .m = + "\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff\x00" + "\x12\x34\x56\x78\x9a\xbc\xde\xf0\x12\x34\x56\x78\x9a\xbc\xde\xf0", + .m_size = 32, + .public_key_vec = true, + .siggen_sigver_test = true, + }, +}; + /* Example vectors below taken from * http://www.oscca.gov.cn/UpFile/20101222141857786.pdf * diff --git a/crypto/xor.c b/crypto/xor.c index ea7349e6ed23..eacbf4f93990 100644 --- a/crypto/xor.c +++ b/crypto/xor.c @@ -54,49 +54,63 @@ EXPORT_SYMBOL(xor_blocks); /* Set of all registered templates. */ static struct xor_block_template *__initdata template_list; -#define BENCH_SIZE (PAGE_SIZE) +#ifndef MODULE +static void __init do_xor_register(struct xor_block_template *tmpl) +{ + tmpl->next = template_list; + template_list = tmpl; +} + +static int __init register_xor_blocks(void) +{ + active_template = XOR_SELECT_TEMPLATE(NULL); + + if (!active_template) { +#define xor_speed do_xor_register + // register all the templates and pick the first as the default + XOR_TRY_TEMPLATES; +#undef xor_speed + active_template = template_list; + } + return 0; +} +#endif + +#define BENCH_SIZE 4096 +#define REPS 800U static void __init do_xor_speed(struct xor_block_template *tmpl, void *b1, void *b2) { int speed; - unsigned long now, j; - int i, count, max; + int i, j; + ktime_t min, start, diff; tmpl->next = template_list; template_list = tmpl; preempt_disable(); - /* - * Count the number of XORs done during a whole jiffy, and use - * this to calculate the speed of checksumming. We use a 2-page - * allocation to have guaranteed color L1-cache layout. - */ - max = 0; - for (i = 0; i < 5; i++) { - j = jiffies; - count = 0; - while ((now = jiffies) == j) - cpu_relax(); - while (time_before(jiffies, now + 1)) { + min = (ktime_t)S64_MAX; + for (i = 0; i < 3; i++) { + start = ktime_get(); + for (j = 0; j < REPS; j++) { mb(); /* prevent loop optimzation */ tmpl->do_2(BENCH_SIZE, b1, b2); mb(); - count++; - mb(); } - if (count > max) - max = count; + diff = ktime_sub(ktime_get(), start); + if (diff < min) + min = diff; } preempt_enable(); - speed = max * (HZ * BENCH_SIZE / 1024); + // bytes/ns == GB/s, multiply by 1000 to get MB/s [not MiB/s] + speed = (1000 * REPS * BENCH_SIZE) / (unsigned int)ktime_to_ns(min); tmpl->speed = speed; - printk(KERN_INFO " %-10s: %5d.%03d MB/sec\n", tmpl->name, - speed / 1000, speed % 1000); + pr_info(" %-16s: %5d MB/sec\n", tmpl->name, speed); } static int __init @@ -129,14 +143,15 @@ calibrate_xor_blocks(void) #define xor_speed(templ) do_xor_speed((templ), b1, b2) printk(KERN_INFO "xor: measuring software checksum speed\n"); + template_list = NULL; XOR_TRY_TEMPLATES; fastest = template_list; for (f = fastest; f; f = f->next) if (f->speed > fastest->speed) fastest = f; - printk(KERN_INFO "xor: using function: %s (%d.%03d MB/sec)\n", - fastest->name, fastest->speed / 1000, fastest->speed % 1000); + pr_info("xor: using function: %s (%d MB/sec)\n", + fastest->name, fastest->speed); #undef xor_speed @@ -150,6 +165,10 @@ static __exit void xor_exit(void) { } MODULE_LICENSE("GPL"); +#ifndef MODULE /* when built-in xor.o must initialize before drivers/md/md.o */ -core_initcall(calibrate_xor_blocks); +core_initcall(register_xor_blocks); +#endif + +module_init(calibrate_xor_blocks); module_exit(xor_exit); diff --git a/drivers/char/hw_random/Kconfig b/drivers/char/hw_random/Kconfig index f976a49e1fb5..e92c4d9469d8 100644 --- a/drivers/char/hw_random/Kconfig +++ b/drivers/char/hw_random/Kconfig @@ -282,6 +282,20 @@ config HW_RANDOM_INGENIC_RNG If unsure, say Y. +config HW_RANDOM_INGENIC_TRNG + tristate "Ingenic True Random Number Generator support" + depends on HW_RANDOM + depends on MACH_X1830 + default HW_RANDOM + help + This driver provides kernel-side support for the True Random Number Generator + hardware found in ingenic X1830 SoC. YSH & ATIL CU1830-Neo uses X1830 SoC. + + To compile this driver as a module, choose M here: the + module will be called ingenic-trng. + + If unsure, say Y. + config HW_RANDOM_NOMADIK tristate "ST-Ericsson Nomadik Random Number Generator support" depends on ARCH_NOMADIK @@ -512,6 +526,16 @@ config HW_RANDOM_CCTRNG will be called cctrng. If unsure, say 'N'. +config HW_RANDOM_XIPHERA + tristate "Xiphera FPGA based True Random Number Generator support" + depends on HAS_IOMEM + help + This driver provides kernel-side support for Xiphera True Random + Number Generator Intellectual Property Core. + + To compile this driver as a module, choose M here: the + module will be called xiphera-trng. + endif # HW_RANDOM config UML_RANDOM diff --git a/drivers/char/hw_random/Makefile b/drivers/char/hw_random/Makefile index 26ae06844f09..5da344509a4d 100644 --- a/drivers/char/hw_random/Makefile +++ b/drivers/char/hw_random/Makefile @@ -24,6 +24,7 @@ obj-$(CONFIG_HW_RANDOM_TX4939) += tx4939-rng.o obj-$(CONFIG_HW_RANDOM_MXC_RNGA) += mxc-rnga.o obj-$(CONFIG_HW_RANDOM_IMX_RNGC) += imx-rngc.o obj-$(CONFIG_HW_RANDOM_INGENIC_RNG) += ingenic-rng.o +obj-$(CONFIG_HW_RANDOM_INGENIC_TRNG) += ingenic-trng.o obj-$(CONFIG_HW_RANDOM_OCTEON) += octeon-rng.o obj-$(CONFIG_HW_RANDOM_NOMADIK) += nomadik-rng.o obj-$(CONFIG_HW_RANDOM_PSERIES) += pseries-rng.o @@ -44,3 +45,4 @@ obj-$(CONFIG_HW_RANDOM_KEYSTONE) += ks-sa-rng.o obj-$(CONFIG_HW_RANDOM_OPTEE) += optee-rng.o obj-$(CONFIG_HW_RANDOM_NPCM) += npcm-rng.o obj-$(CONFIG_HW_RANDOM_CCTRNG) += cctrng.o +obj-$(CONFIG_HW_RANDOM_XIPHERA) += xiphera-trng.o diff --git a/drivers/char/hw_random/cctrng.c b/drivers/char/hw_random/cctrng.c index 619148fb2dc9..7a293f2147a0 100644 --- a/drivers/char/hw_random/cctrng.c +++ b/drivers/char/hw_random/cctrng.c @@ -463,11 +463,10 @@ static int cc_trng_clk_init(struct cctrng_drvdata *drvdata) int rc = 0; clk = devm_clk_get_optional(dev, NULL); - if (IS_ERR(clk)) { - if (PTR_ERR(clk) != -EPROBE_DEFER) - dev_err(dev, "Error getting clock: %pe\n", clk); - return PTR_ERR(clk); - } + if (IS_ERR(clk)) + return dev_err_probe(dev, PTR_ERR(clk), + "Error getting clock\n"); + drvdata->clk = clk; rc = clk_prepare_enable(drvdata->clk); diff --git a/drivers/char/hw_random/imx-rngc.c b/drivers/char/hw_random/imx-rngc.c index 9c47e431ce90..61c844baf26e 100644 --- a/drivers/char/hw_random/imx-rngc.c +++ b/drivers/char/hw_random/imx-rngc.c @@ -285,6 +285,7 @@ static int imx_rngc_probe(struct platform_device *pdev) rngc->rng.init = imx_rngc_init; rngc->rng.read = imx_rngc_read; rngc->rng.cleanup = imx_rngc_cleanup; + rngc->rng.quality = 19; rngc->dev = &pdev->dev; platform_set_drvdata(pdev, rngc); diff --git a/drivers/char/hw_random/ingenic-trng.c b/drivers/char/hw_random/ingenic-trng.c new file mode 100644 index 000000000000..954a8411d67d --- /dev/null +++ b/drivers/char/hw_random/ingenic-trng.c @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Ingenic True Random Number Generator driver + * Copyright (c) 2019 漆鹏振 (Qi Pengzhen) <aric.pzqi@ingenic.com> + * Copyright (c) 2020 周琰杰 (Zhou Yanjie) <zhouyanjie@wanyeetech.com> + */ + +#include <linux/clk.h> +#include <linux/err.h> +#include <linux/kernel.h> +#include <linux/hw_random.h> +#include <linux/io.h> +#include <linux/iopoll.h> +#include <linux/module.h> +#include <linux/of_device.h> +#include <linux/platform_device.h> +#include <linux/slab.h> + +/* DTRNG register offsets */ +#define TRNG_REG_CFG_OFFSET 0x00 +#define TRNG_REG_RANDOMNUM_OFFSET 0x04 +#define TRNG_REG_STATUS_OFFSET 0x08 + +/* bits within the CFG register */ +#define CFG_RDY_CLR BIT(12) +#define CFG_INT_MASK BIT(11) +#define CFG_GEN_EN BIT(0) + +/* bits within the STATUS register */ +#define STATUS_RANDOM_RDY BIT(0) + +struct ingenic_trng { + void __iomem *base; + struct clk *clk; + struct hwrng rng; +}; + +static int ingenic_trng_init(struct hwrng *rng) +{ + struct ingenic_trng *trng = container_of(rng, struct ingenic_trng, rng); + unsigned int ctrl; + + ctrl = readl(trng->base + TRNG_REG_CFG_OFFSET); + ctrl |= CFG_GEN_EN; + writel(ctrl, trng->base + TRNG_REG_CFG_OFFSET); + + return 0; +} + +static void ingenic_trng_cleanup(struct hwrng *rng) +{ + struct ingenic_trng *trng = container_of(rng, struct ingenic_trng, rng); + unsigned int ctrl; + + ctrl = readl(trng->base + TRNG_REG_CFG_OFFSET); + ctrl &= ~CFG_GEN_EN; + writel(ctrl, trng->base + TRNG_REG_CFG_OFFSET); +} + +static int ingenic_trng_read(struct hwrng *rng, void *buf, size_t max, bool wait) +{ + struct ingenic_trng *trng = container_of(rng, struct ingenic_trng, rng); + u32 *data = buf; + u32 status; + int ret; + + ret = readl_poll_timeout(trng->base + TRNG_REG_STATUS_OFFSET, status, + status & STATUS_RANDOM_RDY, 10, 1000); + if (ret == -ETIMEDOUT) { + pr_err("%s: Wait for DTRNG data ready timeout\n", __func__); + return ret; + } + + *data = readl(trng->base + TRNG_REG_RANDOMNUM_OFFSET); + + return 4; +} + +static int ingenic_trng_probe(struct platform_device *pdev) +{ + struct ingenic_trng *trng; + int ret; + + trng = devm_kzalloc(&pdev->dev, sizeof(*trng), GFP_KERNEL); + if (!trng) + return -ENOMEM; + + trng->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(trng->base)) { + pr_err("%s: Failed to map DTRNG registers\n", __func__); + ret = PTR_ERR(trng->base); + return PTR_ERR(trng->base); + } + + trng->clk = devm_clk_get(&pdev->dev, NULL); + if (IS_ERR(trng->clk)) { + ret = PTR_ERR(trng->clk); + pr_crit("%s: Cannot get DTRNG clock\n", __func__); + return PTR_ERR(trng->clk); + } + + ret = clk_prepare_enable(trng->clk); + if (ret) { + pr_crit("%s: Unable to enable DTRNG clock\n", __func__); + return ret; + } + + trng->rng.name = pdev->name; + trng->rng.init = ingenic_trng_init; + trng->rng.cleanup = ingenic_trng_cleanup; + trng->rng.read = ingenic_trng_read; + + ret = hwrng_register(&trng->rng); + if (ret) { + dev_err(&pdev->dev, "Failed to register hwrng\n"); + return ret; + } + + platform_set_drvdata(pdev, trng); + + dev_info(&pdev->dev, "Ingenic DTRNG driver registered\n"); + return 0; +} + +static int ingenic_trng_remove(struct platform_device *pdev) +{ + struct ingenic_trng *trng = platform_get_drvdata(pdev); + unsigned int ctrl; + + hwrng_unregister(&trng->rng); + + ctrl = readl(trng->base + TRNG_REG_CFG_OFFSET); + ctrl &= ~CFG_GEN_EN; + writel(ctrl, trng->base + TRNG_REG_CFG_OFFSET); + + clk_disable_unprepare(trng->clk); + + return 0; +} + +static const struct of_device_id ingenic_trng_of_match[] = { + { .compatible = "ingenic,x1830-dtrng" }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, ingenic_trng_of_match); + +static struct platform_driver ingenic_trng_driver = { + .probe = ingenic_trng_probe, + .remove = ingenic_trng_remove, + .driver = { + .name = "ingenic-trng", + .of_match_table = ingenic_trng_of_match, + }, +}; + +module_platform_driver(ingenic_trng_driver); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("漆鹏振 (Qi Pengzhen) <aric.pzqi@ingenic.com>"); +MODULE_AUTHOR("周琰杰 (Zhou Yanjie) <zhouyanjie@wanyeetech.com>"); +MODULE_DESCRIPTION("Ingenic True Random Number Generator driver"); diff --git a/drivers/char/hw_random/intel-rng.c b/drivers/char/hw_random/intel-rng.c index 9f205bd1acc0..eb7db27f9f19 100644 --- a/drivers/char/hw_random/intel-rng.c +++ b/drivers/char/hw_random/intel-rng.c @@ -330,7 +330,7 @@ static int __init mod_init(void) int err = -ENODEV; int i; struct pci_dev *dev = NULL; - void __iomem *mem = mem; + void __iomem *mem; u8 hw_status; struct intel_rng_hw *intel_rng_hw; diff --git a/drivers/char/hw_random/iproc-rng200.c b/drivers/char/hw_random/iproc-rng200.c index 32d9fe61a225..01583faf9893 100644 --- a/drivers/char/hw_random/iproc-rng200.c +++ b/drivers/char/hw_random/iproc-rng200.c @@ -195,10 +195,10 @@ static int iproc_rng200_probe(struct platform_device *pdev) return PTR_ERR(priv->base); } - priv->rng.name = "iproc-rng200", - priv->rng.read = iproc_rng200_read, - priv->rng.init = iproc_rng200_init, - priv->rng.cleanup = iproc_rng200_cleanup, + priv->rng.name = "iproc-rng200"; + priv->rng.read = iproc_rng200_read; + priv->rng.init = iproc_rng200_init; + priv->rng.cleanup = iproc_rng200_cleanup; /* Register driver */ ret = devm_hwrng_register(dev, &priv->rng); diff --git a/drivers/char/hw_random/mxc-rnga.c b/drivers/char/hw_random/mxc-rnga.c index 025083c838f5..008763c988ed 100644 --- a/drivers/char/hw_random/mxc-rnga.c +++ b/drivers/char/hw_random/mxc-rnga.c @@ -143,9 +143,9 @@ static int __init mxc_rnga_probe(struct platform_device *pdev) mxc_rng->dev = &pdev->dev; mxc_rng->rng.name = "mxc-rnga"; mxc_rng->rng.init = mxc_rnga_init; - mxc_rng->rng.cleanup = mxc_rnga_cleanup, - mxc_rng->rng.data_present = mxc_rnga_data_present, - mxc_rng->rng.data_read = mxc_rnga_data_read, + mxc_rng->rng.cleanup = mxc_rnga_cleanup; + mxc_rng->rng.data_present = mxc_rnga_data_present; + mxc_rng->rng.data_read = mxc_rnga_data_read; mxc_rng->clk = devm_clk_get(&pdev->dev, NULL); if (IS_ERR(mxc_rng->clk)) { diff --git a/drivers/char/hw_random/npcm-rng.c b/drivers/char/hw_random/npcm-rng.c index 5d0d13f891b7..1ec5f267a656 100644 --- a/drivers/char/hw_random/npcm-rng.c +++ b/drivers/char/hw_random/npcm-rng.c @@ -58,24 +58,24 @@ static int npcm_rng_read(struct hwrng *rng, void *buf, size_t max, bool wait) pm_runtime_get_sync((struct device *)priv->rng.priv); - while (max >= sizeof(u32)) { + while (max) { if (wait) { - if (readl_poll_timeout(priv->base + NPCM_RNGCS_REG, + if (readb_poll_timeout(priv->base + NPCM_RNGCS_REG, ready, ready & NPCM_RNG_DATA_VALID, NPCM_RNG_POLL_USEC, NPCM_RNG_TIMEOUT_USEC)) break; } else { - if ((readl(priv->base + NPCM_RNGCS_REG) & + if ((readb(priv->base + NPCM_RNGCS_REG) & NPCM_RNG_DATA_VALID) == 0) break; } - *(u32 *)buf = readl(priv->base + NPCM_RNGD_REG); - retval += sizeof(u32); - buf += sizeof(u32); - max -= sizeof(u32); + *(u8 *)buf = readb(priv->base + NPCM_RNGD_REG); + retval++; + buf++; + max--; } pm_runtime_mark_last_busy((struct device *)priv->rng.priv); diff --git a/drivers/char/hw_random/optee-rng.c b/drivers/char/hw_random/optee-rng.c index 49b2e02537dd..a99d82949981 100644 --- a/drivers/char/hw_random/optee-rng.c +++ b/drivers/char/hw_random/optee-rng.c @@ -122,14 +122,14 @@ static int optee_rng_read(struct hwrng *rng, void *buf, size_t max, bool wait) if (max > MAX_ENTROPY_REQ_SZ) max = MAX_ENTROPY_REQ_SZ; - while (read == 0) { + while (read < max) { rng_size = get_optee_rng_data(pvt_data, data, (max - read)); data += rng_size; read += rng_size; - if (wait) { - if (timeout-- == 0) + if (wait && pvt_data->data_rate) { + if ((timeout-- == 0) || (read == max)) return read; msleep((1000 * (max - read)) / pvt_data->data_rate); } else { diff --git a/drivers/char/hw_random/stm32-rng.c b/drivers/char/hw_random/stm32-rng.c index 38324c2ddda1..bc22178f83e8 100644 --- a/drivers/char/hw_random/stm32-rng.c +++ b/drivers/char/hw_random/stm32-rng.c @@ -145,12 +145,12 @@ static int stm32_rng_probe(struct platform_device *ofdev) dev_set_drvdata(dev, priv); - priv->rng.name = dev_driver_string(dev), + priv->rng.name = dev_driver_string(dev); #ifndef CONFIG_PM - priv->rng.init = stm32_rng_init, - priv->rng.cleanup = stm32_rng_cleanup, + priv->rng.init = stm32_rng_init; + priv->rng.cleanup = stm32_rng_cleanup; #endif - priv->rng.read = stm32_rng_read, + priv->rng.read = stm32_rng_read; priv->rng.priv = (unsigned long) dev; priv->rng.quality = 900; diff --git a/drivers/char/hw_random/xiphera-trng.c b/drivers/char/hw_random/xiphera-trng.c new file mode 100644 index 000000000000..7bdab8c8a6a8 --- /dev/null +++ b/drivers/char/hw_random/xiphera-trng.c @@ -0,0 +1,150 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2020 Xiphera Ltd. */ + +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/mod_devicetable.h> +#include <linux/err.h> +#include <linux/io.h> +#include <linux/hw_random.h> +#include <linux/of_device.h> +#include <linux/platform_device.h> +#include <linux/delay.h> + +#define CONTROL_REG 0x00000000 +#define STATUS_REG 0x00000004 +#define RAND_REG 0x00000000 + +#define HOST_TO_TRNG_RESET 0x00000001 +#define HOST_TO_TRNG_RELEASE_RESET 0x00000002 +#define HOST_TO_TRNG_ENABLE 0x80000000 +#define HOST_TO_TRNG_ZEROIZE 0x80000004 +#define HOST_TO_TRNG_ACK_ZEROIZE 0x80000008 +#define HOST_TO_TRNG_READ 0x8000000F + +/* trng statuses */ +#define TRNG_ACK_RESET 0x000000AC +#define TRNG_SUCCESSFUL_STARTUP 0x00000057 +#define TRNG_FAILED_STARTUP 0x000000FA +#define TRNG_NEW_RAND_AVAILABLE 0x000000ED + +struct xiphera_trng { + void __iomem *mem; + struct hwrng rng; +}; + +static int xiphera_trng_read(struct hwrng *rng, void *buf, size_t max, bool wait) +{ + struct xiphera_trng *trng = container_of(rng, struct xiphera_trng, rng); + int ret = 0; + + while (max >= sizeof(u32)) { + /* check for data */ + if (readl(trng->mem + STATUS_REG) == TRNG_NEW_RAND_AVAILABLE) { + *(u32 *)buf = readl(trng->mem + RAND_REG); + /* + * Inform the trng of the read + * and re-enable it to produce a new random number + */ + writel(HOST_TO_TRNG_READ, trng->mem + CONTROL_REG); + writel(HOST_TO_TRNG_ENABLE, trng->mem + CONTROL_REG); + ret += sizeof(u32); + buf += sizeof(u32); + max -= sizeof(u32); + } else { + break; + } + } + return ret; +} + +static int xiphera_trng_probe(struct platform_device *pdev) +{ + int ret; + struct xiphera_trng *trng; + struct device *dev = &pdev->dev; + struct resource *res; + + trng = devm_kzalloc(dev, sizeof(*trng), GFP_KERNEL); + if (!trng) + return -ENOMEM; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + trng->mem = devm_ioremap_resource(dev, res); + if (IS_ERR(trng->mem)) + return PTR_ERR(trng->mem); + + /* + * the trng needs to be reset first which might not happen in time, + * hence we incorporate a small delay to ensure proper behaviour + */ + writel(HOST_TO_TRNG_RESET, trng->mem + CONTROL_REG); + usleep_range(100, 200); + + if (readl(trng->mem + STATUS_REG) != TRNG_ACK_RESET) { + /* + * there is a small chance the trng is just not ready yet, + * so we try one more time. If the second time fails, we give up + */ + usleep_range(100, 200); + if (readl(trng->mem + STATUS_REG) != TRNG_ACK_RESET) { + dev_err(dev, "failed to reset the trng ip\n"); + return -ENODEV; + } + } + + /* + * once again, to ensure proper behaviour we sleep + * for a while after zeroizing the trng + */ + writel(HOST_TO_TRNG_RELEASE_RESET, trng->mem + CONTROL_REG); + writel(HOST_TO_TRNG_ENABLE, trng->mem + CONTROL_REG); + writel(HOST_TO_TRNG_ZEROIZE, trng->mem + CONTROL_REG); + msleep(20); + + if (readl(trng->mem + STATUS_REG) != TRNG_SUCCESSFUL_STARTUP) { + /* diagnose the reason for the failure */ + if (readl(trng->mem + STATUS_REG) == TRNG_FAILED_STARTUP) { + dev_err(dev, "trng ip startup-tests failed\n"); + return -ENODEV; + } + dev_err(dev, "startup-tests yielded no response\n"); + return -ENODEV; + } + + writel(HOST_TO_TRNG_ACK_ZEROIZE, trng->mem + CONTROL_REG); + + trng->rng.name = pdev->name; + trng->rng.read = xiphera_trng_read; + trng->rng.quality = 900; + + ret = devm_hwrng_register(dev, &trng->rng); + if (ret) { + dev_err(dev, "failed to register rng device: %d\n", ret); + return ret; + } + + platform_set_drvdata(pdev, trng); + + return 0; +} + +static const struct of_device_id xiphera_trng_of_match[] = { + { .compatible = "xiphera,xip8001b-trng", }, + {}, +}; +MODULE_DEVICE_TABLE(of, xiphera_trng_of_match); + +static struct platform_driver xiphera_trng_driver = { + .driver = { + .name = "xiphera-trng", + .of_match_table = xiphera_trng_of_match, + }, + .probe = xiphera_trng_probe, +}; + +module_platform_driver(xiphera_trng_driver); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Atte Tommiska"); +MODULE_DESCRIPTION("Xiphera FPGA-based true random number generator driver"); diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 52a9b7cf6576..37593387164a 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -873,6 +873,7 @@ config CRYPTO_DEV_SA2UL select CRYPTO_AES select CRYPTO_AES_ARM64 select CRYPTO_ALGAPI + select CRYPTO_AUTHENC select CRYPTO_SHA1 select CRYPTO_SHA256 select CRYPTO_SHA512 diff --git a/drivers/crypto/allwinner/Kconfig b/drivers/crypto/allwinner/Kconfig index 12e7c6a85a02..0cdfe0e8cc66 100644 --- a/drivers/crypto/allwinner/Kconfig +++ b/drivers/crypto/allwinner/Kconfig @@ -59,6 +59,32 @@ config CRYPTO_DEV_SUN8I_CE_DEBUG This will create /sys/kernel/debug/sun8i-ce/stats for displaying the number of requests per flow and per algorithm. +config CRYPTO_DEV_SUN8I_CE_HASH + bool "Enable support for hash on sun8i-ce" + depends on CRYPTO_DEV_SUN8I_CE + select MD5 + select SHA1 + select SHA256 + select SHA512 + help + Say y to enable support for hash algorithms. + +config CRYPTO_DEV_SUN8I_CE_PRNG + bool "Support for Allwinner Crypto Engine PRNG" + depends on CRYPTO_DEV_SUN8I_CE + select CRYPTO_RNG + help + Select this option if you want to provide kernel-side support for + the Pseudo-Random Number Generator found in the Crypto Engine. + +config CRYPTO_DEV_SUN8I_CE_TRNG + bool "Support for Allwinner Crypto Engine TRNG" + depends on CRYPTO_DEV_SUN8I_CE + select HW_RANDOM + help + Select this option if you want to provide kernel-side support for + the True Random Number Generator found in the Crypto Engine. + config CRYPTO_DEV_SUN8I_SS tristate "Support for Allwinner Security System cryptographic offloader" select CRYPTO_SKCIPHER @@ -85,3 +111,20 @@ config CRYPTO_DEV_SUN8I_SS_DEBUG Say y to enable sun8i-ss debug stats. This will create /sys/kernel/debug/sun8i-ss/stats for displaying the number of requests per flow and per algorithm. + +config CRYPTO_DEV_SUN8I_SS_PRNG + bool "Support for Allwinner Security System PRNG" + depends on CRYPTO_DEV_SUN8I_SS + select CRYPTO_RNG + help + Select this option if you want to provide kernel-side support for + the Pseudo-Random Number Generator found in the Security System. + +config CRYPTO_DEV_SUN8I_SS_HASH + bool "Enable support for hash on sun8i-ss" + depends on CRYPTO_DEV_SUN8I_SS + select MD5 + select SHA1 + select SHA256 + help + Say y to enable support for hash algorithms. diff --git a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-hash.c b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-hash.c index dc35edd90034..1dff48558f53 100644 --- a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-hash.c +++ b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-hash.c @@ -9,6 +9,7 @@ * You could find the datasheet in Documentation/arm/sunxi.rst */ #include "sun4i-ss.h" +#include <asm/unaligned.h> #include <linux/scatterlist.h> /* This is a totally arbitrary value */ @@ -196,7 +197,7 @@ static int sun4i_hash(struct ahash_request *areq) struct sg_mapping_iter mi; int in_r, err = 0; size_t copied = 0; - __le32 wb = 0; + u32 wb = 0; dev_dbg(ss->dev, "%s %s bc=%llu len=%u mode=%x wl=%u h0=%0x", __func__, crypto_tfm_alg_name(areq->base.tfm), @@ -408,7 +409,7 @@ hash_final: nbw = op->len - 4 * nwait; if (nbw) { - wb = cpu_to_le32(*(u32 *)(op->buf + nwait * 4)); + wb = le32_to_cpup((__le32 *)(op->buf + nwait * 4)); wb &= GENMASK((nbw * 8) - 1, 0); op->byte_count += nbw; @@ -417,7 +418,7 @@ hash_final: /* write the remaining bytes of the nbw buffer */ wb |= ((1 << 7) << (nbw * 8)); - bf[j++] = le32_to_cpu(wb); + ((__le32 *)bf)[j++] = cpu_to_le32(wb); /* * number of space to pad to obtain 64o minus 8(size) minus 4 (final 1) @@ -479,16 +480,16 @@ hash_final: /* Get the hash from the device */ if (op->mode == SS_OP_SHA1) { for (i = 0; i < 5; i++) { + v = readl(ss->base + SS_MD0 + i * 4); if (ss->variant->sha1_in_be) - v = cpu_to_le32(readl(ss->base + SS_MD0 + i * 4)); + put_unaligned_le32(v, areq->result + i * 4); else - v = cpu_to_be32(readl(ss->base + SS_MD0 + i * 4)); - memcpy(areq->result + i * 4, &v, 4); + put_unaligned_be32(v, areq->result + i * 4); } } else { for (i = 0; i < 4; i++) { - v = cpu_to_le32(readl(ss->base + SS_MD0 + i * 4)); - memcpy(areq->result + i * 4, &v, 4); + v = readl(ss->base + SS_MD0 + i * 4); + put_unaligned_le32(v, areq->result + i * 4); } } diff --git a/drivers/crypto/allwinner/sun8i-ce/Makefile b/drivers/crypto/allwinner/sun8i-ce/Makefile index 08b68c3c1ca9..0842eb2d9408 100644 --- a/drivers/crypto/allwinner/sun8i-ce/Makefile +++ b/drivers/crypto/allwinner/sun8i-ce/Makefile @@ -1,2 +1,5 @@ obj-$(CONFIG_CRYPTO_DEV_SUN8I_CE) += sun8i-ce.o sun8i-ce-y += sun8i-ce-core.o sun8i-ce-cipher.o +sun8i-ce-$(CONFIG_CRYPTO_DEV_SUN8I_CE_HASH) += sun8i-ce-hash.o +sun8i-ce-$(CONFIG_CRYPTO_DEV_SUN8I_CE_PRNG) += sun8i-ce-prng.o +sun8i-ce-$(CONFIG_CRYPTO_DEV_SUN8I_CE_TRNG) += sun8i-ce-trng.o diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c index b4d5fea27d20..33707a2e55ff 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c @@ -75,8 +75,9 @@ static int sun8i_ce_cipher_fallback(struct skcipher_request *areq) return err; } -static int sun8i_ce_cipher(struct skcipher_request *areq) +static int sun8i_ce_cipher_prepare(struct crypto_engine *engine, void *async_req) { + struct skcipher_request *areq = container_of(async_req, struct skcipher_request, base); struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); struct sun8i_ce_dev *ce = op->ce; @@ -87,8 +88,6 @@ static int sun8i_ce_cipher(struct skcipher_request *areq) struct ce_task *cet; struct scatterlist *sg; unsigned int todo, len, offset, ivsize; - dma_addr_t addr_iv = 0, addr_key = 0; - void *backup_iv = NULL; u32 common, sym; int flow, i; int nr_sgs = 0; @@ -119,7 +118,7 @@ static int sun8i_ce_cipher(struct skcipher_request *areq) common |= rctx->op_dir | CE_COMM_INT; cet->t_common_ctl = cpu_to_le32(common); /* CTS and recent CE (H6) need length in bytes, in word otherwise */ - if (ce->variant->has_t_dlen_in_bytes) + if (ce->variant->cipher_t_dlen_in_bytes) cet->t_dlen = cpu_to_le32(areq->cryptlen); else cet->t_dlen = cpu_to_le32(areq->cryptlen / 4); @@ -141,41 +140,41 @@ static int sun8i_ce_cipher(struct skcipher_request *areq) cet->t_sym_ctl = cpu_to_le32(sym); cet->t_asym_ctl = 0; - addr_key = dma_map_single(ce->dev, op->key, op->keylen, DMA_TO_DEVICE); - cet->t_key = cpu_to_le32(addr_key); - if (dma_mapping_error(ce->dev, addr_key)) { + rctx->addr_key = dma_map_single(ce->dev, op->key, op->keylen, DMA_TO_DEVICE); + if (dma_mapping_error(ce->dev, rctx->addr_key)) { dev_err(ce->dev, "Cannot DMA MAP KEY\n"); err = -EFAULT; goto theend; } + cet->t_key = cpu_to_le32(rctx->addr_key); ivsize = crypto_skcipher_ivsize(tfm); if (areq->iv && crypto_skcipher_ivsize(tfm) > 0) { - chan->ivlen = ivsize; - chan->bounce_iv = kzalloc(ivsize, GFP_KERNEL | GFP_DMA); - if (!chan->bounce_iv) { + rctx->ivlen = ivsize; + rctx->bounce_iv = kzalloc(ivsize, GFP_KERNEL | GFP_DMA); + if (!rctx->bounce_iv) { err = -ENOMEM; goto theend_key; } if (rctx->op_dir & CE_DECRYPTION) { - backup_iv = kzalloc(ivsize, GFP_KERNEL); - if (!backup_iv) { + rctx->backup_iv = kzalloc(ivsize, GFP_KERNEL); + if (!rctx->backup_iv) { err = -ENOMEM; goto theend_key; } offset = areq->cryptlen - ivsize; - scatterwalk_map_and_copy(backup_iv, areq->src, offset, - ivsize, 0); + scatterwalk_map_and_copy(rctx->backup_iv, areq->src, + offset, ivsize, 0); } - memcpy(chan->bounce_iv, areq->iv, ivsize); - addr_iv = dma_map_single(ce->dev, chan->bounce_iv, chan->ivlen, - DMA_TO_DEVICE); - cet->t_iv = cpu_to_le32(addr_iv); - if (dma_mapping_error(ce->dev, addr_iv)) { + memcpy(rctx->bounce_iv, areq->iv, ivsize); + rctx->addr_iv = dma_map_single(ce->dev, rctx->bounce_iv, rctx->ivlen, + DMA_TO_DEVICE); + if (dma_mapping_error(ce->dev, rctx->addr_iv)) { dev_err(ce->dev, "Cannot DMA MAP IV\n"); err = -ENOMEM; goto theend_iv; } + cet->t_iv = cpu_to_le32(rctx->addr_iv); } if (areq->src == areq->dst) { @@ -235,7 +234,9 @@ static int sun8i_ce_cipher(struct skcipher_request *areq) } chan->timeout = areq->cryptlen; - err = sun8i_ce_run_task(ce, flow, crypto_tfm_alg_name(areq->base.tfm)); + rctx->nr_sgs = nr_sgs; + rctx->nr_sgd = nr_sgd; + return 0; theend_sgs: if (areq->src == areq->dst) { @@ -248,34 +249,83 @@ theend_sgs: theend_iv: if (areq->iv && ivsize > 0) { - if (addr_iv) - dma_unmap_single(ce->dev, addr_iv, chan->ivlen, - DMA_TO_DEVICE); + if (rctx->addr_iv) + dma_unmap_single(ce->dev, rctx->addr_iv, rctx->ivlen, DMA_TO_DEVICE); offset = areq->cryptlen - ivsize; if (rctx->op_dir & CE_DECRYPTION) { - memcpy(areq->iv, backup_iv, ivsize); - kfree_sensitive(backup_iv); + memcpy(areq->iv, rctx->backup_iv, ivsize); + kfree_sensitive(rctx->backup_iv); } else { scatterwalk_map_and_copy(areq->iv, areq->dst, offset, ivsize, 0); } - kfree(chan->bounce_iv); + kfree(rctx->bounce_iv); } theend_key: - dma_unmap_single(ce->dev, addr_key, op->keylen, DMA_TO_DEVICE); + dma_unmap_single(ce->dev, rctx->addr_key, op->keylen, DMA_TO_DEVICE); theend: return err; } -static int sun8i_ce_handle_cipher_request(struct crypto_engine *engine, void *areq) +static int sun8i_ce_cipher_run(struct crypto_engine *engine, void *areq) { - int err; struct skcipher_request *breq = container_of(areq, struct skcipher_request, base); + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(breq); + struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sun8i_ce_dev *ce = op->ce; + struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(breq); + int flow, err; - err = sun8i_ce_cipher(breq); + flow = rctx->flow; + err = sun8i_ce_run_task(ce, flow, crypto_tfm_alg_name(breq->base.tfm)); crypto_finalize_skcipher_request(engine, breq, err); + return 0; +} + +static int sun8i_ce_cipher_unprepare(struct crypto_engine *engine, void *async_req) +{ + struct skcipher_request *areq = container_of(async_req, struct skcipher_request, base); + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sun8i_ce_dev *ce = op->ce; + struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(areq); + struct sun8i_ce_flow *chan; + struct ce_task *cet; + unsigned int ivsize, offset; + int nr_sgs = rctx->nr_sgs; + int nr_sgd = rctx->nr_sgd; + int flow; + + flow = rctx->flow; + chan = &ce->chanlist[flow]; + cet = chan->tl; + ivsize = crypto_skcipher_ivsize(tfm); + + if (areq->src == areq->dst) { + dma_unmap_sg(ce->dev, areq->src, nr_sgs, DMA_BIDIRECTIONAL); + } else { + if (nr_sgs > 0) + dma_unmap_sg(ce->dev, areq->src, nr_sgs, DMA_TO_DEVICE); + dma_unmap_sg(ce->dev, areq->dst, nr_sgd, DMA_FROM_DEVICE); + } + + if (areq->iv && ivsize > 0) { + if (cet->t_iv) + dma_unmap_single(ce->dev, rctx->addr_iv, rctx->ivlen, DMA_TO_DEVICE); + offset = areq->cryptlen - ivsize; + if (rctx->op_dir & CE_DECRYPTION) { + memcpy(areq->iv, rctx->backup_iv, ivsize); + kfree_sensitive(rctx->backup_iv); + } else { + scatterwalk_map_and_copy(areq->iv, areq->dst, offset, + ivsize, 0); + } + kfree(rctx->bounce_iv); + } + + dma_unmap_single(ce->dev, rctx->addr_key, op->keylen, DMA_TO_DEVICE); return 0; } @@ -347,9 +397,9 @@ int sun8i_ce_cipher_init(struct crypto_tfm *tfm) crypto_tfm_alg_driver_name(&sktfm->base), crypto_tfm_alg_driver_name(crypto_skcipher_tfm(op->fallback_tfm))); - op->enginectx.op.do_one_request = sun8i_ce_handle_cipher_request; - op->enginectx.op.prepare_request = NULL; - op->enginectx.op.unprepare_request = NULL; + op->enginectx.op.do_one_request = sun8i_ce_cipher_run; + op->enginectx.op.prepare_request = sun8i_ce_cipher_prepare; + op->enginectx.op.unprepare_request = sun8i_ce_cipher_unprepare; err = pm_runtime_get_sync(op->ce->dev); if (err < 0) @@ -366,10 +416,7 @@ void sun8i_ce_cipher_exit(struct crypto_tfm *tfm) { struct sun8i_cipher_tfm_ctx *op = crypto_tfm_ctx(tfm); - if (op->key) { - memzero_explicit(op->key, op->keylen); - kfree(op->key); - } + kfree_sensitive(op->key); crypto_free_skcipher(op->fallback_tfm); pm_runtime_put_sync_suspend(op->ce->dev); } @@ -391,10 +438,7 @@ int sun8i_ce_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, dev_dbg(ce->dev, "ERROR: Invalid keylen %u\n", keylen); return -EINVAL; } - if (op->key) { - memzero_explicit(op->key, op->keylen); - kfree(op->key); - } + kfree_sensitive(op->key); op->keylen = keylen; op->key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA); if (!op->key) @@ -416,10 +460,7 @@ int sun8i_ce_des3_setkey(struct crypto_skcipher *tfm, const u8 *key, if (err) return err; - if (op->key) { - memzero_explicit(op->key, op->keylen); - kfree(op->key); - } + kfree_sensitive(op->key); op->keylen = keylen; op->key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA); if (!op->key) diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c index 138759dc8190..158422ff5695 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c @@ -22,6 +22,7 @@ #include <linux/platform_device.h> #include <linux/pm_runtime.h> #include <linux/reset.h> +#include <crypto/internal/rng.h> #include <crypto/internal/skcipher.h> #include "sun8i-ce.h" @@ -35,73 +36,108 @@ static const struct ce_variant ce_h3_variant = { .alg_cipher = { CE_ALG_AES, CE_ALG_DES, CE_ALG_3DES, }, + .alg_hash = { CE_ALG_MD5, CE_ALG_SHA1, CE_ALG_SHA224, CE_ALG_SHA256, + CE_ALG_SHA384, CE_ALG_SHA512 + }, .op_mode = { CE_OP_ECB, CE_OP_CBC }, .ce_clks = { { "bus", 0, 200000000 }, { "mod", 50000000, 0 }, - } + }, + .esr = ESR_H3, + .prng = CE_ALG_PRNG, + .trng = CE_ID_NOTSUPP, }; static const struct ce_variant ce_h5_variant = { .alg_cipher = { CE_ALG_AES, CE_ALG_DES, CE_ALG_3DES, }, + .alg_hash = { CE_ALG_MD5, CE_ALG_SHA1, CE_ALG_SHA224, CE_ALG_SHA256, + CE_ID_NOTSUPP, CE_ID_NOTSUPP + }, .op_mode = { CE_OP_ECB, CE_OP_CBC }, .ce_clks = { { "bus", 0, 200000000 }, { "mod", 300000000, 0 }, - } + }, + .esr = ESR_H5, + .prng = CE_ALG_PRNG, + .trng = CE_ID_NOTSUPP, }; static const struct ce_variant ce_h6_variant = { .alg_cipher = { CE_ALG_AES, CE_ALG_DES, CE_ALG_3DES, }, + .alg_hash = { CE_ALG_MD5, CE_ALG_SHA1, CE_ALG_SHA224, CE_ALG_SHA256, + CE_ALG_SHA384, CE_ALG_SHA512 + }, .op_mode = { CE_OP_ECB, CE_OP_CBC }, - .has_t_dlen_in_bytes = true, + .cipher_t_dlen_in_bytes = true, + .hash_t_dlen_in_bits = true, + .prng_t_dlen_in_bytes = true, + .trng_t_dlen_in_bytes = true, .ce_clks = { { "bus", 0, 200000000 }, { "mod", 300000000, 0 }, { "ram", 0, 400000000 }, - } + }, + .esr = ESR_H6, + .prng = CE_ALG_PRNG_V2, + .trng = CE_ALG_TRNG_V2, }; static const struct ce_variant ce_a64_variant = { .alg_cipher = { CE_ALG_AES, CE_ALG_DES, CE_ALG_3DES, }, + .alg_hash = { CE_ALG_MD5, CE_ALG_SHA1, CE_ALG_SHA224, CE_ALG_SHA256, + CE_ID_NOTSUPP, CE_ID_NOTSUPP + }, .op_mode = { CE_OP_ECB, CE_OP_CBC }, .ce_clks = { { "bus", 0, 200000000 }, { "mod", 300000000, 0 }, - } + }, + .esr = ESR_A64, + .prng = CE_ALG_PRNG, + .trng = CE_ID_NOTSUPP, }; static const struct ce_variant ce_r40_variant = { .alg_cipher = { CE_ALG_AES, CE_ALG_DES, CE_ALG_3DES, }, + .alg_hash = { CE_ALG_MD5, CE_ALG_SHA1, CE_ALG_SHA224, CE_ALG_SHA256, + CE_ID_NOTSUPP, CE_ID_NOTSUPP + }, .op_mode = { CE_OP_ECB, CE_OP_CBC }, .ce_clks = { { "bus", 0, 200000000 }, { "mod", 300000000, 0 }, - } + }, + .esr = ESR_R40, + .prng = CE_ALG_PRNG, + .trng = CE_ID_NOTSUPP, }; /* * sun8i_ce_get_engine_number() get the next channel slot * This is a simple round-robin way of getting the next channel + * The flow 3 is reserve for xRNG operations */ int sun8i_ce_get_engine_number(struct sun8i_ce_dev *ce) { - return atomic_inc_return(&ce->flow) % MAXFLOW; + return atomic_inc_return(&ce->flow) % (MAXFLOW - 1); } int sun8i_ce_run_task(struct sun8i_ce_dev *ce, int flow, const char *name) { u32 v; int err = 0; + struct ce_task *cet = ce->chanlist[flow].tl; #ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG ce->chanlist[flow].stat_req++; @@ -120,7 +156,10 @@ int sun8i_ce_run_task(struct sun8i_ce_dev *ce, int flow, const char *name) /* Be sure all data is written before enabling the task */ wmb(); - v = 1 | (ce->chanlist[flow].tl->t_common_ctl & 0x7F) << 8; + /* Only H6 needs to write a part of t_common_ctl along with "1", but since it is ignored + * on older SoCs, we have no reason to complicate things. + */ + v = 1 | ((le32_to_cpu(ce->chanlist[flow].tl->t_common_ctl) & 0x7F) << 8); writel(v, ce->base + CE_TLR); mutex_unlock(&ce->mlock); @@ -128,19 +167,56 @@ int sun8i_ce_run_task(struct sun8i_ce_dev *ce, int flow, const char *name) msecs_to_jiffies(ce->chanlist[flow].timeout)); if (ce->chanlist[flow].status == 0) { - dev_err(ce->dev, "DMA timeout for %s\n", name); + dev_err(ce->dev, "DMA timeout for %s (tm=%d) on flow %d\n", name, + ce->chanlist[flow].timeout, flow); err = -EFAULT; } /* No need to lock for this read, the channel is locked so * nothing could modify the error value for this channel */ v = readl(ce->base + CE_ESR); - if (v) { + switch (ce->variant->esr) { + case ESR_H3: + /* Sadly, the error bit is not per flow */ + if (v) { + dev_err(ce->dev, "CE ERROR: %x for flow %x\n", v, flow); + err = -EFAULT; + print_hex_dump(KERN_INFO, "TASK: ", DUMP_PREFIX_NONE, 16, 4, + cet, sizeof(struct ce_task), false); + } + if (v & CE_ERR_ALGO_NOTSUP) + dev_err(ce->dev, "CE ERROR: algorithm not supported\n"); + if (v & CE_ERR_DATALEN) + dev_err(ce->dev, "CE ERROR: data length error\n"); + if (v & CE_ERR_KEYSRAM) + dev_err(ce->dev, "CE ERROR: keysram access error for AES\n"); + break; + case ESR_A64: + case ESR_H5: + case ESR_R40: v >>= (flow * 4); + v &= 0xF; + if (v) { + dev_err(ce->dev, "CE ERROR: %x for flow %x\n", v, flow); + err = -EFAULT; + print_hex_dump(KERN_INFO, "TASK: ", DUMP_PREFIX_NONE, 16, 4, + cet, sizeof(struct ce_task), false); + } + if (v & CE_ERR_ALGO_NOTSUP) + dev_err(ce->dev, "CE ERROR: algorithm not supported\n"); + if (v & CE_ERR_DATALEN) + dev_err(ce->dev, "CE ERROR: data length error\n"); + if (v & CE_ERR_KEYSRAM) + dev_err(ce->dev, "CE ERROR: keysram access error for AES\n"); + break; + case ESR_H6: + v >>= (flow * 8); v &= 0xFF; if (v) { dev_err(ce->dev, "CE ERROR: %x for flow %x\n", v, flow); err = -EFAULT; + print_hex_dump(KERN_INFO, "TASK: ", DUMP_PREFIX_NONE, 16, 4, + cet, sizeof(struct ce_task), false); } if (v & CE_ERR_ALGO_NOTSUP) dev_err(ce->dev, "CE ERROR: algorithm not supported\n"); @@ -150,7 +226,10 @@ int sun8i_ce_run_task(struct sun8i_ce_dev *ce, int flow, const char *name) dev_err(ce->dev, "CE ERROR: keysram access error for AES\n"); if (v & CE_ERR_ADDR_INVALID) dev_err(ce->dev, "CE ERROR: address invalid\n"); - } + if (v & CE_ERR_KEYLADDER) + dev_err(ce->dev, "CE ERROR: key ladder configuration error\n"); + break; + } return err; } @@ -280,13 +359,214 @@ static struct sun8i_ce_alg_template ce_algs[] = { .decrypt = sun8i_ce_skdecrypt, } }, +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_HASH +{ .type = CRYPTO_ALG_TYPE_AHASH, + .ce_algo_id = CE_ID_HASH_MD5, + .alg.hash = { + .init = sun8i_ce_hash_init, + .update = sun8i_ce_hash_update, + .final = sun8i_ce_hash_final, + .finup = sun8i_ce_hash_finup, + .digest = sun8i_ce_hash_digest, + .export = sun8i_ce_hash_export, + .import = sun8i_ce_hash_import, + .halg = { + .digestsize = MD5_DIGEST_SIZE, + .statesize = sizeof(struct md5_state), + .base = { + .cra_name = "md5", + .cra_driver_name = "md5-sun8i-ce", + .cra_priority = 300, + .cra_alignmask = 3, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = MD5_HMAC_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sun8i_ce_hash_tfm_ctx), + .cra_module = THIS_MODULE, + .cra_init = sun8i_ce_hash_crainit, + .cra_exit = sun8i_ce_hash_craexit, + } + } + } +}, +{ .type = CRYPTO_ALG_TYPE_AHASH, + .ce_algo_id = CE_ID_HASH_SHA1, + .alg.hash = { + .init = sun8i_ce_hash_init, + .update = sun8i_ce_hash_update, + .final = sun8i_ce_hash_final, + .finup = sun8i_ce_hash_finup, + .digest = sun8i_ce_hash_digest, + .export = sun8i_ce_hash_export, + .import = sun8i_ce_hash_import, + .halg = { + .digestsize = SHA1_DIGEST_SIZE, + .statesize = sizeof(struct sha1_state), + .base = { + .cra_name = "sha1", + .cra_driver_name = "sha1-sun8i-ce", + .cra_priority = 300, + .cra_alignmask = 3, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sun8i_ce_hash_tfm_ctx), + .cra_module = THIS_MODULE, + .cra_init = sun8i_ce_hash_crainit, + .cra_exit = sun8i_ce_hash_craexit, + } + } + } +}, +{ .type = CRYPTO_ALG_TYPE_AHASH, + .ce_algo_id = CE_ID_HASH_SHA224, + .alg.hash = { + .init = sun8i_ce_hash_init, + .update = sun8i_ce_hash_update, + .final = sun8i_ce_hash_final, + .finup = sun8i_ce_hash_finup, + .digest = sun8i_ce_hash_digest, + .export = sun8i_ce_hash_export, + .import = sun8i_ce_hash_import, + .halg = { + .digestsize = SHA224_DIGEST_SIZE, + .statesize = sizeof(struct sha256_state), + .base = { + .cra_name = "sha224", + .cra_driver_name = "sha224-sun8i-ce", + .cra_priority = 300, + .cra_alignmask = 3, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sun8i_ce_hash_tfm_ctx), + .cra_module = THIS_MODULE, + .cra_init = sun8i_ce_hash_crainit, + .cra_exit = sun8i_ce_hash_craexit, + } + } + } +}, +{ .type = CRYPTO_ALG_TYPE_AHASH, + .ce_algo_id = CE_ID_HASH_SHA256, + .alg.hash = { + .init = sun8i_ce_hash_init, + .update = sun8i_ce_hash_update, + .final = sun8i_ce_hash_final, + .finup = sun8i_ce_hash_finup, + .digest = sun8i_ce_hash_digest, + .export = sun8i_ce_hash_export, + .import = sun8i_ce_hash_import, + .halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct sha256_state), + .base = { + .cra_name = "sha256", + .cra_driver_name = "sha256-sun8i-ce", + .cra_priority = 300, + .cra_alignmask = 3, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sun8i_ce_hash_tfm_ctx), + .cra_module = THIS_MODULE, + .cra_init = sun8i_ce_hash_crainit, + .cra_exit = sun8i_ce_hash_craexit, + } + } + } +}, +{ .type = CRYPTO_ALG_TYPE_AHASH, + .ce_algo_id = CE_ID_HASH_SHA384, + .alg.hash = { + .init = sun8i_ce_hash_init, + .update = sun8i_ce_hash_update, + .final = sun8i_ce_hash_final, + .finup = sun8i_ce_hash_finup, + .digest = sun8i_ce_hash_digest, + .export = sun8i_ce_hash_export, + .import = sun8i_ce_hash_import, + .halg = { + .digestsize = SHA384_DIGEST_SIZE, + .statesize = sizeof(struct sha512_state), + .base = { + .cra_name = "sha384", + .cra_driver_name = "sha384-sun8i-ce", + .cra_priority = 300, + .cra_alignmask = 3, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA384_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sun8i_ce_hash_tfm_ctx), + .cra_module = THIS_MODULE, + .cra_init = sun8i_ce_hash_crainit, + .cra_exit = sun8i_ce_hash_craexit, + } + } + } +}, +{ .type = CRYPTO_ALG_TYPE_AHASH, + .ce_algo_id = CE_ID_HASH_SHA512, + .alg.hash = { + .init = sun8i_ce_hash_init, + .update = sun8i_ce_hash_update, + .final = sun8i_ce_hash_final, + .finup = sun8i_ce_hash_finup, + .digest = sun8i_ce_hash_digest, + .export = sun8i_ce_hash_export, + .import = sun8i_ce_hash_import, + .halg = { + .digestsize = SHA512_DIGEST_SIZE, + .statesize = sizeof(struct sha512_state), + .base = { + .cra_name = "sha512", + .cra_driver_name = "sha512-sun8i-ce", + .cra_priority = 300, + .cra_alignmask = 3, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sun8i_ce_hash_tfm_ctx), + .cra_module = THIS_MODULE, + .cra_init = sun8i_ce_hash_crainit, + .cra_exit = sun8i_ce_hash_craexit, + } + } + } +}, +#endif +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_PRNG +{ + .type = CRYPTO_ALG_TYPE_RNG, + .alg.rng = { + .base = { + .cra_name = "stdrng", + .cra_driver_name = "sun8i-ce-prng", + .cra_priority = 300, + .cra_ctxsize = sizeof(struct sun8i_ce_rng_tfm_ctx), + .cra_module = THIS_MODULE, + .cra_init = sun8i_ce_prng_init, + .cra_exit = sun8i_ce_prng_exit, + }, + .generate = sun8i_ce_prng_generate, + .seed = sun8i_ce_prng_seed, + .seedsize = PRNG_SEED_SIZE, + } +}, +#endif }; #ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG -static int sun8i_ce_dbgfs_read(struct seq_file *seq, void *v) +static int sun8i_ce_debugfs_show(struct seq_file *seq, void *v) { struct sun8i_ce_dev *ce = seq->private; - int i; + unsigned int i; for (i = 0; i < MAXFLOW; i++) seq_printf(seq, "Channel %d: nreq %lu\n", i, ce->chanlist[i].stat_req); @@ -301,23 +581,28 @@ static int sun8i_ce_dbgfs_read(struct seq_file *seq, void *v) ce_algs[i].alg.skcipher.base.cra_name, ce_algs[i].stat_req, ce_algs[i].stat_fb); break; + case CRYPTO_ALG_TYPE_AHASH: + seq_printf(seq, "%s %s %lu %lu\n", + ce_algs[i].alg.hash.halg.base.cra_driver_name, + ce_algs[i].alg.hash.halg.base.cra_name, + ce_algs[i].stat_req, ce_algs[i].stat_fb); + break; + case CRYPTO_ALG_TYPE_RNG: + seq_printf(seq, "%s %s %lu %lu\n", + ce_algs[i].alg.rng.base.cra_driver_name, + ce_algs[i].alg.rng.base.cra_name, + ce_algs[i].stat_req, ce_algs[i].stat_bytes); + break; } } +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_TRNG + seq_printf(seq, "HWRNG %lu %lu\n", + ce->hwrng_stat_req, ce->hwrng_stat_bytes); +#endif return 0; } -static int sun8i_ce_dbgfs_open(struct inode *inode, struct file *file) -{ - return single_open(file, sun8i_ce_dbgfs_read, inode->i_private); -} - -static const struct file_operations sun8i_ce_debugfs_fops = { - .owner = THIS_MODULE, - .open = sun8i_ce_dbgfs_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; +DEFINE_SHOW_ATTRIBUTE(sun8i_ce_debugfs); #endif static void sun8i_ce_free_chanlist(struct sun8i_ce_dev *ce, int i) @@ -482,7 +767,8 @@ static int sun8i_ce_get_clks(struct sun8i_ce_dev *ce) static int sun8i_ce_register_algs(struct sun8i_ce_dev *ce) { - int ce_method, err, id, i; + int ce_method, err, id; + unsigned int i; for (i = 0; i < ARRAY_SIZE(ce_algs); i++) { ce_algs[i].ce = ce; @@ -515,6 +801,43 @@ static int sun8i_ce_register_algs(struct sun8i_ce_dev *ce) return err; } break; + case CRYPTO_ALG_TYPE_AHASH: + id = ce_algs[i].ce_algo_id; + ce_method = ce->variant->alg_hash[id]; + if (ce_method == CE_ID_NOTSUPP) { + dev_info(ce->dev, + "DEBUG: Algo of %s not supported\n", + ce_algs[i].alg.hash.halg.base.cra_name); + ce_algs[i].ce = NULL; + break; + } + dev_info(ce->dev, "Register %s\n", + ce_algs[i].alg.hash.halg.base.cra_name); + err = crypto_register_ahash(&ce_algs[i].alg.hash); + if (err) { + dev_err(ce->dev, "ERROR: Fail to register %s\n", + ce_algs[i].alg.hash.halg.base.cra_name); + ce_algs[i].ce = NULL; + return err; + } + break; + case CRYPTO_ALG_TYPE_RNG: + if (ce->variant->prng == CE_ID_NOTSUPP) { + dev_info(ce->dev, + "DEBUG: Algo of %s not supported\n", + ce_algs[i].alg.rng.base.cra_name); + ce_algs[i].ce = NULL; + break; + } + dev_info(ce->dev, "Register %s\n", + ce_algs[i].alg.rng.base.cra_name); + err = crypto_register_rng(&ce_algs[i].alg.rng); + if (err) { + dev_err(ce->dev, "Fail to register %s\n", + ce_algs[i].alg.rng.base.cra_name); + ce_algs[i].ce = NULL; + } + break; default: ce_algs[i].ce = NULL; dev_err(ce->dev, "ERROR: tried to register an unknown algo\n"); @@ -525,7 +848,7 @@ static int sun8i_ce_register_algs(struct sun8i_ce_dev *ce) static void sun8i_ce_unregister_algs(struct sun8i_ce_dev *ce) { - int i; + unsigned int i; for (i = 0; i < ARRAY_SIZE(ce_algs); i++) { if (!ce_algs[i].ce) @@ -536,6 +859,16 @@ static void sun8i_ce_unregister_algs(struct sun8i_ce_dev *ce) ce_algs[i].alg.skcipher.base.cra_name); crypto_unregister_skcipher(&ce_algs[i].alg.skcipher); break; + case CRYPTO_ALG_TYPE_AHASH: + dev_info(ce->dev, "Unregister %d %s\n", i, + ce_algs[i].alg.hash.halg.base.cra_name); + crypto_unregister_ahash(&ce_algs[i].alg.hash); + break; + case CRYPTO_ALG_TYPE_RNG: + dev_info(ce->dev, "Unregister %d %s\n", i, + ce_algs[i].alg.rng.base.cra_name); + crypto_unregister_rng(&ce_algs[i].alg.rng); + break; } } } @@ -573,14 +906,12 @@ static int sun8i_ce_probe(struct platform_device *pdev) return irq; ce->reset = devm_reset_control_get(&pdev->dev, NULL); - if (IS_ERR(ce->reset)) { - if (PTR_ERR(ce->reset) == -EPROBE_DEFER) - return PTR_ERR(ce->reset); - dev_err(&pdev->dev, "No reset control found\n"); - return PTR_ERR(ce->reset); - } + if (IS_ERR(ce->reset)) + return dev_err_probe(&pdev->dev, PTR_ERR(ce->reset), + "No reset control found\n"); mutex_init(&ce->mlock); + mutex_init(&ce->rnglock); err = sun8i_ce_allocate_chanlist(ce); if (err) @@ -605,6 +936,10 @@ static int sun8i_ce_probe(struct platform_device *pdev) if (err < 0) goto error_alg; +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_TRNG + sun8i_ce_hwrng_register(ce); +#endif + v = readl(ce->base + CE_CTR); v >>= CE_DIE_ID_SHIFT; v &= CE_DIE_ID_MASK; @@ -634,6 +969,10 @@ static int sun8i_ce_remove(struct platform_device *pdev) { struct sun8i_ce_dev *ce = platform_get_drvdata(pdev); +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_TRNG + sun8i_ce_hwrng_unregister(ce); +#endif + sun8i_ce_unregister_algs(ce); #ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c new file mode 100644 index 000000000000..fa2f1b4fad7b --- /dev/null +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c @@ -0,0 +1,413 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * sun8i-ce-hash.c - hardware cryptographic offloader for + * Allwinner H3/A64/H5/H2+/H6/R40 SoC + * + * Copyright (C) 2015-2020 Corentin Labbe <clabbe@baylibre.com> + * + * This file add support for MD5 and SHA1/SHA224/SHA256/SHA384/SHA512. + * + * You could find the datasheet in Documentation/arm/sunxi/README + */ +#include <linux/dma-mapping.h> +#include <linux/pm_runtime.h> +#include <linux/scatterlist.h> +#include <crypto/internal/hash.h> +#include <crypto/sha.h> +#include <crypto/md5.h> +#include "sun8i-ce.h" + +int sun8i_ce_hash_crainit(struct crypto_tfm *tfm) +{ + struct sun8i_ce_hash_tfm_ctx *op = crypto_tfm_ctx(tfm); + struct ahash_alg *alg = __crypto_ahash_alg(tfm->__crt_alg); + struct sun8i_ce_alg_template *algt; + int err; + + memset(op, 0, sizeof(struct sun8i_ce_hash_tfm_ctx)); + + algt = container_of(alg, struct sun8i_ce_alg_template, alg.hash); + op->ce = algt->ce; + + op->enginectx.op.do_one_request = sun8i_ce_hash_run; + op->enginectx.op.prepare_request = NULL; + op->enginectx.op.unprepare_request = NULL; + + /* FALLBACK */ + op->fallback_tfm = crypto_alloc_ahash(crypto_tfm_alg_name(tfm), 0, + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(op->fallback_tfm)) { + dev_err(algt->ce->dev, "Fallback driver could no be loaded\n"); + return PTR_ERR(op->fallback_tfm); + } + + if (algt->alg.hash.halg.statesize < crypto_ahash_statesize(op->fallback_tfm)) + algt->alg.hash.halg.statesize = crypto_ahash_statesize(op->fallback_tfm); + + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct sun8i_ce_hash_reqctx) + + crypto_ahash_reqsize(op->fallback_tfm)); + + dev_info(op->ce->dev, "Fallback for %s is %s\n", + crypto_tfm_alg_driver_name(tfm), + crypto_tfm_alg_driver_name(&op->fallback_tfm->base)); + err = pm_runtime_get_sync(op->ce->dev); + if (err < 0) + goto error_pm; + return 0; +error_pm: + pm_runtime_put_noidle(op->ce->dev); + crypto_free_ahash(op->fallback_tfm); + return err; +} + +void sun8i_ce_hash_craexit(struct crypto_tfm *tfm) +{ + struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_tfm_ctx(tfm); + + crypto_free_ahash(tfmctx->fallback_tfm); + pm_runtime_put_sync_suspend(tfmctx->ce->dev); +} + +int sun8i_ce_hash_init(struct ahash_request *areq) +{ + struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); + + memset(rctx, 0, sizeof(struct sun8i_ce_hash_reqctx)); + + ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); + rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; + + return crypto_ahash_init(&rctx->fallback_req); +} + +int sun8i_ce_hash_export(struct ahash_request *areq, void *out) +{ + struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); + rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; + + return crypto_ahash_export(&rctx->fallback_req, out); +} + +int sun8i_ce_hash_import(struct ahash_request *areq, const void *in) +{ + struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); + rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; + + return crypto_ahash_import(&rctx->fallback_req, in); +} + +int sun8i_ce_hash_final(struct ahash_request *areq) +{ + struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG + struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg); + struct sun8i_ce_alg_template *algt; +#endif + + ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); + rctx->fallback_req.base.flags = areq->base.flags & + CRYPTO_TFM_REQ_MAY_SLEEP; + rctx->fallback_req.result = areq->result; + +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG + algt = container_of(alg, struct sun8i_ce_alg_template, alg.hash); + algt->stat_fb++; +#endif + + return crypto_ahash_final(&rctx->fallback_req); +} + +int sun8i_ce_hash_update(struct ahash_request *areq) +{ + struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); + rctx->fallback_req.base.flags = areq->base.flags & + CRYPTO_TFM_REQ_MAY_SLEEP; + rctx->fallback_req.nbytes = areq->nbytes; + rctx->fallback_req.src = areq->src; + + return crypto_ahash_update(&rctx->fallback_req); +} + +int sun8i_ce_hash_finup(struct ahash_request *areq) +{ + struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG + struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg); + struct sun8i_ce_alg_template *algt; +#endif + + ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); + rctx->fallback_req.base.flags = areq->base.flags & + CRYPTO_TFM_REQ_MAY_SLEEP; + + rctx->fallback_req.nbytes = areq->nbytes; + rctx->fallback_req.src = areq->src; + rctx->fallback_req.result = areq->result; +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG + algt = container_of(alg, struct sun8i_ce_alg_template, alg.hash); + algt->stat_fb++; +#endif + + return crypto_ahash_finup(&rctx->fallback_req); +} + +static int sun8i_ce_hash_digest_fb(struct ahash_request *areq) +{ + struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG + struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg); + struct sun8i_ce_alg_template *algt; +#endif + + ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); + rctx->fallback_req.base.flags = areq->base.flags & + CRYPTO_TFM_REQ_MAY_SLEEP; + + rctx->fallback_req.nbytes = areq->nbytes; + rctx->fallback_req.src = areq->src; + rctx->fallback_req.result = areq->result; +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG + algt = container_of(alg, struct sun8i_ce_alg_template, alg.hash); + algt->stat_fb++; +#endif + + return crypto_ahash_digest(&rctx->fallback_req); +} + +static bool sun8i_ce_hash_need_fallback(struct ahash_request *areq) +{ + struct scatterlist *sg; + + if (areq->nbytes == 0) + return true; + /* we need to reserve one SG for padding one */ + if (sg_nents(areq->src) > MAX_SG - 1) + return true; + sg = areq->src; + while (sg) { + if (sg->length % 4 || !IS_ALIGNED(sg->offset, sizeof(u32))) + return true; + sg = sg_next(sg); + } + return false; +} + +int sun8i_ce_hash_digest(struct ahash_request *areq) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg); + struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq); + struct sun8i_ce_alg_template *algt; + struct sun8i_ce_dev *ce; + struct crypto_engine *engine; + struct scatterlist *sg; + int nr_sgs, e, i; + + if (sun8i_ce_hash_need_fallback(areq)) + return sun8i_ce_hash_digest_fb(areq); + + nr_sgs = sg_nents(areq->src); + if (nr_sgs > MAX_SG - 1) + return sun8i_ce_hash_digest_fb(areq); + + for_each_sg(areq->src, sg, nr_sgs, i) { + if (sg->length % 4 || !IS_ALIGNED(sg->offset, sizeof(u32))) + return sun8i_ce_hash_digest_fb(areq); + } + + algt = container_of(alg, struct sun8i_ce_alg_template, alg.hash); + ce = algt->ce; + + e = sun8i_ce_get_engine_number(ce); + rctx->flow = e; + engine = ce->chanlist[e].engine; + + return crypto_transfer_hash_request_to_engine(engine, areq); +} + +int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq) +{ + struct ahash_request *areq = container_of(breq, struct ahash_request, base); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg); + struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq); + struct sun8i_ce_alg_template *algt; + struct sun8i_ce_dev *ce; + struct sun8i_ce_flow *chan; + struct ce_task *cet; + struct scatterlist *sg; + int nr_sgs, flow, err; + unsigned int len; + u32 common; + u64 byte_count; + __le32 *bf; + void *buf; + int j, i, todo; + int nbw = 0; + u64 fill, min_fill; + __be64 *bebits; + __le64 *lebits; + void *result; + u64 bs; + int digestsize; + dma_addr_t addr_res, addr_pad; + + algt = container_of(alg, struct sun8i_ce_alg_template, alg.hash); + ce = algt->ce; + + bs = algt->alg.hash.halg.base.cra_blocksize; + digestsize = algt->alg.hash.halg.digestsize; + if (digestsize == SHA224_DIGEST_SIZE) + digestsize = SHA256_DIGEST_SIZE; + if (digestsize == SHA384_DIGEST_SIZE) + digestsize = SHA512_DIGEST_SIZE; + + /* the padding could be up to two block. */ + buf = kzalloc(bs * 2, GFP_KERNEL | GFP_DMA); + if (!buf) + return -ENOMEM; + bf = (__le32 *)buf; + + result = kzalloc(digestsize, GFP_KERNEL | GFP_DMA); + if (!result) + return -ENOMEM; + + flow = rctx->flow; + chan = &ce->chanlist[flow]; + +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG + algt->stat_req++; +#endif + dev_dbg(ce->dev, "%s %s len=%d\n", __func__, crypto_tfm_alg_name(areq->base.tfm), areq->nbytes); + + cet = chan->tl; + memset(cet, 0, sizeof(struct ce_task)); + + cet->t_id = cpu_to_le32(flow); + common = ce->variant->alg_hash[algt->ce_algo_id]; + common |= CE_COMM_INT; + cet->t_common_ctl = cpu_to_le32(common); + + cet->t_sym_ctl = 0; + cet->t_asym_ctl = 0; + + nr_sgs = dma_map_sg(ce->dev, areq->src, sg_nents(areq->src), DMA_TO_DEVICE); + if (nr_sgs <= 0 || nr_sgs > MAX_SG) { + dev_err(ce->dev, "Invalid sg number %d\n", nr_sgs); + err = -EINVAL; + goto theend; + } + + len = areq->nbytes; + for_each_sg(areq->src, sg, nr_sgs, i) { + cet->t_src[i].addr = cpu_to_le32(sg_dma_address(sg)); + todo = min(len, sg_dma_len(sg)); + cet->t_src[i].len = cpu_to_le32(todo / 4); + len -= todo; + } + if (len > 0) { + dev_err(ce->dev, "remaining len %d\n", len); + err = -EINVAL; + goto theend; + } + addr_res = dma_map_single(ce->dev, result, digestsize, DMA_FROM_DEVICE); + cet->t_dst[0].addr = cpu_to_le32(addr_res); + cet->t_dst[0].len = cpu_to_le32(digestsize / 4); + if (dma_mapping_error(ce->dev, addr_res)) { + dev_err(ce->dev, "DMA map dest\n"); + err = -EINVAL; + goto theend; + } + + byte_count = areq->nbytes; + j = 0; + bf[j++] = cpu_to_le32(0x80); + + if (bs == 64) { + fill = 64 - (byte_count % 64); + min_fill = 2 * sizeof(u32) + (nbw ? 0 : sizeof(u32)); + } else { + fill = 128 - (byte_count % 128); + min_fill = 4 * sizeof(u32) + (nbw ? 0 : sizeof(u32)); + } + + if (fill < min_fill) + fill += bs; + + j += (fill - min_fill) / sizeof(u32); + + switch (algt->ce_algo_id) { + case CE_ID_HASH_MD5: + lebits = (__le64 *)&bf[j]; + *lebits = cpu_to_le64(byte_count << 3); + j += 2; + break; + case CE_ID_HASH_SHA1: + case CE_ID_HASH_SHA224: + case CE_ID_HASH_SHA256: + bebits = (__be64 *)&bf[j]; + *bebits = cpu_to_be64(byte_count << 3); + j += 2; + break; + case CE_ID_HASH_SHA384: + case CE_ID_HASH_SHA512: + bebits = (__be64 *)&bf[j]; + *bebits = cpu_to_be64(byte_count >> 61); + j += 2; + bebits = (__be64 *)&bf[j]; + *bebits = cpu_to_be64(byte_count << 3); + j += 2; + break; + } + + addr_pad = dma_map_single(ce->dev, buf, j * 4, DMA_TO_DEVICE); + cet->t_src[i].addr = cpu_to_le32(addr_pad); + cet->t_src[i].len = cpu_to_le32(j); + if (dma_mapping_error(ce->dev, addr_pad)) { + dev_err(ce->dev, "DMA error on padding SG\n"); + err = -EINVAL; + goto theend; + } + + if (ce->variant->hash_t_dlen_in_bits) + cet->t_dlen = cpu_to_le32((areq->nbytes + j * 4) * 8); + else + cet->t_dlen = cpu_to_le32(areq->nbytes / 4 + j); + + chan->timeout = areq->nbytes; + + err = sun8i_ce_run_task(ce, flow, crypto_tfm_alg_name(areq->base.tfm)); + + dma_unmap_single(ce->dev, addr_pad, j * 4, DMA_TO_DEVICE); + dma_unmap_sg(ce->dev, areq->src, nr_sgs, DMA_TO_DEVICE); + dma_unmap_single(ce->dev, addr_res, digestsize, DMA_FROM_DEVICE); + + kfree(buf); + + memcpy(areq->result, result, algt->alg.hash.halg.digestsize); + kfree(result); +theend: + crypto_finalize_hash_request(engine, breq, err); + return 0; +} diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-prng.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-prng.c new file mode 100644 index 000000000000..78503006949c --- /dev/null +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-prng.c @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * sun8i-ce-prng.c - hardware cryptographic offloader for + * Allwinner H3/A64/H5/H2+/H6/R40 SoC + * + * Copyright (C) 2015-2020 Corentin Labbe <clabbe@baylibre.com> + * + * This file handle the PRNG + * + * You could find a link for the datasheet in Documentation/arm/sunxi/README + */ +#include "sun8i-ce.h" +#include <linux/dma-mapping.h> +#include <linux/pm_runtime.h> +#include <crypto/internal/rng.h> + +int sun8i_ce_prng_init(struct crypto_tfm *tfm) +{ + struct sun8i_ce_rng_tfm_ctx *ctx = crypto_tfm_ctx(tfm); + + memset(ctx, 0, sizeof(struct sun8i_ce_rng_tfm_ctx)); + return 0; +} + +void sun8i_ce_prng_exit(struct crypto_tfm *tfm) +{ + struct sun8i_ce_rng_tfm_ctx *ctx = crypto_tfm_ctx(tfm); + + memzero_explicit(ctx->seed, ctx->slen); + kfree(ctx->seed); + ctx->seed = NULL; + ctx->slen = 0; +} + +int sun8i_ce_prng_seed(struct crypto_rng *tfm, const u8 *seed, + unsigned int slen) +{ + struct sun8i_ce_rng_tfm_ctx *ctx = crypto_rng_ctx(tfm); + + if (ctx->seed && ctx->slen != slen) { + memzero_explicit(ctx->seed, ctx->slen); + kfree(ctx->seed); + ctx->slen = 0; + ctx->seed = NULL; + } + if (!ctx->seed) + ctx->seed = kmalloc(slen, GFP_KERNEL | GFP_DMA); + if (!ctx->seed) + return -ENOMEM; + + memcpy(ctx->seed, seed, slen); + ctx->slen = slen; + + return 0; +} + +int sun8i_ce_prng_generate(struct crypto_rng *tfm, const u8 *src, + unsigned int slen, u8 *dst, unsigned int dlen) +{ + struct sun8i_ce_rng_tfm_ctx *ctx = crypto_rng_ctx(tfm); + struct rng_alg *alg = crypto_rng_alg(tfm); + struct sun8i_ce_alg_template *algt; + struct sun8i_ce_dev *ce; + dma_addr_t dma_iv, dma_dst; + int err = 0; + int flow = 3; + unsigned int todo; + struct sun8i_ce_flow *chan; + struct ce_task *cet; + u32 common, sym; + void *d; + + algt = container_of(alg, struct sun8i_ce_alg_template, alg.rng); + ce = algt->ce; + + if (ctx->slen == 0) { + dev_err(ce->dev, "not seeded\n"); + return -EINVAL; + } + + /* we want dlen + seedsize rounded up to a multiple of PRNG_DATA_SIZE */ + todo = dlen + ctx->slen + PRNG_DATA_SIZE * 2; + todo -= todo % PRNG_DATA_SIZE; + + d = kzalloc(todo, GFP_KERNEL | GFP_DMA); + if (!d) { + err = -ENOMEM; + goto err_mem; + } + + dev_dbg(ce->dev, "%s PRNG slen=%u dlen=%u todo=%u multi=%u\n", __func__, + slen, dlen, todo, todo / PRNG_DATA_SIZE); + +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG + algt->stat_req++; + algt->stat_bytes += todo; +#endif + + dma_iv = dma_map_single(ce->dev, ctx->seed, ctx->slen, DMA_TO_DEVICE); + if (dma_mapping_error(ce->dev, dma_iv)) { + dev_err(ce->dev, "Cannot DMA MAP IV\n"); + goto err_iv; + } + + dma_dst = dma_map_single(ce->dev, d, todo, DMA_FROM_DEVICE); + if (dma_mapping_error(ce->dev, dma_dst)) { + dev_err(ce->dev, "Cannot DMA MAP DST\n"); + err = -EFAULT; + goto err_dst; + } + + err = pm_runtime_get_sync(ce->dev); + if (err < 0) { + pm_runtime_put_noidle(ce->dev); + goto err_pm; + } + + mutex_lock(&ce->rnglock); + chan = &ce->chanlist[flow]; + + cet = &chan->tl[0]; + memset(cet, 0, sizeof(struct ce_task)); + + cet->t_id = cpu_to_le32(flow); + common = ce->variant->prng | CE_COMM_INT; + cet->t_common_ctl = cpu_to_le32(common); + + /* recent CE (H6) need length in bytes, in word otherwise */ + if (ce->variant->prng_t_dlen_in_bytes) + cet->t_dlen = cpu_to_le32(todo); + else + cet->t_dlen = cpu_to_le32(todo / 4); + + sym = PRNG_LD; + cet->t_sym_ctl = cpu_to_le32(sym); + cet->t_asym_ctl = 0; + + cet->t_key = cpu_to_le32(dma_iv); + cet->t_iv = cpu_to_le32(dma_iv); + + cet->t_dst[0].addr = cpu_to_le32(dma_dst); + cet->t_dst[0].len = cpu_to_le32(todo / 4); + ce->chanlist[flow].timeout = 2000; + + err = sun8i_ce_run_task(ce, 3, "PRNG"); + mutex_unlock(&ce->rnglock); + + pm_runtime_put(ce->dev); + +err_pm: + dma_unmap_single(ce->dev, dma_dst, todo, DMA_FROM_DEVICE); +err_dst: + dma_unmap_single(ce->dev, dma_iv, ctx->slen, DMA_TO_DEVICE); + + if (!err) { + memcpy(dst, d, dlen); + memcpy(ctx->seed, d + dlen, ctx->slen); + } + memzero_explicit(d, todo); +err_iv: + kfree(d); +err_mem: + return err; +} diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-trng.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-trng.c new file mode 100644 index 000000000000..654328160d19 --- /dev/null +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-trng.c @@ -0,0 +1,127 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * sun8i-ce-trng.c - hardware cryptographic offloader for + * Allwinner H3/A64/H5/H2+/H6/R40 SoC + * + * Copyright (C) 2015-2020 Corentin Labbe <clabbe@baylibre.com> + * + * This file handle the TRNG + * + * You could find a link for the datasheet in Documentation/arm/sunxi/README + */ +#include "sun8i-ce.h" +#include <linux/dma-mapping.h> +#include <linux/pm_runtime.h> +#include <linux/hw_random.h> +/* + * Note that according to the algorithm ID, 2 versions of the TRNG exists, + * The first present in H3/H5/R40/A64 and the second present in H6. + * This file adds support for both, but only the second is working + * reliabily according to rngtest. + **/ + +static int sun8i_ce_trng_read(struct hwrng *rng, void *data, size_t max, bool wait) +{ + struct sun8i_ce_dev *ce; + dma_addr_t dma_dst; + int err = 0; + int flow = 3; + unsigned int todo; + struct sun8i_ce_flow *chan; + struct ce_task *cet; + u32 common; + void *d; + + ce = container_of(rng, struct sun8i_ce_dev, trng); + + /* round the data length to a multiple of 32*/ + todo = max + 32; + todo -= todo % 32; + + d = kzalloc(todo, GFP_KERNEL | GFP_DMA); + if (!d) + return -ENOMEM; + +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG + ce->hwrng_stat_req++; + ce->hwrng_stat_bytes += todo; +#endif + + dma_dst = dma_map_single(ce->dev, d, todo, DMA_FROM_DEVICE); + if (dma_mapping_error(ce->dev, dma_dst)) { + dev_err(ce->dev, "Cannot DMA MAP DST\n"); + err = -EFAULT; + goto err_dst; + } + + err = pm_runtime_get_sync(ce->dev); + if (err < 0) { + pm_runtime_put_noidle(ce->dev); + goto err_pm; + } + + mutex_lock(&ce->rnglock); + chan = &ce->chanlist[flow]; + + cet = &chan->tl[0]; + memset(cet, 0, sizeof(struct ce_task)); + + cet->t_id = cpu_to_le32(flow); + common = ce->variant->trng | CE_COMM_INT; + cet->t_common_ctl = cpu_to_le32(common); + + /* recent CE (H6) need length in bytes, in word otherwise */ + if (ce->variant->trng_t_dlen_in_bytes) + cet->t_dlen = cpu_to_le32(todo); + else + cet->t_dlen = cpu_to_le32(todo / 4); + + cet->t_sym_ctl = 0; + cet->t_asym_ctl = 0; + + cet->t_dst[0].addr = cpu_to_le32(dma_dst); + cet->t_dst[0].len = cpu_to_le32(todo / 4); + ce->chanlist[flow].timeout = todo; + + err = sun8i_ce_run_task(ce, 3, "TRNG"); + mutex_unlock(&ce->rnglock); + + pm_runtime_put(ce->dev); + +err_pm: + dma_unmap_single(ce->dev, dma_dst, todo, DMA_FROM_DEVICE); + + if (!err) { + memcpy(data, d, max); + err = max; + } + memzero_explicit(d, todo); +err_dst: + kfree(d); + return err; +} + +int sun8i_ce_hwrng_register(struct sun8i_ce_dev *ce) +{ + int ret; + + if (ce->variant->trng == CE_ID_NOTSUPP) { + dev_info(ce->dev, "TRNG not supported\n"); + return 0; + } + ce->trng.name = "sun8i Crypto Engine TRNG"; + ce->trng.read = sun8i_ce_trng_read; + ce->trng.quality = 1000; + + ret = hwrng_register(&ce->trng); + if (ret) + dev_err(ce->dev, "Fail to register the TRNG\n"); + return ret; +} + +void sun8i_ce_hwrng_unregister(struct sun8i_ce_dev *ce) +{ + if (ce->variant->trng == CE_ID_NOTSUPP) + return; + hwrng_unregister(&ce->trng); +} diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h index 963645fe4adb..558027516aed 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h @@ -12,6 +12,11 @@ #include <linux/atomic.h> #include <linux/debugfs.h> #include <linux/crypto.h> +#include <linux/hw_random.h> +#include <crypto/internal/hash.h> +#include <crypto/md5.h> +#include <crypto/rng.h> +#include <crypto/sha.h> /* CE Registers */ #define CE_TDQ 0x00 @@ -45,6 +50,16 @@ #define CE_ALG_AES 0 #define CE_ALG_DES 1 #define CE_ALG_3DES 2 +#define CE_ALG_MD5 16 +#define CE_ALG_SHA1 17 +#define CE_ALG_SHA224 18 +#define CE_ALG_SHA256 19 +#define CE_ALG_SHA384 20 +#define CE_ALG_SHA512 21 +#define CE_ALG_TRNG 48 +#define CE_ALG_PRNG 49 +#define CE_ALG_TRNG_V2 0x1c +#define CE_ALG_PRNG_V2 0x1d /* Used in ce_variant */ #define CE_ID_NOTSUPP 0xFF @@ -54,6 +69,14 @@ #define CE_ID_CIPHER_DES3 2 #define CE_ID_CIPHER_MAX 3 +#define CE_ID_HASH_MD5 0 +#define CE_ID_HASH_SHA1 1 +#define CE_ID_HASH_SHA224 2 +#define CE_ID_HASH_SHA256 3 +#define CE_ID_HASH_SHA384 4 +#define CE_ID_HASH_SHA512 5 +#define CE_ID_HASH_MAX 6 + #define CE_ID_OP_ECB 0 #define CE_ID_OP_CBC 1 #define CE_ID_OP_MAX 2 @@ -65,6 +88,16 @@ #define CE_ERR_ADDR_INVALID BIT(5) #define CE_ERR_KEYLADDER BIT(6) +#define ESR_H3 0 +#define ESR_A64 1 +#define ESR_R40 2 +#define ESR_H5 3 +#define ESR_H6 4 + +#define PRNG_DATA_SIZE (160 / 8) +#define PRNG_SEED_SIZE DIV_ROUND_UP(175, 8) +#define PRNG_LD BIT(17) + #define CE_DIE_ID_SHIFT 16 #define CE_DIE_ID_MASK 0x07 @@ -90,16 +123,34 @@ struct ce_clock { * struct ce_variant - Describe CE capability for each variant hardware * @alg_cipher: list of supported ciphers. for each CE_ID_ this will give the * coresponding CE_ALG_XXX value + * @alg_hash: list of supported hashes. for each CE_ID_ this will give the + * corresponding CE_ALG_XXX value * @op_mode: list of supported block modes - * @has_t_dlen_in_bytes: Does the request size for cipher is in + * @cipher_t_dlen_in_bytes: Does the request size for cipher is in + * bytes or words + * @hash_t_dlen_in_bytes: Does the request size for hash is in + * bits or words + * @prng_t_dlen_in_bytes: Does the request size for PRNG is in + * bytes or words + * @trng_t_dlen_in_bytes: Does the request size for TRNG is in * bytes or words * @ce_clks: list of clocks needed by this variant + * @esr: The type of error register + * @prng: The CE_ALG_XXX value for the PRNG + * @trng: The CE_ALG_XXX value for the TRNG */ struct ce_variant { char alg_cipher[CE_ID_CIPHER_MAX]; + char alg_hash[CE_ID_HASH_MAX]; u32 op_mode[CE_ID_OP_MAX]; - bool has_t_dlen_in_bytes; + bool cipher_t_dlen_in_bytes; + bool hash_t_dlen_in_bits; + bool prng_t_dlen_in_bytes; + bool trng_t_dlen_in_bytes; struct ce_clock ce_clks[CE_MAX_CLOCKS]; + int esr; + unsigned char prng; + unsigned char trng; }; struct sginfo { @@ -129,8 +180,6 @@ struct ce_task { /* * struct sun8i_ce_flow - Information used by each flow * @engine: ptr to the crypto_engine for this flow - * @bounce_iv: buffer which contain the IV - * @ivlen: size of bounce_iv * @complete: completion for the current task on this flow * @status: set to 1 by interrupt if task is done * @t_phy: Physical address of task @@ -139,8 +188,6 @@ struct ce_task { */ struct sun8i_ce_flow { struct crypto_engine *engine; - void *bounce_iv; - unsigned int ivlen; struct completion complete; int status; dma_addr_t t_phy; @@ -158,6 +205,7 @@ struct sun8i_ce_flow { * @reset: pointer to reset controller * @dev: the platform device * @mlock: Control access to device registers + * @rnglock: Control access to the RNG (dedicated channel 3) * @chanlist: array of all flow * @flow: flow to use in next request * @variant: pointer to variant specific data @@ -170,6 +218,7 @@ struct sun8i_ce_dev { struct reset_control *reset; struct device *dev; struct mutex mlock; + struct mutex rnglock; struct sun8i_ce_flow *chanlist; atomic_t flow; const struct ce_variant *variant; @@ -177,17 +226,38 @@ struct sun8i_ce_dev { struct dentry *dbgfs_dir; struct dentry *dbgfs_stats; #endif +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_TRNG + struct hwrng trng; +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG + unsigned long hwrng_stat_req; + unsigned long hwrng_stat_bytes; +#endif +#endif }; /* * struct sun8i_cipher_req_ctx - context for a skcipher request * @op_dir: direction (encrypt vs decrypt) for this request * @flow: the flow to use for this request + * @backup_iv: buffer which contain the next IV to store + * @bounce_iv: buffer which contain the IV + * @ivlen: size of bounce_iv + * @nr_sgs: The number of source SG (as given by dma_map_sg()) + * @nr_sgd: The number of destination SG (as given by dma_map_sg()) + * @addr_iv: The IV addr returned by dma_map_single, need to unmap later + * @addr_key: The key addr returned by dma_map_single, need to unmap later * @fallback_req: request struct for invoking the fallback skcipher TFM */ struct sun8i_cipher_req_ctx { u32 op_dir; int flow; + void *backup_iv; + void *bounce_iv; + unsigned int ivlen; + int nr_sgs; + int nr_sgd; + dma_addr_t addr_iv; + dma_addr_t addr_key; struct skcipher_request fallback_req; // keep at the end }; @@ -208,6 +278,38 @@ struct sun8i_cipher_tfm_ctx { }; /* + * struct sun8i_ce_hash_tfm_ctx - context for an ahash TFM + * @enginectx: crypto_engine used by this TFM + * @ce: pointer to the private data of driver handling this TFM + * @fallback_tfm: pointer to the fallback TFM + */ +struct sun8i_ce_hash_tfm_ctx { + struct crypto_engine_ctx enginectx; + struct sun8i_ce_dev *ce; + struct crypto_ahash *fallback_tfm; +}; + +/* + * struct sun8i_ce_hash_reqctx - context for an ahash request + * @fallback_req: pre-allocated fallback request + * @flow: the flow to use for this request + */ +struct sun8i_ce_hash_reqctx { + struct ahash_request fallback_req; + int flow; +}; + +/* + * struct sun8i_ce_prng_ctx - context for PRNG TFM + * @seed: The seed to use + * @slen: The size of the seed + */ +struct sun8i_ce_rng_tfm_ctx { + void *seed; + unsigned int slen; +}; + +/* * struct sun8i_ce_alg_template - crypto_alg template * @type: the CRYPTO_ALG_TYPE for this template * @ce_algo_id: the CE_ID for this template @@ -217,6 +319,7 @@ struct sun8i_cipher_tfm_ctx { * @alg: one of sub struct must be used * @stat_req: number of request done on this template * @stat_fb: number of request which has fallbacked + * @stat_bytes: total data size done by this template */ struct sun8i_ce_alg_template { u32 type; @@ -225,10 +328,13 @@ struct sun8i_ce_alg_template { struct sun8i_ce_dev *ce; union { struct skcipher_alg skcipher; + struct ahash_alg hash; + struct rng_alg rng; } alg; #ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG unsigned long stat_req; unsigned long stat_fb; + unsigned long stat_bytes; #endif }; @@ -246,3 +352,24 @@ int sun8i_ce_skencrypt(struct skcipher_request *areq); int sun8i_ce_get_engine_number(struct sun8i_ce_dev *ce); int sun8i_ce_run_task(struct sun8i_ce_dev *ce, int flow, const char *name); + +int sun8i_ce_hash_crainit(struct crypto_tfm *tfm); +void sun8i_ce_hash_craexit(struct crypto_tfm *tfm); +int sun8i_ce_hash_init(struct ahash_request *areq); +int sun8i_ce_hash_export(struct ahash_request *areq, void *out); +int sun8i_ce_hash_import(struct ahash_request *areq, const void *in); +int sun8i_ce_hash(struct ahash_request *areq); +int sun8i_ce_hash_final(struct ahash_request *areq); +int sun8i_ce_hash_update(struct ahash_request *areq); +int sun8i_ce_hash_finup(struct ahash_request *areq); +int sun8i_ce_hash_digest(struct ahash_request *areq); +int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq); + +int sun8i_ce_prng_generate(struct crypto_rng *tfm, const u8 *src, + unsigned int slen, u8 *dst, unsigned int dlen); +int sun8i_ce_prng_seed(struct crypto_rng *tfm, const u8 *seed, unsigned int slen); +void sun8i_ce_prng_exit(struct crypto_tfm *tfm); +int sun8i_ce_prng_init(struct crypto_tfm *tfm); + +int sun8i_ce_hwrng_register(struct sun8i_ce_dev *ce); +void sun8i_ce_hwrng_unregister(struct sun8i_ce_dev *ce); diff --git a/drivers/crypto/allwinner/sun8i-ss/Makefile b/drivers/crypto/allwinner/sun8i-ss/Makefile index add7b0543fd5..aabfd893c817 100644 --- a/drivers/crypto/allwinner/sun8i-ss/Makefile +++ b/drivers/crypto/allwinner/sun8i-ss/Makefile @@ -1,2 +1,4 @@ obj-$(CONFIG_CRYPTO_DEV_SUN8I_SS) += sun8i-ss.o sun8i-ss-y += sun8i-ss-core.o sun8i-ss-cipher.o +sun8i-ss-$(CONFIG_CRYPTO_DEV_SUN8I_SS_PRNG) += sun8i-ss-prng.o +sun8i-ss-$(CONFIG_CRYPTO_DEV_SUN8I_SS_HASH) += sun8i-ss-hash.o diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c index 7b39b4495571..ed2a69f82e1c 100644 --- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c +++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c @@ -248,7 +248,6 @@ theend_iv: offset = areq->cryptlen - ivsize; if (rctx->op_dir & SS_DECRYPTION) { memcpy(areq->iv, backup_iv, ivsize); - memzero_explicit(backup_iv, ivsize); kfree_sensitive(backup_iv); } else { scatterwalk_map_and_copy(areq->iv, areq->dst, offset, @@ -368,10 +367,7 @@ void sun8i_ss_cipher_exit(struct crypto_tfm *tfm) { struct sun8i_cipher_tfm_ctx *op = crypto_tfm_ctx(tfm); - if (op->key) { - memzero_explicit(op->key, op->keylen); - kfree(op->key); - } + kfree_sensitive(op->key); crypto_free_skcipher(op->fallback_tfm); pm_runtime_put_sync(op->ss->dev); } @@ -393,10 +389,7 @@ int sun8i_ss_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, dev_dbg(ss->dev, "ERROR: Invalid keylen %u\n", keylen); return -EINVAL; } - if (op->key) { - memzero_explicit(op->key, op->keylen); - kfree(op->key); - } + kfree_sensitive(op->key); op->keylen = keylen; op->key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA); if (!op->key) @@ -419,10 +412,7 @@ int sun8i_ss_des3_setkey(struct crypto_skcipher *tfm, const u8 *key, return -EINVAL; } - if (op->key) { - memzero_explicit(op->key, op->keylen); - kfree(op->key); - } + kfree_sensitive(op->key); op->keylen = keylen; op->key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA); if (!op->key) diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c index 9a23515783a6..e0ddc684798d 100644 --- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c +++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c @@ -22,6 +22,7 @@ #include <linux/platform_device.h> #include <linux/pm_runtime.h> #include <linux/reset.h> +#include <crypto/internal/rng.h> #include <crypto/internal/skcipher.h> #include "sun8i-ss.h" @@ -40,6 +41,8 @@ static const struct ss_variant ss_a80_variant = { static const struct ss_variant ss_a83t_variant = { .alg_cipher = { SS_ALG_AES, SS_ALG_DES, SS_ALG_3DES, }, + .alg_hash = { SS_ALG_MD5, SS_ALG_SHA1, SS_ALG_SHA224, SS_ALG_SHA256, + }, .op_mode = { SS_OP_ECB, SS_OP_CBC, }, .ss_clks = { @@ -61,7 +64,7 @@ int sun8i_ss_run_task(struct sun8i_ss_dev *ss, struct sun8i_cipher_req_ctx *rctx const char *name) { int flow = rctx->flow; - u32 v = 1; + u32 v = SS_START; int i; #ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG @@ -264,13 +267,154 @@ static struct sun8i_ss_alg_template ss_algs[] = { .decrypt = sun8i_ss_skdecrypt, } }, +#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_PRNG +{ + .type = CRYPTO_ALG_TYPE_RNG, + .alg.rng = { + .base = { + .cra_name = "stdrng", + .cra_driver_name = "sun8i-ss-prng", + .cra_priority = 300, + .cra_ctxsize = sizeof(struct sun8i_ss_rng_tfm_ctx), + .cra_module = THIS_MODULE, + .cra_init = sun8i_ss_prng_init, + .cra_exit = sun8i_ss_prng_exit, + }, + .generate = sun8i_ss_prng_generate, + .seed = sun8i_ss_prng_seed, + .seedsize = PRNG_SEED_SIZE, + } +}, +#endif +#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_HASH +{ .type = CRYPTO_ALG_TYPE_AHASH, + .ss_algo_id = SS_ID_HASH_MD5, + .alg.hash = { + .init = sun8i_ss_hash_init, + .update = sun8i_ss_hash_update, + .final = sun8i_ss_hash_final, + .finup = sun8i_ss_hash_finup, + .digest = sun8i_ss_hash_digest, + .export = sun8i_ss_hash_export, + .import = sun8i_ss_hash_import, + .halg = { + .digestsize = MD5_DIGEST_SIZE, + .statesize = sizeof(struct md5_state), + .base = { + .cra_name = "md5", + .cra_driver_name = "md5-sun8i-ss", + .cra_priority = 300, + .cra_alignmask = 3, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = MD5_HMAC_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sun8i_ss_hash_tfm_ctx), + .cra_module = THIS_MODULE, + .cra_init = sun8i_ss_hash_crainit, + .cra_exit = sun8i_ss_hash_craexit, + } + } + } +}, +{ .type = CRYPTO_ALG_TYPE_AHASH, + .ss_algo_id = SS_ID_HASH_SHA1, + .alg.hash = { + .init = sun8i_ss_hash_init, + .update = sun8i_ss_hash_update, + .final = sun8i_ss_hash_final, + .finup = sun8i_ss_hash_finup, + .digest = sun8i_ss_hash_digest, + .export = sun8i_ss_hash_export, + .import = sun8i_ss_hash_import, + .halg = { + .digestsize = SHA1_DIGEST_SIZE, + .statesize = sizeof(struct sha1_state), + .base = { + .cra_name = "sha1", + .cra_driver_name = "sha1-sun8i-ss", + .cra_priority = 300, + .cra_alignmask = 3, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sun8i_ss_hash_tfm_ctx), + .cra_module = THIS_MODULE, + .cra_init = sun8i_ss_hash_crainit, + .cra_exit = sun8i_ss_hash_craexit, + } + } + } +}, +{ .type = CRYPTO_ALG_TYPE_AHASH, + .ss_algo_id = SS_ID_HASH_SHA224, + .alg.hash = { + .init = sun8i_ss_hash_init, + .update = sun8i_ss_hash_update, + .final = sun8i_ss_hash_final, + .finup = sun8i_ss_hash_finup, + .digest = sun8i_ss_hash_digest, + .export = sun8i_ss_hash_export, + .import = sun8i_ss_hash_import, + .halg = { + .digestsize = SHA224_DIGEST_SIZE, + .statesize = sizeof(struct sha256_state), + .base = { + .cra_name = "sha224", + .cra_driver_name = "sha224-sun8i-ss", + .cra_priority = 300, + .cra_alignmask = 3, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sun8i_ss_hash_tfm_ctx), + .cra_module = THIS_MODULE, + .cra_init = sun8i_ss_hash_crainit, + .cra_exit = sun8i_ss_hash_craexit, + } + } + } +}, +{ .type = CRYPTO_ALG_TYPE_AHASH, + .ss_algo_id = SS_ID_HASH_SHA256, + .alg.hash = { + .init = sun8i_ss_hash_init, + .update = sun8i_ss_hash_update, + .final = sun8i_ss_hash_final, + .finup = sun8i_ss_hash_finup, + .digest = sun8i_ss_hash_digest, + .export = sun8i_ss_hash_export, + .import = sun8i_ss_hash_import, + .halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct sha256_state), + .base = { + .cra_name = "sha256", + .cra_driver_name = "sha256-sun8i-ss", + .cra_priority = 300, + .cra_alignmask = 3, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sun8i_ss_hash_tfm_ctx), + .cra_module = THIS_MODULE, + .cra_init = sun8i_ss_hash_crainit, + .cra_exit = sun8i_ss_hash_craexit, + } + } + } +}, +#endif }; #ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG -static int sun8i_ss_dbgfs_read(struct seq_file *seq, void *v) +static int sun8i_ss_debugfs_show(struct seq_file *seq, void *v) { struct sun8i_ss_dev *ss = seq->private; - int i; + unsigned int i; for (i = 0; i < MAXFLOW; i++) seq_printf(seq, "Channel %d: nreq %lu\n", i, ss->flows[i].stat_req); @@ -280,28 +424,29 @@ static int sun8i_ss_dbgfs_read(struct seq_file *seq, void *v) continue; switch (ss_algs[i].type) { case CRYPTO_ALG_TYPE_SKCIPHER: - seq_printf(seq, "%s %s %lu %lu\n", + seq_printf(seq, "%s %s reqs=%lu fallback=%lu\n", ss_algs[i].alg.skcipher.base.cra_driver_name, ss_algs[i].alg.skcipher.base.cra_name, ss_algs[i].stat_req, ss_algs[i].stat_fb); break; + case CRYPTO_ALG_TYPE_RNG: + seq_printf(seq, "%s %s reqs=%lu tsize=%lu\n", + ss_algs[i].alg.rng.base.cra_driver_name, + ss_algs[i].alg.rng.base.cra_name, + ss_algs[i].stat_req, ss_algs[i].stat_bytes); + break; + case CRYPTO_ALG_TYPE_AHASH: + seq_printf(seq, "%s %s reqs=%lu fallback=%lu\n", + ss_algs[i].alg.hash.halg.base.cra_driver_name, + ss_algs[i].alg.hash.halg.base.cra_name, + ss_algs[i].stat_req, ss_algs[i].stat_fb); + break; } } return 0; } -static int sun8i_ss_dbgfs_open(struct inode *inode, struct file *file) -{ - return single_open(file, sun8i_ss_dbgfs_read, inode->i_private); -} - -static const struct file_operations sun8i_ss_debugfs_fops = { - .owner = THIS_MODULE, - .open = sun8i_ss_dbgfs_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; +DEFINE_SHOW_ATTRIBUTE(sun8i_ss_debugfs); #endif static void sun8i_ss_free_flows(struct sun8i_ss_dev *ss, int i) @@ -415,7 +560,8 @@ static void sun8i_ss_pm_exit(struct sun8i_ss_dev *ss) static int sun8i_ss_register_algs(struct sun8i_ss_dev *ss) { - int ss_method, err, id, i; + int ss_method, err, id; + unsigned int i; for (i = 0; i < ARRAY_SIZE(ss_algs); i++) { ss_algs[i].ss = ss; @@ -448,6 +594,34 @@ static int sun8i_ss_register_algs(struct sun8i_ss_dev *ss) return err; } break; + case CRYPTO_ALG_TYPE_RNG: + err = crypto_register_rng(&ss_algs[i].alg.rng); + if (err) { + dev_err(ss->dev, "Fail to register %s\n", + ss_algs[i].alg.rng.base.cra_name); + ss_algs[i].ss = NULL; + } + break; + case CRYPTO_ALG_TYPE_AHASH: + id = ss_algs[i].ss_algo_id; + ss_method = ss->variant->alg_hash[id]; + if (ss_method == SS_ID_NOTSUPP) { + dev_info(ss->dev, + "DEBUG: Algo of %s not supported\n", + ss_algs[i].alg.hash.halg.base.cra_name); + ss_algs[i].ss = NULL; + break; + } + dev_info(ss->dev, "Register %s\n", + ss_algs[i].alg.hash.halg.base.cra_name); + err = crypto_register_ahash(&ss_algs[i].alg.hash); + if (err) { + dev_err(ss->dev, "ERROR: Fail to register %s\n", + ss_algs[i].alg.hash.halg.base.cra_name); + ss_algs[i].ss = NULL; + return err; + } + break; default: ss_algs[i].ss = NULL; dev_err(ss->dev, "ERROR: tried to register an unknown algo\n"); @@ -458,7 +632,7 @@ static int sun8i_ss_register_algs(struct sun8i_ss_dev *ss) static void sun8i_ss_unregister_algs(struct sun8i_ss_dev *ss) { - int i; + unsigned int i; for (i = 0; i < ARRAY_SIZE(ss_algs); i++) { if (!ss_algs[i].ss) @@ -469,6 +643,16 @@ static void sun8i_ss_unregister_algs(struct sun8i_ss_dev *ss) ss_algs[i].alg.skcipher.base.cra_name); crypto_unregister_skcipher(&ss_algs[i].alg.skcipher); break; + case CRYPTO_ALG_TYPE_RNG: + dev_info(ss->dev, "Unregister %d %s\n", i, + ss_algs[i].alg.rng.base.cra_name); + crypto_unregister_rng(&ss_algs[i].alg.rng); + break; + case CRYPTO_ALG_TYPE_AHASH: + dev_info(ss->dev, "Unregister %d %s\n", i, + ss_algs[i].alg.hash.halg.base.cra_name); + crypto_unregister_ahash(&ss_algs[i].alg.hash); + break; } } } @@ -545,12 +729,9 @@ static int sun8i_ss_probe(struct platform_device *pdev) return irq; ss->reset = devm_reset_control_get(&pdev->dev, NULL); - if (IS_ERR(ss->reset)) { - if (PTR_ERR(ss->reset) == -EPROBE_DEFER) - return PTR_ERR(ss->reset); - dev_err(&pdev->dev, "No reset control found\n"); - return PTR_ERR(ss->reset); - } + if (IS_ERR(ss->reset)) + return dev_err_probe(&pdev->dev, PTR_ERR(ss->reset), + "No reset control found\n"); mutex_init(&ss->mlock); diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c new file mode 100644 index 000000000000..b6ab2054f217 --- /dev/null +++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c @@ -0,0 +1,444 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * sun8i-ss-hash.c - hardware cryptographic offloader for + * Allwinner A80/A83T SoC + * + * Copyright (C) 2015-2020 Corentin Labbe <clabbe@baylibre.com> + * + * This file add support for MD5 and SHA1/SHA224/SHA256. + * + * You could find the datasheet in Documentation/arm/sunxi.rst + */ +#include <linux/dma-mapping.h> +#include <linux/pm_runtime.h> +#include <linux/scatterlist.h> +#include <crypto/internal/hash.h> +#include <crypto/sha.h> +#include <crypto/md5.h> +#include "sun8i-ss.h" + +int sun8i_ss_hash_crainit(struct crypto_tfm *tfm) +{ + struct sun8i_ss_hash_tfm_ctx *op = crypto_tfm_ctx(tfm); + struct ahash_alg *alg = __crypto_ahash_alg(tfm->__crt_alg); + struct sun8i_ss_alg_template *algt; + int err; + + memset(op, 0, sizeof(struct sun8i_ss_hash_tfm_ctx)); + + algt = container_of(alg, struct sun8i_ss_alg_template, alg.hash); + op->ss = algt->ss; + + op->enginectx.op.do_one_request = sun8i_ss_hash_run; + op->enginectx.op.prepare_request = NULL; + op->enginectx.op.unprepare_request = NULL; + + /* FALLBACK */ + op->fallback_tfm = crypto_alloc_ahash(crypto_tfm_alg_name(tfm), 0, + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(op->fallback_tfm)) { + dev_err(algt->ss->dev, "Fallback driver could no be loaded\n"); + return PTR_ERR(op->fallback_tfm); + } + + if (algt->alg.hash.halg.statesize < crypto_ahash_statesize(op->fallback_tfm)) + algt->alg.hash.halg.statesize = crypto_ahash_statesize(op->fallback_tfm); + + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct sun8i_ss_hash_reqctx) + + crypto_ahash_reqsize(op->fallback_tfm)); + + dev_info(op->ss->dev, "Fallback for %s is %s\n", + crypto_tfm_alg_driver_name(tfm), + crypto_tfm_alg_driver_name(&op->fallback_tfm->base)); + err = pm_runtime_get_sync(op->ss->dev); + if (err < 0) + goto error_pm; + return 0; +error_pm: + pm_runtime_put_noidle(op->ss->dev); + crypto_free_ahash(op->fallback_tfm); + return err; +} + +void sun8i_ss_hash_craexit(struct crypto_tfm *tfm) +{ + struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_tfm_ctx(tfm); + + crypto_free_ahash(tfmctx->fallback_tfm); + pm_runtime_put_sync_suspend(tfmctx->ss->dev); +} + +int sun8i_ss_hash_init(struct ahash_request *areq) +{ + struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); + + memset(rctx, 0, sizeof(struct sun8i_ss_hash_reqctx)); + + ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); + rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; + + return crypto_ahash_init(&rctx->fallback_req); +} + +int sun8i_ss_hash_export(struct ahash_request *areq, void *out) +{ + struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); + rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; + + return crypto_ahash_export(&rctx->fallback_req, out); +} + +int sun8i_ss_hash_import(struct ahash_request *areq, const void *in) +{ + struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); + rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; + + return crypto_ahash_import(&rctx->fallback_req, in); +} + +int sun8i_ss_hash_final(struct ahash_request *areq) +{ + struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); +#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG + struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg); + struct sun8i_ss_alg_template *algt; +#endif + + ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); + rctx->fallback_req.base.flags = areq->base.flags & + CRYPTO_TFM_REQ_MAY_SLEEP; + rctx->fallback_req.result = areq->result; + +#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG + algt = container_of(alg, struct sun8i_ss_alg_template, alg.hash); + algt->stat_fb++; +#endif + + return crypto_ahash_final(&rctx->fallback_req); +} + +int sun8i_ss_hash_update(struct ahash_request *areq) +{ + struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); + rctx->fallback_req.base.flags = areq->base.flags & + CRYPTO_TFM_REQ_MAY_SLEEP; + rctx->fallback_req.nbytes = areq->nbytes; + rctx->fallback_req.src = areq->src; + + return crypto_ahash_update(&rctx->fallback_req); +} + +int sun8i_ss_hash_finup(struct ahash_request *areq) +{ + struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); +#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG + struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg); + struct sun8i_ss_alg_template *algt; +#endif + + ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); + rctx->fallback_req.base.flags = areq->base.flags & + CRYPTO_TFM_REQ_MAY_SLEEP; + + rctx->fallback_req.nbytes = areq->nbytes; + rctx->fallback_req.src = areq->src; + rctx->fallback_req.result = areq->result; +#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG + algt = container_of(alg, struct sun8i_ss_alg_template, alg.hash); + algt->stat_fb++; +#endif + + return crypto_ahash_finup(&rctx->fallback_req); +} + +static int sun8i_ss_hash_digest_fb(struct ahash_request *areq) +{ + struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); +#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG + struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg); + struct sun8i_ss_alg_template *algt; +#endif + + ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); + rctx->fallback_req.base.flags = areq->base.flags & + CRYPTO_TFM_REQ_MAY_SLEEP; + + rctx->fallback_req.nbytes = areq->nbytes; + rctx->fallback_req.src = areq->src; + rctx->fallback_req.result = areq->result; +#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG + algt = container_of(alg, struct sun8i_ss_alg_template, alg.hash); + algt->stat_fb++; +#endif + + return crypto_ahash_digest(&rctx->fallback_req); +} + +static int sun8i_ss_run_hash_task(struct sun8i_ss_dev *ss, + struct sun8i_ss_hash_reqctx *rctx, + const char *name) +{ + int flow = rctx->flow; + u32 v = SS_START; + int i; + +#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG + ss->flows[flow].stat_req++; +#endif + + /* choose between stream0/stream1 */ + if (flow) + v |= SS_FLOW1; + else + v |= SS_FLOW0; + + v |= rctx->method; + + for (i = 0; i < MAX_SG; i++) { + if (!rctx->t_dst[i].addr) + break; + + mutex_lock(&ss->mlock); + if (i > 0) { + v |= BIT(17); + writel(rctx->t_dst[i - 1].addr, ss->base + SS_KEY_ADR_REG); + writel(rctx->t_dst[i - 1].addr, ss->base + SS_IV_ADR_REG); + } + + dev_dbg(ss->dev, + "Processing SG %d on flow %d %s ctl=%x %d to %d method=%x src=%x dst=%x\n", + i, flow, name, v, + rctx->t_src[i].len, rctx->t_dst[i].len, + rctx->method, rctx->t_src[i].addr, rctx->t_dst[i].addr); + + writel(rctx->t_src[i].addr, ss->base + SS_SRC_ADR_REG); + writel(rctx->t_dst[i].addr, ss->base + SS_DST_ADR_REG); + writel(rctx->t_src[i].len, ss->base + SS_LEN_ADR_REG); + writel(BIT(0) | BIT(1), ss->base + SS_INT_CTL_REG); + + reinit_completion(&ss->flows[flow].complete); + ss->flows[flow].status = 0; + wmb(); + + writel(v, ss->base + SS_CTL_REG); + mutex_unlock(&ss->mlock); + wait_for_completion_interruptible_timeout(&ss->flows[flow].complete, + msecs_to_jiffies(2000)); + if (ss->flows[flow].status == 0) { + dev_err(ss->dev, "DMA timeout for %s\n", name); + return -EFAULT; + } + } + + return 0; +} + +static bool sun8i_ss_hash_need_fallback(struct ahash_request *areq) +{ + struct scatterlist *sg; + + if (areq->nbytes == 0) + return true; + /* we need to reserve one SG for the padding one */ + if (sg_nents(areq->src) > MAX_SG - 1) + return true; + sg = areq->src; + while (sg) { + /* SS can operate hash only on full block size + * since SS support only MD5,sha1,sha224 and sha256, blocksize + * is always 64 + * TODO: handle request if last SG is not len%64 + * but this will need to copy data on a new SG of size=64 + */ + if (sg->length % 64 || !IS_ALIGNED(sg->offset, sizeof(u32))) + return true; + sg = sg_next(sg); + } + return false; +} + +int sun8i_ss_hash_digest(struct ahash_request *areq) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg); + struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq); + struct sun8i_ss_alg_template *algt; + struct sun8i_ss_dev *ss; + struct crypto_engine *engine; + struct scatterlist *sg; + int nr_sgs, e, i; + + if (sun8i_ss_hash_need_fallback(areq)) + return sun8i_ss_hash_digest_fb(areq); + + nr_sgs = sg_nents(areq->src); + if (nr_sgs > MAX_SG - 1) + return sun8i_ss_hash_digest_fb(areq); + + for_each_sg(areq->src, sg, nr_sgs, i) { + if (sg->length % 4 || !IS_ALIGNED(sg->offset, sizeof(u32))) + return sun8i_ss_hash_digest_fb(areq); + } + + algt = container_of(alg, struct sun8i_ss_alg_template, alg.hash); + ss = algt->ss; + + e = sun8i_ss_get_engine_number(ss); + rctx->flow = e; + engine = ss->flows[e].engine; + + return crypto_transfer_hash_request_to_engine(engine, areq); +} + +/* sun8i_ss_hash_run - run an ahash request + * Send the data of the request to the SS along with an extra SG with padding + */ +int sun8i_ss_hash_run(struct crypto_engine *engine, void *breq) +{ + struct ahash_request *areq = container_of(breq, struct ahash_request, base); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg); + struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq); + struct sun8i_ss_alg_template *algt; + struct sun8i_ss_dev *ss; + struct scatterlist *sg; + int nr_sgs, err, digestsize; + unsigned int len; + u64 fill, min_fill, byte_count; + void *pad, *result; + int j, i, todo; + __be64 *bebits; + __le64 *lebits; + dma_addr_t addr_res, addr_pad; + __le32 *bf; + + algt = container_of(alg, struct sun8i_ss_alg_template, alg.hash); + ss = algt->ss; + + digestsize = algt->alg.hash.halg.digestsize; + if (digestsize == SHA224_DIGEST_SIZE) + digestsize = SHA256_DIGEST_SIZE; + + /* the padding could be up to two block. */ + pad = kzalloc(algt->alg.hash.halg.base.cra_blocksize * 2, GFP_KERNEL | GFP_DMA); + if (!pad) + return -ENOMEM; + bf = (__le32 *)pad; + + result = kzalloc(digestsize, GFP_KERNEL | GFP_DMA); + if (!result) + return -ENOMEM; + + for (i = 0; i < MAX_SG; i++) { + rctx->t_dst[i].addr = 0; + rctx->t_dst[i].len = 0; + } + +#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG + algt->stat_req++; +#endif + + rctx->method = ss->variant->alg_hash[algt->ss_algo_id]; + + nr_sgs = dma_map_sg(ss->dev, areq->src, sg_nents(areq->src), DMA_TO_DEVICE); + if (nr_sgs <= 0 || nr_sgs > MAX_SG) { + dev_err(ss->dev, "Invalid sg number %d\n", nr_sgs); + err = -EINVAL; + goto theend; + } + + addr_res = dma_map_single(ss->dev, result, digestsize, DMA_FROM_DEVICE); + if (dma_mapping_error(ss->dev, addr_res)) { + dev_err(ss->dev, "DMA map dest\n"); + err = -EINVAL; + goto theend; + } + + len = areq->nbytes; + for_each_sg(areq->src, sg, nr_sgs, i) { + rctx->t_src[i].addr = sg_dma_address(sg); + todo = min(len, sg_dma_len(sg)); + rctx->t_src[i].len = todo / 4; + len -= todo; + rctx->t_dst[i].addr = addr_res; + rctx->t_dst[i].len = digestsize / 4; + } + if (len > 0) { + dev_err(ss->dev, "remaining len %d\n", len); + err = -EINVAL; + goto theend; + } + + byte_count = areq->nbytes; + j = 0; + bf[j++] = cpu_to_le32(0x80); + + fill = 64 - (byte_count % 64); + min_fill = 3 * sizeof(u32); + + if (fill < min_fill) + fill += 64; + + j += (fill - min_fill) / sizeof(u32); + + switch (algt->ss_algo_id) { + case SS_ID_HASH_MD5: + lebits = (__le64 *)&bf[j]; + *lebits = cpu_to_le64(byte_count << 3); + j += 2; + break; + case SS_ID_HASH_SHA1: + case SS_ID_HASH_SHA224: + case SS_ID_HASH_SHA256: + bebits = (__be64 *)&bf[j]; + *bebits = cpu_to_be64(byte_count << 3); + j += 2; + break; + } + + addr_pad = dma_map_single(ss->dev, pad, j * 4, DMA_TO_DEVICE); + rctx->t_src[i].addr = addr_pad; + rctx->t_src[i].len = j; + rctx->t_dst[i].addr = addr_res; + rctx->t_dst[i].len = digestsize / 4; + if (dma_mapping_error(ss->dev, addr_pad)) { + dev_err(ss->dev, "DMA error on padding SG\n"); + err = -EINVAL; + goto theend; + } + + err = sun8i_ss_run_hash_task(ss, rctx, crypto_tfm_alg_name(areq->base.tfm)); + + dma_unmap_single(ss->dev, addr_pad, j * 4, DMA_TO_DEVICE); + dma_unmap_sg(ss->dev, areq->src, nr_sgs, DMA_TO_DEVICE); + dma_unmap_single(ss->dev, addr_res, digestsize, DMA_FROM_DEVICE); + + kfree(pad); + + memcpy(areq->result, result, algt->alg.hash.halg.digestsize); + kfree(result); +theend: + crypto_finalize_hash_request(engine, breq, err); + return 0; +} diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-prng.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-prng.c new file mode 100644 index 000000000000..08a1473b2145 --- /dev/null +++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-prng.c @@ -0,0 +1,173 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * sun8i-ss-prng.c - hardware cryptographic offloader for + * Allwinner A80/A83T SoC + * + * Copyright (C) 2015-2020 Corentin Labbe <clabbe@baylibre.com> + * + * This file handle the PRNG found in the SS + * + * You could find a link for the datasheet in Documentation/arm/sunxi.rst + */ +#include "sun8i-ss.h" +#include <linux/dma-mapping.h> +#include <linux/pm_runtime.h> +#include <crypto/internal/rng.h> + +int sun8i_ss_prng_seed(struct crypto_rng *tfm, const u8 *seed, + unsigned int slen) +{ + struct sun8i_ss_rng_tfm_ctx *ctx = crypto_rng_ctx(tfm); + + if (ctx->seed && ctx->slen != slen) { + memzero_explicit(ctx->seed, ctx->slen); + kfree(ctx->seed); + ctx->slen = 0; + ctx->seed = NULL; + } + if (!ctx->seed) + ctx->seed = kmalloc(slen, GFP_KERNEL | GFP_DMA); + if (!ctx->seed) + return -ENOMEM; + + memcpy(ctx->seed, seed, slen); + ctx->slen = slen; + + return 0; +} + +int sun8i_ss_prng_init(struct crypto_tfm *tfm) +{ + struct sun8i_ss_rng_tfm_ctx *ctx = crypto_tfm_ctx(tfm); + + memset(ctx, 0, sizeof(struct sun8i_ss_rng_tfm_ctx)); + return 0; +} + +void sun8i_ss_prng_exit(struct crypto_tfm *tfm) +{ + struct sun8i_ss_rng_tfm_ctx *ctx = crypto_tfm_ctx(tfm); + + memzero_explicit(ctx->seed, ctx->slen); + kfree(ctx->seed); + ctx->seed = NULL; + ctx->slen = 0; +} + +int sun8i_ss_prng_generate(struct crypto_rng *tfm, const u8 *src, + unsigned int slen, u8 *dst, unsigned int dlen) +{ + struct sun8i_ss_rng_tfm_ctx *ctx = crypto_rng_ctx(tfm); + struct rng_alg *alg = crypto_rng_alg(tfm); + struct sun8i_ss_alg_template *algt; + struct sun8i_ss_dev *ss; + dma_addr_t dma_iv, dma_dst; + unsigned int todo; + int err = 0; + int flow; + void *d; + u32 v; + + algt = container_of(alg, struct sun8i_ss_alg_template, alg.rng); + ss = algt->ss; + + if (ctx->slen == 0) { + dev_err(ss->dev, "The PRNG is not seeded\n"); + return -EINVAL; + } + + /* The SS does not give an updated seed, so we need to get a new one. + * So we will ask for an extra PRNG_SEED_SIZE data. + * We want dlen + seedsize rounded up to a multiple of PRNG_DATA_SIZE + */ + todo = dlen + PRNG_SEED_SIZE + PRNG_DATA_SIZE; + todo -= todo % PRNG_DATA_SIZE; + + d = kzalloc(todo, GFP_KERNEL | GFP_DMA); + if (!d) + return -ENOMEM; + + flow = sun8i_ss_get_engine_number(ss); + +#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG + algt->stat_req++; + algt->stat_bytes += todo; +#endif + + v = SS_ALG_PRNG | SS_PRNG_CONTINUE | SS_START; + if (flow) + v |= SS_FLOW1; + else + v |= SS_FLOW0; + + dma_iv = dma_map_single(ss->dev, ctx->seed, ctx->slen, DMA_TO_DEVICE); + if (dma_mapping_error(ss->dev, dma_iv)) { + dev_err(ss->dev, "Cannot DMA MAP IV\n"); + return -EFAULT; + } + + dma_dst = dma_map_single(ss->dev, d, todo, DMA_FROM_DEVICE); + if (dma_mapping_error(ss->dev, dma_dst)) { + dev_err(ss->dev, "Cannot DMA MAP DST\n"); + err = -EFAULT; + goto err_iv; + } + + err = pm_runtime_get_sync(ss->dev); + if (err < 0) { + pm_runtime_put_noidle(ss->dev); + goto err_pm; + } + err = 0; + + mutex_lock(&ss->mlock); + writel(dma_iv, ss->base + SS_IV_ADR_REG); + /* the PRNG act badly (failing rngtest) without SS_KEY_ADR_REG set */ + writel(dma_iv, ss->base + SS_KEY_ADR_REG); + writel(dma_dst, ss->base + SS_DST_ADR_REG); + writel(todo / 4, ss->base + SS_LEN_ADR_REG); + + reinit_completion(&ss->flows[flow].complete); + ss->flows[flow].status = 0; + /* Be sure all data is written before enabling the task */ + wmb(); + + writel(v, ss->base + SS_CTL_REG); + + wait_for_completion_interruptible_timeout(&ss->flows[flow].complete, + msecs_to_jiffies(todo)); + if (ss->flows[flow].status == 0) { + dev_err(ss->dev, "DMA timeout for PRNG (size=%u)\n", todo); + err = -EFAULT; + } + /* Since cipher and hash use the linux/cryptoengine and that we have + * a cryptoengine per flow, we are sure that they will issue only one + * request per flow. + * Since the cryptoengine wait for completion before submitting a new + * one, the mlock could be left just after the final writel. + * But cryptoengine cannot handle crypto_rng, so we need to be sure + * nothing will use our flow. + * The easiest way is to grab mlock until the hardware end our requests. + * We could have used a per flow lock, but this would increase + * complexity. + * The drawback is that no request could be handled for the other flow. + */ + mutex_unlock(&ss->mlock); + + pm_runtime_put(ss->dev); + +err_pm: + dma_unmap_single(ss->dev, dma_dst, todo, DMA_FROM_DEVICE); +err_iv: + dma_unmap_single(ss->dev, dma_iv, ctx->slen, DMA_TO_DEVICE); + + if (!err) { + memcpy(dst, d, dlen); + /* Update seed */ + memcpy(ctx->seed, d + dlen, ctx->slen); + } + memzero_explicit(d, todo); + kfree(d); + + return err; +} diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss.h b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss.h index 0405767f1f7e..1a66457f4a20 100644 --- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss.h +++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss.h @@ -8,10 +8,16 @@ #include <crypto/aes.h> #include <crypto/des.h> #include <crypto/engine.h> +#include <crypto/rng.h> #include <crypto/skcipher.h> #include <linux/atomic.h> #include <linux/debugfs.h> #include <linux/crypto.h> +#include <crypto/internal/hash.h> +#include <crypto/md5.h> +#include <crypto/sha.h> + +#define SS_START 1 #define SS_ENCRYPTION 0 #define SS_DECRYPTION BIT(6) @@ -19,6 +25,11 @@ #define SS_ALG_AES 0 #define SS_ALG_DES (1 << 2) #define SS_ALG_3DES (2 << 2) +#define SS_ALG_MD5 (3 << 2) +#define SS_ALG_PRNG (4 << 2) +#define SS_ALG_SHA1 (6 << 2) +#define SS_ALG_SHA224 (7 << 2) +#define SS_ALG_SHA256 (8 << 2) #define SS_CTL_REG 0x00 #define SS_INT_CTL_REG 0x04 @@ -47,9 +58,17 @@ #define SS_OP_ECB 0 #define SS_OP_CBC (1 << 13) +#define SS_ID_HASH_MD5 0 +#define SS_ID_HASH_SHA1 1 +#define SS_ID_HASH_SHA224 2 +#define SS_ID_HASH_SHA256 3 +#define SS_ID_HASH_MAX 4 + #define SS_FLOW0 BIT(30) #define SS_FLOW1 BIT(31) +#define SS_PRNG_CONTINUE BIT(18) + #define MAX_SG 8 #define MAXFLOW 2 @@ -59,6 +78,9 @@ #define SS_DIE_ID_SHIFT 20 #define SS_DIE_ID_MASK 0x07 +#define PRNG_DATA_SIZE (160 / 8) +#define PRNG_SEED_SIZE DIV_ROUND_UP(175, 8) + /* * struct ss_clock - Describe clocks used by sun8i-ss * @name: Name of clock needed by this variant @@ -75,11 +97,14 @@ struct ss_clock { * struct ss_variant - Describe SS capability for each variant hardware * @alg_cipher: list of supported ciphers. for each SS_ID_ this will give the * coresponding SS_ALG_XXX value + * @alg_hash: list of supported hashes. for each SS_ID_ this will give the + * corresponding SS_ALG_XXX value * @op_mode: list of supported block modes - * @ss_clks! list of clock needed by this variant + * @ss_clks: list of clock needed by this variant */ struct ss_variant { char alg_cipher[SS_ID_CIPHER_MAX]; + char alg_hash[SS_ID_HASH_MAX]; u32 op_mode[SS_ID_OP_MAX]; struct ss_clock ss_clks[SS_MAX_CLOCKS]; }; @@ -170,6 +195,8 @@ struct sun8i_cipher_req_ctx { * @keylen: len of the key * @ss: pointer to the private data of driver handling this TFM * @fallback_tfm: pointer to the fallback TFM + * + * enginectx must be the first element */ struct sun8i_cipher_tfm_ctx { struct crypto_engine_ctx enginectx; @@ -180,6 +207,46 @@ struct sun8i_cipher_tfm_ctx { }; /* + * struct sun8i_ss_prng_ctx - context for PRNG TFM + * @seed: The seed to use + * @slen: The size of the seed + */ +struct sun8i_ss_rng_tfm_ctx { + void *seed; + unsigned int slen; +}; + +/* + * struct sun8i_ss_hash_tfm_ctx - context for an ahash TFM + * @enginectx: crypto_engine used by this TFM + * @fallback_tfm: pointer to the fallback TFM + * @ss: pointer to the private data of driver handling this TFM + * + * enginectx must be the first element + */ +struct sun8i_ss_hash_tfm_ctx { + struct crypto_engine_ctx enginectx; + struct crypto_ahash *fallback_tfm; + struct sun8i_ss_dev *ss; +}; + +/* + * struct sun8i_ss_hash_reqctx - context for an ahash request + * @t_src: list of DMA address and size for source SGs + * @t_dst: list of DMA address and size for destination SGs + * @fallback_req: pre-allocated fallback request + * @method: the register value for the algorithm used by this request + * @flow: the flow to use for this request + */ +struct sun8i_ss_hash_reqctx { + struct sginfo t_src[MAX_SG]; + struct sginfo t_dst[MAX_SG]; + struct ahash_request fallback_req; + u32 method; + int flow; +}; + +/* * struct sun8i_ss_alg_template - crypto_alg template * @type: the CRYPTO_ALG_TYPE for this template * @ss_algo_id: the SS_ID for this template @@ -189,6 +256,7 @@ struct sun8i_cipher_tfm_ctx { * @alg: one of sub struct must be used * @stat_req: number of request done on this template * @stat_fb: number of request which has fallbacked + * @stat_bytes: total data size done by this template */ struct sun8i_ss_alg_template { u32 type; @@ -197,10 +265,13 @@ struct sun8i_ss_alg_template { struct sun8i_ss_dev *ss; union { struct skcipher_alg skcipher; + struct rng_alg rng; + struct ahash_alg hash; } alg; #ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG unsigned long stat_req; unsigned long stat_fb; + unsigned long stat_bytes; #endif }; @@ -218,3 +289,19 @@ int sun8i_ss_skencrypt(struct skcipher_request *areq); int sun8i_ss_get_engine_number(struct sun8i_ss_dev *ss); int sun8i_ss_run_task(struct sun8i_ss_dev *ss, struct sun8i_cipher_req_ctx *rctx, const char *name); +int sun8i_ss_prng_generate(struct crypto_rng *tfm, const u8 *src, + unsigned int slen, u8 *dst, unsigned int dlen); +int sun8i_ss_prng_seed(struct crypto_rng *tfm, const u8 *seed, unsigned int slen); +int sun8i_ss_prng_init(struct crypto_tfm *tfm); +void sun8i_ss_prng_exit(struct crypto_tfm *tfm); + +int sun8i_ss_hash_crainit(struct crypto_tfm *tfm); +void sun8i_ss_hash_craexit(struct crypto_tfm *tfm); +int sun8i_ss_hash_init(struct ahash_request *areq); +int sun8i_ss_hash_export(struct ahash_request *areq, void *out); +int sun8i_ss_hash_import(struct ahash_request *areq, const void *in); +int sun8i_ss_hash_final(struct ahash_request *areq); +int sun8i_ss_hash_update(struct ahash_request *areq); +int sun8i_ss_hash_finup(struct ahash_request *areq); +int sun8i_ss_hash_digest(struct ahash_request *areq); +int sun8i_ss_hash_run(struct crypto_engine *engine, void *breq); diff --git a/drivers/crypto/amcc/crypto4xx_alg.c b/drivers/crypto/amcc/crypto4xx_alg.c index f7fc0c464125..7729a637fb02 100644 --- a/drivers/crypto/amcc/crypto4xx_alg.c +++ b/drivers/crypto/amcc/crypto4xx_alg.c @@ -55,7 +55,7 @@ static void set_dynamic_sa_command_1(struct dynamic_sa_ctl *sa, u32 cm, sa->sa_command_1.w = 0; sa->sa_command_1.bf.crypto_mode31 = (cm & 4) >> 2; sa->sa_command_1.bf.crypto_mode9_8 = cm & 3; - sa->sa_command_1.bf.feedback_mode = cfb, + sa->sa_command_1.bf.feedback_mode = cfb; sa->sa_command_1.bf.sa_rev = 1; sa->sa_command_1.bf.hmac_muting = hmac_mc; sa->sa_command_1.bf.extended_seq_num = esn; diff --git a/drivers/crypto/amcc/crypto4xx_core.h b/drivers/crypto/amcc/crypto4xx_core.h index 6b6841359190..a4e25b46cd0a 100644 --- a/drivers/crypto/amcc/crypto4xx_core.h +++ b/drivers/crypto/amcc/crypto4xx_core.h @@ -15,6 +15,7 @@ #include <linux/ratelimit.h> #include <linux/mutex.h> +#include <linux/scatterlist.h> #include <crypto/internal/hash.h> #include <crypto/internal/aead.h> #include <crypto/internal/rng.h> diff --git a/drivers/crypto/amlogic/amlogic-gxl-cipher.c b/drivers/crypto/amlogic/amlogic-gxl-cipher.c index d93210726697..8b5e07316352 100644 --- a/drivers/crypto/amlogic/amlogic-gxl-cipher.c +++ b/drivers/crypto/amlogic/amlogic-gxl-cipher.c @@ -99,7 +99,7 @@ static int meson_cipher(struct skcipher_request *areq) unsigned int keyivlen, ivsize, offset, tloffset; dma_addr_t phykeyiv; void *backup_iv = NULL, *bkeyiv; - __le32 v; + u32 v; algt = container_of(alg, struct meson_alg_template, alg.skcipher); @@ -340,10 +340,7 @@ void meson_cipher_exit(struct crypto_tfm *tfm) { struct meson_cipher_tfm_ctx *op = crypto_tfm_ctx(tfm); - if (op->key) { - memzero_explicit(op->key, op->keylen); - kfree(op->key); - } + kfree_sensitive(op->key); crypto_free_skcipher(op->fallback_tfm); } @@ -367,10 +364,7 @@ int meson_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, dev_dbg(mc->dev, "ERROR: Invalid keylen %u\n", keylen); return -EINVAL; } - if (op->key) { - memzero_explicit(op->key, op->keylen); - kfree(op->key); - } + kfree_sensitive(op->key); op->keylen = keylen; op->key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA); if (!op->key) diff --git a/drivers/crypto/amlogic/amlogic-gxl-core.c b/drivers/crypto/amlogic/amlogic-gxl-core.c index 466552acbbbb..5bbeff433c8c 100644 --- a/drivers/crypto/amlogic/amlogic-gxl-core.c +++ b/drivers/crypto/amlogic/amlogic-gxl-core.c @@ -98,7 +98,7 @@ static struct meson_alg_template mc_algs[] = { }; #ifdef CONFIG_CRYPTO_DEV_AMLOGIC_GXL_DEBUG -static int meson_dbgfs_read(struct seq_file *seq, void *v) +static int meson_debugfs_show(struct seq_file *seq, void *v) { struct meson_dev *mc = seq->private; int i; @@ -118,19 +118,7 @@ static int meson_dbgfs_read(struct seq_file *seq, void *v) } return 0; } - -static int meson_dbgfs_open(struct inode *inode, struct file *file) -{ - return single_open(file, meson_dbgfs_read, inode->i_private); -} - -static const struct file_operations meson_debugfs_fops = { - .owner = THIS_MODULE, - .open = meson_dbgfs_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; +DEFINE_SHOW_ATTRIBUTE(meson_debugfs); #endif static void meson_free_chanlist(struct meson_dev *mc, int i) diff --git a/drivers/crypto/atmel-aes.c b/drivers/crypto/atmel-aes.c index a6e14491e080..b1d286004295 100644 --- a/drivers/crypto/atmel-aes.c +++ b/drivers/crypto/atmel-aes.c @@ -1539,7 +1539,7 @@ static int atmel_aes_gcm_length(struct atmel_aes_dev *dd) /* Write incr32(J0) into IV. */ j0_lsw = j0[3]; - j0[3] = cpu_to_be32(be32_to_cpu(j0[3]) + 1); + be32_add_cpu(&j0[3], 1); atmel_aes_write_block(dd, AES_IVR(0), j0); j0[3] = j0_lsw; diff --git a/drivers/crypto/atmel-tdes.c b/drivers/crypto/atmel-tdes.c index ed40dbb98c6b..4d63cb13a54f 100644 --- a/drivers/crypto/atmel-tdes.c +++ b/drivers/crypto/atmel-tdes.c @@ -912,7 +912,7 @@ static void atmel_tdes_skcipher_alg_init(struct skcipher_alg *alg) { alg->base.cra_priority = ATMEL_TDES_PRIORITY; alg->base.cra_flags = CRYPTO_ALG_ASYNC; - alg->base.cra_ctxsize = sizeof(struct atmel_tdes_ctx), + alg->base.cra_ctxsize = sizeof(struct atmel_tdes_ctx); alg->base.cra_module = THIS_MODULE; alg->init = atmel_tdes_init_tfm; diff --git a/drivers/crypto/bcm/cipher.c b/drivers/crypto/bcm/cipher.c index 8a7fa1ae1ade..50d169e61b41 100644 --- a/drivers/crypto/bcm/cipher.c +++ b/drivers/crypto/bcm/cipher.c @@ -165,10 +165,6 @@ spu_skcipher_rx_sg_create(struct brcm_message *mssg, return -EFAULT; } - if (ctx->cipher.alg == CIPHER_ALG_RC4) - /* Add buffer to catch 260-byte SUPDT field for RC4 */ - sg_set_buf(sg++, rctx->msg_buf.c.supdt_tweak, SPU_SUPDT_LEN); - if (stat_pad_len) sg_set_buf(sg++, rctx->msg_buf.rx_stat_pad, stat_pad_len); @@ -317,7 +313,6 @@ static int handle_skcipher_req(struct iproc_reqctx_s *rctx) u8 local_iv_ctr[MAX_IV_SIZE]; u32 stat_pad_len; /* num bytes to align status field */ u32 pad_len; /* total length of all padding */ - bool update_key = false; struct brcm_message *mssg; /* mailbox message */ /* number of entries in src and dst sg in mailbox message. */ @@ -391,28 +386,6 @@ static int handle_skcipher_req(struct iproc_reqctx_s *rctx) } } - if (ctx->cipher.alg == CIPHER_ALG_RC4) { - rx_frag_num++; - if (chunk_start) { - /* - * for non-first RC4 chunks, use SUPDT from previous - * response as key for this chunk. - */ - cipher_parms.key_buf = rctx->msg_buf.c.supdt_tweak; - update_key = true; - cipher_parms.type = CIPHER_TYPE_UPDT; - } else if (!rctx->is_encrypt) { - /* - * First RC4 chunk. For decrypt, key in pre-built msg - * header may have been changed if encrypt required - * multiple chunks. So revert the key to the - * ctx->enckey value. - */ - update_key = true; - cipher_parms.type = CIPHER_TYPE_INIT; - } - } - if (ctx->max_payload == SPU_MAX_PAYLOAD_INF) flow_log("max_payload infinite\n"); else @@ -425,14 +398,9 @@ static int handle_skcipher_req(struct iproc_reqctx_s *rctx) memcpy(rctx->msg_buf.bcm_spu_req_hdr, ctx->bcm_spu_req_hdr, sizeof(rctx->msg_buf.bcm_spu_req_hdr)); - /* - * Pass SUPDT field as key. Key field in finish() call is only used - * when update_key has been set above for RC4. Will be ignored in - * all other cases. - */ spu->spu_cipher_req_finish(rctx->msg_buf.bcm_spu_req_hdr + BCM_HDR_LEN, ctx->spu_req_hdr_len, !(rctx->is_encrypt), - &cipher_parms, update_key, chunksize); + &cipher_parms, chunksize); atomic64_add(chunksize, &iproc_priv.bytes_out); @@ -527,9 +495,6 @@ static void handle_skcipher_resp(struct iproc_reqctx_s *rctx) __func__, rctx->total_received, payload_len); dump_sg(req->dst, rctx->total_received, payload_len); - if (ctx->cipher.alg == CIPHER_ALG_RC4) - packet_dump(" supdt ", rctx->msg_buf.c.supdt_tweak, - SPU_SUPDT_LEN); rctx->total_received += payload_len; if (rctx->total_received == rctx->total_todo) { @@ -1853,26 +1818,6 @@ static int aes_setkey(struct crypto_skcipher *cipher, const u8 *key, return 0; } -static int rc4_setkey(struct crypto_skcipher *cipher, const u8 *key, - unsigned int keylen) -{ - struct iproc_ctx_s *ctx = crypto_skcipher_ctx(cipher); - int i; - - ctx->enckeylen = ARC4_MAX_KEY_SIZE + ARC4_STATE_SIZE; - - ctx->enckey[0] = 0x00; /* 0x00 */ - ctx->enckey[1] = 0x00; /* i */ - ctx->enckey[2] = 0x00; /* 0x00 */ - ctx->enckey[3] = 0x00; /* j */ - for (i = 0; i < ARC4_MAX_KEY_SIZE; i++) - ctx->enckey[i + ARC4_STATE_SIZE] = key[i % keylen]; - - ctx->cipher_type = CIPHER_TYPE_INIT; - - return 0; -} - static int skcipher_setkey(struct crypto_skcipher *cipher, const u8 *key, unsigned int keylen) { @@ -1895,9 +1840,6 @@ static int skcipher_setkey(struct crypto_skcipher *cipher, const u8 *key, case CIPHER_ALG_AES: err = aes_setkey(cipher, key, keylen); break; - case CIPHER_ALG_RC4: - err = rc4_setkey(cipher, key, keylen); - break; default: pr_err("%s() Error: unknown cipher alg\n", __func__); err = -EINVAL; @@ -1905,11 +1847,9 @@ static int skcipher_setkey(struct crypto_skcipher *cipher, const u8 *key, if (err) return err; - /* RC4 already populated ctx->enkey */ - if (ctx->cipher.alg != CIPHER_ALG_RC4) { - memcpy(ctx->enckey, key, keylen); - ctx->enckeylen = keylen; - } + memcpy(ctx->enckey, key, keylen); + ctx->enckeylen = keylen; + /* SPU needs XTS keys in the reverse order the crypto API presents */ if ((ctx->cipher.alg == CIPHER_ALG_AES) && (ctx->cipher.mode == CIPHER_MODE_XTS)) { @@ -2872,9 +2812,6 @@ static int aead_authenc_setkey(struct crypto_aead *cipher, goto badkey; } break; - case CIPHER_ALG_RC4: - ctx->cipher_type = CIPHER_TYPE_INIT; - break; default: pr_err("%s() Error: Unknown cipher alg\n", __func__); return -EINVAL; @@ -2930,7 +2867,6 @@ static int aead_gcm_ccm_setkey(struct crypto_aead *cipher, ctx->enckeylen = keylen; ctx->authkeylen = 0; - memcpy(ctx->enckey, key, ctx->enckeylen); switch (ctx->enckeylen) { case AES_KEYSIZE_128: @@ -2946,6 +2882,8 @@ static int aead_gcm_ccm_setkey(struct crypto_aead *cipher, goto badkey; } + memcpy(ctx->enckey, key, ctx->enckeylen); + flow_log(" enckeylen:%u authkeylen:%u\n", ctx->enckeylen, ctx->authkeylen); flow_dump(" enc: ", ctx->enckey, ctx->enckeylen); @@ -3000,6 +2938,10 @@ static int aead_gcm_esp_setkey(struct crypto_aead *cipher, struct iproc_ctx_s *ctx = crypto_aead_ctx(cipher); flow_log("%s\n", __func__); + + if (keylen < GCM_ESP_SALT_SIZE) + return -EINVAL; + ctx->salt_len = GCM_ESP_SALT_SIZE; ctx->salt_offset = GCM_ESP_SALT_OFFSET; memcpy(ctx->salt, key + keylen - GCM_ESP_SALT_SIZE, GCM_ESP_SALT_SIZE); @@ -3028,6 +2970,10 @@ static int rfc4543_gcm_esp_setkey(struct crypto_aead *cipher, struct iproc_ctx_s *ctx = crypto_aead_ctx(cipher); flow_log("%s\n", __func__); + + if (keylen < GCM_ESP_SALT_SIZE) + return -EINVAL; + ctx->salt_len = GCM_ESP_SALT_SIZE; ctx->salt_offset = GCM_ESP_SALT_OFFSET; memcpy(ctx->salt, key + keylen - GCM_ESP_SALT_SIZE, GCM_ESP_SALT_SIZE); @@ -3057,6 +3003,10 @@ static int aead_ccm_esp_setkey(struct crypto_aead *cipher, struct iproc_ctx_s *ctx = crypto_aead_ctx(cipher); flow_log("%s\n", __func__); + + if (keylen < CCM_ESP_SALT_SIZE) + return -EINVAL; + ctx->salt_len = CCM_ESP_SALT_SIZE; ctx->salt_offset = CCM_ESP_SALT_OFFSET; memcpy(ctx->salt, key + keylen - CCM_ESP_SALT_SIZE, CCM_ESP_SALT_SIZE); @@ -3606,25 +3556,6 @@ static struct iproc_alg_s driver_algs[] = { { .type = CRYPTO_ALG_TYPE_SKCIPHER, .alg.skcipher = { - .base.cra_name = "ecb(arc4)", - .base.cra_driver_name = "ecb-arc4-iproc", - .base.cra_blocksize = ARC4_BLOCK_SIZE, - .min_keysize = ARC4_MIN_KEY_SIZE, - .max_keysize = ARC4_MAX_KEY_SIZE, - .ivsize = 0, - }, - .cipher_info = { - .alg = CIPHER_ALG_RC4, - .mode = CIPHER_MODE_NONE, - }, - .auth_info = { - .alg = HASH_ALG_NONE, - .mode = HASH_MODE_NONE, - }, - }, - { - .type = CRYPTO_ALG_TYPE_SKCIPHER, - .alg.skcipher = { .base.cra_name = "ofb(des)", .base.cra_driver_name = "ofb-des-iproc", .base.cra_blocksize = DES_BLOCK_SIZE, @@ -4526,15 +4457,9 @@ static void spu_counters_init(void) static int spu_register_skcipher(struct iproc_alg_s *driver_alg) { - struct spu_hw *spu = &iproc_priv.spu; struct skcipher_alg *crypto = &driver_alg->alg.skcipher; int err; - /* SPU2 does not support RC4 */ - if ((driver_alg->cipher_info.alg == CIPHER_ALG_RC4) && - (spu->spu_type == SPU_TYPE_SPU2)) - return 0; - crypto->base.cra_module = THIS_MODULE; crypto->base.cra_priority = cipher_pri; crypto->base.cra_alignmask = 0; diff --git a/drivers/crypto/bcm/cipher.h b/drivers/crypto/bcm/cipher.h index b6d83e3aa46c..035c8389cb3d 100644 --- a/drivers/crypto/bcm/cipher.h +++ b/drivers/crypto/bcm/cipher.h @@ -388,7 +388,6 @@ struct spu_hw { u16 spu_req_hdr_len, unsigned int is_inbound, struct spu_cipher_parms *cipher_parms, - bool update_key, unsigned int data_size); void (*spu_request_pad)(u8 *pad_start, u32 gcm_padding, u32 hash_pad_len, enum hash_alg auth_alg, diff --git a/drivers/crypto/bcm/spu.c b/drivers/crypto/bcm/spu.c index e7562e9bf396..fe126f95c702 100644 --- a/drivers/crypto/bcm/spu.c +++ b/drivers/crypto/bcm/spu.c @@ -222,10 +222,6 @@ void spum_dump_msg_hdr(u8 *buf, unsigned int buf_len) cipher_key_len = 24; name = "3DES"; break; - case CIPHER_ALG_RC4: - cipher_key_len = 260; - name = "ARC4"; - break; case CIPHER_ALG_AES: switch (cipher_type) { case CIPHER_TYPE_AES128: @@ -919,21 +915,16 @@ u16 spum_cipher_req_init(u8 *spu_hdr, struct spu_cipher_parms *cipher_parms) * @spu_req_hdr_len: Length in bytes of the SPU request header * @isInbound: 0 encrypt, 1 decrypt * @cipher_parms: Parameters describing cipher operation to be performed - * @update_key: If true, rewrite the cipher key in SCTX * @data_size: Length of the data in the BD field * * Assumes much of the header was already filled in at setkey() time in * spum_cipher_req_init(). - * spum_cipher_req_init() fills in the encryption key. For RC4, when submitting - * a request for a non-first chunk, we use the 260-byte SUPDT field from the - * previous response as the key. update_key is true for this case. Unused in all - * other cases. + * spum_cipher_req_init() fills in the encryption key. */ void spum_cipher_req_finish(u8 *spu_hdr, u16 spu_req_hdr_len, unsigned int is_inbound, struct spu_cipher_parms *cipher_parms, - bool update_key, unsigned int data_size) { struct SPUHEADER *spuh; @@ -948,11 +939,6 @@ void spum_cipher_req_finish(u8 *spu_hdr, flow_log(" in: %u\n", is_inbound); flow_log(" cipher alg: %u, cipher_type: %u\n", cipher_parms->alg, cipher_parms->type); - if (update_key) { - flow_log(" cipher key len: %u\n", cipher_parms->key_len); - flow_dump(" key: ", cipher_parms->key_buf, - cipher_parms->key_len); - } /* * In XTS mode, API puts "i" parameter (block tweak) in IV. For @@ -981,13 +967,6 @@ void spum_cipher_req_finish(u8 *spu_hdr, else cipher_bits &= ~CIPHER_INBOUND; - /* update encryption key for RC4 on non-first chunk */ - if (update_key) { - spuh->sa.cipher_flags |= - cipher_parms->type << CIPHER_TYPE_SHIFT; - memcpy(spuh + 1, cipher_parms->key_buf, cipher_parms->key_len); - } - if (cipher_parms->alg && cipher_parms->iv_buf && cipher_parms->iv_len) /* cipher iv provided so put it in here */ memcpy(bdesc_ptr - cipher_parms->iv_len, cipher_parms->iv_buf, diff --git a/drivers/crypto/bcm/spu.h b/drivers/crypto/bcm/spu.h index b247bc5b9354..dd132389bcaa 100644 --- a/drivers/crypto/bcm/spu.h +++ b/drivers/crypto/bcm/spu.h @@ -251,7 +251,6 @@ void spum_cipher_req_finish(u8 *spu_hdr, u16 spu_req_hdr_len, unsigned int is_inbound, struct spu_cipher_parms *cipher_parms, - bool update_key, unsigned int data_size); void spum_request_pad(u8 *pad_start, diff --git a/drivers/crypto/bcm/spu2.c b/drivers/crypto/bcm/spu2.c index 59abb5ecefa4..c860ffb0b4c3 100644 --- a/drivers/crypto/bcm/spu2.c +++ b/drivers/crypto/bcm/spu2.c @@ -1170,21 +1170,16 @@ u16 spu2_cipher_req_init(u8 *spu_hdr, struct spu_cipher_parms *cipher_parms) * @spu_req_hdr_len: Length in bytes of the SPU request header * @isInbound: 0 encrypt, 1 decrypt * @cipher_parms: Parameters describing cipher operation to be performed - * @update_key: If true, rewrite the cipher key in SCTX * @data_size: Length of the data in the BD field * * Assumes much of the header was already filled in at setkey() time in * spu_cipher_req_init(). - * spu_cipher_req_init() fills in the encryption key. For RC4, when submitting a - * request for a non-first chunk, we use the 260-byte SUPDT field from the - * previous response as the key. update_key is true for this case. Unused in all - * other cases. + * spu_cipher_req_init() fills in the encryption key. */ void spu2_cipher_req_finish(u8 *spu_hdr, u16 spu_req_hdr_len, unsigned int is_inbound, struct spu_cipher_parms *cipher_parms, - bool update_key, unsigned int data_size) { struct SPU2_FMD *fmd; @@ -1196,11 +1191,6 @@ void spu2_cipher_req_finish(u8 *spu_hdr, flow_log(" in: %u\n", is_inbound); flow_log(" cipher alg: %u, cipher_type: %u\n", cipher_parms->alg, cipher_parms->type); - if (update_key) { - flow_log(" cipher key len: %u\n", cipher_parms->key_len); - flow_dump(" key: ", cipher_parms->key_buf, - cipher_parms->key_len); - } flow_log(" iv len: %d\n", cipher_parms->iv_len); flow_dump(" iv: ", cipher_parms->iv_buf, cipher_parms->iv_len); flow_log(" data_size: %u\n", data_size); diff --git a/drivers/crypto/bcm/spu2.h b/drivers/crypto/bcm/spu2.h index 03af6c38df7f..6e666bfb3cfc 100644 --- a/drivers/crypto/bcm/spu2.h +++ b/drivers/crypto/bcm/spu2.h @@ -200,7 +200,6 @@ void spu2_cipher_req_finish(u8 *spu_hdr, u16 spu_req_hdr_len, unsigned int is_inbound, struct spu_cipher_parms *cipher_parms, - bool update_key, unsigned int data_size); void spu2_request_pad(u8 *pad_start, u32 gcm_padding, u32 hash_pad_len, enum hash_alg auth_alg, enum hash_mode auth_mode, diff --git a/drivers/crypto/caam/Kconfig b/drivers/crypto/caam/Kconfig index bc35aa0ec07a..84ea7cba5ee5 100644 --- a/drivers/crypto/caam/Kconfig +++ b/drivers/crypto/caam/Kconfig @@ -101,6 +101,7 @@ config CRYPTO_DEV_FSL_CAAM_CRYPTO_API select CRYPTO_AUTHENC select CRYPTO_SKCIPHER select CRYPTO_LIB_DES + select CRYPTO_XTS help Selecting this will offload crypto for users of the scatterlist crypto API (such as the linux native IPSec @@ -114,6 +115,7 @@ config CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI select CRYPTO_AUTHENC select CRYPTO_SKCIPHER select CRYPTO_DES + select CRYPTO_XTS help Selecting this will use CAAM Queue Interface (QI) for sending & receiving crypto jobs to/from CAAM. This gives better performance @@ -165,6 +167,7 @@ config CRYPTO_DEV_FSL_DPAA2_CAAM select CRYPTO_AEAD select CRYPTO_HASH select CRYPTO_DES + select CRYPTO_XTS help CAAM driver for QorIQ Data Path Acceleration Architecture 2. It handles DPSECI DPAA2 objects that sit on the Management Complex diff --git a/drivers/crypto/caam/Makefile b/drivers/crypto/caam/Makefile index 68d5cc0f28e2..3570286eb9ce 100644 --- a/drivers/crypto/caam/Makefile +++ b/drivers/crypto/caam/Makefile @@ -27,6 +27,8 @@ ifneq ($(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI),) ccflags-y += -DCONFIG_CAAM_QI endif +caam-$(CONFIG_DEBUG_FS) += debugfs.o + obj-$(CONFIG_CRYPTO_DEV_FSL_DPAA2_CAAM) += dpaa2_caam.o dpaa2_caam-y := caamalg_qi2.o dpseci.o diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index 91feda5b63f6..cf5bd7666dfc 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -57,6 +57,8 @@ #include "key_gen.h" #include "caamalg_desc.h" #include <crypto/engine.h> +#include <crypto/xts.h> +#include <asm/unaligned.h> /* * crypto alg @@ -114,10 +116,13 @@ struct caam_ctx { struct alginfo adata; struct alginfo cdata; unsigned int authsize; + bool xts_key_fallback; + struct crypto_skcipher *fallback; }; struct caam_skcipher_req_ctx { struct skcipher_edesc *edesc; + struct skcipher_request fallback_req; }; struct caam_aead_req_ctx { @@ -829,11 +834,23 @@ static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, { struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); struct device *jrdev = ctx->jrdev; + struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); u32 *desc; + int err; - if (keylen != 2 * AES_MIN_KEY_SIZE && keylen != 2 * AES_MAX_KEY_SIZE) { + err = xts_verify_key(skcipher, key, keylen); + if (err) { dev_dbg(jrdev, "key size mismatch\n"); - return -EINVAL; + return err; + } + + if (keylen != 2 * AES_KEYSIZE_128 && keylen != 2 * AES_KEYSIZE_256) + ctx->xts_key_fallback = true; + + if (ctrlpriv->era <= 8 || ctx->xts_key_fallback) { + err = crypto_skcipher_setkey(ctx->fallback, key, keylen); + if (err) + return err; } ctx->cdata.keylen = keylen; @@ -1755,6 +1772,14 @@ static int skcipher_do_one_req(struct crypto_engine *engine, void *areq) return ret; } +static inline bool xts_skcipher_ivsize(struct skcipher_request *req) +{ + struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); + unsigned int ivsize = crypto_skcipher_ivsize(skcipher); + + return !!get_unaligned((u64 *)(req->iv + (ivsize / 2))); +} + static inline int skcipher_crypt(struct skcipher_request *req, bool encrypt) { struct skcipher_edesc *edesc; @@ -1762,12 +1787,34 @@ static inline int skcipher_crypt(struct skcipher_request *req, bool encrypt) struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); struct device *jrdev = ctx->jrdev; struct caam_drv_private_jr *jrpriv = dev_get_drvdata(jrdev); + struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); u32 *desc; int ret = 0; - if (!req->cryptlen) + /* + * XTS is expected to return an error even for input length = 0 + * Note that the case input length < block size will be caught during + * HW offloading and return an error. + */ + if (!req->cryptlen && !ctx->fallback) return 0; + if (ctx->fallback && ((ctrlpriv->era <= 8 && xts_skcipher_ivsize(req)) || + ctx->xts_key_fallback)) { + struct caam_skcipher_req_ctx *rctx = skcipher_request_ctx(req); + + skcipher_request_set_tfm(&rctx->fallback_req, ctx->fallback); + skcipher_request_set_callback(&rctx->fallback_req, + req->base.flags, + req->base.complete, + req->base.data); + skcipher_request_set_crypt(&rctx->fallback_req, req->src, + req->dst, req->cryptlen, req->iv); + + return encrypt ? crypto_skcipher_encrypt(&rctx->fallback_req) : + crypto_skcipher_decrypt(&rctx->fallback_req); + } + /* allocate extended descriptor */ edesc = skcipher_edesc_alloc(req, DESC_JOB_IO_LEN * CAAM_CMD_SZ); if (IS_ERR(edesc)) @@ -1905,6 +1952,7 @@ static struct caam_skcipher_alg driver_algs[] = { .base = { .cra_name = "xts(aes)", .cra_driver_name = "xts-aes-caam", + .cra_flags = CRYPTO_ALG_NEED_FALLBACK, .cra_blocksize = AES_BLOCK_SIZE, }, .setkey = xts_skcipher_setkey, @@ -3344,13 +3392,35 @@ static int caam_cra_init(struct crypto_skcipher *tfm) struct caam_skcipher_alg *caam_alg = container_of(alg, typeof(*caam_alg), skcipher); struct caam_ctx *ctx = crypto_skcipher_ctx(tfm); - - crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_skcipher_req_ctx)); + u32 alg_aai = caam_alg->caam.class1_alg_type & OP_ALG_AAI_MASK; + int ret = 0; ctx->enginectx.op.do_one_request = skcipher_do_one_req; - return caam_init_common(crypto_skcipher_ctx(tfm), &caam_alg->caam, - false); + if (alg_aai == OP_ALG_AAI_XTS) { + const char *tfm_name = crypto_tfm_alg_name(&tfm->base); + struct crypto_skcipher *fallback; + + fallback = crypto_alloc_skcipher(tfm_name, 0, + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(fallback)) { + dev_err(ctx->jrdev, "Failed to allocate %s fallback: %ld\n", + tfm_name, PTR_ERR(fallback)); + return PTR_ERR(fallback); + } + + ctx->fallback = fallback; + crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_skcipher_req_ctx) + + crypto_skcipher_reqsize(fallback)); + } else { + crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_skcipher_req_ctx)); + } + + ret = caam_init_common(ctx, &caam_alg->caam, false); + if (ret && ctx->fallback) + crypto_free_skcipher(ctx->fallback); + + return ret; } static int caam_aead_init(struct crypto_aead *tfm) @@ -3378,7 +3448,11 @@ static void caam_exit_common(struct caam_ctx *ctx) static void caam_cra_exit(struct crypto_skcipher *tfm) { - caam_exit_common(crypto_skcipher_ctx(tfm)); + struct caam_ctx *ctx = crypto_skcipher_ctx(tfm); + + if (ctx->fallback) + crypto_free_skcipher(ctx->fallback); + caam_exit_common(ctx); } static void caam_aead_exit(struct crypto_aead *tfm) @@ -3412,8 +3486,8 @@ static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg) alg->base.cra_module = THIS_MODULE; alg->base.cra_priority = CAAM_CRA_PRIORITY; alg->base.cra_ctxsize = sizeof(struct caam_ctx); - alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | - CRYPTO_ALG_KERN_DRIVER_ONLY; + alg->base.cra_flags |= (CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY); alg->init = caam_cra_init; alg->exit = caam_cra_exit; diff --git a/drivers/crypto/caam/caamalg_desc.c b/drivers/crypto/caam/caamalg_desc.c index d6c58184bb57..7571e1ac913b 100644 --- a/drivers/crypto/caam/caamalg_desc.c +++ b/drivers/crypto/caam/caamalg_desc.c @@ -373,6 +373,7 @@ EXPORT_SYMBOL(cnstr_shdsc_aead_encap); * with OP_ALG_AAI_HMAC_PRECOMP. * @ivsize: initialization vector size * @icvsize: integrity check value (ICV) size (truncated or full) + * @geniv: whether to generate Encrypted Chain IV * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template * @nonce: pointer to rfc3686 nonce * @ctx1_iv_off: IV offset in CONTEXT1 register @@ -1550,13 +1551,14 @@ void cnstr_shdsc_xts_skcipher_encap(u32 * const desc, struct alginfo *cdata) set_jump_tgt_here(desc, key_jump_cmd); /* - * create sequence for loading the sector index - * Upper 8B of IV - will be used as sector index - * Lower 8B of IV - will be discarded + * create sequence for loading the sector index / 16B tweak value + * Lower 8B of IV - sector index / tweak lower half + * Upper 8B of IV - upper half of 16B tweak */ append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | (0x20 << LDST_OFFSET_SHIFT)); - append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP); + append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | + (0x30 << LDST_OFFSET_SHIFT)); /* Load operation */ append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | @@ -1565,9 +1567,11 @@ void cnstr_shdsc_xts_skcipher_encap(u32 * const desc, struct alginfo *cdata) /* Perform operation */ skcipher_append_src_dst(desc); - /* Store upper 8B of IV */ + /* Store lower 8B and upper 8B of IV */ append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | (0x20 << LDST_OFFSET_SHIFT)); + append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | + (0x30 << LDST_OFFSET_SHIFT)); print_hex_dump_debug("xts skcipher enc shdesc@" __stringify(__LINE__) ": ", DUMP_PREFIX_ADDRESS, 16, 4, @@ -1609,23 +1613,25 @@ void cnstr_shdsc_xts_skcipher_decap(u32 * const desc, struct alginfo *cdata) set_jump_tgt_here(desc, key_jump_cmd); /* - * create sequence for loading the sector index - * Upper 8B of IV - will be used as sector index - * Lower 8B of IV - will be discarded + * create sequence for loading the sector index / 16B tweak value + * Lower 8B of IV - sector index / tweak lower half + * Upper 8B of IV - upper half of 16B tweak */ append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | (0x20 << LDST_OFFSET_SHIFT)); - append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP); - + append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | + (0x30 << LDST_OFFSET_SHIFT)); /* Load operation */ append_dec_op1(desc, cdata->algtype); /* Perform operation */ skcipher_append_src_dst(desc); - /* Store upper 8B of IV */ + /* Store lower 8B and upper 8B of IV */ append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | (0x20 << LDST_OFFSET_SHIFT)); + append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | + (0x30 << LDST_OFFSET_SHIFT)); print_hex_dump_debug("xts skcipher dec shdesc@" __stringify(__LINE__) ": ", DUMP_PREFIX_ADDRESS, 16, 4, desc, diff --git a/drivers/crypto/caam/caamalg_qi.c b/drivers/crypto/caam/caamalg_qi.c index bb1c0106a95c..66f60d78bdc8 100644 --- a/drivers/crypto/caam/caamalg_qi.c +++ b/drivers/crypto/caam/caamalg_qi.c @@ -18,6 +18,8 @@ #include "qi.h" #include "jr.h" #include "caamalg_desc.h" +#include <crypto/xts.h> +#include <asm/unaligned.h> /* * crypto alg @@ -67,6 +69,12 @@ struct caam_ctx { struct device *qidev; spinlock_t lock; /* Protects multiple init of driver context */ struct caam_drv_ctx *drv_ctx[NUM_OP]; + bool xts_key_fallback; + struct crypto_skcipher *fallback; +}; + +struct caam_skcipher_req_ctx { + struct skcipher_request fallback_req; }; static int aead_set_sh_desc(struct crypto_aead *aead) @@ -725,11 +733,23 @@ static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, { struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); struct device *jrdev = ctx->jrdev; + struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); int ret = 0; + int err; - if (keylen != 2 * AES_MIN_KEY_SIZE && keylen != 2 * AES_MAX_KEY_SIZE) { + err = xts_verify_key(skcipher, key, keylen); + if (err) { dev_dbg(jrdev, "key size mismatch\n"); - return -EINVAL; + return err; + } + + if (keylen != 2 * AES_KEYSIZE_128 && keylen != 2 * AES_KEYSIZE_256) + ctx->xts_key_fallback = true; + + if (ctrlpriv->era <= 8 || ctx->xts_key_fallback) { + err = crypto_skcipher_setkey(ctx->fallback, key, keylen); + if (err) + return err; } ctx->cdata.keylen = keylen; @@ -1373,16 +1393,46 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, return edesc; } +static inline bool xts_skcipher_ivsize(struct skcipher_request *req) +{ + struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); + unsigned int ivsize = crypto_skcipher_ivsize(skcipher); + + return !!get_unaligned((u64 *)(req->iv + (ivsize / 2))); +} + static inline int skcipher_crypt(struct skcipher_request *req, bool encrypt) { struct skcipher_edesc *edesc; struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); + struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent); int ret; - if (!req->cryptlen) + /* + * XTS is expected to return an error even for input length = 0 + * Note that the case input length < block size will be caught during + * HW offloading and return an error. + */ + if (!req->cryptlen && !ctx->fallback) return 0; + if (ctx->fallback && ((ctrlpriv->era <= 8 && xts_skcipher_ivsize(req)) || + ctx->xts_key_fallback)) { + struct caam_skcipher_req_ctx *rctx = skcipher_request_ctx(req); + + skcipher_request_set_tfm(&rctx->fallback_req, ctx->fallback); + skcipher_request_set_callback(&rctx->fallback_req, + req->base.flags, + req->base.complete, + req->base.data); + skcipher_request_set_crypt(&rctx->fallback_req, req->src, + req->dst, req->cryptlen, req->iv); + + return encrypt ? crypto_skcipher_encrypt(&rctx->fallback_req) : + crypto_skcipher_decrypt(&rctx->fallback_req); + } + if (unlikely(caam_congested)) return -EAGAIN; @@ -1507,6 +1557,7 @@ static struct caam_skcipher_alg driver_algs[] = { .base = { .cra_name = "xts(aes)", .cra_driver_name = "xts-aes-caam-qi", + .cra_flags = CRYPTO_ALG_NEED_FALLBACK, .cra_blocksize = AES_BLOCK_SIZE, }, .setkey = xts_skcipher_setkey, @@ -2440,9 +2491,32 @@ static int caam_cra_init(struct crypto_skcipher *tfm) struct skcipher_alg *alg = crypto_skcipher_alg(tfm); struct caam_skcipher_alg *caam_alg = container_of(alg, typeof(*caam_alg), skcipher); + struct caam_ctx *ctx = crypto_skcipher_ctx(tfm); + u32 alg_aai = caam_alg->caam.class1_alg_type & OP_ALG_AAI_MASK; + int ret = 0; + + if (alg_aai == OP_ALG_AAI_XTS) { + const char *tfm_name = crypto_tfm_alg_name(&tfm->base); + struct crypto_skcipher *fallback; + + fallback = crypto_alloc_skcipher(tfm_name, 0, + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(fallback)) { + dev_err(ctx->jrdev, "Failed to allocate %s fallback: %ld\n", + tfm_name, PTR_ERR(fallback)); + return PTR_ERR(fallback); + } + + ctx->fallback = fallback; + crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_skcipher_req_ctx) + + crypto_skcipher_reqsize(fallback)); + } + + ret = caam_init_common(ctx, &caam_alg->caam, false); + if (ret && ctx->fallback) + crypto_free_skcipher(ctx->fallback); - return caam_init_common(crypto_skcipher_ctx(tfm), &caam_alg->caam, - false); + return ret; } static int caam_aead_init(struct crypto_aead *tfm) @@ -2468,7 +2542,11 @@ static void caam_exit_common(struct caam_ctx *ctx) static void caam_cra_exit(struct crypto_skcipher *tfm) { - caam_exit_common(crypto_skcipher_ctx(tfm)); + struct caam_ctx *ctx = crypto_skcipher_ctx(tfm); + + if (ctx->fallback) + crypto_free_skcipher(ctx->fallback); + caam_exit_common(ctx); } static void caam_aead_exit(struct crypto_aead *tfm) @@ -2502,8 +2580,8 @@ static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg) alg->base.cra_module = THIS_MODULE; alg->base.cra_priority = CAAM_CRA_PRIORITY; alg->base.cra_ctxsize = sizeof(struct caam_ctx); - alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | - CRYPTO_ALG_KERN_DRIVER_ONLY; + alg->base.cra_flags |= (CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY); alg->init = caam_cra_init; alg->exit = caam_cra_exit; diff --git a/drivers/crypto/caam/caamalg_qi2.c b/drivers/crypto/caam/caamalg_qi2.c index 66ae1d581168..98c1ff1744bb 100644 --- a/drivers/crypto/caam/caamalg_qi2.c +++ b/drivers/crypto/caam/caamalg_qi2.c @@ -19,6 +19,8 @@ #include <linux/fsl/mc.h> #include <soc/fsl/dpaa2-io.h> #include <soc/fsl/dpaa2-fd.h> +#include <crypto/xts.h> +#include <asm/unaligned.h> #define CAAM_CRA_PRIORITY 2000 @@ -59,7 +61,7 @@ struct caam_skcipher_alg { }; /** - * caam_ctx - per-session context + * struct caam_ctx - per-session context * @flc: Flow Contexts array * @key: [authentication key], encryption key * @flc_dma: I/O virtual addresses of the Flow Contexts @@ -80,6 +82,8 @@ struct caam_ctx { struct alginfo adata; struct alginfo cdata; unsigned int authsize; + bool xts_key_fallback; + struct crypto_skcipher *fallback; }; static void *dpaa2_caam_iova_to_virt(struct dpaa2_caam_priv *priv, @@ -1054,12 +1058,24 @@ static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, { struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); struct device *dev = ctx->dev; + struct dpaa2_caam_priv *priv = dev_get_drvdata(dev); struct caam_flc *flc; u32 *desc; + int err; - if (keylen != 2 * AES_MIN_KEY_SIZE && keylen != 2 * AES_MAX_KEY_SIZE) { + err = xts_verify_key(skcipher, key, keylen); + if (err) { dev_dbg(dev, "key size mismatch\n"); - return -EINVAL; + return err; + } + + if (keylen != 2 * AES_KEYSIZE_128 && keylen != 2 * AES_KEYSIZE_256) + ctx->xts_key_fallback = true; + + if (priv->sec_attr.era <= 8 || ctx->xts_key_fallback) { + err = crypto_skcipher_setkey(ctx->fallback, key, keylen); + if (err) + return err; } ctx->cdata.keylen = keylen; @@ -1443,17 +1459,44 @@ static void skcipher_decrypt_done(void *cbk_ctx, u32 status) skcipher_request_complete(req, ecode); } +static inline bool xts_skcipher_ivsize(struct skcipher_request *req) +{ + struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); + unsigned int ivsize = crypto_skcipher_ivsize(skcipher); + + return !!get_unaligned((u64 *)(req->iv + (ivsize / 2))); +} + static int skcipher_encrypt(struct skcipher_request *req) { struct skcipher_edesc *edesc; struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); struct caam_request *caam_req = skcipher_request_ctx(req); + struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev); int ret; - if (!req->cryptlen) + /* + * XTS is expected to return an error even for input length = 0 + * Note that the case input length < block size will be caught during + * HW offloading and return an error. + */ + if (!req->cryptlen && !ctx->fallback) return 0; + if (ctx->fallback && ((priv->sec_attr.era <= 8 && xts_skcipher_ivsize(req)) || + ctx->xts_key_fallback)) { + skcipher_request_set_tfm(&caam_req->fallback_req, ctx->fallback); + skcipher_request_set_callback(&caam_req->fallback_req, + req->base.flags, + req->base.complete, + req->base.data); + skcipher_request_set_crypt(&caam_req->fallback_req, req->src, + req->dst, req->cryptlen, req->iv); + + return crypto_skcipher_encrypt(&caam_req->fallback_req); + } + /* allocate extended descriptor */ edesc = skcipher_edesc_alloc(req); if (IS_ERR(edesc)) @@ -1480,10 +1523,30 @@ static int skcipher_decrypt(struct skcipher_request *req) struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); struct caam_request *caam_req = skcipher_request_ctx(req); + struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev); int ret; - if (!req->cryptlen) + /* + * XTS is expected to return an error even for input length = 0 + * Note that the case input length < block size will be caught during + * HW offloading and return an error. + */ + if (!req->cryptlen && !ctx->fallback) return 0; + + if (ctx->fallback && ((priv->sec_attr.era <= 8 && xts_skcipher_ivsize(req)) || + ctx->xts_key_fallback)) { + skcipher_request_set_tfm(&caam_req->fallback_req, ctx->fallback); + skcipher_request_set_callback(&caam_req->fallback_req, + req->base.flags, + req->base.complete, + req->base.data); + skcipher_request_set_crypt(&caam_req->fallback_req, req->src, + req->dst, req->cryptlen, req->iv); + + return crypto_skcipher_decrypt(&caam_req->fallback_req); + } + /* allocate extended descriptor */ edesc = skcipher_edesc_alloc(req); if (IS_ERR(edesc)) @@ -1537,9 +1600,34 @@ static int caam_cra_init_skcipher(struct crypto_skcipher *tfm) struct skcipher_alg *alg = crypto_skcipher_alg(tfm); struct caam_skcipher_alg *caam_alg = container_of(alg, typeof(*caam_alg), skcipher); + struct caam_ctx *ctx = crypto_skcipher_ctx(tfm); + u32 alg_aai = caam_alg->caam.class1_alg_type & OP_ALG_AAI_MASK; + int ret = 0; - crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_request)); - return caam_cra_init(crypto_skcipher_ctx(tfm), &caam_alg->caam, false); + if (alg_aai == OP_ALG_AAI_XTS) { + const char *tfm_name = crypto_tfm_alg_name(&tfm->base); + struct crypto_skcipher *fallback; + + fallback = crypto_alloc_skcipher(tfm_name, 0, + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(fallback)) { + dev_err(ctx->dev, "Failed to allocate %s fallback: %ld\n", + tfm_name, PTR_ERR(fallback)); + return PTR_ERR(fallback); + } + + ctx->fallback = fallback; + crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_request) + + crypto_skcipher_reqsize(fallback)); + } else { + crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_request)); + } + + ret = caam_cra_init(ctx, &caam_alg->caam, false); + if (ret && ctx->fallback) + crypto_free_skcipher(ctx->fallback); + + return ret; } static int caam_cra_init_aead(struct crypto_aead *tfm) @@ -1562,7 +1650,11 @@ static void caam_exit_common(struct caam_ctx *ctx) static void caam_cra_exit(struct crypto_skcipher *tfm) { - caam_exit_common(crypto_skcipher_ctx(tfm)); + struct caam_ctx *ctx = crypto_skcipher_ctx(tfm); + + if (ctx->fallback) + crypto_free_skcipher(ctx->fallback); + caam_exit_common(ctx); } static void caam_cra_exit_aead(struct crypto_aead *tfm) @@ -1665,6 +1757,7 @@ static struct caam_skcipher_alg driver_algs[] = { .base = { .cra_name = "xts(aes)", .cra_driver_name = "xts-aes-caam-qi2", + .cra_flags = CRYPTO_ALG_NEED_FALLBACK, .cra_blocksize = AES_BLOCK_SIZE, }, .setkey = xts_skcipher_setkey, @@ -2912,8 +3005,8 @@ static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg) alg->base.cra_module = THIS_MODULE; alg->base.cra_priority = CAAM_CRA_PRIORITY; alg->base.cra_ctxsize = sizeof(struct caam_ctx); - alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | - CRYPTO_ALG_KERN_DRIVER_ONLY; + alg->base.cra_flags |= (CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY); alg->init = caam_cra_init_skcipher; alg->exit = caam_cra_exit; @@ -2951,7 +3044,7 @@ enum hash_optype { }; /** - * caam_hash_ctx - ahash per-session context + * struct caam_hash_ctx - ahash per-session context * @flc: Flow Contexts array * @key: authentication key * @flc_dma: I/O virtual addresses of the Flow Contexts @@ -5115,8 +5208,7 @@ static int dpaa2_caam_probe(struct fsl_mc_device *dpseci_dev) /* DPIO */ err = dpaa2_dpseci_dpio_setup(priv); if (err) { - if (err != -EPROBE_DEFER) - dev_err(dev, "dpaa2_dpseci_dpio_setup() failed\n"); + dev_err_probe(dev, err, "dpaa2_dpseci_dpio_setup() failed\n"); goto err_dpio_setup; } diff --git a/drivers/crypto/caam/caamalg_qi2.h b/drivers/crypto/caam/caamalg_qi2.h index f29cb7bd7dd3..d35253407ade 100644 --- a/drivers/crypto/caam/caamalg_qi2.h +++ b/drivers/crypto/caam/caamalg_qi2.h @@ -13,6 +13,7 @@ #include <linux/netdevice.h> #include "dpseci.h" #include "desc_constr.h" +#include <crypto/skcipher.h> #define DPAA2_CAAM_STORE_SIZE 16 /* NAPI weight *must* be a multiple of the store size. */ @@ -186,6 +187,7 @@ struct caam_request { void (*cbk)(void *ctx, u32 err); void *ctx; void *edesc; + struct skcipher_request fallback_req; }; /** diff --git a/drivers/crypto/caam/ctrl.c b/drivers/crypto/caam/ctrl.c index 94502f1d4b48..ca0361b2dbb0 100644 --- a/drivers/crypto/caam/ctrl.c +++ b/drivers/crypto/caam/ctrl.c @@ -13,6 +13,7 @@ #include <linux/fsl/mc.h> #include "compat.h" +#include "debugfs.h" #include "regs.h" #include "intern.h" #include "jr.h" @@ -332,11 +333,10 @@ static int instantiate_rng(struct device *ctrldev, int state_handle_mask, kfree(desc); - if (!ret) - ret = devm_add_action_or_reset(ctrldev, devm_deinstantiate_rng, - ctrldev); + if (ret) + return ret; - return ret; + return devm_add_action_or_reset(ctrldev, devm_deinstantiate_rng, ctrldev); } /* @@ -443,7 +443,9 @@ static int caam_get_era_from_hw(struct caam_ctrl __iomem *ctrl) * by u-boot. * In case this property is not passed an attempt to retrieve the CAAM * era via register reads will be made. - **/ + * + * @ctrl: controller region + */ static int caam_get_era(struct caam_ctrl __iomem *ctrl) { struct device_node *caam_node; @@ -582,12 +584,10 @@ static int init_clocks(struct device *dev, const struct caam_imx_data *data) return devm_add_action_or_reset(dev, disable_clocks, ctrlpriv); } -#ifdef CONFIG_DEBUG_FS static void caam_remove_debugfs(void *root) { debugfs_remove_recursive(root); } -#endif #ifdef CONFIG_FSL_MC_BUS static bool check_version(struct fsl_mc_version *mc_version, u32 major, @@ -619,10 +619,7 @@ static int caam_probe(struct platform_device *pdev) struct device_node *nprop, *np; struct caam_ctrl __iomem *ctrl; struct caam_drv_private *ctrlpriv; -#ifdef CONFIG_DEBUG_FS - struct caam_perfmon *perfmon; struct dentry *dfs_root; -#endif u32 scfgr, comp_params; u8 rng_vid; int pg_size; @@ -777,21 +774,15 @@ static int caam_probe(struct platform_device *pdev) ctrlpriv->era = caam_get_era(ctrl); ctrlpriv->domain = iommu_get_domain_for_dev(dev); -#ifdef CONFIG_DEBUG_FS - /* - * FIXME: needs better naming distinction, as some amalgamation of - * "caam" and nprop->full_name. The OF name isn't distinctive, - * but does separate instances - */ - perfmon = (struct caam_perfmon __force *)&ctrl->perfmon; - dfs_root = debugfs_create_dir(dev_name(dev), NULL); - ret = devm_add_action_or_reset(dev, caam_remove_debugfs, dfs_root); - if (ret) - return ret; + if (IS_ENABLED(CONFIG_DEBUG_FS)) { + ret = devm_add_action_or_reset(dev, caam_remove_debugfs, + dfs_root); + if (ret) + return ret; + } - ctrlpriv->ctl = debugfs_create_dir("ctl", dfs_root); -#endif + caam_debugfs_init(ctrlpriv, dfs_root); /* Check to see if (DPAA 1.x) QI present. If so, enable */ if (ctrlpriv->qi_present && !caam_dpaa2) { @@ -912,57 +903,6 @@ static int caam_probe(struct platform_device *pdev) dev_info(dev, "job rings = %d, qi = %d\n", ctrlpriv->total_jobrs, ctrlpriv->qi_present); -#ifdef CONFIG_DEBUG_FS - debugfs_create_file("rq_dequeued", S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->req_dequeued, - &caam_fops_u64_ro); - debugfs_create_file("ob_rq_encrypted", S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->ob_enc_req, - &caam_fops_u64_ro); - debugfs_create_file("ib_rq_decrypted", S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->ib_dec_req, - &caam_fops_u64_ro); - debugfs_create_file("ob_bytes_encrypted", S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->ob_enc_bytes, - &caam_fops_u64_ro); - debugfs_create_file("ob_bytes_protected", S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->ob_prot_bytes, - &caam_fops_u64_ro); - debugfs_create_file("ib_bytes_decrypted", S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->ib_dec_bytes, - &caam_fops_u64_ro); - debugfs_create_file("ib_bytes_validated", S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->ib_valid_bytes, - &caam_fops_u64_ro); - - /* Controller level - global status values */ - debugfs_create_file("fault_addr", S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->faultaddr, - &caam_fops_u32_ro); - debugfs_create_file("fault_detail", S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->faultdetail, - &caam_fops_u32_ro); - debugfs_create_file("fault_status", S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->status, - &caam_fops_u32_ro); - - /* Internal covering keys (useful in non-secure mode only) */ - ctrlpriv->ctl_kek_wrap.data = (__force void *)&ctrlpriv->ctrl->kek[0]; - ctrlpriv->ctl_kek_wrap.size = KEK_KEY_SIZE * sizeof(u32); - debugfs_create_blob("kek", S_IRUSR | S_IRGRP | S_IROTH, ctrlpriv->ctl, - &ctrlpriv->ctl_kek_wrap); - - ctrlpriv->ctl_tkek_wrap.data = (__force void *)&ctrlpriv->ctrl->tkek[0]; - ctrlpriv->ctl_tkek_wrap.size = KEK_KEY_SIZE * sizeof(u32); - debugfs_create_blob("tkek", S_IRUSR | S_IRGRP | S_IROTH, ctrlpriv->ctl, - &ctrlpriv->ctl_tkek_wrap); - - ctrlpriv->ctl_tdsk_wrap.data = (__force void *)&ctrlpriv->ctrl->tdsk[0]; - ctrlpriv->ctl_tdsk_wrap.size = KEK_KEY_SIZE * sizeof(u32); - debugfs_create_blob("tdsk", S_IRUSR | S_IRGRP | S_IROTH, ctrlpriv->ctl, - &ctrlpriv->ctl_tdsk_wrap); -#endif - ret = devm_of_platform_populate(dev); if (ret) dev_err(dev, "JR platform devices creation error\n"); diff --git a/drivers/crypto/caam/debugfs.c b/drivers/crypto/caam/debugfs.c new file mode 100644 index 000000000000..8ebf18398166 --- /dev/null +++ b/drivers/crypto/caam/debugfs.c @@ -0,0 +1,96 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) +/* Copyright 2019 NXP */ + +#include <linux/debugfs.h> +#include "compat.h" +#include "debugfs.h" +#include "regs.h" +#include "intern.h" + +static int caam_debugfs_u64_get(void *data, u64 *val) +{ + *val = caam64_to_cpu(*(u64 *)data); + return 0; +} + +static int caam_debugfs_u32_get(void *data, u64 *val) +{ + *val = caam32_to_cpu(*(u32 *)data); + return 0; +} + +DEFINE_SIMPLE_ATTRIBUTE(caam_fops_u32_ro, caam_debugfs_u32_get, NULL, "%llu\n"); +DEFINE_SIMPLE_ATTRIBUTE(caam_fops_u64_ro, caam_debugfs_u64_get, NULL, "%llu\n"); + +#ifdef CONFIG_CAAM_QI +/* + * This is a counter for the number of times the congestion group (where all + * the request and response queueus are) reached congestion. Incremented + * each time the congestion callback is called with congested == true. + */ +static u64 times_congested; + +void caam_debugfs_qi_congested(void) +{ + times_congested++; +} + +void caam_debugfs_qi_init(struct caam_drv_private *ctrlpriv) +{ + debugfs_create_file("qi_congested", 0444, ctrlpriv->ctl, + ×_congested, &caam_fops_u64_ro); +} +#endif + +void caam_debugfs_init(struct caam_drv_private *ctrlpriv, struct dentry *root) +{ + struct caam_perfmon *perfmon; + + /* + * FIXME: needs better naming distinction, as some amalgamation of + * "caam" and nprop->full_name. The OF name isn't distinctive, + * but does separate instances + */ + perfmon = (struct caam_perfmon __force *)&ctrlpriv->ctrl->perfmon; + + ctrlpriv->ctl = debugfs_create_dir("ctl", root); + + debugfs_create_file("rq_dequeued", 0444, ctrlpriv->ctl, + &perfmon->req_dequeued, &caam_fops_u64_ro); + debugfs_create_file("ob_rq_encrypted", 0444, ctrlpriv->ctl, + &perfmon->ob_enc_req, &caam_fops_u64_ro); + debugfs_create_file("ib_rq_decrypted", 0444, ctrlpriv->ctl, + &perfmon->ib_dec_req, &caam_fops_u64_ro); + debugfs_create_file("ob_bytes_encrypted", 0444, ctrlpriv->ctl, + &perfmon->ob_enc_bytes, &caam_fops_u64_ro); + debugfs_create_file("ob_bytes_protected", 0444, ctrlpriv->ctl, + &perfmon->ob_prot_bytes, &caam_fops_u64_ro); + debugfs_create_file("ib_bytes_decrypted", 0444, ctrlpriv->ctl, + &perfmon->ib_dec_bytes, &caam_fops_u64_ro); + debugfs_create_file("ib_bytes_validated", 0444, ctrlpriv->ctl, + &perfmon->ib_valid_bytes, &caam_fops_u64_ro); + + /* Controller level - global status values */ + debugfs_create_file("fault_addr", 0444, ctrlpriv->ctl, + &perfmon->faultaddr, &caam_fops_u32_ro); + debugfs_create_file("fault_detail", 0444, ctrlpriv->ctl, + &perfmon->faultdetail, &caam_fops_u32_ro); + debugfs_create_file("fault_status", 0444, ctrlpriv->ctl, + &perfmon->status, &caam_fops_u32_ro); + + /* Internal covering keys (useful in non-secure mode only) */ + ctrlpriv->ctl_kek_wrap.data = (__force void *)&ctrlpriv->ctrl->kek[0]; + ctrlpriv->ctl_kek_wrap.size = KEK_KEY_SIZE * sizeof(u32); + debugfs_create_blob("kek", 0444, ctrlpriv->ctl, + &ctrlpriv->ctl_kek_wrap); + + ctrlpriv->ctl_tkek_wrap.data = (__force void *)&ctrlpriv->ctrl->tkek[0]; + ctrlpriv->ctl_tkek_wrap.size = KEK_KEY_SIZE * sizeof(u32); + debugfs_create_blob("tkek", 0444, ctrlpriv->ctl, + &ctrlpriv->ctl_tkek_wrap); + + ctrlpriv->ctl_tdsk_wrap.data = (__force void *)&ctrlpriv->ctrl->tdsk[0]; + ctrlpriv->ctl_tdsk_wrap.size = KEK_KEY_SIZE * sizeof(u32); + debugfs_create_blob("tdsk", 0444, ctrlpriv->ctl, + &ctrlpriv->ctl_tdsk_wrap); +} diff --git a/drivers/crypto/caam/debugfs.h b/drivers/crypto/caam/debugfs.h new file mode 100644 index 000000000000..661d768acdbf --- /dev/null +++ b/drivers/crypto/caam/debugfs.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) */ +/* Copyright 2019 NXP */ + +#ifndef CAAM_DEBUGFS_H +#define CAAM_DEBUGFS_H + +struct dentry; +struct caam_drv_private; + +#ifdef CONFIG_DEBUG_FS +void caam_debugfs_init(struct caam_drv_private *ctrlpriv, struct dentry *root); +#else +static inline void caam_debugfs_init(struct caam_drv_private *ctrlpriv, + struct dentry *root) +{} +#endif + +#if defined(CONFIG_DEBUG_FS) && defined(CONFIG_CAAM_QI) +void caam_debugfs_qi_congested(void); +void caam_debugfs_qi_init(struct caam_drv_private *ctrlpriv); +#else +static inline void caam_debugfs_qi_congested(void) {} +static inline void caam_debugfs_qi_init(struct caam_drv_private *ctrlpriv) {} +#endif + +#endif /* CAAM_DEBUGFS_H */ diff --git a/drivers/crypto/caam/dpseci-debugfs.c b/drivers/crypto/caam/dpseci-debugfs.c index c5bfc923abd8..0eca8c2fd916 100644 --- a/drivers/crypto/caam/dpseci-debugfs.c +++ b/drivers/crypto/caam/dpseci-debugfs.c @@ -44,33 +44,14 @@ static int dpseci_dbg_fqs_show(struct seq_file *file, void *offset) return 0; } -static int dpseci_dbg_fqs_open(struct inode *inode, struct file *file) -{ - int err; - struct dpaa2_caam_priv *priv; - - priv = (struct dpaa2_caam_priv *)inode->i_private; - - err = single_open(file, dpseci_dbg_fqs_show, priv); - if (err < 0) - dev_err(priv->dev, "single_open() failed\n"); - - return err; -} - -static const struct file_operations dpseci_dbg_fq_ops = { - .open = dpseci_dbg_fqs_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; +DEFINE_SHOW_ATTRIBUTE(dpseci_dbg_fqs); void dpaa2_dpseci_debugfs_init(struct dpaa2_caam_priv *priv) { priv->dfs_root = debugfs_create_dir(dev_name(priv->dev), NULL); debugfs_create_file("fq_stats", 0444, priv->dfs_root, priv, - &dpseci_dbg_fq_ops); + &dpseci_dbg_fqs_fops); } void dpaa2_dpseci_debugfs_exit(struct dpaa2_caam_priv *priv) diff --git a/drivers/crypto/caam/intern.h b/drivers/crypto/caam/intern.h index 402d6a362e8c..9112279a4de0 100644 --- a/drivers/crypto/caam/intern.h +++ b/drivers/crypto/caam/intern.h @@ -195,23 +195,6 @@ static inline void caam_qi_algapi_exit(void) #endif /* CONFIG_CAAM_QI */ -#ifdef CONFIG_DEBUG_FS -static int caam_debugfs_u64_get(void *data, u64 *val) -{ - *val = caam64_to_cpu(*(u64 *)data); - return 0; -} - -static int caam_debugfs_u32_get(void *data, u64 *val) -{ - *val = caam32_to_cpu(*(u32 *)data); - return 0; -} - -DEFINE_SIMPLE_ATTRIBUTE(caam_fops_u32_ro, caam_debugfs_u32_get, NULL, "%llu\n"); -DEFINE_SIMPLE_ATTRIBUTE(caam_fops_u64_ro, caam_debugfs_u64_get, NULL, "%llu\n"); -#endif - static inline u64 caam_get_dma_mask(struct device *dev) { struct device_node *nprop = dev->of_node; diff --git a/drivers/crypto/caam/jr.c b/drivers/crypto/caam/jr.c index bf6b03b17251..6f669966ba2c 100644 --- a/drivers/crypto/caam/jr.c +++ b/drivers/crypto/caam/jr.c @@ -324,7 +324,7 @@ EXPORT_SYMBOL(caam_jr_alloc); /** * caam_jr_free() - Free the Job Ring - * @rdev - points to the dev that identifies the Job ring to + * @rdev: points to the dev that identifies the Job ring to * be released. **/ void caam_jr_free(struct device *rdev) @@ -349,15 +349,15 @@ EXPORT_SYMBOL(caam_jr_free); * of this request. This has the form: * callback(struct device *dev, u32 *desc, u32 stat, void *arg) * where: - * @dev: contains the job ring device that processed this + * dev: contains the job ring device that processed this * response. - * @desc: descriptor that initiated the request, same as + * desc: descriptor that initiated the request, same as * "desc" being argued to caam_jr_enqueue(). - * @status: untranslated status received from CAAM. See the + * status: untranslated status received from CAAM. See the * reference manual for a detailed description of * error meaning, or see the JRSTA definitions in the * register header file - * @areq: optional pointer to an argument passed with the + * areq: optional pointer to an argument passed with the * original request * @areq: optional pointer to a user argument for use at callback * time. diff --git a/drivers/crypto/caam/qi.c b/drivers/crypto/caam/qi.c index b390b935db6d..ec53528d8205 100644 --- a/drivers/crypto/caam/qi.c +++ b/drivers/crypto/caam/qi.c @@ -11,6 +11,7 @@ #include <linux/kthread.h> #include <soc/fsl/qman.h> +#include "debugfs.h" #include "regs.h" #include "qi.h" #include "desc.h" @@ -73,15 +74,6 @@ static struct caam_qi_priv qipriv ____cacheline_aligned; bool caam_congested __read_mostly; EXPORT_SYMBOL(caam_congested); -#ifdef CONFIG_DEBUG_FS -/* - * This is a counter for the number of times the congestion group (where all - * the request and response queueus are) reached congestion. Incremented - * each time the congestion callback is called with congested == true. - */ -static u64 times_congested; -#endif - /* * This is a a cache of buffers, from which the users of CAAM QI driver * can allocate short (CAAM_QI_MEMCACHE_SIZE) buffers. It's faster than @@ -544,9 +536,8 @@ static void cgr_cb(struct qman_portal *qm, struct qman_cgr *cgr, int congested) caam_congested = congested; if (congested) { -#ifdef CONFIG_DEBUG_FS - times_congested++; -#endif + caam_debugfs_qi_congested(); + pr_debug_ratelimited("CAAM entered congestion\n"); } else { @@ -775,10 +766,7 @@ int caam_qi_init(struct platform_device *caam_pdev) return -ENOMEM; } -#ifdef CONFIG_DEBUG_FS - debugfs_create_file("qi_congested", 0444, ctrlpriv->ctl, - ×_congested, &caam_fops_u64_ro); -#endif + caam_debugfs_qi_init(ctrlpriv); err = devm_add_action_or_reset(qidev, caam_qi_shutdown, ctrlpriv); if (err) diff --git a/drivers/crypto/cavium/cpt/cptvf_algs.c b/drivers/crypto/cavium/cpt/cptvf_algs.c index 5af0dc2a8909..ce3b91c612f0 100644 --- a/drivers/crypto/cavium/cpt/cptvf_algs.c +++ b/drivers/crypto/cavium/cpt/cptvf_algs.c @@ -451,13 +451,7 @@ static struct skcipher_alg algs[] = { { static inline int cav_register_algs(void) { - int err = 0; - - err = crypto_register_skciphers(algs, ARRAY_SIZE(algs)); - if (err) - return err; - - return 0; + return crypto_register_skciphers(algs, ARRAY_SIZE(algs)); } static inline void cav_unregister_algs(void) diff --git a/drivers/crypto/cavium/nitrox/nitrox_main.c b/drivers/crypto/cavium/nitrox/nitrox_main.c index cee2a2713038..9d14be97e381 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_main.c +++ b/drivers/crypto/cavium/nitrox/nitrox_main.c @@ -451,6 +451,7 @@ static int nitrox_probe(struct pci_dev *pdev, err = pci_request_mem_regions(pdev, nitrox_driver_name); if (err) { pci_disable_device(pdev); + dev_err(&pdev->dev, "Failed to request mem regions!\n"); return err; } pci_set_master(pdev); diff --git a/drivers/crypto/cavium/zip/zip_main.c b/drivers/crypto/cavium/zip/zip_main.c index 194624b4855b..d35216e2f6cd 100644 --- a/drivers/crypto/cavium/zip/zip_main.c +++ b/drivers/crypto/cavium/zip/zip_main.c @@ -460,7 +460,7 @@ static void zip_unregister_compression_device(void) #include <linux/debugfs.h> /* Displays ZIP device statistics */ -static int zip_show_stats(struct seq_file *s, void *unused) +static int zip_stats_show(struct seq_file *s, void *unused) { u64 val = 0ull; u64 avg_chunk = 0ull, avg_cr = 0ull; @@ -523,7 +523,7 @@ static int zip_show_stats(struct seq_file *s, void *unused) } /* Clears stats data */ -static int zip_clear_stats(struct seq_file *s, void *unused) +static int zip_clear_show(struct seq_file *s, void *unused) { int index = 0; @@ -558,7 +558,7 @@ static struct zip_registers zipregs[64] = { }; /* Prints registers' contents */ -static int zip_print_regs(struct seq_file *s, void *unused) +static int zip_regs_show(struct seq_file *s, void *unused) { u64 val = 0; int i = 0, index = 0; @@ -584,41 +584,9 @@ static int zip_print_regs(struct seq_file *s, void *unused) return 0; } -static int zip_stats_open(struct inode *inode, struct file *file) -{ - return single_open(file, zip_show_stats, NULL); -} - -static const struct file_operations zip_stats_fops = { - .owner = THIS_MODULE, - .open = zip_stats_open, - .read = seq_read, - .release = single_release, -}; - -static int zip_clear_open(struct inode *inode, struct file *file) -{ - return single_open(file, zip_clear_stats, NULL); -} - -static const struct file_operations zip_clear_fops = { - .owner = THIS_MODULE, - .open = zip_clear_open, - .read = seq_read, - .release = single_release, -}; - -static int zip_regs_open(struct inode *inode, struct file *file) -{ - return single_open(file, zip_print_regs, NULL); -} - -static const struct file_operations zip_regs_fops = { - .owner = THIS_MODULE, - .open = zip_regs_open, - .read = seq_read, - .release = single_release, -}; +DEFINE_SHOW_ATTRIBUTE(zip_stats); +DEFINE_SHOW_ATTRIBUTE(zip_clear); +DEFINE_SHOW_ATTRIBUTE(zip_regs); /* Root directory for thunderx_zip debugfs entry */ static struct dentry *zip_debugfs_root; diff --git a/drivers/crypto/ccp/ccp-ops.c b/drivers/crypto/ccp/ccp-ops.c index bd270e66185e..d6a8f4e4b14a 100644 --- a/drivers/crypto/ccp/ccp-ops.c +++ b/drivers/crypto/ccp/ccp-ops.c @@ -8,6 +8,7 @@ * Author: Gary R Hook <gary.hook@amd.com> */ +#include <linux/dma-mapping.h> #include <linux/module.h> #include <linux/kernel.h> #include <linux/interrupt.h> @@ -1744,7 +1745,7 @@ ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) break; default: ret = -EINVAL; - goto e_ctx; + goto e_data; } } else { /* Stash the context */ diff --git a/drivers/crypto/ccree/cc_cipher.c b/drivers/crypto/ccree/cc_cipher.c index d77ae981b64b..dafa6577a845 100644 --- a/drivers/crypto/ccree/cc_cipher.c +++ b/drivers/crypto/ccree/cc_cipher.c @@ -75,8 +75,7 @@ static int validate_keys_sizes(struct cc_cipher_ctx *ctx_p, u32 size) switch (size) { case CC_AES_128_BIT_KEY_SIZE: case CC_AES_192_BIT_KEY_SIZE: - if (ctx_p->cipher_mode != DRV_CIPHER_XTS && - ctx_p->cipher_mode != DRV_CIPHER_BITLOCKER) + if (ctx_p->cipher_mode != DRV_CIPHER_XTS) return 0; break; case CC_AES_256_BIT_KEY_SIZE: @@ -84,8 +83,7 @@ static int validate_keys_sizes(struct cc_cipher_ctx *ctx_p, u32 size) case (CC_AES_192_BIT_KEY_SIZE * 2): case (CC_AES_256_BIT_KEY_SIZE * 2): if (ctx_p->cipher_mode == DRV_CIPHER_XTS || - ctx_p->cipher_mode == DRV_CIPHER_ESSIV || - ctx_p->cipher_mode == DRV_CIPHER_BITLOCKER) + ctx_p->cipher_mode == DRV_CIPHER_ESSIV) return 0; break; default: @@ -122,7 +120,6 @@ static int validate_data_size(struct cc_cipher_ctx *ctx_p, case DRV_CIPHER_ECB: case DRV_CIPHER_CBC: case DRV_CIPHER_ESSIV: - case DRV_CIPHER_BITLOCKER: if (IS_ALIGNED(size, AES_BLOCK_SIZE)) return 0; break; @@ -348,8 +345,7 @@ static int cc_cipher_sethkey(struct crypto_skcipher *sktfm, const u8 *key, } if (ctx_p->cipher_mode == DRV_CIPHER_XTS || - ctx_p->cipher_mode == DRV_CIPHER_ESSIV || - ctx_p->cipher_mode == DRV_CIPHER_BITLOCKER) { + ctx_p->cipher_mode == DRV_CIPHER_ESSIV) { if (hki.hw_key1 == hki.hw_key2) { dev_err(dev, "Illegal hw key numbers (%d,%d)\n", hki.hw_key1, hki.hw_key2); @@ -547,7 +543,6 @@ static void cc_setup_readiv_desc(struct crypto_tfm *tfm, break; case DRV_CIPHER_XTS: case DRV_CIPHER_ESSIV: - case DRV_CIPHER_BITLOCKER: /* IV */ hw_desc_init(&desc[*seq_size]); set_setup_mode(&desc[*seq_size], SETUP_WRITE_STATE1); @@ -602,7 +597,6 @@ static void cc_setup_state_desc(struct crypto_tfm *tfm, break; case DRV_CIPHER_XTS: case DRV_CIPHER_ESSIV: - case DRV_CIPHER_BITLOCKER: break; default: dev_err(dev, "Unsupported cipher mode (%d)\n", cipher_mode); @@ -624,16 +618,8 @@ static void cc_setup_xex_state_desc(struct crypto_tfm *tfm, dma_addr_t key_dma_addr = ctx_p->user.key_dma_addr; unsigned int key_len = (ctx_p->keylen / 2); dma_addr_t iv_dma_addr = req_ctx->gen_ctx.iv_dma_addr; - unsigned int du_size = nbytes; unsigned int key_offset = key_len; - struct cc_crypto_alg *cc_alg = - container_of(tfm->__crt_alg, struct cc_crypto_alg, - skcipher_alg.base); - - if (cc_alg->data_unit) - du_size = cc_alg->data_unit; - switch (cipher_mode) { case DRV_CIPHER_ECB: break; @@ -644,7 +630,6 @@ static void cc_setup_xex_state_desc(struct crypto_tfm *tfm, break; case DRV_CIPHER_XTS: case DRV_CIPHER_ESSIV: - case DRV_CIPHER_BITLOCKER: if (cipher_mode == DRV_CIPHER_ESSIV) key_len = SHA256_DIGEST_SIZE; @@ -661,7 +646,7 @@ static void cc_setup_xex_state_desc(struct crypto_tfm *tfm, (key_dma_addr + key_offset), key_len, NS_BIT); } - set_xex_data_unit_size(&desc[*seq_size], du_size); + set_xex_data_unit_size(&desc[*seq_size], nbytes); set_flow_mode(&desc[*seq_size], S_DIN_to_AES2); set_key_size_aes(&desc[*seq_size], key_len); set_setup_mode(&desc[*seq_size], SETUP_LOAD_XEX_KEY); @@ -758,7 +743,6 @@ static void cc_setup_key_desc(struct crypto_tfm *tfm, break; case DRV_CIPHER_XTS: case DRV_CIPHER_ESSIV: - case DRV_CIPHER_BITLOCKER: /* Load AES key */ hw_desc_init(&desc[*seq_size]); set_cipher_mode(&desc[*seq_size], cipher_mode); @@ -1039,44 +1023,6 @@ static const struct cc_alg_template skcipher_algs[] = { .sec_func = true, }, { - .name = "xts512(paes)", - .driver_name = "xts-paes-du512-ccree", - .blocksize = 1, - .template_skcipher = { - .setkey = cc_cipher_sethkey, - .encrypt = cc_cipher_encrypt, - .decrypt = cc_cipher_decrypt, - .min_keysize = CC_HW_KEY_SIZE, - .max_keysize = CC_HW_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - }, - .cipher_mode = DRV_CIPHER_XTS, - .flow_mode = S_DIN_to_AES, - .data_unit = 512, - .min_hw_rev = CC_HW_REV_712, - .std_body = CC_STD_NIST, - .sec_func = true, - }, - { - .name = "xts4096(paes)", - .driver_name = "xts-paes-du4096-ccree", - .blocksize = 1, - .template_skcipher = { - .setkey = cc_cipher_sethkey, - .encrypt = cc_cipher_encrypt, - .decrypt = cc_cipher_decrypt, - .min_keysize = CC_HW_KEY_SIZE, - .max_keysize = CC_HW_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - }, - .cipher_mode = DRV_CIPHER_XTS, - .flow_mode = S_DIN_to_AES, - .data_unit = 4096, - .min_hw_rev = CC_HW_REV_712, - .std_body = CC_STD_NIST, - .sec_func = true, - }, - { .name = "essiv(cbc(paes),sha256)", .driver_name = "essiv-paes-ccree", .blocksize = AES_BLOCK_SIZE, @@ -1095,100 +1041,6 @@ static const struct cc_alg_template skcipher_algs[] = { .sec_func = true, }, { - .name = "essiv512(cbc(paes),sha256)", - .driver_name = "essiv-paes-du512-ccree", - .blocksize = AES_BLOCK_SIZE, - .template_skcipher = { - .setkey = cc_cipher_sethkey, - .encrypt = cc_cipher_encrypt, - .decrypt = cc_cipher_decrypt, - .min_keysize = CC_HW_KEY_SIZE, - .max_keysize = CC_HW_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - }, - .cipher_mode = DRV_CIPHER_ESSIV, - .flow_mode = S_DIN_to_AES, - .data_unit = 512, - .min_hw_rev = CC_HW_REV_712, - .std_body = CC_STD_NIST, - .sec_func = true, - }, - { - .name = "essiv4096(cbc(paes),sha256)", - .driver_name = "essiv-paes-du4096-ccree", - .blocksize = AES_BLOCK_SIZE, - .template_skcipher = { - .setkey = cc_cipher_sethkey, - .encrypt = cc_cipher_encrypt, - .decrypt = cc_cipher_decrypt, - .min_keysize = CC_HW_KEY_SIZE, - .max_keysize = CC_HW_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - }, - .cipher_mode = DRV_CIPHER_ESSIV, - .flow_mode = S_DIN_to_AES, - .data_unit = 4096, - .min_hw_rev = CC_HW_REV_712, - .std_body = CC_STD_NIST, - .sec_func = true, - }, - { - .name = "bitlocker(paes)", - .driver_name = "bitlocker-paes-ccree", - .blocksize = AES_BLOCK_SIZE, - .template_skcipher = { - .setkey = cc_cipher_sethkey, - .encrypt = cc_cipher_encrypt, - .decrypt = cc_cipher_decrypt, - .min_keysize = CC_HW_KEY_SIZE, - .max_keysize = CC_HW_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - }, - .cipher_mode = DRV_CIPHER_BITLOCKER, - .flow_mode = S_DIN_to_AES, - .min_hw_rev = CC_HW_REV_712, - .std_body = CC_STD_NIST, - .sec_func = true, - }, - { - .name = "bitlocker512(paes)", - .driver_name = "bitlocker-paes-du512-ccree", - .blocksize = AES_BLOCK_SIZE, - .template_skcipher = { - .setkey = cc_cipher_sethkey, - .encrypt = cc_cipher_encrypt, - .decrypt = cc_cipher_decrypt, - .min_keysize = CC_HW_KEY_SIZE, - .max_keysize = CC_HW_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - }, - .cipher_mode = DRV_CIPHER_BITLOCKER, - .flow_mode = S_DIN_to_AES, - .data_unit = 512, - .min_hw_rev = CC_HW_REV_712, - .std_body = CC_STD_NIST, - .sec_func = true, - }, - { - .name = "bitlocker4096(paes)", - .driver_name = "bitlocker-paes-du4096-ccree", - .blocksize = AES_BLOCK_SIZE, - .template_skcipher = { - .setkey = cc_cipher_sethkey, - .encrypt = cc_cipher_encrypt, - .decrypt = cc_cipher_decrypt, - .min_keysize = CC_HW_KEY_SIZE, - .max_keysize = CC_HW_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - }, - .cipher_mode = DRV_CIPHER_BITLOCKER, - .flow_mode = S_DIN_to_AES, - .data_unit = 4096, - .min_hw_rev = CC_HW_REV_712, - .std_body = CC_STD_NIST, - .sec_func = true, - }, - { .name = "ecb(paes)", .driver_name = "ecb-paes-ccree", .blocksize = AES_BLOCK_SIZE, @@ -1300,42 +1152,6 @@ static const struct cc_alg_template skcipher_algs[] = { .std_body = CC_STD_NIST, }, { - .name = "xts512(aes)", - .driver_name = "xts-aes-du512-ccree", - .blocksize = 1, - .template_skcipher = { - .setkey = cc_cipher_setkey, - .encrypt = cc_cipher_encrypt, - .decrypt = cc_cipher_decrypt, - .min_keysize = AES_MIN_KEY_SIZE * 2, - .max_keysize = AES_MAX_KEY_SIZE * 2, - .ivsize = AES_BLOCK_SIZE, - }, - .cipher_mode = DRV_CIPHER_XTS, - .flow_mode = S_DIN_to_AES, - .data_unit = 512, - .min_hw_rev = CC_HW_REV_712, - .std_body = CC_STD_NIST, - }, - { - .name = "xts4096(aes)", - .driver_name = "xts-aes-du4096-ccree", - .blocksize = 1, - .template_skcipher = { - .setkey = cc_cipher_setkey, - .encrypt = cc_cipher_encrypt, - .decrypt = cc_cipher_decrypt, - .min_keysize = AES_MIN_KEY_SIZE * 2, - .max_keysize = AES_MAX_KEY_SIZE * 2, - .ivsize = AES_BLOCK_SIZE, - }, - .cipher_mode = DRV_CIPHER_XTS, - .flow_mode = S_DIN_to_AES, - .data_unit = 4096, - .min_hw_rev = CC_HW_REV_712, - .std_body = CC_STD_NIST, - }, - { .name = "essiv(cbc(aes),sha256)", .driver_name = "essiv-aes-ccree", .blocksize = AES_BLOCK_SIZE, @@ -1353,95 +1169,6 @@ static const struct cc_alg_template skcipher_algs[] = { .std_body = CC_STD_NIST, }, { - .name = "essiv512(cbc(aes),sha256)", - .driver_name = "essiv-aes-du512-ccree", - .blocksize = AES_BLOCK_SIZE, - .template_skcipher = { - .setkey = cc_cipher_setkey, - .encrypt = cc_cipher_encrypt, - .decrypt = cc_cipher_decrypt, - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - }, - .cipher_mode = DRV_CIPHER_ESSIV, - .flow_mode = S_DIN_to_AES, - .data_unit = 512, - .min_hw_rev = CC_HW_REV_712, - .std_body = CC_STD_NIST, - }, - { - .name = "essiv4096(cbc(aes),sha256)", - .driver_name = "essiv-aes-du4096-ccree", - .blocksize = AES_BLOCK_SIZE, - .template_skcipher = { - .setkey = cc_cipher_setkey, - .encrypt = cc_cipher_encrypt, - .decrypt = cc_cipher_decrypt, - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - }, - .cipher_mode = DRV_CIPHER_ESSIV, - .flow_mode = S_DIN_to_AES, - .data_unit = 4096, - .min_hw_rev = CC_HW_REV_712, - .std_body = CC_STD_NIST, - }, - { - .name = "bitlocker(aes)", - .driver_name = "bitlocker-aes-ccree", - .blocksize = AES_BLOCK_SIZE, - .template_skcipher = { - .setkey = cc_cipher_setkey, - .encrypt = cc_cipher_encrypt, - .decrypt = cc_cipher_decrypt, - .min_keysize = AES_MIN_KEY_SIZE * 2, - .max_keysize = AES_MAX_KEY_SIZE * 2, - .ivsize = AES_BLOCK_SIZE, - }, - .cipher_mode = DRV_CIPHER_BITLOCKER, - .flow_mode = S_DIN_to_AES, - .min_hw_rev = CC_HW_REV_712, - .std_body = CC_STD_NIST, - }, - { - .name = "bitlocker512(aes)", - .driver_name = "bitlocker-aes-du512-ccree", - .blocksize = AES_BLOCK_SIZE, - .template_skcipher = { - .setkey = cc_cipher_setkey, - .encrypt = cc_cipher_encrypt, - .decrypt = cc_cipher_decrypt, - .min_keysize = AES_MIN_KEY_SIZE * 2, - .max_keysize = AES_MAX_KEY_SIZE * 2, - .ivsize = AES_BLOCK_SIZE, - }, - .cipher_mode = DRV_CIPHER_BITLOCKER, - .flow_mode = S_DIN_to_AES, - .data_unit = 512, - .min_hw_rev = CC_HW_REV_712, - .std_body = CC_STD_NIST, - }, - { - .name = "bitlocker4096(aes)", - .driver_name = "bitlocker-aes-du4096-ccree", - .blocksize = AES_BLOCK_SIZE, - .template_skcipher = { - .setkey = cc_cipher_setkey, - .encrypt = cc_cipher_encrypt, - .decrypt = cc_cipher_decrypt, - .min_keysize = AES_MIN_KEY_SIZE * 2, - .max_keysize = AES_MAX_KEY_SIZE * 2, - .ivsize = AES_BLOCK_SIZE, - }, - .cipher_mode = DRV_CIPHER_BITLOCKER, - .flow_mode = S_DIN_to_AES, - .data_unit = 4096, - .min_hw_rev = CC_HW_REV_712, - .std_body = CC_STD_NIST, - }, - { .name = "ecb(aes)", .driver_name = "ecb-aes-ccree", .blocksize = AES_BLOCK_SIZE, @@ -1712,7 +1439,6 @@ static struct cc_crypto_alg *cc_create_alg(const struct cc_alg_template *tmpl, t_alg->cipher_mode = tmpl->cipher_mode; t_alg->flow_mode = tmpl->flow_mode; - t_alg->data_unit = tmpl->data_unit; return t_alg; } diff --git a/drivers/crypto/ccree/cc_crypto_ctx.h b/drivers/crypto/ccree/cc_crypto_ctx.h index ccf960a0d989..bd9a1c0896b3 100644 --- a/drivers/crypto/ccree/cc_crypto_ctx.h +++ b/drivers/crypto/ccree/cc_crypto_ctx.h @@ -108,7 +108,6 @@ enum drv_cipher_mode { DRV_CIPHER_CBC_CTS = 11, DRV_CIPHER_GCTR = 12, DRV_CIPHER_ESSIV = 13, - DRV_CIPHER_BITLOCKER = 14, DRV_CIPHER_RESERVE32B = S32_MAX }; diff --git a/drivers/crypto/ccree/cc_driver.c b/drivers/crypto/ccree/cc_driver.c index 2d50991b9a17..6f519d3e896c 100644 --- a/drivers/crypto/ccree/cc_driver.c +++ b/drivers/crypto/ccree/cc_driver.c @@ -300,11 +300,8 @@ static int init_cc_resources(struct platform_device *plat_dev) new_drvdata->plat_dev = plat_dev; clk = devm_clk_get_optional(dev, NULL); - if (IS_ERR(clk)) { - if (PTR_ERR(clk) != -EPROBE_DEFER) - dev_err(dev, "Error getting clock: %pe\n", clk); - return PTR_ERR(clk); - } + if (IS_ERR(clk)) + return dev_err_probe(dev, PTR_ERR(clk), "Error getting clock\n"); new_drvdata->clk = clk; new_drvdata->coherent = of_dma_is_coherent(np); diff --git a/drivers/crypto/ccree/cc_driver.h b/drivers/crypto/ccree/cc_driver.h index d938886390d2..af77b2020350 100644 --- a/drivers/crypto/ccree/cc_driver.h +++ b/drivers/crypto/ccree/cc_driver.h @@ -162,7 +162,6 @@ struct cc_crypto_alg { int cipher_mode; int flow_mode; /* Note: currently, refers to the cipher mode only. */ int auth_mode; - unsigned int data_unit; struct cc_drvdata *drvdata; struct skcipher_alg skcipher_alg; struct aead_alg aead_alg; diff --git a/drivers/crypto/ccree/cc_pm.c b/drivers/crypto/ccree/cc_pm.c index d39e1664fc7e..3c65bf070c90 100644 --- a/drivers/crypto/ccree/cc_pm.c +++ b/drivers/crypto/ccree/cc_pm.c @@ -65,8 +65,12 @@ const struct dev_pm_ops ccree_pm = { int cc_pm_get(struct device *dev) { int rc = pm_runtime_get_sync(dev); + if (rc < 0) { + pm_runtime_put_noidle(dev); + return rc; + } - return (rc == 1 ? 0 : rc); + return 0; } void cc_pm_put_suspend(struct device *dev) diff --git a/drivers/crypto/chelsio/chcr_core.c b/drivers/crypto/chelsio/chcr_core.c index bd8dac806e7a..ed7989cf151e 100644 --- a/drivers/crypto/chelsio/chcr_core.c +++ b/drivers/crypto/chelsio/chcr_core.c @@ -148,7 +148,7 @@ static void chcr_dev_init(struct uld_ctx *u_ctx) static int chcr_dev_move(struct uld_ctx *u_ctx) { - mutex_lock(&drv_data.drv_mutex); + mutex_lock(&drv_data.drv_mutex); if (drv_data.last_dev == u_ctx) { if (list_is_last(&drv_data.last_dev->entry, &drv_data.act_dev)) drv_data.last_dev = list_first_entry(&drv_data.act_dev, diff --git a/drivers/crypto/hifn_795x.c b/drivers/crypto/hifn_795x.c index 354836468c5d..7e7a8f01ea6b 100644 --- a/drivers/crypto/hifn_795x.c +++ b/drivers/crypto/hifn_795x.c @@ -780,8 +780,8 @@ static int hifn_register_rng(struct hifn_device *dev) dev->pk_clk_freq) * 256; dev->rng.name = dev->name; - dev->rng.data_present = hifn_rng_data_present, - dev->rng.data_read = hifn_rng_data_read, + dev->rng.data_present = hifn_rng_data_present; + dev->rng.data_read = hifn_rng_data_read; dev->rng.priv = (unsigned long)dev; return hwrng_register(&dev->rng); @@ -1235,7 +1235,8 @@ static int hifn_setup_src_desc(struct hifn_device *dev, struct page *page, int idx; dma_addr_t addr; - addr = pci_map_page(dev->pdev, page, offset, size, PCI_DMA_TODEVICE); + addr = dma_map_page(&dev->pdev->dev, page, offset, size, + DMA_TO_DEVICE); idx = dma->srci; @@ -1293,7 +1294,8 @@ static void hifn_setup_dst_desc(struct hifn_device *dev, struct page *page, int idx; dma_addr_t addr; - addr = pci_map_page(dev->pdev, page, offset, size, PCI_DMA_FROMDEVICE); + addr = dma_map_page(&dev->pdev->dev, page, offset, size, + DMA_FROM_DEVICE); idx = dma->dsti; dma->dstr[idx].p = __cpu_to_le32(addr); @@ -2470,7 +2472,7 @@ static int hifn_probe(struct pci_dev *pdev, const struct pci_device_id *id) return err; pci_set_master(pdev); - err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); + err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32)); if (err) goto err_out_disable_pci_device; @@ -2514,8 +2516,9 @@ static int hifn_probe(struct pci_dev *pdev, const struct pci_device_id *id) } } - dev->desc_virt = pci_zalloc_consistent(pdev, sizeof(struct hifn_dma), - &dev->desc_dma); + dev->desc_virt = dma_alloc_coherent(&pdev->dev, + sizeof(struct hifn_dma), + &dev->desc_dma, GFP_KERNEL); if (!dev->desc_virt) { dev_err(&pdev->dev, "Failed to allocate descriptor rings.\n"); err = -ENOMEM; @@ -2572,8 +2575,8 @@ err_out_free_irq: free_irq(dev->irq, dev); tasklet_kill(&dev->tasklet); err_out_free_desc: - pci_free_consistent(pdev, sizeof(struct hifn_dma), - dev->desc_virt, dev->desc_dma); + dma_free_coherent(&pdev->dev, sizeof(struct hifn_dma), dev->desc_virt, + dev->desc_dma); err_out_unmap_bars: for (i = 0; i < 3; ++i) @@ -2610,8 +2613,8 @@ static void hifn_remove(struct pci_dev *pdev) hifn_flush(dev); - pci_free_consistent(pdev, sizeof(struct hifn_dma), - dev->desc_virt, dev->desc_dma); + dma_free_coherent(&pdev->dev, sizeof(struct hifn_dma), + dev->desc_virt, dev->desc_dma); for (i = 0; i < 3; ++i) if (dev->bar[i]) iounmap(dev->bar[i]); @@ -2642,9 +2645,6 @@ static int __init hifn_init(void) unsigned int freq; int err; - /* HIFN supports only 32-bit addresses */ - BUILD_BUG_ON(sizeof(dma_addr_t) != 4); - if (strncmp(hifn_pll_ref, "ext", 3) && strncmp(hifn_pll_ref, "pci", 3)) { pr_err("hifn795x: invalid hifn_pll_ref clock, must be pci or ext"); diff --git a/drivers/crypto/hisilicon/hpre/hpre.h b/drivers/crypto/hisilicon/hpre/hpre.h index ed730d173e95..f69252b24671 100644 --- a/drivers/crypto/hisilicon/hpre/hpre.h +++ b/drivers/crypto/hisilicon/hpre/hpre.h @@ -56,7 +56,6 @@ struct hpre_dfx { * Just relevant for PF. */ struct hpre_debug { - struct dentry *debug_root; struct hpre_dfx dfx[HPRE_DFX_FILE_NUM]; struct hpre_debugfs_file files[HPRE_DEBUGFS_FILE_NUM]; }; diff --git a/drivers/crypto/hisilicon/hpre/hpre_crypto.c b/drivers/crypto/hisilicon/hpre/hpre_crypto.c index 7b5cb27d473d..a87f9904087a 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_crypto.c +++ b/drivers/crypto/hisilicon/hpre/hpre_crypto.c @@ -98,9 +98,6 @@ struct hpre_asym_request { struct timespec64 req_time; }; -static DEFINE_MUTEX(hpre_alg_lock); -static unsigned int hpre_active_devs; - static int hpre_alloc_req_id(struct hpre_ctx *ctx) { unsigned long flags; @@ -191,8 +188,7 @@ static int hpre_get_data_dma_addr(struct hpre_asym_request *hpre_req, hpre_req->dst = NULL; dma_dir = DMA_FROM_DEVICE; } - *tmp = dma_map_single(dev, sg_virt(data), - len, dma_dir); + *tmp = dma_map_single(dev, sg_virt(data), len, dma_dir); if (unlikely(dma_mapping_error(dev, *tmp))) { dev_err(dev, "dma map data err!\n"); return -ENOMEM; @@ -242,8 +238,8 @@ static int hpre_hw_data_init(struct hpre_asym_request *hpre_req, ((is_dh && !is_src) || !is_dh)) ret = hpre_get_data_dma_addr(hpre_req, data, len, is_src, &tmp); else - ret = hpre_prepare_dma_buf(hpre_req, data, len, - is_src, &tmp); + ret = hpre_prepare_dma_buf(hpre_req, data, len, is_src, &tmp); + if (unlikely(ret)) return ret; @@ -270,11 +266,9 @@ static void hpre_hw_data_clr_all(struct hpre_ctx *ctx, if (src) { if (req->src) - dma_free_coherent(dev, ctx->key_sz, - req->src, tmp); + dma_free_coherent(dev, ctx->key_sz, req->src, tmp); else - dma_unmap_single(dev, tmp, - ctx->key_sz, DMA_TO_DEVICE); + dma_unmap_single(dev, tmp, ctx->key_sz, DMA_TO_DEVICE); } tmp = le64_to_cpu(sqe->out); @@ -477,7 +471,7 @@ static int hpre_msg_request_set(struct hpre_ctx *ctx, void *req, bool is_rsa) h_req->areq.dh = kreq; msg = &h_req->req; memset(msg, 0, sizeof(*msg)); - msg->key = cpu_to_le64((u64)ctx->dh.dma_xa_p); + msg->key = cpu_to_le64(ctx->dh.dma_xa_p); } msg->dw0 |= cpu_to_le32(0x1 << HPRE_SQE_DONE_SHIFT); @@ -534,6 +528,8 @@ static int hpre_dh_compute_value(struct kpp_request *req) ret = hpre_hw_data_init(hpre_req, req->src, req->src_len, 1, 1); if (unlikely(ret)) goto clear_all; + } else { + msg->in = cpu_to_le64(ctx->dh.dma_g); } ret = hpre_hw_data_init(hpre_req, req->dst, req->dst_len, 0, 1); @@ -743,7 +739,7 @@ static int hpre_rsa_enc(struct akcipher_request *req) return ret; msg->dw0 |= cpu_to_le32(HPRE_ALG_NC_NCRT); - msg->key = cpu_to_le64((u64)ctx->rsa.dma_pubkey); + msg->key = cpu_to_le64(ctx->rsa.dma_pubkey); ret = hpre_hw_data_init(hpre_req, req->src, req->src_len, 1, 0); if (unlikely(ret)) @@ -791,11 +787,11 @@ static int hpre_rsa_dec(struct akcipher_request *req) return ret; if (ctx->crt_g2_mode) { - msg->key = cpu_to_le64((u64)ctx->rsa.dma_crt_prikey); + msg->key = cpu_to_le64(ctx->rsa.dma_crt_prikey); msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_NC_CRT); } else { - msg->key = cpu_to_le64((u64)ctx->rsa.dma_prikey); + msg->key = cpu_to_le64(ctx->rsa.dma_prikey); msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_NC_NCRT); } @@ -1160,36 +1156,25 @@ static struct kpp_alg dh = { int hpre_algs_register(void) { - int ret = 0; - - mutex_lock(&hpre_alg_lock); - if (++hpre_active_devs == 1) { - rsa.base.cra_flags = 0; - ret = crypto_register_akcipher(&rsa); - if (ret) - goto unlock; + int ret; + + rsa.base.cra_flags = 0; + ret = crypto_register_akcipher(&rsa); + if (ret) + return ret; #ifdef CONFIG_CRYPTO_DH - ret = crypto_register_kpp(&dh); - if (ret) { - crypto_unregister_akcipher(&rsa); - goto unlock; - } + ret = crypto_register_kpp(&dh); + if (ret) + crypto_unregister_akcipher(&rsa); #endif - } -unlock: - mutex_unlock(&hpre_alg_lock); return ret; } void hpre_algs_unregister(void) { - mutex_lock(&hpre_alg_lock); - if (--hpre_active_devs == 0) { - crypto_unregister_akcipher(&rsa); + crypto_unregister_akcipher(&rsa); #ifdef CONFIG_CRYPTO_DH - crypto_unregister_kpp(&dh); + crypto_unregister_kpp(&dh); #endif - } - mutex_unlock(&hpre_alg_lock); } diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c index b135c74fb619..a33394d91bbf 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_main.c +++ b/drivers/crypto/hisilicon/hpre/hpre_main.c @@ -90,7 +90,6 @@ #define HPRE_SQE_MASK_OFFSET 8 #define HPRE_SQE_MASK_LEN 24 -static struct hisi_qm_list hpre_devices; static const char hpre_name[] = "hisi_hpre"; static struct dentry *hpre_debugfs_root; static const struct pci_device_id hpre_dev_ids[] = { @@ -106,6 +105,11 @@ struct hpre_hw_error { const char *msg; }; +static struct hisi_qm_list hpre_devices = { + .register_to_crypto = hpre_algs_register, + .unregister_from_crypto = hpre_algs_unregister, +}; + static const char * const hpre_debug_file_name[] = { [HPRE_CURRENT_QM] = "current_qm", [HPRE_CLEAR_ENABLE] = "rdclr_en", @@ -186,7 +190,7 @@ static const struct kernel_param_ops hpre_pf_q_num_ops = { static u32 pf_q_num = HPRE_PF_DEF_Q_NUM; module_param_cb(pf_q_num, &hpre_pf_q_num_ops, &pf_q_num, 0444); -MODULE_PARM_DESC(pf_q_num, "Number of queues in PF of CS(1-1024)"); +MODULE_PARM_DESC(pf_q_num, "Number of queues in PF of CS(2-1024)"); static const struct kernel_param_ops vfs_num_ops = { .set = vfs_num_set, @@ -864,9 +868,7 @@ static int hpre_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (ret) dev_warn(&pdev->dev, "init debugfs fail!\n"); - hisi_qm_add_to_list(qm, &hpre_devices); - - ret = hpre_algs_register(); + ret = hisi_qm_alg_register(qm, &hpre_devices); if (ret < 0) { pci_err(pdev, "fail to register algs to crypto!\n"); goto err_with_qm_start; @@ -875,18 +877,17 @@ static int hpre_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (qm->fun_type == QM_HW_PF && vfs_num) { ret = hisi_qm_sriov_enable(pdev, vfs_num); if (ret < 0) - goto err_with_crypto_register; + goto err_with_alg_register; } return 0; -err_with_crypto_register: - hpre_algs_unregister(); +err_with_alg_register: + hisi_qm_alg_unregister(qm, &hpre_devices); err_with_qm_start: - hisi_qm_del_from_list(qm, &hpre_devices); hpre_debugfs_exit(qm); - hisi_qm_stop(qm); + hisi_qm_stop(qm, QM_NORMAL); err_with_err_init: hisi_qm_dev_err_uninit(qm); @@ -899,14 +900,13 @@ err_with_qm_init: static void hpre_remove(struct pci_dev *pdev) { - struct hpre *hpre = pci_get_drvdata(pdev); - struct hisi_qm *qm = &hpre->qm; + struct hisi_qm *qm = pci_get_drvdata(pdev); int ret; - hpre_algs_unregister(); - hisi_qm_del_from_list(qm, &hpre_devices); + hisi_qm_wait_task_finish(qm, &hpre_devices); + hisi_qm_alg_unregister(qm, &hpre_devices); if (qm->fun_type == QM_HW_PF && qm->vfs_num) { - ret = hisi_qm_sriov_disable(pdev); + ret = hisi_qm_sriov_disable(pdev, qm->is_frozen); if (ret) { pci_err(pdev, "Disable SRIOV fail!\n"); return; @@ -918,7 +918,7 @@ static void hpre_remove(struct pci_dev *pdev) } hpre_debugfs_exit(qm); - hisi_qm_stop(qm); + hisi_qm_stop(qm, QM_NORMAL); hisi_qm_dev_err_uninit(qm); hisi_qm_uninit(qm); } @@ -939,6 +939,7 @@ static struct pci_driver hpre_pci_driver = { .sriov_configure = IS_ENABLED(CONFIG_PCI_IOV) ? hisi_qm_sriov_configure : NULL, .err_handler = &hpre_err_handler, + .shutdown = hisi_qm_dev_shutdown, }; static void hpre_register_debugfs(void) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 6527c53b073f..530f23116d7c 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -180,7 +180,10 @@ #define QM_DBG_TMP_BUF_LEN 22 #define QM_PCI_COMMAND_INVALID ~0 +#define WAIT_PERIOD 20 +#define REMOVE_WAIT_DELAY 10 #define QM_SQE_ADDR_MASK GENMASK(7, 0) +#define QM_EQ_DEPTH (1024 * 2) #define QM_MK_CQC_DW3_V1(hop_num, pg_sz, buf_sz, cqe_sz) \ (((hop_num) << QM_CQ_HOP_NUM_SHIFT) | \ @@ -652,7 +655,7 @@ static void qm_work_process(struct work_struct *work) qp = qm_to_hisi_qp(qm, eqe); qm_poll_qp(qp, qm); - if (qm->status.eq_head == QM_Q_DEPTH - 1) { + if (qm->status.eq_head == QM_EQ_DEPTH - 1) { qm->status.eqc_phase = !qm->status.eqc_phase; eqe = qm->eqe; qm->status.eq_head = 0; @@ -661,7 +664,7 @@ static void qm_work_process(struct work_struct *work) qm->status.eq_head++; } - if (eqe_num == QM_Q_DEPTH / 2 - 1) { + if (eqe_num == QM_EQ_DEPTH / 2 - 1) { eqe_num = 0; qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0); } @@ -754,7 +757,7 @@ static void qm_init_qp_status(struct hisi_qp *qp) qp_status->sq_tail = 0; qp_status->cq_head = 0; qp_status->cqc_phase = true; - atomic_set(&qp_status->flags, 0); + atomic_set(&qp_status->used, 0); } static void qm_vft_data_cfg(struct hisi_qm *qm, enum vft_type type, u32 base, @@ -1046,17 +1049,7 @@ static int qm_regs_show(struct seq_file *s, void *unused) return 0; } -static int qm_regs_open(struct inode *inode, struct file *file) -{ - return single_open(file, qm_regs_show, inode->i_private); -} - -static const struct file_operations qm_regs_fops = { - .owner = THIS_MODULE, - .open = qm_regs_open, - .read = seq_read, - .release = single_release, -}; +DEFINE_SHOW_ATTRIBUTE(qm_regs); static ssize_t qm_cmd_read(struct file *filp, char __user *buffer, size_t count, loff_t *pos) @@ -1370,7 +1363,13 @@ static int qm_eq_aeq_dump(struct hisi_qm *qm, const char *s, return -EINVAL; ret = kstrtou32(s, 0, &xeqe_id); - if (ret || xeqe_id >= QM_Q_DEPTH) { + if (ret) + return -EINVAL; + + if (!strcmp(name, "EQE") && xeqe_id >= QM_EQ_DEPTH) { + dev_err(dev, "Please input eqe num (0-%d)", QM_EQ_DEPTH - 1); + return -EINVAL; + } else if (!strcmp(name, "AEQE") && xeqe_id >= QM_Q_DEPTH) { dev_err(dev, "Please input aeqe num (0-%d)", QM_Q_DEPTH - 1); return -EINVAL; } @@ -1420,17 +1419,18 @@ static int qm_dbg_help(struct hisi_qm *qm, char *s) static int qm_cmd_write_dump(struct hisi_qm *qm, const char *cmd_buf) { struct device *dev = &qm->pdev->dev; - char *presult, *s; + char *presult, *s, *s_tmp; int ret; s = kstrdup(cmd_buf, GFP_KERNEL); if (!s) return -ENOMEM; + s_tmp = s; presult = strsep(&s, " "); if (!presult) { - kfree(s); - return -EINVAL; + ret = -EINVAL; + goto err_buffer_free; } if (!strcmp(presult, "sqc")) @@ -1459,7 +1459,8 @@ static int qm_cmd_write_dump(struct hisi_qm *qm, const char *cmd_buf) if (ret) dev_info(dev, "Please echo help\n"); - kfree(s); +err_buffer_free: + kfree(s_tmp); return ret; } @@ -1644,7 +1645,7 @@ static void *qm_get_avail_sqe(struct hisi_qp *qp) struct hisi_qp_status *qp_status = &qp->qp_status; u16 sq_tail = qp_status->sq_tail; - if (unlikely(atomic_read(&qp->qp_status.used) == QM_Q_DEPTH)) + if (unlikely(atomic_read(&qp->qp_status.used) == QM_Q_DEPTH - 1)) return NULL; return qp->sqe + sq_tail * qp->qm->sqe_size; @@ -1981,7 +1982,7 @@ int hisi_qp_send(struct hisi_qp *qp, const void *msg) if (unlikely(atomic_read(&qp->qp_status.flags) == QP_STOP || atomic_read(&qp->qm->status.flags) == QM_STOP || qp->is_resetting)) { - dev_info(&qp->qm->pdev->dev, "QP is stopped or resetting\n"); + dev_info_ratelimited(&qp->qm->pdev->dev, "QP is stopped or resetting\n"); return -EAGAIN; } @@ -2215,6 +2216,82 @@ static int qm_alloc_uacce(struct hisi_qm *qm) } /** + * qm_frozen() - Try to froze QM to cut continuous queue request. If + * there is user on the QM, return failure without doing anything. + * @qm: The qm needed to be fronzen. + * + * This function frozes QM, then we can do SRIOV disabling. + */ +static int qm_frozen(struct hisi_qm *qm) +{ + down_write(&qm->qps_lock); + + if (qm->is_frozen) { + up_write(&qm->qps_lock); + return 0; + } + + if (!qm->qp_in_used) { + qm->qp_in_used = qm->qp_num; + qm->is_frozen = true; + up_write(&qm->qps_lock); + return 0; + } + + up_write(&qm->qps_lock); + + return -EBUSY; +} + +static int qm_try_frozen_vfs(struct pci_dev *pdev, + struct hisi_qm_list *qm_list) +{ + struct hisi_qm *qm, *vf_qm; + struct pci_dev *dev; + int ret = 0; + + if (!qm_list || !pdev) + return -EINVAL; + + /* Try to frozen all the VFs as disable SRIOV */ + mutex_lock(&qm_list->lock); + list_for_each_entry(qm, &qm_list->list, list) { + dev = qm->pdev; + if (dev == pdev) + continue; + if (pci_physfn(dev) == pdev) { + vf_qm = pci_get_drvdata(dev); + ret = qm_frozen(vf_qm); + if (ret) + goto frozen_fail; + } + } + +frozen_fail: + mutex_unlock(&qm_list->lock); + + return ret; +} + +/** + * hisi_qm_wait_task_finish() - Wait until the task is finished + * when removing the driver. + * @qm: The qm needed to wait for the task to finish. + * @qm_list: The list of all available devices. + */ +void hisi_qm_wait_task_finish(struct hisi_qm *qm, struct hisi_qm_list *qm_list) +{ + while (qm_frozen(qm) || + ((qm->fun_type == QM_HW_PF) && + qm_try_frozen_vfs(qm->pdev, qm_list))) { + msleep(WAIT_PERIOD); + } + + udelay(REMOVE_WAIT_DELAY); +} +EXPORT_SYMBOL_GPL(hisi_qm_wait_task_finish); + +/** * hisi_qm_get_free_qp_num() - Get free number of qp in qm. * @qm: The qm which want to get free qp. * @@ -2282,7 +2359,7 @@ static int hisi_qm_memory_init(struct hisi_qm *qm) } while (0) idr_init(&qm->qp_idr); - qm->qdma.size = QMC_ALIGN(sizeof(struct qm_eqe) * QM_Q_DEPTH) + + qm->qdma.size = QMC_ALIGN(sizeof(struct qm_eqe) * QM_EQ_DEPTH) + QMC_ALIGN(sizeof(struct qm_aeqe) * QM_Q_DEPTH) + QMC_ALIGN(sizeof(struct qm_sqc) * qm->qp_num) + QMC_ALIGN(sizeof(struct qm_cqc) * qm->qp_num); @@ -2292,7 +2369,7 @@ static int hisi_qm_memory_init(struct hisi_qm *qm) if (!qm->qdma.va) return -ENOMEM; - QM_INIT_BUF(qm, eqe, QM_Q_DEPTH); + QM_INIT_BUF(qm, eqe, QM_EQ_DEPTH); QM_INIT_BUF(qm, aeqe, QM_Q_DEPTH); QM_INIT_BUF(qm, sqc, qm->qp_num); QM_INIT_BUF(qm, cqc, qm->qp_num); @@ -2338,6 +2415,7 @@ static void hisi_qm_pre_init(struct hisi_qm *qm) mutex_init(&qm->mailbox_lock); init_rwsem(&qm->qps_lock); qm->qp_in_used = 0; + qm->is_frozen = false; } /** @@ -2462,7 +2540,7 @@ static int qm_eq_ctx_cfg(struct hisi_qm *qm) eqc->base_h = cpu_to_le32(upper_32_bits(qm->eqe_dma)); if (qm->ver == QM_HW_V1) eqc->dw3 = cpu_to_le32(QM_EQE_AEQE_SIZE); - eqc->dw6 = cpu_to_le32((QM_Q_DEPTH - 1) | (1 << QM_EQC_PHASE_SHIFT)); + eqc->dw6 = cpu_to_le32((QM_EQ_DEPTH - 1) | (1 << QM_EQC_PHASE_SHIFT)); ret = qm_mb(qm, QM_MB_CMD_EQC, eqc_dma, 0, 0); dma_unmap_single(dev, eqc_dma, sizeof(struct qm_eqc), DMA_TO_DEVICE); kfree(eqc); @@ -2633,18 +2711,20 @@ static void qm_clear_queues(struct hisi_qm *qm) /** * hisi_qm_stop() - Stop a qm. * @qm: The qm which will be stopped. + * @r: The reason to stop qm. * * This function stops qm and its qps, then qm can not accept request. * Related resources are not released at this state, we can use hisi_qm_start * to let qm start again. */ -int hisi_qm_stop(struct hisi_qm *qm) +int hisi_qm_stop(struct hisi_qm *qm, enum qm_stop_reason r) { struct device *dev = &qm->pdev->dev; int ret = 0; down_write(&qm->qps_lock); + qm->status.stop_reason = r; if (!qm_avail_state(qm, QM_STOP)) { ret = -EPERM; goto err_unlock; @@ -3081,11 +3161,12 @@ EXPORT_SYMBOL_GPL(hisi_qm_sriov_enable); /** * hisi_qm_sriov_disable - disable virtual functions - * @pdev: the PCI device + * @pdev: the PCI device. + * @is_frozen: true when all the VFs are frozen. * - * Return failure if there are VFs assigned already. + * Return failure if there are VFs assigned already or VF is in used. */ -int hisi_qm_sriov_disable(struct pci_dev *pdev) +int hisi_qm_sriov_disable(struct pci_dev *pdev, bool is_frozen) { struct hisi_qm *qm = pci_get_drvdata(pdev); @@ -3094,7 +3175,12 @@ int hisi_qm_sriov_disable(struct pci_dev *pdev) return -EPERM; } - /* remove in hpre_pci_driver will be called to free VF resources */ + /* While VF is in used, SRIOV cannot be disabled. */ + if (!is_frozen && qm_try_frozen_vfs(pdev, qm->qm_list)) { + pci_err(pdev, "Task is using its VF!\n"); + return -EBUSY; + } + pci_disable_sriov(pdev); return qm_clear_vft_config(qm); } @@ -3110,7 +3196,7 @@ EXPORT_SYMBOL_GPL(hisi_qm_sriov_disable); int hisi_qm_sriov_configure(struct pci_dev *pdev, int num_vfs) { if (num_vfs == 0) - return hisi_qm_sriov_disable(pdev); + return hisi_qm_sriov_disable(pdev, 0); else return hisi_qm_sriov_enable(pdev, num_vfs); } @@ -3290,10 +3376,10 @@ static int qm_set_msi(struct hisi_qm *qm, bool set) return 0; } -static int qm_vf_reset_prepare(struct hisi_qm *qm) +static int qm_vf_reset_prepare(struct hisi_qm *qm, + enum qm_stop_reason stop_reason) { struct hisi_qm_list *qm_list = qm->qm_list; - int stop_reason = qm->status.stop_reason; struct pci_dev *pdev = qm->pdev; struct pci_dev *virtfn; struct hisi_qm *vf_qm; @@ -3306,8 +3392,10 @@ static int qm_vf_reset_prepare(struct hisi_qm *qm) continue; if (pci_physfn(virtfn) == pdev) { - vf_qm->status.stop_reason = stop_reason; - ret = hisi_qm_stop(vf_qm); + /* save VFs PCIE BAR configuration */ + pci_save_state(virtfn); + + ret = hisi_qm_stop(vf_qm, stop_reason); if (ret) goto stop_fail; } @@ -3346,15 +3434,14 @@ static int qm_controller_reset_prepare(struct hisi_qm *qm) } if (qm->vfs_num) { - ret = qm_vf_reset_prepare(qm); + ret = qm_vf_reset_prepare(qm, QM_SOFT_RESET); if (ret) { pci_err(pdev, "Fails to stop VFs!\n"); return ret; } } - qm->status.stop_reason = QM_SOFT_RESET; - ret = hisi_qm_stop(qm); + ret = hisi_qm_stop(qm, QM_SOFT_RESET); if (ret) { pci_err(pdev, "Fails to stop QM!\n"); return ret; @@ -3471,6 +3558,9 @@ static int qm_vf_reset_done(struct hisi_qm *qm) continue; if (pci_physfn(virtfn) == pdev) { + /* enable VFs PCIE BAR configuration */ + pci_restore_state(virtfn); + ret = qm_restart(vf_qm); if (ret) goto restart_fail; @@ -3695,7 +3785,7 @@ void hisi_qm_reset_prepare(struct pci_dev *pdev) } if (qm->vfs_num) { - ret = qm_vf_reset_prepare(qm); + ret = qm_vf_reset_prepare(qm, QM_FLR); if (ret) { pci_err(pdev, "Failed to prepare reset, ret = %d.\n", ret); @@ -3703,7 +3793,7 @@ void hisi_qm_reset_prepare(struct pci_dev *pdev) } } - ret = hisi_qm_stop(qm); + ret = hisi_qm_stop(qm, QM_FLR); if (ret) { pci_err(pdev, "Failed to stop QM, ret = %d.\n", ret); return; @@ -3821,6 +3911,23 @@ err_aeq_irq: return ret; } +/** + * hisi_qm_dev_shutdown() - Shutdown device. + * @pdev: The device will be shutdown. + * + * This function will stop qm when OS shutdown or rebooting. + */ +void hisi_qm_dev_shutdown(struct pci_dev *pdev) +{ + struct hisi_qm *qm = pci_get_drvdata(pdev); + int ret; + + ret = hisi_qm_stop(qm, QM_NORMAL); + if (ret) + dev_err(&pdev->dev, "Fail to stop qm in shutdown!\n"); +} +EXPORT_SYMBOL_GPL(hisi_qm_dev_shutdown); + static void hisi_qm_controller_reset(struct work_struct *rst_work) { struct hisi_qm *qm = container_of(rst_work, struct hisi_qm, rst_work); @@ -3834,6 +3941,58 @@ static void hisi_qm_controller_reset(struct work_struct *rst_work) } /** + * hisi_qm_alg_register() - Register alg to crypto and add qm to qm_list. + * @qm: The qm needs add. + * @qm_list: The qm list. + * + * This function adds qm to qm list, and will register algorithm to + * crypto when the qm list is empty. + */ +int hisi_qm_alg_register(struct hisi_qm *qm, struct hisi_qm_list *qm_list) +{ + int flag = 0; + int ret = 0; + + mutex_lock(&qm_list->lock); + if (list_empty(&qm_list->list)) + flag = 1; + list_add_tail(&qm->list, &qm_list->list); + mutex_unlock(&qm_list->lock); + + if (flag) { + ret = qm_list->register_to_crypto(); + if (ret) { + mutex_lock(&qm_list->lock); + list_del(&qm->list); + mutex_unlock(&qm_list->lock); + } + } + + return ret; +} +EXPORT_SYMBOL_GPL(hisi_qm_alg_register); + +/** + * hisi_qm_alg_unregister() - Unregister alg from crypto and delete qm from + * qm list. + * @qm: The qm needs delete. + * @qm_list: The qm list. + * + * This function deletes qm from qm list, and will unregister algorithm + * from crypto when the qm list is empty. + */ +void hisi_qm_alg_unregister(struct hisi_qm *qm, struct hisi_qm_list *qm_list) +{ + mutex_lock(&qm_list->lock); + list_del(&qm->list); + mutex_unlock(&qm_list->lock); + + if (list_empty(&qm_list->list)) + qm_list->unregister_from_crypto(); +} +EXPORT_SYMBOL_GPL(hisi_qm_alg_unregister); + +/** * hisi_qm_init() - Initialize configures about qm. * @qm: The qm needing init. * diff --git a/drivers/crypto/hisilicon/qm.h b/drivers/crypto/hisilicon/qm.h index 6c1d3c7d64ee..0420f4ce7197 100644 --- a/drivers/crypto/hisilicon/qm.h +++ b/drivers/crypto/hisilicon/qm.h @@ -79,7 +79,7 @@ #define QM_BASE_CE QM_ECC_1BIT #define QM_Q_DEPTH 1024 - +#define QM_MIN_QNUM 2 #define HISI_ACC_SGL_SGE_NR_MAX 255 /* page number for queue file region */ @@ -193,6 +193,8 @@ struct hisi_qm_err_ini { struct hisi_qm_list { struct mutex lock; struct list_head list; + int (*register_to_crypto)(void); + void (*unregister_from_crypto)(void); }; struct hisi_qm { @@ -243,6 +245,7 @@ struct hisi_qm { const char *algs; bool use_sva; + bool is_frozen; resource_size_t phys_base; resource_size_t phys_size; struct uacce_device *uacce; @@ -306,7 +309,7 @@ static inline int q_num_set(const char *val, const struct kernel_param *kp, } ret = kstrtou32(val, 10, &n); - if (ret || !n || n > q_num) + if (ret || n < QM_MIN_QNUM || n > q_num) return -EINVAL; return param_set_int(val, kp); @@ -336,26 +339,10 @@ static inline void hisi_qm_init_list(struct hisi_qm_list *qm_list) mutex_init(&qm_list->lock); } -static inline void hisi_qm_add_to_list(struct hisi_qm *qm, - struct hisi_qm_list *qm_list) -{ - mutex_lock(&qm_list->lock); - list_add_tail(&qm->list, &qm_list->list); - mutex_unlock(&qm_list->lock); -} - -static inline void hisi_qm_del_from_list(struct hisi_qm *qm, - struct hisi_qm_list *qm_list) -{ - mutex_lock(&qm_list->lock); - list_del(&qm->list); - mutex_unlock(&qm_list->lock); -} - int hisi_qm_init(struct hisi_qm *qm); void hisi_qm_uninit(struct hisi_qm *qm); int hisi_qm_start(struct hisi_qm *qm); -int hisi_qm_stop(struct hisi_qm *qm); +int hisi_qm_stop(struct hisi_qm *qm, enum qm_stop_reason r); struct hisi_qp *hisi_qm_create_qp(struct hisi_qm *qm, u8 alg_type); int hisi_qm_start_qp(struct hisi_qp *qp, unsigned long arg); int hisi_qm_stop_qp(struct hisi_qp *qp); @@ -367,7 +354,7 @@ int hisi_qm_debug_init(struct hisi_qm *qm); enum qm_hw_ver hisi_qm_get_hw_version(struct pci_dev *pdev); void hisi_qm_debug_regs_clear(struct hisi_qm *qm); int hisi_qm_sriov_enable(struct pci_dev *pdev, int max_vfs); -int hisi_qm_sriov_disable(struct pci_dev *pdev); +int hisi_qm_sriov_disable(struct pci_dev *pdev, bool is_frozen); int hisi_qm_sriov_configure(struct pci_dev *pdev, int num_vfs); void hisi_qm_dev_err_init(struct hisi_qm *qm); void hisi_qm_dev_err_uninit(struct hisi_qm *qm); @@ -390,4 +377,8 @@ void hisi_acc_free_sgl_pool(struct device *dev, int hisi_qm_alloc_qps_node(struct hisi_qm_list *qm_list, int qp_num, u8 alg_type, int node, struct hisi_qp **qps); void hisi_qm_free_qps(struct hisi_qp **qps, int qp_num); +void hisi_qm_dev_shutdown(struct pci_dev *pdev); +void hisi_qm_wait_task_finish(struct hisi_qm *qm, struct hisi_qm_list *qm_list); +int hisi_qm_alg_register(struct hisi_qm *qm, struct hisi_qm_list *qm_list); +void hisi_qm_alg_unregister(struct hisi_qm *qm, struct hisi_qm_list *qm_list); #endif diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.c b/drivers/crypto/hisilicon/sec2/sec_crypto.c index 497969ae8b23..bb493423668c 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.c +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.c @@ -66,8 +66,6 @@ #define SEC_SQE_AEAD_FLAG 3 #define SEC_SQE_DONE 0x1 -static atomic_t sec_active_devs; - /* Get an en/de-cipher queue cyclically to balance load over queues of TFM */ static inline int sec_alloc_queue_id(struct sec_ctx *ctx, struct sec_req *req) { @@ -342,11 +340,14 @@ static int sec_alg_resource_alloc(struct sec_ctx *ctx, ret = sec_alloc_pbuf_resource(dev, res); if (ret) { dev_err(dev, "fail to alloc pbuf dma resource!\n"); - goto alloc_fail; + goto alloc_pbuf_fail; } } return 0; +alloc_pbuf_fail: + if (ctx->alg_type == SEC_AEAD) + sec_free_mac_resource(dev, qp_ctx->res); alloc_fail: sec_free_civ_resource(dev, res); @@ -457,8 +458,10 @@ static int sec_ctx_base_init(struct sec_ctx *ctx) ctx->fake_req_limit = QM_Q_DEPTH >> 1; ctx->qp_ctx = kcalloc(sec->ctx_q_num, sizeof(struct sec_qp_ctx), GFP_KERNEL); - if (!ctx->qp_ctx) - return -ENOMEM; + if (!ctx->qp_ctx) { + ret = -ENOMEM; + goto err_destroy_qps; + } for (i = 0; i < sec->ctx_q_num; i++) { ret = sec_create_qp_ctx(&sec->qm, ctx, i, 0); @@ -467,12 +470,15 @@ static int sec_ctx_base_init(struct sec_ctx *ctx) } return 0; + err_sec_release_qp_ctx: for (i = i - 1; i >= 0; i--) sec_release_qp_ctx(ctx, &ctx->qp_ctx[i]); - sec_destroy_qps(ctx->qps, sec->ctx_q_num); kfree(ctx->qp_ctx); +err_destroy_qps: + sec_destroy_qps(ctx->qps, sec->ctx_q_num); + return ret; } @@ -1633,33 +1639,24 @@ static struct aead_alg sec_aeads[] = { int sec_register_to_crypto(void) { - int ret = 0; + int ret; /* To avoid repeat register */ - if (atomic_add_return(1, &sec_active_devs) == 1) { - ret = crypto_register_skciphers(sec_skciphers, - ARRAY_SIZE(sec_skciphers)); - if (ret) - return ret; - - ret = crypto_register_aeads(sec_aeads, ARRAY_SIZE(sec_aeads)); - if (ret) - goto reg_aead_fail; - } - - return ret; - -reg_aead_fail: - crypto_unregister_skciphers(sec_skciphers, ARRAY_SIZE(sec_skciphers)); + ret = crypto_register_skciphers(sec_skciphers, + ARRAY_SIZE(sec_skciphers)); + if (ret) + return ret; + ret = crypto_register_aeads(sec_aeads, ARRAY_SIZE(sec_aeads)); + if (ret) + crypto_unregister_skciphers(sec_skciphers, + ARRAY_SIZE(sec_skciphers)); return ret; } void sec_unregister_from_crypto(void) { - if (atomic_sub_return(1, &sec_active_devs) == 0) { - crypto_unregister_skciphers(sec_skciphers, - ARRAY_SIZE(sec_skciphers)); - crypto_unregister_aeads(sec_aeads, ARRAY_SIZE(sec_aeads)); - } + crypto_unregister_skciphers(sec_skciphers, + ARRAY_SIZE(sec_skciphers)); + crypto_unregister_aeads(sec_aeads, ARRAY_SIZE(sec_aeads)); } diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c index 2297425486cb..548896394c4b 100644 --- a/drivers/crypto/hisilicon/sec2/sec_main.c +++ b/drivers/crypto/hisilicon/sec2/sec_main.c @@ -99,7 +99,11 @@ struct sec_dfx_item { static const char sec_name[] = "hisi_sec2"; static struct dentry *sec_debugfs_root; -static struct hisi_qm_list sec_devices; + +static struct hisi_qm_list sec_devices = { + .register_to_crypto = sec_register_to_crypto, + .unregister_from_crypto = sec_unregister_from_crypto, +}; static const struct sec_hw_error sec_hw_errors[] = { {.int_msk = BIT(0), .msg = "sec_axi_rresp_err_rint"}, @@ -165,7 +169,7 @@ static const struct kernel_param_ops sec_pf_q_num_ops = { static u32 pf_q_num = SEC_PF_DEF_Q_NUM; module_param_cb(pf_q_num, &sec_pf_q_num_ops, &pf_q_num, 0444); -MODULE_PARM_DESC(pf_q_num, "Number of queues in PF(v1 0-4096, v2 0-1024)"); +MODULE_PARM_DESC(pf_q_num, "Number of queues in PF(v1 2-4096, v2 2-1024)"); static int sec_ctx_q_num_set(const char *val, const struct kernel_param *kp) { @@ -879,29 +883,26 @@ static int sec_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (ret) pci_warn(pdev, "Failed to init debugfs!\n"); - hisi_qm_add_to_list(qm, &sec_devices); - - ret = sec_register_to_crypto(); + ret = hisi_qm_alg_register(qm, &sec_devices); if (ret < 0) { pr_err("Failed to register driver to crypto.\n"); - goto err_remove_from_list; + goto err_qm_stop; } if (qm->fun_type == QM_HW_PF && vfs_num) { ret = hisi_qm_sriov_enable(pdev, vfs_num); if (ret < 0) - goto err_crypto_unregister; + goto err_alg_unregister; } return 0; -err_crypto_unregister: - sec_unregister_from_crypto(); +err_alg_unregister: + hisi_qm_alg_unregister(qm, &sec_devices); -err_remove_from_list: - hisi_qm_del_from_list(qm, &sec_devices); +err_qm_stop: sec_debugfs_exit(qm); - hisi_qm_stop(qm); + hisi_qm_stop(qm, QM_NORMAL); err_probe_uninit: sec_probe_uninit(qm); @@ -914,19 +915,16 @@ err_qm_uninit: static void sec_remove(struct pci_dev *pdev) { - struct sec_dev *sec = pci_get_drvdata(pdev); - struct hisi_qm *qm = &sec->qm; - - sec_unregister_from_crypto(); - - hisi_qm_del_from_list(qm, &sec_devices); + struct hisi_qm *qm = pci_get_drvdata(pdev); + hisi_qm_wait_task_finish(qm, &sec_devices); + hisi_qm_alg_unregister(qm, &sec_devices); if (qm->fun_type == QM_HW_PF && qm->vfs_num) - hisi_qm_sriov_disable(pdev); + hisi_qm_sriov_disable(pdev, qm->is_frozen); sec_debugfs_exit(qm); - (void)hisi_qm_stop(qm); + (void)hisi_qm_stop(qm, QM_NORMAL); if (qm->fun_type == QM_HW_PF) sec_debug_regs_clear(qm); @@ -950,6 +948,7 @@ static struct pci_driver sec_pci_driver = { .remove = sec_remove, .err_handler = &sec_err_handler, .sriov_configure = hisi_qm_sriov_configure, + .shutdown = hisi_qm_dev_shutdown, }; static void sec_register_debugfs(void) diff --git a/drivers/crypto/hisilicon/zip/zip.h b/drivers/crypto/hisilicon/zip/zip.h index 4484be13812b..92397f993e23 100644 --- a/drivers/crypto/hisilicon/zip/zip.h +++ b/drivers/crypto/hisilicon/zip/zip.h @@ -9,20 +9,6 @@ #include <linux/list.h> #include "../qm.h" -/* hisi_zip_sqe dw3 */ -#define HZIP_BD_STATUS_M GENMASK(7, 0) -/* hisi_zip_sqe dw7 */ -#define HZIP_IN_SGE_DATA_OFFSET_M GENMASK(23, 0) -/* hisi_zip_sqe dw8 */ -#define HZIP_OUT_SGE_DATA_OFFSET_M GENMASK(23, 0) -/* hisi_zip_sqe dw9 */ -#define HZIP_REQ_TYPE_M GENMASK(7, 0) -#define HZIP_ALG_TYPE_ZLIB 0x02 -#define HZIP_ALG_TYPE_GZIP 0x03 -#define HZIP_BUF_TYPE_M GENMASK(11, 8) -#define HZIP_PBUFFER 0x0 -#define HZIP_SGL 0x1 - enum hisi_zip_error_type { /* negative compression */ HZIP_NC_ERR = 0x0d, @@ -39,7 +25,6 @@ struct hisi_zip_ctrl; struct hisi_zip { struct hisi_qm qm; - struct list_head list; struct hisi_zip_ctrl *ctrl; struct hisi_zip_dfx dfx; }; diff --git a/drivers/crypto/hisilicon/zip/zip_crypto.c b/drivers/crypto/hisilicon/zip/zip_crypto.c index 01fd6a78111d..08b4660b014c 100644 --- a/drivers/crypto/hisilicon/zip/zip_crypto.c +++ b/drivers/crypto/hisilicon/zip/zip_crypto.c @@ -6,6 +6,20 @@ #include <linux/scatterlist.h> #include "zip.h" +/* hisi_zip_sqe dw3 */ +#define HZIP_BD_STATUS_M GENMASK(7, 0) +/* hisi_zip_sqe dw7 */ +#define HZIP_IN_SGE_DATA_OFFSET_M GENMASK(23, 0) +/* hisi_zip_sqe dw8 */ +#define HZIP_OUT_SGE_DATA_OFFSET_M GENMASK(23, 0) +/* hisi_zip_sqe dw9 */ +#define HZIP_REQ_TYPE_M GENMASK(7, 0) +#define HZIP_ALG_TYPE_ZLIB 0x02 +#define HZIP_ALG_TYPE_GZIP 0x03 +#define HZIP_BUF_TYPE_M GENMASK(11, 8) +#define HZIP_PBUFFER 0x0 +#define HZIP_SGL 0x1 + #define HZIP_ZLIB_HEAD_SIZE 2 #define HZIP_GZIP_HEAD_SIZE 10 @@ -16,22 +30,29 @@ #define GZIP_HEAD_FLG_SHIFT 3 #define GZIP_HEAD_FEXTRA_SHIFT 10 -#define GZIP_HEAD_FEXTRA_XLEN 2 +#define GZIP_HEAD_FEXTRA_XLEN 2UL #define GZIP_HEAD_FHCRC_SIZE 2 -#define HZIP_CTX_Q_NUM 2 #define HZIP_GZIP_HEAD_BUF 256 #define HZIP_ALG_PRIORITY 300 #define HZIP_SGL_SGE_NR 10 static const u8 zlib_head[HZIP_ZLIB_HEAD_SIZE] = {0x78, 0x9c}; -static const u8 gzip_head[HZIP_GZIP_HEAD_SIZE] = {0x1f, 0x8b, 0x08, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x03}; +static const u8 gzip_head[HZIP_GZIP_HEAD_SIZE] = { + 0x1f, 0x8b, 0x08, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x03 +}; + enum hisi_zip_alg_type { HZIP_ALG_TYPE_COMP = 0, HZIP_ALG_TYPE_DECOMP = 1, }; +enum { + HZIP_QPC_COMP, + HZIP_QPC_DECOMP, + HZIP_CTX_Q_NUM +}; + #define COMP_NAME_TO_TYPE(alg_name) \ (!strcmp((alg_name), "zlib-deflate") ? HZIP_ALG_TYPE_ZLIB : \ !strcmp((alg_name), "gzip") ? HZIP_ALG_TYPE_GZIP : 0) \ @@ -46,13 +67,13 @@ enum hisi_zip_alg_type { struct hisi_zip_req { struct acomp_req *req; - int sskip; - int dskip; + u32 sskip; + u32 dskip; struct hisi_acc_hw_sgl *hw_src; struct hisi_acc_hw_sgl *hw_dst; dma_addr_t dma_src; dma_addr_t dma_dst; - int req_id; + u16 req_id; }; struct hisi_zip_req_q { @@ -71,8 +92,6 @@ struct hisi_zip_qp_ctx { }; struct hisi_zip_ctx { -#define QPC_COMP 0 -#define QPC_DECOMP 1 struct hisi_zip_qp_ctx qp_ctx[HZIP_CTX_Q_NUM]; }; @@ -116,7 +135,7 @@ static void hisi_zip_config_tag(struct hisi_zip_sqe *sqe, u32 tag) static void hisi_zip_fill_sqe(struct hisi_zip_sqe *sqe, u8 req_type, dma_addr_t s_addr, dma_addr_t d_addr, u32 slen, - u32 dlen, int sskip, int dskip) + u32 dlen, u32 sskip, u32 dskip) { memset(sqe, 0, sizeof(struct hisi_zip_sqe)); @@ -143,7 +162,7 @@ static int hisi_zip_start_qp(struct hisi_qp *qp, struct hisi_zip_qp_ctx *ctx, ret = hisi_qm_start_qp(qp, 0); if (ret < 0) { - dev_err(dev, "start qp failed!\n"); + dev_err(dev, "failed to start qp (%d)!\n", ret); return ret; } @@ -166,7 +185,7 @@ static int hisi_zip_ctx_init(struct hisi_zip_ctx *hisi_zip_ctx, u8 req_type, int ret = zip_create_qps(qps, HZIP_CTX_Q_NUM, node); if (ret) { - pr_err("Can not create zip qps!\n"); + pr_err("failed to create zip qps (%d)!\n", ret); return -ENODEV; } @@ -264,11 +283,11 @@ static int hisi_zip_create_req_q(struct hisi_zip_ctx *ctx) return 0; err_free_loop1: - kfree(ctx->qp_ctx[QPC_DECOMP].req_q.req_bitmap); + kfree(ctx->qp_ctx[HZIP_QPC_DECOMP].req_q.req_bitmap); err_free_loop0: - kfree(ctx->qp_ctx[QPC_COMP].req_q.q); + kfree(ctx->qp_ctx[HZIP_QPC_COMP].req_q.q); err_free_bitmap: - kfree(ctx->qp_ctx[QPC_COMP].req_q.req_bitmap); + kfree(ctx->qp_ctx[HZIP_QPC_COMP].req_q.req_bitmap); return ret; } @@ -303,8 +322,8 @@ static int hisi_zip_create_sgl_pool(struct hisi_zip_ctx *ctx) return 0; err_free_sgl_pool0: - hisi_acc_free_sgl_pool(&ctx->qp_ctx[QPC_COMP].qp->qm->pdev->dev, - ctx->qp_ctx[QPC_COMP].sgl_pool); + hisi_acc_free_sgl_pool(&ctx->qp_ctx[HZIP_QPC_COMP].qp->qm->pdev->dev, + ctx->qp_ctx[HZIP_QPC_COMP].sgl_pool); return -ENOMEM; } @@ -342,7 +361,6 @@ static void hisi_zip_acomp_cb(struct hisi_qp *qp, void *data) atomic64_inc(&dfx->recv_cnt); status = sqe->dw3 & HZIP_BD_STATUS_M; - if (status != 0 && status != HZIP_NC_ERR) { dev_err(dev, "%scompress fail in qp%u: %u, output: %u\n", (qp->alg_type == 0) ? "" : "de", qp->qp_id, status, @@ -377,19 +395,28 @@ static int hisi_zip_acomp_init(struct crypto_acomp *tfm) { const char *alg_name = crypto_tfm_alg_name(&tfm->base); struct hisi_zip_ctx *ctx = crypto_tfm_ctx(&tfm->base); + struct device *dev; int ret; ret = hisi_zip_ctx_init(ctx, COMP_NAME_TO_TYPE(alg_name), tfm->base.node); - if (ret) + if (ret) { + pr_err("failed to init ctx (%d)!\n", ret); return ret; + } + + dev = &ctx->qp_ctx[0].qp->qm->pdev->dev; ret = hisi_zip_create_req_q(ctx); - if (ret) + if (ret) { + dev_err(dev, "failed to create request queue (%d)!\n", ret); goto err_ctx_exit; + } ret = hisi_zip_create_sgl_pool(ctx); - if (ret) + if (ret) { + dev_err(dev, "failed to create sgl pool (%d)!\n", ret); goto err_release_req_q; + } hisi_zip_set_acomp_cb(ctx, hisi_zip_acomp_cb); @@ -419,13 +446,15 @@ static int add_comp_head(struct scatterlist *dst, u8 req_type) int ret; ret = sg_copy_from_buffer(dst, sg_nents(dst), head, head_size); - if (ret != head_size) + if (ret != head_size) { + pr_err("the head size of buffer is wrong (%d)!\n", ret); return -ENOMEM; + } return head_size; } -static size_t get_gzip_head_size(struct scatterlist *sgl) +static size_t __maybe_unused get_gzip_head_size(struct scatterlist *sgl) { char buf[HZIP_GZIP_HEAD_BUF]; @@ -434,13 +463,20 @@ static size_t get_gzip_head_size(struct scatterlist *sgl) return __get_gzip_head_size(buf); } -static size_t get_comp_head_size(struct scatterlist *src, u8 req_type) +static int get_comp_head_size(struct acomp_req *acomp_req, u8 req_type) { + if (!acomp_req->src || !acomp_req->slen) + return -EINVAL; + + if ((req_type == HZIP_ALG_TYPE_GZIP) && + (acomp_req->slen < GZIP_HEAD_FEXTRA_SHIFT)) + return -EINVAL; + switch (req_type) { case HZIP_ALG_TYPE_ZLIB: return TO_HEAD_SIZE(HZIP_ALG_TYPE_ZLIB); case HZIP_ALG_TYPE_GZIP: - return get_gzip_head_size(src); + return TO_HEAD_SIZE(HZIP_ALG_TYPE_GZIP); default: pr_err("request type does not support!\n"); return -EINVAL; @@ -462,7 +498,7 @@ static struct hisi_zip_req *hisi_zip_create_req(struct acomp_req *req, if (req_id >= req_q->size) { write_unlock(&req_q->req_lock); dev_dbg(&qp_ctx->qp->qm->pdev->dev, "req cache is full!\n"); - return ERR_PTR(-EBUSY); + return ERR_PTR(-EAGAIN); } set_bit(req_id, req_q->req_bitmap); @@ -492,8 +528,7 @@ static int hisi_zip_do_work(struct hisi_zip_req *req, struct hisi_acc_sgl_pool *pool = qp_ctx->sgl_pool; struct hisi_zip_dfx *dfx = &qp_ctx->zip_dev->dfx; struct hisi_zip_sqe zip_sqe; - dma_addr_t input; - dma_addr_t output; + dma_addr_t input, output; int ret; if (!a_req->src || !a_req->slen || !a_req->dst || !a_req->dlen) @@ -501,8 +536,11 @@ static int hisi_zip_do_work(struct hisi_zip_req *req, req->hw_src = hisi_acc_sg_buf_map_to_hw_sgl(dev, a_req->src, pool, req->req_id << 1, &input); - if (IS_ERR(req->hw_src)) + if (IS_ERR(req->hw_src)) { + dev_err(dev, "failed to map the src buffer to hw sgl (%ld)!\n", + PTR_ERR(req->hw_src)); return PTR_ERR(req->hw_src); + } req->dma_src = input; req->hw_dst = hisi_acc_sg_buf_map_to_hw_sgl(dev, a_req->dst, pool, @@ -510,6 +548,8 @@ static int hisi_zip_do_work(struct hisi_zip_req *req, &output); if (IS_ERR(req->hw_dst)) { ret = PTR_ERR(req->hw_dst); + dev_err(dev, "failed to map the dst buffer to hw slg (%d)!\n", + ret); goto err_unmap_input; } req->dma_dst = output; @@ -524,6 +564,8 @@ static int hisi_zip_do_work(struct hisi_zip_req *req, ret = hisi_qp_send(qp, &zip_sqe); if (ret < 0) { atomic64_inc(&dfx->send_busy_cnt); + ret = -EAGAIN; + dev_dbg_ratelimited(dev, "failed to send request!\n"); goto err_unmap_output; } @@ -539,23 +581,29 @@ err_unmap_input: static int hisi_zip_acompress(struct acomp_req *acomp_req) { struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm); - struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[QPC_COMP]; + struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[HZIP_QPC_COMP]; + struct device *dev = &qp_ctx->qp->qm->pdev->dev; struct hisi_zip_req *req; int head_size; int ret; /* let's output compression head now */ head_size = add_comp_head(acomp_req->dst, qp_ctx->qp->req_type); - if (head_size < 0) - return -ENOMEM; + if (head_size < 0) { + dev_err_ratelimited(dev, "failed to add comp head (%d)!\n", + head_size); + return head_size; + } - req = hisi_zip_create_req(acomp_req, qp_ctx, (size_t)head_size, true); + req = hisi_zip_create_req(acomp_req, qp_ctx, head_size, true); if (IS_ERR(req)) return PTR_ERR(req); ret = hisi_zip_do_work(req, qp_ctx); - if (ret != -EINPROGRESS) + if (ret != -EINPROGRESS) { + dev_info_ratelimited(dev, "failed to do compress (%d)!\n", ret); hisi_zip_remove_req(qp_ctx, req); + } return ret; } @@ -563,20 +611,28 @@ static int hisi_zip_acompress(struct acomp_req *acomp_req) static int hisi_zip_adecompress(struct acomp_req *acomp_req) { struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm); - struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[QPC_DECOMP]; + struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[HZIP_QPC_DECOMP]; + struct device *dev = &qp_ctx->qp->qm->pdev->dev; struct hisi_zip_req *req; - size_t head_size; - int ret; + int head_size, ret; - head_size = get_comp_head_size(acomp_req->src, qp_ctx->qp->req_type); + head_size = get_comp_head_size(acomp_req, qp_ctx->qp->req_type); + if (head_size < 0) { + dev_err_ratelimited(dev, "failed to get comp head size (%d)!\n", + head_size); + return head_size; + } req = hisi_zip_create_req(acomp_req, qp_ctx, head_size, false); if (IS_ERR(req)) return PTR_ERR(req); ret = hisi_zip_do_work(req, qp_ctx); - if (ret != -EINPROGRESS) + if (ret != -EINPROGRESS) { + dev_info_ratelimited(dev, "failed to do decompress (%d)!\n", + ret); hisi_zip_remove_req(qp_ctx, req); + } return ret; } @@ -611,17 +667,17 @@ static struct acomp_alg hisi_zip_acomp_gzip = { int hisi_zip_register_to_crypto(void) { - int ret = 0; + int ret; ret = crypto_register_acomp(&hisi_zip_acomp_zlib); if (ret) { - pr_err("Zlib acomp algorithm registration failed\n"); + pr_err("failed to register to zlib (%d)!\n", ret); return ret; } ret = crypto_register_acomp(&hisi_zip_acomp_gzip); if (ret) { - pr_err("Gzip acomp algorithm registration failed\n"); + pr_err("failed to register to gzip (%d)!\n", ret); crypto_unregister_acomp(&hisi_zip_acomp_zlib); } diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c index e2845b2c963d..4bd2c811abba 100644 --- a/drivers/crypto/hisilicon/zip/zip_main.c +++ b/drivers/crypto/hisilicon/zip/zip_main.c @@ -17,7 +17,6 @@ #define PCI_DEVICE_ID_ZIP_PF 0xa250 #define PCI_DEVICE_ID_ZIP_VF 0xa251 -#define HZIP_VF_NUM 63 #define HZIP_QUEUE_NUM_V1 4096 #define HZIP_QUEUE_NUM_V2 1024 @@ -30,18 +29,18 @@ #define DECOMP3_ENABLE BIT(5) #define DECOMP4_ENABLE BIT(6) #define DECOMP5_ENABLE BIT(7) -#define ALL_COMP_DECOMP_EN (COMP0_ENABLE | COMP1_ENABLE | \ +#define HZIP_ALL_COMP_DECOMP_EN (COMP0_ENABLE | COMP1_ENABLE | \ DECOMP0_ENABLE | DECOMP1_ENABLE | \ DECOMP2_ENABLE | DECOMP3_ENABLE | \ DECOMP4_ENABLE | DECOMP5_ENABLE) -#define DECOMP_CHECK_ENABLE BIT(16) +#define HZIP_DECOMP_CHECK_ENABLE BIT(16) #define HZIP_FSM_MAX_CNT 0x301008 #define HZIP_PORT_ARCA_CHE_0 0x301040 #define HZIP_PORT_ARCA_CHE_1 0x301044 #define HZIP_PORT_AWCA_CHE_0 0x301060 #define HZIP_PORT_AWCA_CHE_1 0x301064 -#define CACHE_ALL_EN 0xffffffff +#define HZIP_CACHE_ALL_EN 0xffffffff #define HZIP_BD_RUSER_32_63 0x301110 #define HZIP_SGL_RUSER_32_63 0x30111c @@ -83,7 +82,7 @@ #define HZIP_PF_DEF_Q_BASE 0 #define HZIP_SOFT_CTRL_CNT_CLR_CE 0x301000 -#define SOFT_CTRL_CNT_CLR_CE_BIT BIT(0) +#define HZIP_SOFT_CTRL_CNT_CLR_CE_BIT BIT(0) #define HZIP_SOFT_CTRL_ZIP_CONTROL 0x30100C #define HZIP_AXI_SHUTDOWN_ENABLE BIT(14) #define HZIP_WR_PORT BIT(11) @@ -92,9 +91,13 @@ #define HZIP_SQE_MASK_OFFSET 64 #define HZIP_SQE_MASK_LEN 48 +#define HZIP_CNT_CLR_CE_EN BIT(0) +#define HZIP_RO_CNT_CLR_CE_EN BIT(2) +#define HZIP_RD_CNT_CLR_CE_EN (HZIP_CNT_CLR_CE_EN | \ + HZIP_RO_CNT_CLR_CE_EN) + static const char hisi_zip_name[] = "hisi_zip"; static struct dentry *hzip_debugfs_root; -static struct hisi_qm_list zip_devices; struct hisi_zip_hw_error { u32 int_msk; @@ -106,6 +109,11 @@ struct zip_dfx_item { u32 offset; }; +static struct hisi_qm_list zip_devices = { + .register_to_crypto = hisi_zip_register_to_crypto, + .unregister_from_crypto = hisi_zip_unregister_from_crypto, +}; + static struct zip_dfx_item zip_dfx_files[] = { {"send_cnt", offsetof(struct hisi_zip_dfx, send_cnt)}, {"recv_cnt", offsetof(struct hisi_zip_dfx, recv_cnt)}, @@ -153,7 +161,6 @@ struct ctrl_debug_file { */ struct hisi_zip_ctrl { struct hisi_zip *hisi_zip; - struct dentry *debug_root; struct ctrl_debug_file files[HZIP_DEBUG_FILE_NUM]; }; @@ -216,7 +223,7 @@ static const struct kernel_param_ops pf_q_num_ops = { static u32 pf_q_num = HZIP_PF_DEF_Q_NUM; module_param_cb(pf_q_num, &pf_q_num_ops, &pf_q_num, 0444); -MODULE_PARM_DESC(pf_q_num, "Number of queues in PF(v1 1-4096, v2 1-1024)"); +MODULE_PARM_DESC(pf_q_num, "Number of queues in PF(v1 2-4096, v2 2-1024)"); static const struct kernel_param_ops vfs_num_ops = { .set = vfs_num_set, @@ -256,15 +263,16 @@ static int hisi_zip_set_user_domain_and_cache(struct hisi_qm *qm) /* qm cache */ writel(AXI_M_CFG, base + QM_AXI_M_CFG); writel(AXI_M_CFG_ENABLE, base + QM_AXI_M_CFG_ENABLE); + /* disable FLR triggered by BME(bus master enable) */ writel(PEH_AXUSER_CFG, base + QM_PEH_AXUSER_CFG); writel(PEH_AXUSER_CFG_ENABLE, base + QM_PEH_AXUSER_CFG_ENABLE); /* cache */ - writel(CACHE_ALL_EN, base + HZIP_PORT_ARCA_CHE_0); - writel(CACHE_ALL_EN, base + HZIP_PORT_ARCA_CHE_1); - writel(CACHE_ALL_EN, base + HZIP_PORT_AWCA_CHE_0); - writel(CACHE_ALL_EN, base + HZIP_PORT_AWCA_CHE_1); + writel(HZIP_CACHE_ALL_EN, base + HZIP_PORT_ARCA_CHE_0); + writel(HZIP_CACHE_ALL_EN, base + HZIP_PORT_ARCA_CHE_1); + writel(HZIP_CACHE_ALL_EN, base + HZIP_PORT_AWCA_CHE_0); + writel(HZIP_CACHE_ALL_EN, base + HZIP_PORT_AWCA_CHE_1); /* user domain configurations */ writel(AXUSER_BASE, base + HZIP_BD_RUSER_32_63); @@ -280,10 +288,10 @@ static int hisi_zip_set_user_domain_and_cache(struct hisi_qm *qm) } /* let's open all compression/decompression cores */ - writel(DECOMP_CHECK_ENABLE | ALL_COMP_DECOMP_EN, + writel(HZIP_DECOMP_CHECK_ENABLE | HZIP_ALL_COMP_DECOMP_EN, base + HZIP_CLOCK_GATE_CTRL); - /* enable sqc writeback */ + /* enable sqc,cqc writeback */ writel(SQC_CACHE_ENABLE | CQC_CACHE_ENABLE | SQC_CACHE_WB_ENABLE | CQC_CACHE_WB_ENABLE | FIELD_PREP(SQC_CACHE_WB_THRD, 1) | FIELD_PREP(CQC_CACHE_WB_THRD, 1), base + QM_CACHE_CTL); @@ -309,7 +317,7 @@ static void hisi_zip_hw_error_enable(struct hisi_qm *qm) writel(0x1, qm->io_base + HZIP_CORE_INT_RAS_CE_ENB); writel(0x0, qm->io_base + HZIP_CORE_INT_RAS_FE_ENB); writel(HZIP_CORE_INT_RAS_NFE_ENABLE, - qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB); + qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB); /* enable ZIP hw error interrupts */ writel(0, qm->io_base + HZIP_CORE_INT_MASK_REG); @@ -356,7 +364,7 @@ static int current_qm_write(struct ctrl_debug_file *file, u32 val) if (val > qm->vfs_num) return -EINVAL; - /* Calculate curr_qm_qp_num and store */ + /* According PF or VF Dev ID to calculation curr_qm_qp_num and store */ if (val == 0) { qm->debug.curr_qm_qp_num = qm->qp_num; } else { @@ -387,7 +395,7 @@ static u32 clear_enable_read(struct ctrl_debug_file *file) struct hisi_qm *qm = file_to_qm(file); return readl(qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE) & - SOFT_CTRL_CNT_CLR_CE_BIT; + HZIP_SOFT_CTRL_CNT_CLR_CE_BIT; } static int clear_enable_write(struct ctrl_debug_file *file, u32 val) @@ -399,14 +407,14 @@ static int clear_enable_write(struct ctrl_debug_file *file, u32 val) return -EINVAL; tmp = (readl(qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE) & - ~SOFT_CTRL_CNT_CLR_CE_BIT) | val; + ~HZIP_SOFT_CTRL_CNT_CLR_CE_BIT) | val; writel(tmp, qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE); return 0; } -static ssize_t ctrl_debug_read(struct file *filp, char __user *buf, - size_t count, loff_t *pos) +static ssize_t hisi_zip_ctrl_debug_read(struct file *filp, char __user *buf, + size_t count, loff_t *pos) { struct ctrl_debug_file *file = filp->private_data; char tbuf[HZIP_BUF_SIZE]; @@ -426,12 +434,13 @@ static ssize_t ctrl_debug_read(struct file *filp, char __user *buf, return -EINVAL; } spin_unlock_irq(&file->lock); - ret = sprintf(tbuf, "%u\n", val); + ret = scnprintf(tbuf, sizeof(tbuf), "%u\n", val); return simple_read_from_buffer(buf, count, pos, tbuf, ret); } -static ssize_t ctrl_debug_write(struct file *filp, const char __user *buf, - size_t count, loff_t *pos) +static ssize_t hisi_zip_ctrl_debug_write(struct file *filp, + const char __user *buf, + size_t count, loff_t *pos) { struct ctrl_debug_file *file = filp->private_data; char tbuf[HZIP_BUF_SIZE]; @@ -480,11 +489,10 @@ err_input: static const struct file_operations ctrl_debug_fops = { .owner = THIS_MODULE, .open = simple_open, - .read = ctrl_debug_read, - .write = ctrl_debug_write, + .read = hisi_zip_ctrl_debug_read, + .write = hisi_zip_ctrl_debug_write, }; - static int zip_debugfs_atomic64_set(void *data, u64 val) { if (val) @@ -505,10 +513,8 @@ static int zip_debugfs_atomic64_get(void *data, u64 *val) DEFINE_DEBUGFS_ATTRIBUTE(zip_atomic64_ops, zip_debugfs_atomic64_get, zip_debugfs_atomic64_set, "%llu\n"); -static int hisi_zip_core_debug_init(struct hisi_zip_ctrl *ctrl) +static int hisi_zip_core_debug_init(struct hisi_qm *qm) { - struct hisi_zip *hisi_zip = ctrl->hisi_zip; - struct hisi_qm *qm = &hisi_zip->qm; struct device *dev = &qm->pdev->dev; struct debugfs_regset32 *regset; struct dentry *tmp_d; @@ -517,9 +523,10 @@ static int hisi_zip_core_debug_init(struct hisi_zip_ctrl *ctrl) for (i = 0; i < HZIP_CORE_NUM; i++) { if (i < HZIP_COMP_CORE_NUM) - sprintf(buf, "comp_core%d", i); + scnprintf(buf, sizeof(buf), "comp_core%d", i); else - sprintf(buf, "decomp_core%d", i - HZIP_COMP_CORE_NUM); + scnprintf(buf, sizeof(buf), "decomp_core%d", + i - HZIP_COMP_CORE_NUM); regset = devm_kzalloc(dev, sizeof(*regset), GFP_KERNEL); if (!regset) @@ -529,7 +536,7 @@ static int hisi_zip_core_debug_init(struct hisi_zip_ctrl *ctrl) regset->nregs = ARRAY_SIZE(hzip_dfx_regs); regset->base = qm->io_base + core_offsets[i]; - tmp_d = debugfs_create_dir(buf, ctrl->debug_root); + tmp_d = debugfs_create_dir(buf, qm->debug.debug_root); debugfs_create_regset32("regs", 0444, tmp_d, regset); } @@ -548,33 +555,32 @@ static void hisi_zip_dfx_debug_init(struct hisi_qm *qm) for (i = 0; i < ARRAY_SIZE(zip_dfx_files); i++) { data = (atomic64_t *)((uintptr_t)dfx + zip_dfx_files[i].offset); debugfs_create_file(zip_dfx_files[i].name, - 0644, - tmp_dir, - data, - &zip_atomic64_ops); + 0644, tmp_dir, data, + &zip_atomic64_ops); } } -static int hisi_zip_ctrl_debug_init(struct hisi_zip_ctrl *ctrl) +static int hisi_zip_ctrl_debug_init(struct hisi_qm *qm) { + struct hisi_zip *zip = container_of(qm, struct hisi_zip, qm); int i; for (i = HZIP_CURRENT_QM; i < HZIP_DEBUG_FILE_NUM; i++) { - spin_lock_init(&ctrl->files[i].lock); - ctrl->files[i].ctrl = ctrl; - ctrl->files[i].index = i; + spin_lock_init(&zip->ctrl->files[i].lock); + zip->ctrl->files[i].ctrl = zip->ctrl; + zip->ctrl->files[i].index = i; debugfs_create_file(ctrl_debug_file_name[i], 0600, - ctrl->debug_root, ctrl->files + i, + qm->debug.debug_root, + zip->ctrl->files + i, &ctrl_debug_fops); } - return hisi_zip_core_debug_init(ctrl); + return hisi_zip_core_debug_init(qm); } -static int hisi_zip_debugfs_init(struct hisi_zip *hisi_zip) +static int hisi_zip_debugfs_init(struct hisi_qm *qm) { - struct hisi_qm *qm = &hisi_zip->qm; struct device *dev = &qm->pdev->dev; struct dentry *dev_d; int ret; @@ -589,8 +595,7 @@ static int hisi_zip_debugfs_init(struct hisi_zip *hisi_zip) goto failed_to_create; if (qm->fun_type == QM_HW_PF) { - hisi_zip->ctrl->debug_root = dev_d; - ret = hisi_zip_ctrl_debug_init(hisi_zip->ctrl); + ret = hisi_zip_ctrl_debug_init(qm); if (ret) goto failed_to_create; } @@ -604,25 +609,36 @@ failed_to_create: return ret; } -static void hisi_zip_debug_regs_clear(struct hisi_zip *hisi_zip) +/* hisi_zip_debug_regs_clear() - clear the zip debug regs */ +static void hisi_zip_debug_regs_clear(struct hisi_qm *qm) { - struct hisi_qm *qm = &hisi_zip->qm; + int i, j; + /* clear current_qm */ writel(0x0, qm->io_base + QM_DFX_MB_CNT_VF); writel(0x0, qm->io_base + QM_DFX_DB_CNT_VF); + + /* enable register read_clear bit */ + writel(HZIP_RD_CNT_CLR_CE_EN, qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE); + for (i = 0; i < ARRAY_SIZE(core_offsets); i++) + for (j = 0; j < ARRAY_SIZE(hzip_dfx_regs); j++) + readl(qm->io_base + core_offsets[i] + + hzip_dfx_regs[j].offset); + + /* disable register read_clear bit */ writel(0x0, qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE); hisi_qm_debug_regs_clear(qm); } -static void hisi_zip_debugfs_exit(struct hisi_zip *hisi_zip) +static void hisi_zip_debugfs_exit(struct hisi_qm *qm) { - struct hisi_qm *qm = &hisi_zip->qm; - debugfs_remove_recursive(qm->debug.debug_root); - if (qm->fun_type == QM_HW_PF) - hisi_zip_debug_regs_clear(hisi_zip); + if (qm->fun_type == QM_HW_PF) { + hisi_zip_debug_regs_clear(qm); + qm->debug.curr_qm_qp_num = 0; + } } static void hisi_zip_log_hw_error(struct hisi_qm *qm, u32 err_sts) @@ -634,7 +650,7 @@ static void hisi_zip_log_hw_error(struct hisi_qm *qm, u32 err_sts) while (err->msg) { if (err->int_msk & err_sts) { dev_err(dev, "%s [error status=0x%x] found\n", - err->msg, err->int_msk); + err->msg, err->int_msk); if (err->int_msk & HZIP_CORE_INT_STATUS_M_ECC) { err_val = readl(qm->io_base + @@ -642,9 +658,6 @@ static void hisi_zip_log_hw_error(struct hisi_qm *qm, u32 err_sts) dev_err(dev, "hisi-zip multi ecc sram num=0x%x\n", ((err_val >> HZIP_SRAM_ECC_ERR_NUM_SHIFT) & 0xFF)); - dev_err(dev, "hisi-zip multi ecc sram addr=0x%x\n", - (err_val >> - HZIP_SRAM_ECC_ERR_ADDR_SHIFT)); } } err++; @@ -729,7 +742,7 @@ static int hisi_zip_pf_probe_init(struct hisi_zip *hisi_zip) hisi_zip_set_user_domain_and_cache(qm); hisi_qm_dev_err_init(qm); - hisi_zip_debug_regs_clear(hisi_zip); + hisi_zip_debug_regs_clear(qm); return 0; } @@ -747,6 +760,7 @@ static int hisi_zip_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) if (qm->fun_type == QM_HW_PF) { qm->qp_base = HZIP_PF_DEF_Q_BASE; qm->qp_num = pf_q_num; + qm->debug.curr_qm_qp_num = pf_q_num; qm->qm_list = &zip_devices; } else if (qm->fun_type == QM_HW_VF && qm->ver == QM_HW_V1) { /* @@ -803,32 +817,44 @@ static int hisi_zip_probe(struct pci_dev *pdev, const struct pci_device_id *id) ret = hisi_qm_start(qm); if (ret) - goto err_qm_uninit; + goto err_dev_err_uninit; - ret = hisi_zip_debugfs_init(hisi_zip); + ret = hisi_zip_debugfs_init(qm); if (ret) - dev_err(&pdev->dev, "Failed to init debugfs (%d)!\n", ret); + pci_err(pdev, "failed to init debugfs (%d)!\n", ret); - hisi_qm_add_to_list(qm, &zip_devices); + ret = hisi_qm_alg_register(qm, &zip_devices); + if (ret < 0) { + pci_err(pdev, "failed to register driver to crypto!\n"); + goto err_qm_stop; + } if (qm->uacce) { ret = uacce_register(qm->uacce); - if (ret) - goto err_qm_uninit; + if (ret) { + pci_err(pdev, "failed to register uacce (%d)!\n", ret); + goto err_qm_alg_unregister; + } } if (qm->fun_type == QM_HW_PF && vfs_num > 0) { ret = hisi_qm_sriov_enable(pdev, vfs_num); if (ret < 0) - goto err_remove_from_list; + goto err_qm_alg_unregister; } return 0; -err_remove_from_list: - hisi_qm_del_from_list(qm, &zip_devices); - hisi_zip_debugfs_exit(hisi_zip); - hisi_qm_stop(qm); +err_qm_alg_unregister: + hisi_qm_alg_unregister(qm, &zip_devices); + +err_qm_stop: + hisi_zip_debugfs_exit(qm); + hisi_qm_stop(qm, QM_NORMAL); + +err_dev_err_uninit: + hisi_qm_dev_err_uninit(qm); + err_qm_uninit: hisi_qm_uninit(qm); @@ -837,18 +863,18 @@ err_qm_uninit: static void hisi_zip_remove(struct pci_dev *pdev) { - struct hisi_zip *hisi_zip = pci_get_drvdata(pdev); - struct hisi_qm *qm = &hisi_zip->qm; + struct hisi_qm *qm = pci_get_drvdata(pdev); - if (qm->fun_type == QM_HW_PF && qm->vfs_num) - hisi_qm_sriov_disable(pdev); + hisi_qm_wait_task_finish(qm, &zip_devices); + hisi_qm_alg_unregister(qm, &zip_devices); - hisi_zip_debugfs_exit(hisi_zip); - hisi_qm_stop(qm); + if (qm->fun_type == QM_HW_PF && qm->vfs_num) + hisi_qm_sriov_disable(pdev, qm->is_frozen); + hisi_zip_debugfs_exit(qm); + hisi_qm_stop(qm, QM_NORMAL); hisi_qm_dev_err_uninit(qm); hisi_qm_uninit(qm); - hisi_qm_del_from_list(qm, &zip_devices); } static const struct pci_error_handlers hisi_zip_err_handler = { @@ -866,6 +892,7 @@ static struct pci_driver hisi_zip_pci_driver = { .sriov_configure = IS_ENABLED(CONFIG_PCI_IOV) ? hisi_qm_sriov_configure : NULL, .err_handler = &hisi_zip_err_handler, + .shutdown = hisi_qm_dev_shutdown, }; static void hisi_zip_register_debugfs(void) @@ -890,29 +917,15 @@ static int __init hisi_zip_init(void) ret = pci_register_driver(&hisi_zip_pci_driver); if (ret < 0) { + hisi_zip_unregister_debugfs(); pr_err("Failed to register pci driver.\n"); - goto err_pci; } - ret = hisi_zip_register_to_crypto(); - if (ret < 0) { - pr_err("Failed to register driver to crypto.\n"); - goto err_crypto; - } - - return 0; - -err_crypto: - pci_unregister_driver(&hisi_zip_pci_driver); -err_pci: - hisi_zip_unregister_debugfs(); - return ret; } static void __exit hisi_zip_exit(void) { - hisi_zip_unregister_from_crypto(); pci_unregister_driver(&hisi_zip_pci_driver); hisi_zip_unregister_debugfs(); } diff --git a/drivers/crypto/img-hash.c b/drivers/crypto/img-hash.c index 87226b7c2795..91f555ccbb31 100644 --- a/drivers/crypto/img-hash.c +++ b/drivers/crypto/img-hash.c @@ -7,6 +7,7 @@ */ #include <linux/clk.h> +#include <linux/dma-mapping.h> #include <linux/dmaengine.h> #include <linux/interrupt.h> #include <linux/io.h> diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index fa7398e68858..eb2418450f12 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -304,6 +304,11 @@ static void eip197_init_firmware(struct safexcel_crypto_priv *priv) /* Enable access to all IFPP program memories */ writel(EIP197_PE_ICE_RAM_CTRL_FPP_PROG_EN, EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL(pe)); + + /* bypass the OCE, if present */ + if (priv->flags & EIP197_OCE) + writel(EIP197_DEBUG_OCE_BYPASS, EIP197_PE(priv) + + EIP197_PE_DEBUG(pe)); } } @@ -1495,6 +1500,9 @@ static int safexcel_probe_generic(void *pdev, hwopt = readl(EIP197_GLOBAL(priv) + EIP197_OPTIONS); hiaopt = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_OPTIONS); + priv->hwconfig.icever = 0; + priv->hwconfig.ocever = 0; + priv->hwconfig.psever = 0; if (priv->flags & SAFEXCEL_HW_EIP197) { /* EIP197 */ peopt = readl(EIP197_PE(priv) + EIP197_PE_OPTIONS(0)); @@ -1513,8 +1521,37 @@ static int safexcel_probe_generic(void *pdev, EIP197_N_RINGS_MASK; if (hiaopt & EIP197_HIA_OPT_HAS_PE_ARB) priv->flags |= EIP197_PE_ARB; - if (EIP206_OPT_ICE_TYPE(peopt) == 1) + if (EIP206_OPT_ICE_TYPE(peopt) == 1) { priv->flags |= EIP197_ICE; + /* Detect ICE EIP207 class. engine and version */ + version = readl(EIP197_PE(priv) + + EIP197_PE_ICE_VERSION(0)); + if (EIP197_REG_LO16(version) != EIP207_VERSION_LE) { + dev_err(dev, "EIP%d: ICE EIP207 not detected.\n", + peid); + return -ENODEV; + } + priv->hwconfig.icever = EIP197_VERSION_MASK(version); + } + if (EIP206_OPT_OCE_TYPE(peopt) == 1) { + priv->flags |= EIP197_OCE; + /* Detect EIP96PP packet stream editor and version */ + version = readl(EIP197_PE(priv) + EIP197_PE_PSE_VERSION(0)); + if (EIP197_REG_LO16(version) != EIP96_VERSION_LE) { + dev_err(dev, "EIP%d: EIP96PP not detected.\n", peid); + return -ENODEV; + } + priv->hwconfig.psever = EIP197_VERSION_MASK(version); + /* Detect OCE EIP207 class. engine and version */ + version = readl(EIP197_PE(priv) + + EIP197_PE_ICE_VERSION(0)); + if (EIP197_REG_LO16(version) != EIP207_VERSION_LE) { + dev_err(dev, "EIP%d: OCE EIP207 not detected.\n", + peid); + return -ENODEV; + } + priv->hwconfig.ocever = EIP197_VERSION_MASK(version); + } /* If not a full TRC, then assume simple TRC */ if (!(hwopt & EIP197_OPT_HAS_TRC)) priv->flags |= EIP197_SIMPLE_TRC; @@ -1552,13 +1589,14 @@ static int safexcel_probe_generic(void *pdev, EIP197_PE_EIP96_OPTIONS(0)); /* Print single info line describing what we just detected */ - dev_info(priv->dev, "EIP%d:%x(%d,%d,%d,%d)-HIA:%x(%d,%d,%d),PE:%x/%x,alg:%08x\n", + dev_info(priv->dev, "EIP%d:%x(%d,%d,%d,%d)-HIA:%x(%d,%d,%d),PE:%x/%x(alg:%08x)/%x/%x/%x\n", peid, priv->hwconfig.hwver, hwctg, priv->hwconfig.hwnumpes, priv->hwconfig.hwnumrings, priv->hwconfig.hwnumraic, priv->hwconfig.hiaver, priv->hwconfig.hwdataw, priv->hwconfig.hwcfsize, priv->hwconfig.hwrfsize, priv->hwconfig.ppver, priv->hwconfig.pever, - priv->hwconfig.algo_flags); + priv->hwconfig.algo_flags, priv->hwconfig.icever, + priv->hwconfig.ocever, priv->hwconfig.psever); safexcel_configure(priv); diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index 7c5fe382d272..9045f2d7f4c6 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -12,7 +12,9 @@ #include <crypto/algapi.h> #include <crypto/internal/hash.h> #include <crypto/sha.h> +#include <crypto/sha3.h> #include <crypto/skcipher.h> +#include <linux/types.h> #define EIP197_HIA_VERSION_BE 0xca35 #define EIP197_HIA_VERSION_LE 0x35ca @@ -22,6 +24,7 @@ #define EIP96_VERSION_LE 0x9f60 #define EIP201_VERSION_LE 0x36c9 #define EIP206_VERSION_LE 0x31ce +#define EIP207_VERSION_LE 0x30cf #define EIP197_REG_LO16(reg) (reg & 0xffff) #define EIP197_REG_HI16(reg) ((reg >> 16) & 0xffff) #define EIP197_VERSION_MASK(reg) ((reg >> 16) & 0xfff) @@ -34,6 +37,7 @@ /* EIP206 OPTIONS ENCODING */ #define EIP206_OPT_ICE_TYPE(n) ((n>>8)&3) +#define EIP206_OPT_OCE_TYPE(n) ((n>>10)&3) /* EIP197 OPTIONS ENCODING */ #define EIP197_OPT_HAS_TRC BIT(31) @@ -168,6 +172,7 @@ #define EIP197_PE_ICE_FPP_CTRL(n) (0x0d80 + (0x2000 * (n))) #define EIP197_PE_ICE_PPTF_CTRL(n) (0x0e00 + (0x2000 * (n))) #define EIP197_PE_ICE_RAM_CTRL(n) (0x0ff0 + (0x2000 * (n))) +#define EIP197_PE_ICE_VERSION(n) (0x0ffc + (0x2000 * (n))) #define EIP197_PE_EIP96_TOKEN_CTRL(n) (0x1000 + (0x2000 * (n))) #define EIP197_PE_EIP96_FUNCTION_EN(n) (0x1004 + (0x2000 * (n))) #define EIP197_PE_EIP96_CONTEXT_CTRL(n) (0x1008 + (0x2000 * (n))) @@ -176,8 +181,11 @@ #define EIP197_PE_EIP96_FUNCTION2_EN(n) (0x1030 + (0x2000 * (n))) #define EIP197_PE_EIP96_OPTIONS(n) (0x13f8 + (0x2000 * (n))) #define EIP197_PE_EIP96_VERSION(n) (0x13fc + (0x2000 * (n))) +#define EIP197_PE_OCE_VERSION(n) (0x1bfc + (0x2000 * (n))) #define EIP197_PE_OUT_DBUF_THRES(n) (0x1c00 + (0x2000 * (n))) #define EIP197_PE_OUT_TBUF_THRES(n) (0x1d00 + (0x2000 * (n))) +#define EIP197_PE_PSE_VERSION(n) (0x1efc + (0x2000 * (n))) +#define EIP197_PE_DEBUG(n) (0x1ff4 + (0x2000 * (n))) #define EIP197_PE_OPTIONS(n) (0x1ff8 + (0x2000 * (n))) #define EIP197_PE_VERSION(n) (0x1ffc + (0x2000 * (n))) #define EIP197_MST_CTRL 0xfff4 @@ -352,6 +360,9 @@ /* EIP197_PE_EIP96_TOKEN_CTRL2 */ #define EIP197_PE_EIP96_TOKEN_CTRL2_CTX_DONE BIT(3) +/* EIP197_PE_DEBUG */ +#define EIP197_DEBUG_OCE_BYPASS BIT(1) + /* EIP197_STRC_CONFIG */ #define EIP197_STRC_CONFIG_INIT BIT(31) #define EIP197_STRC_CONFIG_LARGE_REC(s) (s<<8) @@ -776,6 +787,7 @@ enum safexcel_flags { EIP197_PE_ARB = BIT(2), EIP197_ICE = BIT(3), EIP197_SIMPLE_TRC = BIT(4), + EIP197_OCE = BIT(5), }; struct safexcel_hwconfig { @@ -783,7 +795,10 @@ struct safexcel_hwconfig { int hwver; int hiaver; int ppver; + int icever; int pever; + int ocever; + int psever; int hwdataw; int hwcfsize; int hwrfsize; @@ -819,8 +834,16 @@ struct safexcel_context { struct crypto_async_request *req, bool *complete, int *ret); struct safexcel_context_record *ctxr; + struct safexcel_crypto_priv *priv; dma_addr_t ctxr_dma; + union { + __le32 le[SHA3_512_BLOCK_SIZE / 4]; + __be32 be[SHA3_512_BLOCK_SIZE / 4]; + u32 word[SHA3_512_BLOCK_SIZE / 4]; + u8 byte[SHA3_512_BLOCK_SIZE]; + } ipad, opad; + int ring; bool needs_inv; bool exit_inv; @@ -898,8 +921,9 @@ void safexcel_rdr_req_set(struct safexcel_crypto_priv *priv, inline struct crypto_async_request * safexcel_rdr_req_get(struct safexcel_crypto_priv *priv, int ring); void safexcel_inv_complete(struct crypto_async_request *req, int error); -int safexcel_hmac_setkey(const char *alg, const u8 *key, unsigned int keylen, - void *istate, void *ostate); +int safexcel_hmac_setkey(struct safexcel_context *base, const u8 *key, + unsigned int keylen, const char *alg, + unsigned int state_sz); /* available algorithms */ extern struct safexcel_alg_template safexcel_alg_ecb_des; diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c index 1ac3253b7903..9bcfb79a030f 100644 --- a/drivers/crypto/inside-secure/safexcel_cipher.c +++ b/drivers/crypto/inside-secure/safexcel_cipher.c @@ -61,8 +61,6 @@ struct safexcel_cipher_ctx { /* All the below is AEAD specific */ u32 hash_alg; u32 state_sz; - __be32 ipad[SHA512_DIGEST_SIZE / sizeof(u32)]; - __be32 opad[SHA512_DIGEST_SIZE / sizeof(u32)]; struct crypto_cipher *hkaes; struct crypto_aead *fback; @@ -375,7 +373,7 @@ static int safexcel_skcipher_aes_setkey(struct crypto_skcipher *ctfm, { struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); - struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_crypto_priv *priv = ctx->base.priv; struct crypto_aes_ctx aes; int ret, i; @@ -406,11 +404,11 @@ static int safexcel_aead_setkey(struct crypto_aead *ctfm, const u8 *key, { struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); - struct safexcel_ahash_export_state istate, ostate; - struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_crypto_priv *priv = ctx->base.priv; struct crypto_authenc_keys keys; struct crypto_aes_ctx aes; int err = -EINVAL, i; + const char *alg; if (unlikely(crypto_authenc_extractkeys(&keys, key, len))) goto badkey; @@ -465,53 +463,37 @@ static int safexcel_aead_setkey(struct crypto_aead *ctfm, const u8 *key, /* Auth key */ switch (ctx->hash_alg) { case CONTEXT_CONTROL_CRYPTO_ALG_SHA1: - if (safexcel_hmac_setkey("safexcel-sha1", keys.authkey, - keys.authkeylen, &istate, &ostate)) - goto badkey; + alg = "safexcel-sha1"; break; case CONTEXT_CONTROL_CRYPTO_ALG_SHA224: - if (safexcel_hmac_setkey("safexcel-sha224", keys.authkey, - keys.authkeylen, &istate, &ostate)) - goto badkey; + alg = "safexcel-sha224"; break; case CONTEXT_CONTROL_CRYPTO_ALG_SHA256: - if (safexcel_hmac_setkey("safexcel-sha256", keys.authkey, - keys.authkeylen, &istate, &ostate)) - goto badkey; + alg = "safexcel-sha256"; break; case CONTEXT_CONTROL_CRYPTO_ALG_SHA384: - if (safexcel_hmac_setkey("safexcel-sha384", keys.authkey, - keys.authkeylen, &istate, &ostate)) - goto badkey; + alg = "safexcel-sha384"; break; case CONTEXT_CONTROL_CRYPTO_ALG_SHA512: - if (safexcel_hmac_setkey("safexcel-sha512", keys.authkey, - keys.authkeylen, &istate, &ostate)) - goto badkey; + alg = "safexcel-sha512"; break; case CONTEXT_CONTROL_CRYPTO_ALG_SM3: - if (safexcel_hmac_setkey("safexcel-sm3", keys.authkey, - keys.authkeylen, &istate, &ostate)) - goto badkey; + alg = "safexcel-sm3"; break; default: dev_err(priv->dev, "aead: unsupported hash algorithm\n"); goto badkey; } - if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma && - (memcmp(ctx->ipad, istate.state, ctx->state_sz) || - memcmp(ctx->opad, ostate.state, ctx->state_sz))) - ctx->base.needs_inv = true; + if (safexcel_hmac_setkey(&ctx->base, keys.authkey, keys.authkeylen, + alg, ctx->state_sz)) + goto badkey; /* Now copy the keys into the context */ for (i = 0; i < keys.enckeylen / sizeof(u32); i++) ctx->key[i] = cpu_to_le32(((u32 *)keys.enckey)[i]); ctx->key_len = keys.enckeylen; - memcpy(ctx->ipad, &istate.state, ctx->state_sz); - memcpy(ctx->opad, &ostate.state, ctx->state_sz); - memzero_explicit(&keys, sizeof(keys)); return 0; @@ -525,7 +507,7 @@ static int safexcel_context_control(struct safexcel_cipher_ctx *ctx, struct safexcel_cipher_req *sreq, struct safexcel_command_desc *cdesc) { - struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_crypto_priv *priv = ctx->base.priv; int ctrl_size = ctx->key_len / sizeof(u32); cdesc->control_data.control1 = ctx->mode; @@ -692,7 +674,7 @@ static int safexcel_send_req(struct crypto_async_request *base, int ring, struct skcipher_request *areq = skcipher_request_cast(base); struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(areq); struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm); - struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_crypto_priv *priv = ctx->base.priv; struct safexcel_command_desc *cdesc; struct safexcel_command_desc *first_cdesc = NULL; struct safexcel_result_desc *rdesc, *first_rdesc = NULL; @@ -718,10 +700,10 @@ static int safexcel_send_req(struct crypto_async_request *base, int ring, totlen_dst += digestsize; memcpy(ctx->base.ctxr->data + ctx->key_len / sizeof(u32), - ctx->ipad, ctx->state_sz); + &ctx->base.ipad, ctx->state_sz); if (!ctx->xcm) memcpy(ctx->base.ctxr->data + (ctx->key_len + - ctx->state_sz) / sizeof(u32), ctx->opad, + ctx->state_sz) / sizeof(u32), &ctx->base.opad, ctx->state_sz); } else if ((ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) && (sreq->direction == SAFEXCEL_DECRYPT)) { @@ -1020,7 +1002,7 @@ static int safexcel_cipher_send_inv(struct crypto_async_request *base, int ring, int *commands, int *results) { struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm); - struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_crypto_priv *priv = ctx->base.priv; int ret; ret = safexcel_invalidate_cache(base, priv, ctx->base.ctxr_dma, ring); @@ -1039,7 +1021,7 @@ static int safexcel_skcipher_send(struct crypto_async_request *async, int ring, struct skcipher_request *req = skcipher_request_cast(async); struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm); struct safexcel_cipher_req *sreq = skcipher_request_ctx(req); - struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_crypto_priv *priv = ctx->base.priv; int ret; BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && sreq->needs_inv); @@ -1072,7 +1054,7 @@ static int safexcel_aead_send(struct crypto_async_request *async, int ring, struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm); struct safexcel_cipher_req *sreq = aead_request_ctx(req); - struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_crypto_priv *priv = ctx->base.priv; int ret; BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && sreq->needs_inv); @@ -1094,7 +1076,7 @@ static int safexcel_cipher_exit_inv(struct crypto_tfm *tfm, struct safexcel_inv_result *result) { struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); - struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_crypto_priv *priv = ctx->base.priv; int ring = ctx->base.ring; init_completion(&result->completion); @@ -1157,7 +1139,7 @@ static int safexcel_queue_req(struct crypto_async_request *base, enum safexcel_cipher_direction dir) { struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm); - struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_crypto_priv *priv = ctx->base.priv; int ret, ring; sreq->needs_inv = false; @@ -1211,7 +1193,7 @@ static int safexcel_skcipher_cra_init(struct crypto_tfm *tfm) crypto_skcipher_set_reqsize(__crypto_skcipher_cast(tfm), sizeof(struct safexcel_cipher_req)); - ctx->priv = tmpl->priv; + ctx->base.priv = tmpl->priv; ctx->base.send = safexcel_skcipher_send; ctx->base.handle_result = safexcel_skcipher_handle_result; @@ -1237,7 +1219,7 @@ static int safexcel_cipher_cra_exit(struct crypto_tfm *tfm) static void safexcel_skcipher_cra_exit(struct crypto_tfm *tfm) { struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); - struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_crypto_priv *priv = ctx->base.priv; int ret; if (safexcel_cipher_cra_exit(tfm)) @@ -1257,7 +1239,7 @@ static void safexcel_skcipher_cra_exit(struct crypto_tfm *tfm) static void safexcel_aead_cra_exit(struct crypto_tfm *tfm) { struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); - struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_crypto_priv *priv = ctx->base.priv; int ret; if (safexcel_cipher_cra_exit(tfm)) @@ -1431,7 +1413,7 @@ static int safexcel_skcipher_aesctr_setkey(struct crypto_skcipher *ctfm, { struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); - struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_crypto_priv *priv = ctx->base.priv; struct crypto_aes_ctx aes; int ret, i; unsigned int keylen; @@ -1505,7 +1487,7 @@ static int safexcel_des_setkey(struct crypto_skcipher *ctfm, const u8 *key, unsigned int len) { struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(ctfm); - struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_crypto_priv *priv = ctx->base.priv; int ret; ret = verify_skcipher_des_key(ctfm, key); @@ -1604,7 +1586,7 @@ static int safexcel_des3_ede_setkey(struct crypto_skcipher *ctfm, const u8 *key, unsigned int len) { struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(ctfm); - struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_crypto_priv *priv = ctx->base.priv; int err; err = verify_skcipher_des3_key(ctfm, key); @@ -1723,7 +1705,7 @@ static int safexcel_aead_cra_init(struct crypto_tfm *tfm) crypto_aead_set_reqsize(__crypto_aead_cast(tfm), sizeof(struct safexcel_cipher_req)); - ctx->priv = tmpl->priv; + ctx->base.priv = tmpl->priv; ctx->alg = SAFEXCEL_AES; /* default */ ctx->blocksz = AES_BLOCK_SIZE; @@ -2466,7 +2448,7 @@ static int safexcel_skcipher_aesxts_setkey(struct crypto_skcipher *ctfm, { struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); - struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_crypto_priv *priv = ctx->base.priv; struct crypto_aes_ctx aes; int ret, i; unsigned int keylen; @@ -2580,7 +2562,7 @@ static int safexcel_aead_gcm_setkey(struct crypto_aead *ctfm, const u8 *key, { struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); - struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_crypto_priv *priv = ctx->base.priv; struct crypto_aes_ctx aes; u32 hashkey[AES_BLOCK_SIZE >> 2]; int ret, i; @@ -2618,7 +2600,7 @@ static int safexcel_aead_gcm_setkey(struct crypto_aead *ctfm, const u8 *key, if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++) { - if (be32_to_cpu(ctx->ipad[i]) != hashkey[i]) { + if (be32_to_cpu(ctx->base.ipad.be[i]) != hashkey[i]) { ctx->base.needs_inv = true; break; } @@ -2626,7 +2608,7 @@ static int safexcel_aead_gcm_setkey(struct crypto_aead *ctfm, const u8 *key, } for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++) - ctx->ipad[i] = cpu_to_be32(hashkey[i]); + ctx->base.ipad.be[i] = cpu_to_be32(hashkey[i]); memzero_explicit(hashkey, AES_BLOCK_SIZE); memzero_explicit(&aes, sizeof(aes)); @@ -2693,7 +2675,7 @@ static int safexcel_aead_ccm_setkey(struct crypto_aead *ctfm, const u8 *key, { struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); - struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_crypto_priv *priv = ctx->base.priv; struct crypto_aes_ctx aes; int ret, i; @@ -2714,7 +2696,7 @@ static int safexcel_aead_ccm_setkey(struct crypto_aead *ctfm, const u8 *key, for (i = 0; i < len / sizeof(u32); i++) { ctx->key[i] = cpu_to_le32(aes.key_enc[i]); - ctx->ipad[i + 2 * AES_BLOCK_SIZE / sizeof(u32)] = + ctx->base.ipad.be[i + 2 * AES_BLOCK_SIZE / sizeof(u32)] = cpu_to_be32(aes.key_enc[i]); } @@ -2815,7 +2797,7 @@ struct safexcel_alg_template safexcel_alg_ccm = { static void safexcel_chacha20_setkey(struct safexcel_cipher_ctx *ctx, const u8 *key) { - struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_crypto_priv *priv = ctx->base.priv; if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) if (memcmp(ctx->key, key, CHACHA_KEY_SIZE)) @@ -3084,7 +3066,7 @@ static int safexcel_skcipher_sm4_setkey(struct crypto_skcipher *ctfm, { struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); - struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_crypto_priv *priv = ctx->base.priv; if (len != SM4_KEY_SIZE) return -EINVAL; diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c index 16a467969d8e..56d5ccb5cc00 100644 --- a/drivers/crypto/inside-secure/safexcel_hash.c +++ b/drivers/crypto/inside-secure/safexcel_hash.c @@ -20,7 +20,6 @@ struct safexcel_ahash_ctx { struct safexcel_context base; - struct safexcel_crypto_priv *priv; u32 alg; u8 key_sz; @@ -29,9 +28,6 @@ struct safexcel_ahash_ctx { bool fb_init_done; bool fb_do_setkey; - __le32 ipad[SHA3_512_BLOCK_SIZE / sizeof(__le32)]; - __le32 opad[SHA3_512_BLOCK_SIZE / sizeof(__le32)]; - struct crypto_cipher *kaes; struct crypto_ahash *fback; struct crypto_shash *shpre; @@ -111,7 +107,7 @@ static void safexcel_context_control(struct safexcel_ahash_ctx *ctx, struct safexcel_ahash_req *req, struct safexcel_command_desc *cdesc) { - struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_crypto_priv *priv = ctx->base.priv; u64 count = 0; cdesc->control_data.control0 = ctx->alg; @@ -124,7 +120,7 @@ static void safexcel_context_control(struct safexcel_ahash_ctx *ctx, */ if (unlikely(req->digest == CONTEXT_CONTROL_DIGEST_XCM)) { if (req->xcbcmac) - memcpy(ctx->base.ctxr->data, ctx->ipad, ctx->key_sz); + memcpy(ctx->base.ctxr->data, &ctx->base.ipad, ctx->key_sz); else memcpy(ctx->base.ctxr->data, req->state, req->state_sz); @@ -206,7 +202,7 @@ static void safexcel_context_control(struct safexcel_ahash_ctx *ctx, } else { /* HMAC */ /* Need outer digest for HMAC finalization */ memcpy(ctx->base.ctxr->data + (req->state_sz >> 2), - ctx->opad, req->state_sz); + &ctx->base.opad, req->state_sz); /* Single pass HMAC - no digest count */ cdesc->control_data.control0 |= @@ -275,7 +271,7 @@ static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, memcpy(sreq->cache, sreq->state, crypto_ahash_digestsize(ahash)); - memcpy(sreq->state, ctx->opad, sreq->digest_sz); + memcpy(sreq->state, &ctx->base.opad, sreq->digest_sz); sreq->len = sreq->block_sz + crypto_ahash_digestsize(ahash); @@ -316,7 +312,7 @@ static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring, struct ahash_request *areq = ahash_request_cast(async); struct safexcel_ahash_req *req = ahash_request_ctx(areq); struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); - struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_crypto_priv *priv = ctx->base.priv; struct safexcel_command_desc *cdesc, *first_cdesc = NULL; struct safexcel_result_desc *rdesc; struct scatterlist *sg; @@ -379,10 +375,14 @@ static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring, // 10- padding for XCBCMAC & CMAC req->cache[cache_len + skip] = 0x80; // HW will use K2 iso K3 - compensate! - for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++) - ((__be32 *)req->cache)[i] ^= - cpu_to_be32(le32_to_cpu( - ctx->ipad[i] ^ ctx->ipad[i + 4])); + for (i = 0; i < AES_BLOCK_SIZE / 4; i++) { + u32 *cache = (void *)req->cache; + u32 *ipad = ctx->base.ipad.word; + u32 x; + + x = ipad[i] ^ ipad[i + 4]; + cache[i] ^= swab(x); + } } cache_len = AES_BLOCK_SIZE; queued = queued + extra; @@ -591,7 +591,7 @@ static int safexcel_ahash_send_inv(struct crypto_async_request *async, struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); int ret; - ret = safexcel_invalidate_cache(async, ctx->priv, + ret = safexcel_invalidate_cache(async, ctx->base.priv, ctx->base.ctxr_dma, ring); if (unlikely(ret)) return ret; @@ -620,7 +620,7 @@ static int safexcel_ahash_send(struct crypto_async_request *async, static int safexcel_ahash_exit_inv(struct crypto_tfm *tfm) { struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); - struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_crypto_priv *priv = ctx->base.priv; EIP197_REQUEST_ON_STACK(req, ahash, EIP197_AHASH_REQ_SIZE); struct safexcel_ahash_req *rctx = ahash_request_ctx(req); struct safexcel_inv_result result = {}; @@ -688,7 +688,7 @@ static int safexcel_ahash_enqueue(struct ahash_request *areq) { struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); struct safexcel_ahash_req *req = ahash_request_ctx(areq); - struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_crypto_priv *priv = ctx->base.priv; int ret, ring; req->needs_inv = false; @@ -702,7 +702,7 @@ static int safexcel_ahash_enqueue(struct ahash_request *areq) /* invalidate for HMAC finish with odigest changed */ (req->finish && req->hmac && memcmp(ctx->base.ctxr->data + (req->state_sz>>2), - ctx->opad, req->state_sz)))) + &ctx->base.opad, req->state_sz)))) /* * We're still setting needs_inv here, even though it is * cleared right away, because the needs_inv flag can be @@ -803,7 +803,7 @@ static int safexcel_ahash_final(struct ahash_request *areq) ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_MD5 && req->len == sizeof(u32) && !areq->nbytes)) { /* Zero length CRC32 */ - memcpy(areq->result, ctx->ipad, sizeof(u32)); + memcpy(areq->result, &ctx->base.ipad, sizeof(u32)); return 0; } else if (unlikely(ctx->cbcmac && req->len == AES_BLOCK_SIZE && !areq->nbytes)) { @@ -815,9 +815,12 @@ static int safexcel_ahash_final(struct ahash_request *areq) /* Zero length (X)CBC/CMAC */ int i; - for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++) - ((__be32 *)areq->result)[i] = - cpu_to_be32(le32_to_cpu(ctx->ipad[i + 4]));//K3 + for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++) { + u32 *result = (void *)areq->result; + + /* K3 */ + result[i] = swab(ctx->base.ipad.word[i + 4]); + } areq->result[0] ^= 0x80; // 10- padding crypto_cipher_encrypt_one(ctx->kaes, areq->result, areq->result); return 0; @@ -917,7 +920,7 @@ static int safexcel_ahash_cra_init(struct crypto_tfm *tfm) container_of(__crypto_ahash_alg(tfm->__crt_alg), struct safexcel_alg_template, alg.ahash); - ctx->priv = tmpl->priv; + ctx->base.priv = tmpl->priv; ctx->base.send = safexcel_ahash_send; ctx->base.handle_result = safexcel_handle_result; ctx->fb_do_setkey = false; @@ -956,7 +959,7 @@ static int safexcel_sha1_digest(struct ahash_request *areq) static void safexcel_ahash_cra_exit(struct crypto_tfm *tfm) { struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); - struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_crypto_priv *priv = ctx->base.priv; int ret; /* context not allocated, skip invalidation */ @@ -1012,7 +1015,7 @@ static int safexcel_hmac_sha1_init(struct ahash_request *areq) memset(req, 0, sizeof(*req)); /* Start from ipad precompute */ - memcpy(req->state, ctx->ipad, SHA1_DIGEST_SIZE); + memcpy(req->state, &ctx->base.ipad, SHA1_DIGEST_SIZE); /* Already processed the key^ipad part now! */ req->len = SHA1_BLOCK_SIZE; req->processed = SHA1_BLOCK_SIZE; @@ -1082,8 +1085,7 @@ static int safexcel_hmac_init_pad(struct ahash_request *areq, } /* Avoid leaking */ - memzero_explicit(keydup, keylen); - kfree(keydup); + kfree_sensitive(keydup); if (ret) return ret; @@ -1135,8 +1137,9 @@ static int safexcel_hmac_init_iv(struct ahash_request *areq, return crypto_ahash_export(areq, state); } -int safexcel_hmac_setkey(const char *alg, const u8 *key, unsigned int keylen, - void *istate, void *ostate) +static int __safexcel_hmac_setkey(const char *alg, const u8 *key, + unsigned int keylen, + void *istate, void *ostate) { struct ahash_request *areq; struct crypto_ahash *tfm; @@ -1185,30 +1188,38 @@ free_ahash: return ret; } -static int safexcel_hmac_alg_setkey(struct crypto_ahash *tfm, const u8 *key, - unsigned int keylen, const char *alg, - unsigned int state_sz) +int safexcel_hmac_setkey(struct safexcel_context *base, const u8 *key, + unsigned int keylen, const char *alg, + unsigned int state_sz) { - struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); - struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_crypto_priv *priv = base->priv; struct safexcel_ahash_export_state istate, ostate; int ret; - ret = safexcel_hmac_setkey(alg, key, keylen, &istate, &ostate); + ret = __safexcel_hmac_setkey(alg, key, keylen, &istate, &ostate); if (ret) return ret; - if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr && - (memcmp(ctx->ipad, istate.state, state_sz) || - memcmp(ctx->opad, ostate.state, state_sz))) - ctx->base.needs_inv = true; + if (priv->flags & EIP197_TRC_CACHE && base->ctxr && + (memcmp(&base->ipad, istate.state, state_sz) || + memcmp(&base->opad, ostate.state, state_sz))) + base->needs_inv = true; - memcpy(ctx->ipad, &istate.state, state_sz); - memcpy(ctx->opad, &ostate.state, state_sz); + memcpy(&base->ipad, &istate.state, state_sz); + memcpy(&base->opad, &ostate.state, state_sz); return 0; } +static int safexcel_hmac_alg_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int keylen, const char *alg, + unsigned int state_sz) +{ + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + + return safexcel_hmac_setkey(&ctx->base, key, keylen, alg, state_sz); +} + static int safexcel_hmac_sha1_setkey(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen) { @@ -1377,7 +1388,7 @@ static int safexcel_hmac_sha224_init(struct ahash_request *areq) memset(req, 0, sizeof(*req)); /* Start from ipad precompute */ - memcpy(req->state, ctx->ipad, SHA256_DIGEST_SIZE); + memcpy(req->state, &ctx->base.ipad, SHA256_DIGEST_SIZE); /* Already processed the key^ipad part now! */ req->len = SHA256_BLOCK_SIZE; req->processed = SHA256_BLOCK_SIZE; @@ -1449,7 +1460,7 @@ static int safexcel_hmac_sha256_init(struct ahash_request *areq) memset(req, 0, sizeof(*req)); /* Start from ipad precompute */ - memcpy(req->state, ctx->ipad, SHA256_DIGEST_SIZE); + memcpy(req->state, &ctx->base.ipad, SHA256_DIGEST_SIZE); /* Already processed the key^ipad part now! */ req->len = SHA256_BLOCK_SIZE; req->processed = SHA256_BLOCK_SIZE; @@ -1635,7 +1646,7 @@ static int safexcel_hmac_sha512_init(struct ahash_request *areq) memset(req, 0, sizeof(*req)); /* Start from ipad precompute */ - memcpy(req->state, ctx->ipad, SHA512_DIGEST_SIZE); + memcpy(req->state, &ctx->base.ipad, SHA512_DIGEST_SIZE); /* Already processed the key^ipad part now! */ req->len = SHA512_BLOCK_SIZE; req->processed = SHA512_BLOCK_SIZE; @@ -1707,7 +1718,7 @@ static int safexcel_hmac_sha384_init(struct ahash_request *areq) memset(req, 0, sizeof(*req)); /* Start from ipad precompute */ - memcpy(req->state, ctx->ipad, SHA512_DIGEST_SIZE); + memcpy(req->state, &ctx->base.ipad, SHA512_DIGEST_SIZE); /* Already processed the key^ipad part now! */ req->len = SHA512_BLOCK_SIZE; req->processed = SHA512_BLOCK_SIZE; @@ -1829,7 +1840,7 @@ static int safexcel_hmac_md5_init(struct ahash_request *areq) memset(req, 0, sizeof(*req)); /* Start from ipad precompute */ - memcpy(req->state, ctx->ipad, MD5_DIGEST_SIZE); + memcpy(req->state, &ctx->base.ipad, MD5_DIGEST_SIZE); /* Already processed the key^ipad part now! */ req->len = MD5_HMAC_BLOCK_SIZE; req->processed = MD5_HMAC_BLOCK_SIZE; @@ -1900,7 +1911,7 @@ static int safexcel_crc32_cra_init(struct crypto_tfm *tfm) int ret = safexcel_ahash_cra_init(tfm); /* Default 'key' is all zeroes */ - memset(ctx->ipad, 0, sizeof(u32)); + memset(&ctx->base.ipad, 0, sizeof(u32)); return ret; } @@ -1912,7 +1923,7 @@ static int safexcel_crc32_init(struct ahash_request *areq) memset(req, 0, sizeof(*req)); /* Start from loaded key */ - req->state[0] = (__force __le32)le32_to_cpu(~ctx->ipad[0]); + req->state[0] = cpu_to_le32(~ctx->base.ipad.word[0]); /* Set processed to non-zero to enable invalidation detection */ req->len = sizeof(u32); req->processed = sizeof(u32); @@ -1934,7 +1945,7 @@ static int safexcel_crc32_setkey(struct crypto_ahash *tfm, const u8 *key, if (keylen != sizeof(u32)) return -EINVAL; - memcpy(ctx->ipad, key, sizeof(u32)); + memcpy(&ctx->base.ipad, key, sizeof(u32)); return 0; } @@ -1984,7 +1995,7 @@ static int safexcel_cbcmac_init(struct ahash_request *areq) memset(req, 0, sizeof(*req)); /* Start from loaded keys */ - memcpy(req->state, ctx->ipad, ctx->key_sz); + memcpy(req->state, &ctx->base.ipad, ctx->key_sz); /* Set processed to non-zero to enable invalidation detection */ req->len = AES_BLOCK_SIZE; req->processed = AES_BLOCK_SIZE; @@ -2009,9 +2020,9 @@ static int safexcel_cbcmac_setkey(struct crypto_ahash *tfm, const u8 *key, if (ret) return ret; - memset(ctx->ipad, 0, 2 * AES_BLOCK_SIZE); + memset(&ctx->base.ipad, 0, 2 * AES_BLOCK_SIZE); for (i = 0; i < len / sizeof(u32); i++) - ctx->ipad[i + 8] = (__force __le32)cpu_to_be32(aes.key_enc[i]); + ctx->base.ipad.be[i + 8] = cpu_to_be32(aes.key_enc[i]); if (len == AES_KEYSIZE_192) { ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC192; @@ -2093,8 +2104,7 @@ static int safexcel_xcbcmac_setkey(struct crypto_ahash *tfm, const u8 *key, crypto_cipher_encrypt_one(ctx->kaes, (u8 *)key_tmp + AES_BLOCK_SIZE, "\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3"); for (i = 0; i < 3 * AES_BLOCK_SIZE / sizeof(u32); i++) - ctx->ipad[i] = - cpu_to_le32((__force u32)cpu_to_be32(key_tmp[i])); + ctx->base.ipad.word[i] = swab(key_tmp[i]); crypto_cipher_clear_flags(ctx->kaes, CRYPTO_TFM_REQ_MASK); crypto_cipher_set_flags(ctx->kaes, crypto_ahash_get_flags(tfm) & @@ -2177,8 +2187,7 @@ static int safexcel_cmac_setkey(struct crypto_ahash *tfm, const u8 *key, return ret; for (i = 0; i < len / sizeof(u32); i++) - ctx->ipad[i + 8] = - cpu_to_le32((__force u32)cpu_to_be32(aes.key_enc[i])); + ctx->base.ipad.word[i + 8] = swab(aes.key_enc[i]); /* precompute the CMAC key material */ crypto_cipher_clear_flags(ctx->kaes, CRYPTO_TFM_REQ_MASK); @@ -2209,7 +2218,7 @@ static int safexcel_cmac_setkey(struct crypto_ahash *tfm, const u8 *key, /* end of code borrowed from crypto/cmac.c */ for (i = 0; i < 2 * AES_BLOCK_SIZE / sizeof(u32); i++) - ctx->ipad[i] = (__force __le32)cpu_to_be32(((u32 *)consts)[i]); + ctx->base.ipad.be[i] = cpu_to_be32(((u32 *)consts)[i]); if (len == AES_KEYSIZE_192) { ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC192; @@ -2331,7 +2340,7 @@ static int safexcel_hmac_sm3_init(struct ahash_request *areq) memset(req, 0, sizeof(*req)); /* Start from ipad precompute */ - memcpy(req->state, ctx->ipad, SM3_DIGEST_SIZE); + memcpy(req->state, &ctx->base.ipad, SM3_DIGEST_SIZE); /* Already processed the key^ipad part now! */ req->len = SM3_BLOCK_SIZE; req->processed = SM3_BLOCK_SIZE; @@ -2424,11 +2433,11 @@ static int safexcel_sha3_fbcheck(struct ahash_request *req) /* Set fallback cipher HMAC key */ u8 key[SHA3_224_BLOCK_SIZE]; - memcpy(key, ctx->ipad, + memcpy(key, &ctx->base.ipad, crypto_ahash_blocksize(ctx->fback) / 2); memcpy(key + crypto_ahash_blocksize(ctx->fback) / 2, - ctx->opad, + &ctx->base.opad, crypto_ahash_blocksize(ctx->fback) / 2); ret = crypto_ahash_setkey(ctx->fback, key, crypto_ahash_blocksize(ctx->fback)); @@ -2801,7 +2810,7 @@ static int safexcel_hmac_sha3_setkey(struct crypto_ahash *tfm, const u8 *key, * first using our fallback cipher */ ret = crypto_shash_digest(ctx->shdesc, key, keylen, - (u8 *)ctx->ipad); + ctx->base.ipad.byte); keylen = crypto_shash_digestsize(ctx->shpre); /* @@ -2810,8 +2819,8 @@ static int safexcel_hmac_sha3_setkey(struct crypto_ahash *tfm, const u8 *key, */ if (keylen > crypto_ahash_blocksize(tfm) / 2) /* Buffers overlap, need to use memmove iso memcpy! */ - memmove(ctx->opad, - (u8 *)ctx->ipad + + memmove(&ctx->base.opad, + ctx->base.ipad.byte + crypto_ahash_blocksize(tfm) / 2, keylen - crypto_ahash_blocksize(tfm) / 2); } else { @@ -2821,11 +2830,11 @@ static int safexcel_hmac_sha3_setkey(struct crypto_ahash *tfm, const u8 *key, * to match the existing HMAC driver infrastructure. */ if (keylen <= crypto_ahash_blocksize(tfm) / 2) { - memcpy(ctx->ipad, key, keylen); + memcpy(&ctx->base.ipad, key, keylen); } else { - memcpy(ctx->ipad, key, + memcpy(&ctx->base.ipad, key, crypto_ahash_blocksize(tfm) / 2); - memcpy(ctx->opad, + memcpy(&ctx->base.opad, key + crypto_ahash_blocksize(tfm) / 2, keylen - crypto_ahash_blocksize(tfm) / 2); } @@ -2833,11 +2842,11 @@ static int safexcel_hmac_sha3_setkey(struct crypto_ahash *tfm, const u8 *key, /* Pad key with zeroes */ if (keylen <= crypto_ahash_blocksize(tfm) / 2) { - memset((u8 *)ctx->ipad + keylen, 0, + memset(ctx->base.ipad.byte + keylen, 0, crypto_ahash_blocksize(tfm) / 2 - keylen); - memset(ctx->opad, 0, crypto_ahash_blocksize(tfm) / 2); + memset(&ctx->base.opad, 0, crypto_ahash_blocksize(tfm) / 2); } else { - memset((u8 *)ctx->opad + keylen - + memset(ctx->base.opad.byte + keylen - crypto_ahash_blocksize(tfm) / 2, 0, crypto_ahash_blocksize(tfm) - keylen); } @@ -2856,7 +2865,7 @@ static int safexcel_hmac_sha3_224_init(struct ahash_request *areq) memset(req, 0, sizeof(*req)); /* Copy (half of) the key */ - memcpy(req->state, ctx->ipad, SHA3_224_BLOCK_SIZE / 2); + memcpy(req->state, &ctx->base.ipad, SHA3_224_BLOCK_SIZE / 2); /* Start of HMAC should have len == processed == blocksize */ req->len = SHA3_224_BLOCK_SIZE; req->processed = SHA3_224_BLOCK_SIZE; @@ -2927,7 +2936,7 @@ static int safexcel_hmac_sha3_256_init(struct ahash_request *areq) memset(req, 0, sizeof(*req)); /* Copy (half of) the key */ - memcpy(req->state, ctx->ipad, SHA3_256_BLOCK_SIZE / 2); + memcpy(req->state, &ctx->base.ipad, SHA3_256_BLOCK_SIZE / 2); /* Start of HMAC should have len == processed == blocksize */ req->len = SHA3_256_BLOCK_SIZE; req->processed = SHA3_256_BLOCK_SIZE; @@ -2998,7 +3007,7 @@ static int safexcel_hmac_sha3_384_init(struct ahash_request *areq) memset(req, 0, sizeof(*req)); /* Copy (half of) the key */ - memcpy(req->state, ctx->ipad, SHA3_384_BLOCK_SIZE / 2); + memcpy(req->state, &ctx->base.ipad, SHA3_384_BLOCK_SIZE / 2); /* Start of HMAC should have len == processed == blocksize */ req->len = SHA3_384_BLOCK_SIZE; req->processed = SHA3_384_BLOCK_SIZE; @@ -3069,7 +3078,7 @@ static int safexcel_hmac_sha3_512_init(struct ahash_request *areq) memset(req, 0, sizeof(*req)); /* Copy (half of) the key */ - memcpy(req->state, ctx->ipad, SHA3_512_BLOCK_SIZE / 2); + memcpy(req->state, &ctx->base.ipad, SHA3_512_BLOCK_SIZE / 2); /* Start of HMAC should have len == processed == blocksize */ req->len = SHA3_512_BLOCK_SIZE; req->processed = SHA3_512_BLOCK_SIZE; diff --git a/drivers/crypto/inside-secure/safexcel_ring.c b/drivers/crypto/inside-secure/safexcel_ring.c index e454c3d44f07..90f15032c8df 100644 --- a/drivers/crypto/inside-secure/safexcel_ring.c +++ b/drivers/crypto/inside-secure/safexcel_ring.c @@ -236,8 +236,8 @@ struct safexcel_result_desc *safexcel_add_rdesc(struct safexcel_crypto_priv *pri rdesc->particle_size = len; rdesc->rsvd0 = 0; - rdesc->descriptor_overflow = 0; - rdesc->buffer_overflow = 0; + rdesc->descriptor_overflow = 1; /* assume error */ + rdesc->buffer_overflow = 1; /* assume error */ rdesc->last_seg = last; rdesc->first_seg = first; rdesc->result_size = EIP197_RD64_RESULT_SIZE; @@ -245,9 +245,10 @@ struct safexcel_result_desc *safexcel_add_rdesc(struct safexcel_crypto_priv *pri rdesc->data_lo = lower_32_bits(data); rdesc->data_hi = upper_32_bits(data); - /* Clear length & error code in result token */ + /* Clear length in result token */ rtoken->packet_length = 0; - rtoken->error_code = 0; + /* Assume errors - HW will clear if not the case */ + rtoken->error_code = 0x7fff; return rdesc; } diff --git a/drivers/crypto/ixp4xx_crypto.c b/drivers/crypto/ixp4xx_crypto.c index f478bb0a566a..276012e7c482 100644 --- a/drivers/crypto/ixp4xx_crypto.c +++ b/drivers/crypto/ixp4xx_crypto.c @@ -528,7 +528,7 @@ static void release_ixp_crypto(struct device *dev) if (crypt_virt) { dma_free_coherent(dev, - NPE_QLEN_TOTAL * sizeof( struct crypt_ctl), + NPE_QLEN * sizeof(struct crypt_ctl), crypt_virt, crypt_phys); } } diff --git a/drivers/crypto/marvell/cesa/cesa.c b/drivers/crypto/marvell/cesa/cesa.c index d63bca9718dc..06211858bf2e 100644 --- a/drivers/crypto/marvell/cesa/cesa.c +++ b/drivers/crypto/marvell/cesa/cesa.c @@ -437,7 +437,6 @@ static int mv_cesa_probe(struct platform_device *pdev) struct device *dev = &pdev->dev; struct mv_cesa_dev *cesa; struct mv_cesa_engine *engines; - struct resource *res; int irq, ret, i, cpu; u32 sram_size; @@ -475,8 +474,7 @@ static int mv_cesa_probe(struct platform_device *pdev) spin_lock_init(&cesa->lock); - res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "regs"); - cesa->regs = devm_ioremap_resource(dev, res); + cesa->regs = devm_platform_ioremap_resource_byname(pdev, "regs"); if (IS_ERR(cesa->regs)) return PTR_ERR(cesa->regs); diff --git a/drivers/crypto/marvell/cesa/cesa.h b/drivers/crypto/marvell/cesa/cesa.h index 0c9cbb681e49..fabfaaccca87 100644 --- a/drivers/crypto/marvell/cesa/cesa.h +++ b/drivers/crypto/marvell/cesa/cesa.h @@ -2,12 +2,10 @@ #ifndef __MARVELL_CESA_H__ #define __MARVELL_CESA_H__ -#include <crypto/algapi.h> -#include <crypto/hash.h> #include <crypto/internal/hash.h> #include <crypto/internal/skcipher.h> -#include <linux/crypto.h> +#include <linux/dma-direction.h> #include <linux/dmapool.h> #define CESA_ENGINE_OFF(i) (((i) * 0x2000)) @@ -239,7 +237,7 @@ struct mv_cesa_sec_accel_desc { * Context associated to a cipher operation. */ struct mv_cesa_skcipher_op_ctx { - u32 key[8]; + __le32 key[8]; u32 iv[4]; }; @@ -252,7 +250,7 @@ struct mv_cesa_skcipher_op_ctx { */ struct mv_cesa_hash_op_ctx { u32 iv[16]; - u32 hash[8]; + __le32 hash[8]; }; /** @@ -300,8 +298,14 @@ struct mv_cesa_op_ctx { */ struct mv_cesa_tdma_desc { __le32 byte_cnt; - __le32 src; - __le32 dst; + union { + __le32 src; + dma_addr_t src_dma; + }; + union { + __le32 dst; + dma_addr_t dst_dma; + }; __le32 next_dma; /* Software state */ @@ -506,7 +510,7 @@ struct mv_cesa_hash_ctx { */ struct mv_cesa_hmac_ctx { struct mv_cesa_ctx base; - u32 iv[16]; + __be32 iv[16]; }; /** diff --git a/drivers/crypto/marvell/cesa/cipher.c b/drivers/crypto/marvell/cesa/cipher.c index 45b4d7a29833..b4a6ff9dd6d5 100644 --- a/drivers/crypto/marvell/cesa/cipher.c +++ b/drivers/crypto/marvell/cesa/cipher.c @@ -11,6 +11,8 @@ #include <crypto/aes.h> #include <crypto/internal/des.h> +#include <linux/device.h> +#include <linux/dma-mapping.h> #include "cesa.h" @@ -262,8 +264,7 @@ static int mv_cesa_aes_setkey(struct crypto_skcipher *cipher, const u8 *key, remaining = (ctx->aes.key_length - 16) / 4; offset = ctx->aes.key_length + 24 - remaining; for (i = 0; i < remaining; i++) - ctx->aes.key_dec[4 + i] = - cpu_to_le32(ctx->aes.key_enc[offset + i]); + ctx->aes.key_dec[4 + i] = ctx->aes.key_enc[offset + i]; return 0; } diff --git a/drivers/crypto/marvell/cesa/hash.c b/drivers/crypto/marvell/cesa/hash.c index f2a2fc111164..add7ea011c98 100644 --- a/drivers/crypto/marvell/cesa/hash.c +++ b/drivers/crypto/marvell/cesa/hash.c @@ -12,6 +12,8 @@ #include <crypto/hmac.h> #include <crypto/md5.h> #include <crypto/sha.h> +#include <linux/device.h> +#include <linux/dma-mapping.h> #include "cesa.h" @@ -222,9 +224,11 @@ static void mv_cesa_ahash_std_step(struct ahash_request *req) CESA_SA_DATA_SRAM_OFFSET + len, new_cache_ptr); } else { - len += mv_cesa_ahash_pad_req(creq, - engine->sram + len + - CESA_SA_DATA_SRAM_OFFSET); + i = mv_cesa_ahash_pad_req(creq, creq->cache); + len += i; + memcpy_toio(engine->sram + len + + CESA_SA_DATA_SRAM_OFFSET, + creq->cache, i); } if (frag_mode == CESA_SA_DESC_CFG_LAST_FRAG) @@ -342,7 +346,7 @@ static void mv_cesa_ahash_complete(struct crypto_async_request *req) */ data = creq->base.chain.last->op->ctx.hash.hash; for (i = 0; i < digsize / 4; i++) - creq->state[i] = cpu_to_le32(data[i]); + creq->state[i] = le32_to_cpu(data[i]); memcpy(ahashreq->result, data, digsize); } else { @@ -1265,10 +1269,10 @@ static int mv_cesa_ahmac_md5_setkey(struct crypto_ahash *tfm, const u8 *key, return ret; for (i = 0; i < ARRAY_SIZE(istate.hash); i++) - ctx->iv[i] = be32_to_cpu(istate.hash[i]); + ctx->iv[i] = cpu_to_be32(istate.hash[i]); for (i = 0; i < ARRAY_SIZE(ostate.hash); i++) - ctx->iv[i + 8] = be32_to_cpu(ostate.hash[i]); + ctx->iv[i + 8] = cpu_to_be32(ostate.hash[i]); return 0; } @@ -1336,10 +1340,10 @@ static int mv_cesa_ahmac_sha1_setkey(struct crypto_ahash *tfm, const u8 *key, return ret; for (i = 0; i < ARRAY_SIZE(istate.state); i++) - ctx->iv[i] = be32_to_cpu(istate.state[i]); + ctx->iv[i] = cpu_to_be32(istate.state[i]); for (i = 0; i < ARRAY_SIZE(ostate.state); i++) - ctx->iv[i + 8] = be32_to_cpu(ostate.state[i]); + ctx->iv[i + 8] = cpu_to_be32(ostate.state[i]); return 0; } @@ -1394,10 +1398,10 @@ static int mv_cesa_ahmac_sha256_setkey(struct crypto_ahash *tfm, const u8 *key, return ret; for (i = 0; i < ARRAY_SIZE(istate.state); i++) - ctx->iv[i] = be32_to_cpu(istate.state[i]); + ctx->iv[i] = cpu_to_be32(istate.state[i]); for (i = 0; i < ARRAY_SIZE(ostate.state); i++) - ctx->iv[i + 8] = be32_to_cpu(ostate.state[i]); + ctx->iv[i + 8] = cpu_to_be32(ostate.state[i]); return 0; } diff --git a/drivers/crypto/marvell/cesa/tdma.c b/drivers/crypto/marvell/cesa/tdma.c index b81ee276fe0e..5d9c48fb72b2 100644 --- a/drivers/crypto/marvell/cesa/tdma.c +++ b/drivers/crypto/marvell/cesa/tdma.c @@ -83,10 +83,10 @@ void mv_cesa_dma_prepare(struct mv_cesa_req *dreq, for (tdma = dreq->chain.first; tdma; tdma = tdma->next) { if (tdma->flags & CESA_TDMA_DST_IN_SRAM) - tdma->dst = cpu_to_le32(tdma->dst + engine->sram_dma); + tdma->dst = cpu_to_le32(tdma->dst_dma + engine->sram_dma); if (tdma->flags & CESA_TDMA_SRC_IN_SRAM) - tdma->src = cpu_to_le32(tdma->src + engine->sram_dma); + tdma->src = cpu_to_le32(tdma->src_dma + engine->sram_dma); if ((tdma->flags & CESA_TDMA_TYPE_MSK) == CESA_TDMA_OP) mv_cesa_adjust_op(engine, tdma->op); @@ -114,7 +114,7 @@ void mv_cesa_tdma_chain(struct mv_cesa_engine *engine, */ if (!(last->flags & CESA_TDMA_BREAK_CHAIN) && !(dreq->chain.first->flags & CESA_TDMA_SET_STATE)) - last->next_dma = dreq->chain.first->cur_dma; + last->next_dma = cpu_to_le32(dreq->chain.first->cur_dma); } } @@ -237,8 +237,8 @@ int mv_cesa_dma_add_result_op(struct mv_cesa_tdma_chain *chain, dma_addr_t src, return -EIO; tdma->byte_cnt = cpu_to_le32(size | BIT(31)); - tdma->src = src; - tdma->dst = op_desc->src; + tdma->src_dma = src; + tdma->dst_dma = op_desc->src_dma; tdma->op = op_desc->op; flags &= (CESA_TDMA_DST_IN_SRAM | CESA_TDMA_SRC_IN_SRAM); @@ -272,7 +272,7 @@ struct mv_cesa_op_ctx *mv_cesa_dma_add_op(struct mv_cesa_tdma_chain *chain, tdma->op = op; tdma->byte_cnt = cpu_to_le32(size | BIT(31)); tdma->src = cpu_to_le32(dma_handle); - tdma->dst = CESA_SA_CFG_SRAM_OFFSET; + tdma->dst_dma = CESA_SA_CFG_SRAM_OFFSET; tdma->flags = CESA_TDMA_DST_IN_SRAM | CESA_TDMA_OP; return op; @@ -289,8 +289,8 @@ int mv_cesa_dma_add_data_transfer(struct mv_cesa_tdma_chain *chain, return PTR_ERR(tdma); tdma->byte_cnt = cpu_to_le32(size | BIT(31)); - tdma->src = src; - tdma->dst = dst; + tdma->src_dma = src; + tdma->dst_dma = dst; flags &= (CESA_TDMA_DST_IN_SRAM | CESA_TDMA_SRC_IN_SRAM); tdma->flags = flags | CESA_TDMA_DATA; diff --git a/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c b/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c index cc103b1bc224..40b482198ebc 100644 --- a/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c +++ b/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c @@ -824,18 +824,12 @@ static ssize_t eng_grp_info_show(struct device *dev, static int create_sysfs_eng_grps_info(struct device *dev, struct otx_cpt_eng_grp_info *eng_grp) { - int ret; - eng_grp->info_attr.show = eng_grp_info_show; eng_grp->info_attr.store = NULL; eng_grp->info_attr.attr.name = eng_grp->sysfs_info_name; eng_grp->info_attr.attr.mode = 0440; sysfs_attr_init(&eng_grp->info_attr.attr); - ret = device_create_file(dev, &eng_grp->info_attr); - if (ret) - return ret; - - return 0; + return device_create_file(dev, &eng_grp->info_attr); } static void ucode_unload(struct device *dev, struct otx_cpt_ucode *ucode) diff --git a/drivers/crypto/mediatek/mtk-aes.c b/drivers/crypto/mediatek/mtk-aes.c index 4ad3571ab6af..7323066724c3 100644 --- a/drivers/crypto/mediatek/mtk-aes.c +++ b/drivers/crypto/mediatek/mtk-aes.c @@ -126,7 +126,7 @@ struct mtk_aes_ctx { struct mtk_aes_ctr_ctx { struct mtk_aes_base_ctx base; - u32 iv[AES_BLOCK_SIZE / sizeof(u32)]; + __be32 iv[AES_BLOCK_SIZE / sizeof(u32)]; size_t offset; struct scatterlist src[2]; struct scatterlist dst[2]; @@ -242,22 +242,6 @@ static inline void mtk_aes_restore_sg(const struct mtk_aes_dma *dma) sg->length += dma->remainder; } -static inline void mtk_aes_write_state_le(__le32 *dst, const u32 *src, u32 size) -{ - int i; - - for (i = 0; i < SIZE_IN_WORDS(size); i++) - dst[i] = cpu_to_le32(src[i]); -} - -static inline void mtk_aes_write_state_be(__be32 *dst, const u32 *src, u32 size) -{ - int i; - - for (i = 0; i < SIZE_IN_WORDS(size); i++) - dst[i] = cpu_to_be32(src[i]); -} - static inline int mtk_aes_complete(struct mtk_cryp *cryp, struct mtk_aes_rec *aes, int err) @@ -321,7 +305,7 @@ static int mtk_aes_xmit(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) /* Prepare enough space for authenticated tag */ if (aes->flags & AES_FLAGS_GCM) - res->hdr += AES_BLOCK_SIZE; + le32_add_cpu(&res->hdr, AES_BLOCK_SIZE); /* * Make sure that all changes to the DMA ring are done before we @@ -449,10 +433,10 @@ static void mtk_aes_info_init(struct mtk_cryp *cryp, struct mtk_aes_rec *aes, return; } - mtk_aes_write_state_le(info->state + ctx->keylen, (void *)req->iv, - AES_BLOCK_SIZE); + memcpy(info->state + ctx->keylen, req->iv, AES_BLOCK_SIZE); ctr: - info->tfm[0] += AES_TFM_SIZE(SIZE_IN_WORDS(AES_BLOCK_SIZE)); + le32_add_cpu(&info->tfm[0], + le32_to_cpu(AES_TFM_SIZE(SIZE_IN_WORDS(AES_BLOCK_SIZE)))); info->tfm[1] |= AES_TFM_FULL_IV; info->cmd[cnt++] = AES_CMD2; ecb: @@ -601,8 +585,7 @@ static int mtk_aes_ctr_transfer(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) scatterwalk_ffwd(cctx->dst, req->dst, cctx->offset)); /* Write IVs into transform state buffer. */ - mtk_aes_write_state_le(ctx->info.state + ctx->keylen, cctx->iv, - AES_BLOCK_SIZE); + memcpy(ctx->info.state + ctx->keylen, cctx->iv, AES_BLOCK_SIZE); if (unlikely(fragmented)) { /* @@ -654,7 +637,7 @@ static int mtk_aes_setkey(struct crypto_skcipher *tfm, } ctx->keylen = SIZE_IN_WORDS(keylen); - mtk_aes_write_state_le(ctx->key, (const u32 *)key, keylen); + memcpy(ctx->key, key, keylen); return 0; } @@ -848,7 +831,7 @@ mtk_aes_gcm_ctx_cast(struct mtk_aes_base_ctx *ctx) static int mtk_aes_gcm_tag_verify(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) { - u32 status = cryp->ring[aes->id]->res_prev->ct; + __le32 status = cryp->ring[aes->id]->res_prev->ct; return mtk_aes_complete(cryp, aes, (status & AES_AUTH_TAG_ERR) ? -EBADMSG : 0); @@ -866,7 +849,7 @@ static void mtk_aes_gcm_info_init(struct mtk_cryp *cryp, u32 ivsize = crypto_aead_ivsize(crypto_aead_reqtfm(req)); u32 cnt = 0; - ctx->ct_hdr = AES_CT_CTRL_HDR | len; + ctx->ct_hdr = AES_CT_CTRL_HDR | cpu_to_le32(len); info->cmd[cnt++] = AES_GCM_CMD0 | cpu_to_le32(req->assoclen); info->cmd[cnt++] = AES_GCM_CMD1 | cpu_to_le32(req->assoclen); @@ -889,8 +872,8 @@ static void mtk_aes_gcm_info_init(struct mtk_cryp *cryp, info->tfm[1] = AES_TFM_CTR_INIT | AES_TFM_IV_CTR_MODE | AES_TFM_3IV | AES_TFM_ENC_HASH; - mtk_aes_write_state_le(info->state + ctx->keylen + SIZE_IN_WORDS( - AES_BLOCK_SIZE), (const u32 *)req->iv, ivsize); + memcpy(info->state + ctx->keylen + SIZE_IN_WORDS(AES_BLOCK_SIZE), + req->iv, ivsize); } static int mtk_aes_gcm_dma(struct mtk_cryp *cryp, struct mtk_aes_rec *aes, @@ -994,9 +977,13 @@ static int mtk_aes_gcm_setkey(struct crypto_aead *aead, const u8 *key, u32 keylen) { struct mtk_aes_base_ctx *ctx = crypto_aead_ctx(aead); - u8 hash[AES_BLOCK_SIZE] __aligned(4) = {}; + union { + u32 x32[SIZE_IN_WORDS(AES_BLOCK_SIZE)]; + u8 x8[AES_BLOCK_SIZE]; + } hash = {}; struct crypto_aes_ctx aes_ctx; int err; + int i; switch (keylen) { case AES_KEYSIZE_128: @@ -1019,12 +1006,16 @@ static int mtk_aes_gcm_setkey(struct crypto_aead *aead, const u8 *key, if (err) return err; - aes_encrypt(&aes_ctx, hash, hash); + aes_encrypt(&aes_ctx, hash.x8, hash.x8); memzero_explicit(&aes_ctx, sizeof(aes_ctx)); - mtk_aes_write_state_le(ctx->key, (const u32 *)key, keylen); - mtk_aes_write_state_be(ctx->key + ctx->keylen, (const u32 *)hash, - AES_BLOCK_SIZE); + memcpy(ctx->key, key, keylen); + + /* Why do we need to do this? */ + for (i = 0; i < SIZE_IN_WORDS(AES_BLOCK_SIZE); i++) + hash.x32[i] = swab32(hash.x32[i]); + + memcpy(ctx->key + ctx->keylen, &hash, AES_BLOCK_SIZE); return 0; } diff --git a/drivers/crypto/mediatek/mtk-platform.c b/drivers/crypto/mediatek/mtk-platform.c index 7e3ad085b5bd..9d878620e5c9 100644 --- a/drivers/crypto/mediatek/mtk-platform.c +++ b/drivers/crypto/mediatek/mtk-platform.c @@ -185,8 +185,6 @@ static int mtk_dfe_dse_state_check(struct mtk_cryp *cryp) static int mtk_dfe_dse_reset(struct mtk_cryp *cryp) { - int err; - /* Reset DSE/DFE and correct system priorities for all rings. */ writel(MTK_DFSE_THR_CTRL_RESET, cryp->base + DFE_THR_CTRL); writel(0, cryp->base + DFE_PRIO_0); @@ -200,11 +198,7 @@ static int mtk_dfe_dse_reset(struct mtk_cryp *cryp) writel(0, cryp->base + DSE_PRIO_2); writel(0, cryp->base + DSE_PRIO_3); - err = mtk_dfe_dse_state_check(cryp); - if (err) - return err; - - return 0; + return mtk_dfe_dse_state_check(cryp); } static void mtk_cmd_desc_ring_setup(struct mtk_cryp *cryp, @@ -442,7 +436,7 @@ static void mtk_desc_dma_free(struct mtk_cryp *cryp) static int mtk_desc_ring_alloc(struct mtk_cryp *cryp) { struct mtk_ring **ring = cryp->ring; - int i, err = ENOMEM; + int i; for (i = 0; i < MTK_RING_MAX; i++) { ring[i] = kzalloc(sizeof(**ring), GFP_KERNEL); @@ -469,14 +463,14 @@ static int mtk_desc_ring_alloc(struct mtk_cryp *cryp) return 0; err_cleanup: - for (; i--; ) { + do { dma_free_coherent(cryp->dev, MTK_DESC_RING_SZ, ring[i]->res_base, ring[i]->res_dma); dma_free_coherent(cryp->dev, MTK_DESC_RING_SZ, ring[i]->cmd_base, ring[i]->cmd_dma); kfree(ring[i]); - } - return err; + } while (i--); + return -ENOMEM; } static int mtk_crypto_probe(struct platform_device *pdev) diff --git a/drivers/crypto/mediatek/mtk-sha.c b/drivers/crypto/mediatek/mtk-sha.c index da3f0b8814aa..3d5d7d68b03b 100644 --- a/drivers/crypto/mediatek/mtk-sha.c +++ b/drivers/crypto/mediatek/mtk-sha.c @@ -239,7 +239,7 @@ static int mtk_sha_append_sg(struct mtk_sha_reqctx *ctx) static void mtk_sha_fill_padding(struct mtk_sha_reqctx *ctx, u32 len) { u32 index, padlen; - u64 bits[2]; + __be64 bits[2]; u64 size = ctx->digcnt; size += ctx->bufcnt; diff --git a/drivers/crypto/n2_core.c b/drivers/crypto/n2_core.c index d8aec5153b21..3642bf83d809 100644 --- a/drivers/crypto/n2_core.c +++ b/drivers/crypto/n2_core.c @@ -249,7 +249,7 @@ static inline bool n2_should_run_async(struct spu_queue *qp, int this_len) struct n2_ahash_alg { struct list_head entry; const u8 *hash_zero; - const u32 *hash_init; + const u8 *hash_init; u8 hw_op_hashsz; u8 digest_size; u8 auth_type; @@ -662,7 +662,6 @@ struct n2_skcipher_context { u8 aes[AES_MAX_KEY_SIZE]; u8 des[DES_KEY_SIZE]; u8 des3[3 * DES_KEY_SIZE]; - u8 arc4[258]; /* S-box, X, Y */ } key; }; @@ -789,36 +788,6 @@ static int n2_3des_setkey(struct crypto_skcipher *skcipher, const u8 *key, return 0; } -static int n2_arc4_setkey(struct crypto_skcipher *skcipher, const u8 *key, - unsigned int keylen) -{ - struct crypto_tfm *tfm = crypto_skcipher_tfm(skcipher); - struct n2_skcipher_context *ctx = crypto_tfm_ctx(tfm); - struct n2_skcipher_alg *n2alg = n2_skcipher_alg(skcipher); - u8 *s = ctx->key.arc4; - u8 *x = s + 256; - u8 *y = x + 1; - int i, j, k; - - ctx->enc_type = n2alg->enc_type; - - j = k = 0; - *x = 0; - *y = 0; - for (i = 0; i < 256; i++) - s[i] = i; - for (i = 0; i < 256; i++) { - u8 a = s[i]; - j = (j + key[k] + a) & 0xff; - s[i] = s[j]; - s[j] = a; - if (++k >= keylen) - k = 0; - } - - return 0; -} - static inline int skcipher_descriptor_len(int nbytes, unsigned int block_size) { int this_len = nbytes; @@ -1122,21 +1091,6 @@ struct n2_skcipher_tmpl { }; static const struct n2_skcipher_tmpl skcipher_tmpls[] = { - /* ARC4: only ECB is supported (chaining bits ignored) */ - { .name = "ecb(arc4)", - .drv_name = "ecb-arc4", - .block_size = 1, - .enc_type = (ENC_TYPE_ALG_RC4_STREAM | - ENC_TYPE_CHAINING_ECB), - .skcipher = { - .min_keysize = 1, - .max_keysize = 256, - .setkey = n2_arc4_setkey, - .encrypt = n2_encrypt_ecb, - .decrypt = n2_decrypt_ecb, - }, - }, - /* DES: ECB CBC and CFB are supported */ { .name = "ecb(des)", .drv_name = "ecb-des", @@ -1271,7 +1225,7 @@ static LIST_HEAD(skcipher_algs); struct n2_hash_tmpl { const char *name; const u8 *hash_zero; - const u32 *hash_init; + const u8 *hash_init; u8 hw_op_hashsz; u8 digest_size; u8 block_size; @@ -1279,7 +1233,7 @@ struct n2_hash_tmpl { u8 hmac_type; }; -static const u32 n2_md5_init[MD5_HASH_WORDS] = { +static const __le32 n2_md5_init[MD5_HASH_WORDS] = { cpu_to_le32(MD5_H0), cpu_to_le32(MD5_H1), cpu_to_le32(MD5_H2), @@ -1300,7 +1254,7 @@ static const u32 n2_sha224_init[SHA256_DIGEST_SIZE / 4] = { static const struct n2_hash_tmpl hash_tmpls[] = { { .name = "md5", .hash_zero = md5_zero_message_hash, - .hash_init = n2_md5_init, + .hash_init = (u8 *)n2_md5_init, .auth_type = AUTH_TYPE_MD5, .hmac_type = AUTH_TYPE_HMAC_MD5, .hw_op_hashsz = MD5_DIGEST_SIZE, @@ -1308,7 +1262,7 @@ static const struct n2_hash_tmpl hash_tmpls[] = { .block_size = MD5_HMAC_BLOCK_SIZE }, { .name = "sha1", .hash_zero = sha1_zero_message_hash, - .hash_init = n2_sha1_init, + .hash_init = (u8 *)n2_sha1_init, .auth_type = AUTH_TYPE_SHA1, .hmac_type = AUTH_TYPE_HMAC_SHA1, .hw_op_hashsz = SHA1_DIGEST_SIZE, @@ -1316,7 +1270,7 @@ static const struct n2_hash_tmpl hash_tmpls[] = { .block_size = SHA1_BLOCK_SIZE }, { .name = "sha256", .hash_zero = sha256_zero_message_hash, - .hash_init = n2_sha256_init, + .hash_init = (u8 *)n2_sha256_init, .auth_type = AUTH_TYPE_SHA256, .hmac_type = AUTH_TYPE_HMAC_SHA256, .hw_op_hashsz = SHA256_DIGEST_SIZE, @@ -1324,7 +1278,7 @@ static const struct n2_hash_tmpl hash_tmpls[] = { .block_size = SHA256_BLOCK_SIZE }, { .name = "sha224", .hash_zero = sha224_zero_message_hash, - .hash_init = n2_sha224_init, + .hash_init = (u8 *)n2_sha224_init, .auth_type = AUTH_TYPE_SHA256, .hmac_type = AUTH_TYPE_RESERVED, .hw_op_hashsz = SHA256_DIGEST_SIZE, diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c index 954d703f2981..a3b38d2c92e7 100644 --- a/drivers/crypto/omap-sham.c +++ b/drivers/crypto/omap-sham.c @@ -39,6 +39,7 @@ #include <crypto/hash.h> #include <crypto/hmac.h> #include <crypto/internal/hash.h> +#include <crypto/engine.h> #define MD5_DIGEST_SIZE 16 @@ -100,7 +101,6 @@ #define DEFAULT_AUTOSUSPEND_DELAY 1000 /* mostly device flags */ -#define FLAGS_BUSY 0 #define FLAGS_FINAL 1 #define FLAGS_DMA_ACTIVE 2 #define FLAGS_OUTPUT_READY 3 @@ -144,7 +144,7 @@ struct omap_sham_dev; struct omap_sham_reqctx { struct omap_sham_dev *dd; unsigned long flags; - unsigned long op; + u8 op; u8 digest[SHA512_DIGEST_SIZE] OMAP_ALIGNED; size_t digcnt; @@ -168,6 +168,7 @@ struct omap_sham_hmac_ctx { }; struct omap_sham_ctx { + struct crypto_engine_ctx enginectx; unsigned long flags; /* fallback stuff */ @@ -219,7 +220,6 @@ struct omap_sham_dev { struct device *dev; void __iomem *io_base; int irq; - spinlock_t lock; int err; struct dma_chan *dma_lch; struct tasklet_struct done_task; @@ -230,6 +230,7 @@ struct omap_sham_dev { int fallback_sz; struct crypto_queue queue; struct ahash_request *req; + struct crypto_engine *engine; const struct omap_sham_pdata *pdata; }; @@ -245,6 +246,9 @@ static struct omap_sham_drv sham = { .lock = __SPIN_LOCK_UNLOCKED(sham.lock), }; +static int omap_sham_enqueue(struct ahash_request *req, unsigned int op); +static void omap_sham_finish_req(struct ahash_request *req, int err); + static inline u32 omap_sham_read(struct omap_sham_dev *dd, u32 offset) { return __raw_readl(dd->io_base + offset); @@ -456,6 +460,9 @@ static void omap_sham_write_ctrl_omap4(struct omap_sham_dev *dd, size_t length, struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req); u32 val, mask; + if (likely(ctx->digcnt)) + omap_sham_write(dd, SHA_REG_DIGCNT(dd), ctx->digcnt); + /* * Setting ALGO_CONST only for the first iteration and * CLOSE_HASH only for the last one. Note that flags mode bits @@ -854,13 +861,16 @@ static int omap_sham_align_sgs(struct scatterlist *sg, return 0; } -static int omap_sham_prepare_request(struct ahash_request *req, bool update) +static int omap_sham_prepare_request(struct crypto_engine *engine, void *areq) { + struct ahash_request *req = container_of(areq, struct ahash_request, + base); struct omap_sham_reqctx *rctx = ahash_request_ctx(req); int bs; int ret; unsigned int nbytes; bool final = rctx->flags & BIT(FLAGS_FINUP); + bool update = rctx->op == OP_UPDATE; int hash_later; bs = get_block_size(rctx); @@ -1021,7 +1031,7 @@ static int omap_sham_update_req(struct omap_sham_dev *dd) struct omap_sham_reqctx *ctx = ahash_request_ctx(req); int err; bool final = (ctx->flags & BIT(FLAGS_FINUP)) && - !(dd->flags & BIT(FLAGS_HUGE)); + !(dd->flags & BIT(FLAGS_HUGE)); dev_dbg(dd->dev, "update_req: total: %u, digcnt: %zd, final: %d", ctx->total, ctx->digcnt, final); @@ -1069,6 +1079,39 @@ static int omap_sham_final_req(struct omap_sham_dev *dd) return err; } +static int omap_sham_hash_one_req(struct crypto_engine *engine, void *areq) +{ + struct ahash_request *req = container_of(areq, struct ahash_request, + base); + struct omap_sham_reqctx *ctx = ahash_request_ctx(req); + struct omap_sham_dev *dd = ctx->dd; + int err; + bool final = (ctx->flags & BIT(FLAGS_FINUP)) && + !(dd->flags & BIT(FLAGS_HUGE)); + + dev_dbg(dd->dev, "hash-one: op: %u, total: %u, digcnt: %zd, final: %d", + ctx->op, ctx->total, ctx->digcnt, final); + + dd->req = req; + + err = omap_sham_hw_init(dd); + if (err) + return err; + + if (ctx->digcnt) + dd->pdata->copy_hash(req, 0); + + if (ctx->op == OP_UPDATE) + err = omap_sham_update_req(dd); + else if (ctx->op == OP_FINAL) + err = omap_sham_final_req(dd); + + if (err != -EINPROGRESS) + omap_sham_finish_req(req, err); + + return 0; +} + static int omap_sham_finish_hmac(struct ahash_request *req) { struct omap_sham_ctx *tctx = crypto_tfm_ctx(req->base.tfm); @@ -1116,25 +1159,20 @@ static void omap_sham_finish_req(struct ahash_request *req, int err) ctx->sg = NULL; - dd->flags &= ~(BIT(FLAGS_SGS_ALLOCED) | BIT(FLAGS_SGS_COPIED)); + dd->flags &= ~(BIT(FLAGS_SGS_ALLOCED) | BIT(FLAGS_SGS_COPIED) | + BIT(FLAGS_CPU) | BIT(FLAGS_DMA_READY) | + BIT(FLAGS_OUTPUT_READY)); + + if (!err) + dd->pdata->copy_hash(req, 1); if (dd->flags & BIT(FLAGS_HUGE)) { - dd->flags &= ~(BIT(FLAGS_CPU) | BIT(FLAGS_DMA_READY) | - BIT(FLAGS_OUTPUT_READY) | BIT(FLAGS_HUGE)); - omap_sham_prepare_request(req, ctx->op == OP_UPDATE); - if (ctx->op == OP_UPDATE || (dd->flags & BIT(FLAGS_HUGE))) { - err = omap_sham_update_req(dd); - if (err != -EINPROGRESS && - (ctx->flags & BIT(FLAGS_FINUP))) - err = omap_sham_final_req(dd); - } else if (ctx->op == OP_FINAL) { - omap_sham_final_req(dd); - } + /* Re-enqueue the request */ + omap_sham_enqueue(req, ctx->op); return; } if (!err) { - dd->pdata->copy_hash(req, 1); if (test_bit(FLAGS_FINAL, &dd->flags)) err = omap_sham_finish(req); } else { @@ -1142,7 +1180,7 @@ static void omap_sham_finish_req(struct ahash_request *req, int err) } /* atomic operation is not needed here */ - dd->flags &= ~(BIT(FLAGS_BUSY) | BIT(FLAGS_FINAL) | BIT(FLAGS_CPU) | + dd->flags &= ~(BIT(FLAGS_FINAL) | BIT(FLAGS_CPU) | BIT(FLAGS_DMA_READY) | BIT(FLAGS_OUTPUT_READY)); pm_runtime_mark_last_busy(dd->dev); @@ -1150,81 +1188,13 @@ static void omap_sham_finish_req(struct ahash_request *req, int err) ctx->offset = 0; - if (req->base.complete) - req->base.complete(&req->base, err); + crypto_finalize_hash_request(dd->engine, req, err); } static int omap_sham_handle_queue(struct omap_sham_dev *dd, struct ahash_request *req) { - struct crypto_async_request *async_req, *backlog; - struct omap_sham_reqctx *ctx; - unsigned long flags; - int err = 0, ret = 0; - -retry: - spin_lock_irqsave(&dd->lock, flags); - if (req) - ret = ahash_enqueue_request(&dd->queue, req); - if (test_bit(FLAGS_BUSY, &dd->flags)) { - spin_unlock_irqrestore(&dd->lock, flags); - return ret; - } - backlog = crypto_get_backlog(&dd->queue); - async_req = crypto_dequeue_request(&dd->queue); - if (async_req) - set_bit(FLAGS_BUSY, &dd->flags); - spin_unlock_irqrestore(&dd->lock, flags); - - if (!async_req) - return ret; - - if (backlog) - backlog->complete(backlog, -EINPROGRESS); - - req = ahash_request_cast(async_req); - dd->req = req; - ctx = ahash_request_ctx(req); - - err = omap_sham_prepare_request(req, ctx->op == OP_UPDATE); - if (err || !ctx->total) - goto err1; - - dev_dbg(dd->dev, "handling new req, op: %lu, nbytes: %d\n", - ctx->op, req->nbytes); - - err = omap_sham_hw_init(dd); - if (err) - goto err1; - - if (ctx->digcnt) - /* request has changed - restore hash */ - dd->pdata->copy_hash(req, 0); - - if (ctx->op == OP_UPDATE || (dd->flags & BIT(FLAGS_HUGE))) { - err = omap_sham_update_req(dd); - if (err != -EINPROGRESS && (ctx->flags & BIT(FLAGS_FINUP))) - /* no final() after finup() */ - err = omap_sham_final_req(dd); - } else if (ctx->op == OP_FINAL) { - err = omap_sham_final_req(dd); - } -err1: - dev_dbg(dd->dev, "exit, err: %d\n", err); - - if (err != -EINPROGRESS) { - /* done_task will not finish it, so do it here */ - omap_sham_finish_req(req, err); - req = NULL; - - /* - * Execute next request immediately if there is anything - * in queue. - */ - goto retry; - } - - return ret; + return crypto_transfer_hash_request_to_engine(dd->engine, req); } static int omap_sham_enqueue(struct ahash_request *req, unsigned int op) @@ -1394,6 +1364,10 @@ static int omap_sham_cra_init_alg(struct crypto_tfm *tfm, const char *alg_base) } + tctx->enginectx.op.do_one_request = omap_sham_hash_one_req; + tctx->enginectx.op.prepare_request = omap_sham_prepare_request; + tctx->enginectx.op.unprepare_request = NULL; + return 0; } @@ -1757,11 +1731,6 @@ static void omap_sham_done_task(unsigned long data) dev_dbg(dd->dev, "%s: flags=%lx\n", __func__, dd->flags); - if (!test_bit(FLAGS_BUSY, &dd->flags)) { - omap_sham_handle_queue(dd, NULL); - return; - } - if (test_bit(FLAGS_CPU, &dd->flags)) { if (test_and_clear_bit(FLAGS_OUTPUT_READY, &dd->flags)) goto finish; @@ -1786,20 +1755,12 @@ finish: dev_dbg(dd->dev, "update done: err: %d\n", err); /* finish curent request */ omap_sham_finish_req(dd->req, err); - - /* If we are not busy, process next req */ - if (!test_bit(FLAGS_BUSY, &dd->flags)) - omap_sham_handle_queue(dd, NULL); } static irqreturn_t omap_sham_irq_common(struct omap_sham_dev *dd) { - if (!test_bit(FLAGS_BUSY, &dd->flags)) { - dev_warn(dd->dev, "Interrupt when no active requests.\n"); - } else { - set_bit(FLAGS_OUTPUT_READY, &dd->flags); - tasklet_schedule(&dd->done_task); - } + set_bit(FLAGS_OUTPUT_READY, &dd->flags); + tasklet_schedule(&dd->done_task); return IRQ_HANDLED; } @@ -2072,7 +2033,6 @@ static ssize_t queue_len_store(struct device *dev, struct omap_sham_dev *dd = dev_get_drvdata(dev); ssize_t status; long value; - unsigned long flags; status = kstrtol(buf, 0, &value); if (status) @@ -2086,9 +2046,7 @@ static ssize_t queue_len_store(struct device *dev, * than current size, it will just not accept new entries until * it has shrank enough. */ - spin_lock_irqsave(&dd->lock, flags); dd->queue.max_qlen = value; - spin_unlock_irqrestore(&dd->lock, flags); return size; } @@ -2125,7 +2083,6 @@ static int omap_sham_probe(struct platform_device *pdev) platform_set_drvdata(pdev, dd); INIT_LIST_HEAD(&dd->list); - spin_lock_init(&dd->lock); tasklet_init(&dd->done_task, omap_sham_done_task, (unsigned long)dd); crypto_init_queue(&dd->queue, OMAP_SHAM_QUEUE_LENGTH); @@ -2190,6 +2147,16 @@ static int omap_sham_probe(struct platform_device *pdev) list_add_tail(&dd->list, &sham.dev_list); spin_unlock(&sham.lock); + dd->engine = crypto_engine_alloc_init(dev, 1); + if (!dd->engine) { + err = -ENOMEM; + goto err_engine; + } + + err = crypto_engine_start(dd->engine); + if (err) + goto err_engine_start; + for (i = 0; i < dd->pdata->algs_info_size; i++) { if (dd->pdata->algs_info[i].registered) break; @@ -2223,6 +2190,12 @@ err_algs: for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--) crypto_unregister_ahash( &dd->pdata->algs_info[i].algs_list[j]); +err_engine_start: + crypto_engine_exit(dd->engine); +err_engine: + spin_lock(&sham.lock); + list_del(&dd->list); + spin_unlock(&sham.lock); err_pm: pm_runtime_disable(dev); if (!dd->polling_mode) diff --git a/drivers/crypto/padlock-aes.c b/drivers/crypto/padlock-aes.c index 62c6fe88b212..1be549a07a21 100644 --- a/drivers/crypto/padlock-aes.c +++ b/drivers/crypto/padlock-aes.c @@ -18,6 +18,7 @@ #include <linux/errno.h> #include <linux/interrupt.h> #include <linux/kernel.h> +#include <linux/mm.h> #include <linux/percpu.h> #include <linux/smp.h> #include <linux/slab.h> diff --git a/drivers/crypto/picoxcell_crypto.c b/drivers/crypto/picoxcell_crypto.c index dac6eb37fff9..fb34bf92861d 100644 --- a/drivers/crypto/picoxcell_crypto.c +++ b/drivers/crypto/picoxcell_crypto.c @@ -1685,11 +1685,6 @@ static int spacc_probe(struct platform_device *pdev) goto err_clk_put; } - ret = device_create_file(&pdev->dev, &dev_attr_stat_irq_thresh); - if (ret) - goto err_clk_disable; - - /* * Use an IRQ threshold of 50% as a default. This seems to be a * reasonable trade off of latency against throughput but can be @@ -1697,6 +1692,10 @@ static int spacc_probe(struct platform_device *pdev) */ engine->stat_irq_thresh = (engine->fifo_sz / 2); + ret = device_create_file(&pdev->dev, &dev_attr_stat_irq_thresh); + if (ret) + goto err_clk_disable; + /* * Configure the interrupts. We only use the STAT_CNT interrupt as we * only submit a new packet for processing when we complete another in diff --git a/drivers/crypto/qat/qat_c3xxx/adf_drv.c b/drivers/crypto/qat/qat_c3xxx/adf_drv.c index 020d099409e5..ed0e8e33fe4b 100644 --- a/drivers/crypto/qat/qat_c3xxx/adf_drv.c +++ b/drivers/crypto/qat/qat_c3xxx/adf_drv.c @@ -18,12 +18,9 @@ #include <adf_cfg.h> #include "adf_c3xxx_hw_data.h" -#define ADF_SYSTEM_DEVICE(device_id) \ - {PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)} - static const struct pci_device_id adf_pci_tbl[] = { - ADF_SYSTEM_DEVICE(ADF_C3XXX_PCI_DEVICE_ID), - {0,} + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_C3XXX), }, + { } }; MODULE_DEVICE_TABLE(pci, adf_pci_tbl); @@ -58,7 +55,7 @@ static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) if (accel_dev->hw_device) { switch (accel_pci_dev->pci_dev->device) { - case ADF_C3XXX_PCI_DEVICE_ID: + case PCI_DEVICE_ID_INTEL_QAT_C3XXX: adf_clean_hw_data_c3xxx(accel_dev->hw_device); break; default: @@ -83,7 +80,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) int ret; switch (ent->device) { - case ADF_C3XXX_PCI_DEVICE_ID: + case PCI_DEVICE_ID_INTEL_QAT_C3XXX: break; default: dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device); @@ -143,10 +140,8 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) } /* Create dev top level debugfs entry */ - snprintf(name, sizeof(name), "%s%s_%02x:%02d.%d", - ADF_DEVICE_NAME_PREFIX, hw_data->dev_class->name, - pdev->bus->number, PCI_SLOT(pdev->devfn), - PCI_FUNC(pdev->devfn)); + snprintf(name, sizeof(name), "%s%s_%s", ADF_DEVICE_NAME_PREFIX, + hw_data->dev_class->name, pci_name(pdev)); accel_dev->debugfs_dir = debugfs_create_dir(name, NULL); @@ -203,7 +198,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) } pci_set_master(pdev); - if (adf_enable_aer(accel_dev, &adf_driver)) { + if (adf_enable_aer(accel_dev)) { dev_err(&pdev->dev, "Failed to enable aer\n"); ret = -EFAULT; goto out_err_free_reg; diff --git a/drivers/crypto/qat/qat_c3xxxvf/adf_drv.c b/drivers/crypto/qat/qat_c3xxxvf/adf_drv.c index 11039fe55f61..456979b136a2 100644 --- a/drivers/crypto/qat/qat_c3xxxvf/adf_drv.c +++ b/drivers/crypto/qat/qat_c3xxxvf/adf_drv.c @@ -18,12 +18,9 @@ #include <adf_cfg.h> #include "adf_c3xxxvf_hw_data.h" -#define ADF_SYSTEM_DEVICE(device_id) \ - {PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)} - static const struct pci_device_id adf_pci_tbl[] = { - ADF_SYSTEM_DEVICE(ADF_C3XXXIOV_PCI_DEVICE_ID), - {0,} + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_C3XXX_VF), }, + { } }; MODULE_DEVICE_TABLE(pci, adf_pci_tbl); @@ -58,7 +55,7 @@ static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) if (accel_dev->hw_device) { switch (accel_pci_dev->pci_dev->device) { - case ADF_C3XXXIOV_PCI_DEVICE_ID: + case PCI_DEVICE_ID_INTEL_QAT_C3XXX_VF: adf_clean_hw_data_c3xxxiov(accel_dev->hw_device); break; default: @@ -85,7 +82,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) int ret; switch (ent->device) { - case ADF_C3XXXIOV_PCI_DEVICE_ID: + case PCI_DEVICE_ID_INTEL_QAT_C3XXX_VF: break; default: dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device); @@ -127,10 +124,8 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) accel_pci_dev->sku = hw_data->get_sku(hw_data); /* Create dev top level debugfs entry */ - snprintf(name, sizeof(name), "%s%s_%02x:%02d.%d", - ADF_DEVICE_NAME_PREFIX, hw_data->dev_class->name, - pdev->bus->number, PCI_SLOT(pdev->devfn), - PCI_FUNC(pdev->devfn)); + snprintf(name, sizeof(name), "%s%s_%s", ADF_DEVICE_NAME_PREFIX, + hw_data->dev_class->name, pci_name(pdev)); accel_dev->debugfs_dir = debugfs_create_dir(name, NULL); diff --git a/drivers/crypto/qat/qat_c62x/adf_drv.c b/drivers/crypto/qat/qat_c62x/adf_drv.c index 4ba9c14383af..d8e7c9c25590 100644 --- a/drivers/crypto/qat/qat_c62x/adf_drv.c +++ b/drivers/crypto/qat/qat_c62x/adf_drv.c @@ -18,12 +18,9 @@ #include <adf_cfg.h> #include "adf_c62x_hw_data.h" -#define ADF_SYSTEM_DEVICE(device_id) \ - {PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)} - static const struct pci_device_id adf_pci_tbl[] = { - ADF_SYSTEM_DEVICE(ADF_C62X_PCI_DEVICE_ID), - {0,} + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_C62X), }, + { } }; MODULE_DEVICE_TABLE(pci, adf_pci_tbl); @@ -58,7 +55,7 @@ static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) if (accel_dev->hw_device) { switch (accel_pci_dev->pci_dev->device) { - case ADF_C62X_PCI_DEVICE_ID: + case PCI_DEVICE_ID_INTEL_QAT_C62X: adf_clean_hw_data_c62x(accel_dev->hw_device); break; default: @@ -83,7 +80,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) int ret; switch (ent->device) { - case ADF_C62X_PCI_DEVICE_ID: + case PCI_DEVICE_ID_INTEL_QAT_C62X: break; default: dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device); @@ -143,10 +140,8 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) } /* Create dev top level debugfs entry */ - snprintf(name, sizeof(name), "%s%s_%02x:%02d.%d", - ADF_DEVICE_NAME_PREFIX, hw_data->dev_class->name, - pdev->bus->number, PCI_SLOT(pdev->devfn), - PCI_FUNC(pdev->devfn)); + snprintf(name, sizeof(name), "%s%s_%s", ADF_DEVICE_NAME_PREFIX, + hw_data->dev_class->name, pci_name(pdev)); accel_dev->debugfs_dir = debugfs_create_dir(name, NULL); @@ -203,7 +198,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) } pci_set_master(pdev); - if (adf_enable_aer(accel_dev, &adf_driver)) { + if (adf_enable_aer(accel_dev)) { dev_err(&pdev->dev, "Failed to enable aer\n"); ret = -EFAULT; goto out_err_free_reg; diff --git a/drivers/crypto/qat/qat_c62xvf/adf_drv.c b/drivers/crypto/qat/qat_c62xvf/adf_drv.c index b8b021d54bb5..b9810f79eb84 100644 --- a/drivers/crypto/qat/qat_c62xvf/adf_drv.c +++ b/drivers/crypto/qat/qat_c62xvf/adf_drv.c @@ -18,12 +18,9 @@ #include <adf_cfg.h> #include "adf_c62xvf_hw_data.h" -#define ADF_SYSTEM_DEVICE(device_id) \ - {PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)} - static const struct pci_device_id adf_pci_tbl[] = { - ADF_SYSTEM_DEVICE(ADF_C62XIOV_PCI_DEVICE_ID), - {0,} + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_C62X_VF), }, + { } }; MODULE_DEVICE_TABLE(pci, adf_pci_tbl); @@ -58,7 +55,7 @@ static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) if (accel_dev->hw_device) { switch (accel_pci_dev->pci_dev->device) { - case ADF_C62XIOV_PCI_DEVICE_ID: + case PCI_DEVICE_ID_INTEL_QAT_C62X_VF: adf_clean_hw_data_c62xiov(accel_dev->hw_device); break; default: @@ -85,7 +82,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) int ret; switch (ent->device) { - case ADF_C62XIOV_PCI_DEVICE_ID: + case PCI_DEVICE_ID_INTEL_QAT_C62X_VF: break; default: dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device); @@ -127,10 +124,8 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) accel_pci_dev->sku = hw_data->get_sku(hw_data); /* Create dev top level debugfs entry */ - snprintf(name, sizeof(name), "%s%s_%02x:%02d.%d", - ADF_DEVICE_NAME_PREFIX, hw_data->dev_class->name, - pdev->bus->number, PCI_SLOT(pdev->devfn), - PCI_FUNC(pdev->devfn)); + snprintf(name, sizeof(name), "%s%s_%s", ADF_DEVICE_NAME_PREFIX, + hw_data->dev_class->name, pci_name(pdev)); accel_dev->debugfs_dir = debugfs_create_dir(name, NULL); diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index c1db8c26afb6..06952ece53d9 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -15,12 +15,6 @@ #define ADF_C62XVF_DEVICE_NAME "c6xxvf" #define ADF_C3XXX_DEVICE_NAME "c3xxx" #define ADF_C3XXXVF_DEVICE_NAME "c3xxxvf" -#define ADF_DH895XCC_PCI_DEVICE_ID 0x435 -#define ADF_DH895XCCIOV_PCI_DEVICE_ID 0x443 -#define ADF_C62X_PCI_DEVICE_ID 0x37c8 -#define ADF_C62XIOV_PCI_DEVICE_ID 0x37c9 -#define ADF_C3XXX_PCI_DEVICE_ID 0x19e2 -#define ADF_C3XXXIOV_PCI_DEVICE_ID 0x19e3 #define ADF_ERRSOU3 (0x3A000 + 0x0C) #define ADF_ERRSOU5 (0x3A000 + 0xD8) #define ADF_DEVICE_FUSECTL_OFFSET 0x40 diff --git a/drivers/crypto/qat/qat_common/adf_aer.c b/drivers/crypto/qat/qat_common/adf_aer.c index 32102e27e559..d2ae293d0df6 100644 --- a/drivers/crypto/qat/qat_common/adf_aer.c +++ b/drivers/crypto/qat/qat_common/adf_aer.c @@ -175,7 +175,6 @@ static const struct pci_error_handlers adf_err_handler = { /** * adf_enable_aer() - Enable Advance Error Reporting for acceleration device * @accel_dev: Pointer to acceleration device. - * @adf: PCI device driver owning the given acceleration device. * * Function enables PCI Advance Error Reporting for the * QAT acceleration device accel_dev. @@ -183,11 +182,12 @@ static const struct pci_error_handlers adf_err_handler = { * * Return: 0 on success, error code otherwise. */ -int adf_enable_aer(struct adf_accel_dev *accel_dev, struct pci_driver *adf) +int adf_enable_aer(struct adf_accel_dev *accel_dev) { struct pci_dev *pdev = accel_to_pci_dev(accel_dev); + struct pci_driver *pdrv = pdev->driver; - adf->err_handler = &adf_err_handler; + pdrv->err_handler = &adf_err_handler; pci_enable_pcie_error_reporting(pdev); return 0; } diff --git a/drivers/crypto/qat/qat_common/adf_cfg.c b/drivers/crypto/qat/qat_common/adf_cfg.c index ac462796cefc..22ae32838113 100644 --- a/drivers/crypto/qat/qat_common/adf_cfg.c +++ b/drivers/crypto/qat/qat_common/adf_cfg.c @@ -52,24 +52,7 @@ static const struct seq_operations qat_dev_cfg_sops = { .show = qat_dev_cfg_show }; -static int qat_dev_cfg_open(struct inode *inode, struct file *file) -{ - int ret = seq_open(file, &qat_dev_cfg_sops); - - if (!ret) { - struct seq_file *seq_f = file->private_data; - - seq_f->private = inode->i_private; - } - return ret; -} - -static const struct file_operations qat_dev_cfg_fops = { - .open = qat_dev_cfg_open, - .read = seq_read, - .llseek = seq_lseek, - .release = seq_release -}; +DEFINE_SEQ_ATTRIBUTE(qat_dev_cfg); /** * adf_cfg_dev_add() - Create an acceleration device configuration table. diff --git a/drivers/crypto/qat/qat_common/adf_common_drv.h b/drivers/crypto/qat/qat_common/adf_common_drv.h index ebfcb4ea618d..f22342f612c1 100644 --- a/drivers/crypto/qat/qat_common/adf_common_drv.h +++ b/drivers/crypto/qat/qat_common/adf_common_drv.h @@ -95,7 +95,7 @@ void adf_ae_fw_release(struct adf_accel_dev *accel_dev); int adf_ae_start(struct adf_accel_dev *accel_dev); int adf_ae_stop(struct adf_accel_dev *accel_dev); -int adf_enable_aer(struct adf_accel_dev *accel_dev, struct pci_driver *adf); +int adf_enable_aer(struct adf_accel_dev *accel_dev); void adf_disable_aer(struct adf_accel_dev *accel_dev); void adf_reset_sbr(struct adf_accel_dev *accel_dev); void adf_reset_flr(struct adf_accel_dev *accel_dev); diff --git a/drivers/crypto/qat/qat_common/adf_ctl_drv.c b/drivers/crypto/qat/qat_common/adf_ctl_drv.c index 71d0c44aacca..eb9b3be9d8eb 100644 --- a/drivers/crypto/qat/qat_common/adf_ctl_drv.c +++ b/drivers/crypto/qat/qat_common/adf_ctl_drv.c @@ -416,8 +416,6 @@ static long adf_ctl_ioctl(struct file *fp, unsigned int cmd, unsigned long arg) static int __init adf_register_ctl_device_driver(void) { - mutex_init(&adf_ctl_lock); - if (adf_chr_drv_create()) goto err_chr_dev; diff --git a/drivers/crypto/qat/qat_common/adf_dev_mgr.c b/drivers/crypto/qat/qat_common/adf_dev_mgr.c index 72753af056b3..92ec035576df 100644 --- a/drivers/crypto/qat/qat_common/adf_dev_mgr.c +++ b/drivers/crypto/qat/qat_common/adf_dev_mgr.c @@ -285,7 +285,7 @@ struct adf_accel_dev *adf_devmgr_get_first(void) /** * adf_devmgr_pci_to_accel_dev() - Get accel_dev associated with the pci_dev. - * @accel_dev: Pointer to pci device. + * @pci_dev: Pointer to pci device. * * Function returns acceleration device associated with the given pci device. * To be used by QAT device specific drivers. diff --git a/drivers/crypto/qat/qat_common/adf_sriov.c b/drivers/crypto/qat/qat_common/adf_sriov.c index 8827aa139f96..963b2bea78f2 100644 --- a/drivers/crypto/qat/qat_common/adf_sriov.c +++ b/drivers/crypto/qat/qat_common/adf_sriov.c @@ -173,10 +173,14 @@ EXPORT_SYMBOL_GPL(adf_disable_sriov); /** * adf_sriov_configure() - Enable SRIOV for the device * @pdev: Pointer to pci device. + * @numvfs: Number of virtual functions (VFs) to enable. + * + * Note that the @numvfs parameter is ignored and all VFs supported by the + * device are enabled due to the design of the hardware. * * Function enables SRIOV for the pci device. * - * Return: 0 on success, error code otherwise. + * Return: number of VFs enabled on success, error code otherwise. */ int adf_sriov_configure(struct pci_dev *pdev, int numvfs) { diff --git a/drivers/crypto/qat/qat_common/adf_transport_debug.c b/drivers/crypto/qat/qat_common/adf_transport_debug.c index 2a2eccbf56ec..dac25ba47260 100644 --- a/drivers/crypto/qat/qat_common/adf_transport_debug.c +++ b/drivers/crypto/qat/qat_common/adf_transport_debug.c @@ -77,31 +77,14 @@ static void adf_ring_stop(struct seq_file *sfile, void *v) mutex_unlock(&ring_read_lock); } -static const struct seq_operations adf_ring_sops = { +static const struct seq_operations adf_ring_debug_sops = { .start = adf_ring_start, .next = adf_ring_next, .stop = adf_ring_stop, .show = adf_ring_show }; -static int adf_ring_open(struct inode *inode, struct file *file) -{ - int ret = seq_open(file, &adf_ring_sops); - - if (!ret) { - struct seq_file *seq_f = file->private_data; - - seq_f->private = inode->i_private; - } - return ret; -} - -static const struct file_operations adf_ring_debug_fops = { - .open = adf_ring_open, - .read = seq_read, - .llseek = seq_lseek, - .release = seq_release -}; +DEFINE_SEQ_ATTRIBUTE(adf_ring_debug); int adf_ring_debugfs_add(struct adf_etr_ring_data *ring, const char *name) { @@ -188,31 +171,14 @@ static void adf_bank_stop(struct seq_file *sfile, void *v) mutex_unlock(&bank_read_lock); } -static const struct seq_operations adf_bank_sops = { +static const struct seq_operations adf_bank_debug_sops = { .start = adf_bank_start, .next = adf_bank_next, .stop = adf_bank_stop, .show = adf_bank_show }; -static int adf_bank_open(struct inode *inode, struct file *file) -{ - int ret = seq_open(file, &adf_bank_sops); - - if (!ret) { - struct seq_file *seq_f = file->private_data; - - seq_f->private = inode->i_private; - } - return ret; -} - -static const struct file_operations adf_bank_debug_fops = { - .open = adf_bank_open, - .read = seq_read, - .llseek = seq_lseek, - .release = seq_release -}; +DEFINE_SEQ_ATTRIBUTE(adf_bank_debug); int adf_bank_debugfs_add(struct adf_etr_bank_data *bank) { diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c index 72753b84dc95..d552dbcfe0a0 100644 --- a/drivers/crypto/qat/qat_common/qat_algs.c +++ b/drivers/crypto/qat/qat_common/qat_algs.c @@ -828,6 +828,11 @@ static int qat_alg_aead_dec(struct aead_request *areq) struct icp_qat_fw_la_bulk_req *msg; int digst_size = crypto_aead_authsize(aead_tfm); int ret, ctr = 0; + u32 cipher_len; + + cipher_len = areq->cryptlen - digst_size; + if (cipher_len % AES_BLOCK_SIZE != 0) + return -EINVAL; ret = qat_alg_sgl_to_bufl(ctx->inst, areq->src, areq->dst, qat_req); if (unlikely(ret)) @@ -842,7 +847,7 @@ static int qat_alg_aead_dec(struct aead_request *areq) qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp; qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp; cipher_param = (void *)&qat_req->req.serv_specif_rqpars; - cipher_param->cipher_length = areq->cryptlen - digst_size; + cipher_param->cipher_length = cipher_len; cipher_param->cipher_offset = areq->assoclen; memcpy(cipher_param->u.cipher_IV_array, areq->iv, AES_BLOCK_SIZE); auth_param = (void *)((u8 *)cipher_param + sizeof(*cipher_param)); @@ -871,6 +876,9 @@ static int qat_alg_aead_enc(struct aead_request *areq) u8 *iv = areq->iv; int ret, ctr = 0; + if (areq->cryptlen % AES_BLOCK_SIZE != 0) + return -EINVAL; + ret = qat_alg_sgl_to_bufl(ctx->inst, areq->src, areq->dst, qat_req); if (unlikely(ret)) return ret; diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index fa467e0f8285..6b9d47682d04 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -2,6 +2,7 @@ /* Copyright(c) 2014 - 2020 Intel Corporation */ #include <linux/slab.h> #include <linux/delay.h> +#include <linux/pci_ids.h> #include "adf_accel_devices.h" #include "adf_common_drv.h" @@ -412,7 +413,7 @@ static int qat_hal_init_esram(struct icp_qat_fw_loader_handle *handle) unsigned int csr_val; int times = 30; - if (handle->pci_dev->device != ADF_DH895XCC_PCI_DEVICE_ID) + if (handle->pci_dev->device != PCI_DEVICE_ID_INTEL_QAT_DH895XCC) return 0; csr_val = ADF_CSR_RD(csr_addr, 0); @@ -672,13 +673,13 @@ int qat_hal_init(struct adf_accel_dev *accel_dev) (void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v + LOCAL_TO_XFER_REG_OFFSET); handle->pci_dev = pci_info->pci_dev; - if (handle->pci_dev->device == ADF_DH895XCC_PCI_DEVICE_ID) { + if (handle->pci_dev->device == PCI_DEVICE_ID_INTEL_QAT_DH895XCC) { sram_bar = &pci_info->pci_bars[hw_data->get_sram_bar_id(hw_data)]; handle->hal_sram_addr_v = sram_bar->virt_addr; } handle->fw_auth = (handle->pci_dev->device == - ADF_DH895XCC_PCI_DEVICE_ID) ? false : true; + PCI_DEVICE_ID_INTEL_QAT_DH895XCC) ? false : true; handle->hal_handle = kzalloc(sizeof(*handle->hal_handle), GFP_KERNEL); if (!handle->hal_handle) goto out_hal_handle; diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index 00c615f9f9a8..5d1f28cd6680 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -4,6 +4,7 @@ #include <linux/ctype.h> #include <linux/kernel.h> #include <linux/delay.h> +#include <linux/pci_ids.h> #include "adf_accel_devices.h" #include "adf_common_drv.h" #include "icp_qat_uclo.h" @@ -711,11 +712,11 @@ static unsigned int qat_uclo_get_dev_type(struct icp_qat_fw_loader_handle *handle) { switch (handle->pci_dev->device) { - case ADF_DH895XCC_PCI_DEVICE_ID: + case PCI_DEVICE_ID_INTEL_QAT_DH895XCC: return ICP_QAT_AC_895XCC_DEV_TYPE; - case ADF_C62X_PCI_DEVICE_ID: + case PCI_DEVICE_ID_INTEL_QAT_C62X: return ICP_QAT_AC_C62X_DEV_TYPE; - case ADF_C3XXX_PCI_DEVICE_ID: + case PCI_DEVICE_ID_INTEL_QAT_C3XXX: return ICP_QAT_AC_C3XXX_DEV_TYPE; default: pr_err("QAT: unsupported device 0x%x\n", @@ -1391,7 +1392,7 @@ int qat_uclo_wr_mimage(struct icp_qat_fw_loader_handle *handle, status = qat_uclo_auth_fw(handle, desc); qat_uclo_ummap_auth_fw(handle, &desc); } else { - if (handle->pci_dev->device == ADF_C3XXX_PCI_DEVICE_ID) { + if (handle->pci_dev->device == PCI_DEVICE_ID_INTEL_QAT_C3XXX) { pr_err("QAT: C3XXX doesn't support unsigned MMP\n"); return -EINVAL; } diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_drv.c b/drivers/crypto/qat/qat_dh895xcc/adf_drv.c index 4e877b75822b..ecb4f6f20e22 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_drv.c +++ b/drivers/crypto/qat/qat_dh895xcc/adf_drv.c @@ -18,12 +18,9 @@ #include <adf_cfg.h> #include "adf_dh895xcc_hw_data.h" -#define ADF_SYSTEM_DEVICE(device_id) \ - {PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)} - static const struct pci_device_id adf_pci_tbl[] = { - ADF_SYSTEM_DEVICE(ADF_DH895XCC_PCI_DEVICE_ID), - {0,} + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_DH895XCC), }, + { } }; MODULE_DEVICE_TABLE(pci, adf_pci_tbl); @@ -58,7 +55,7 @@ static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) if (accel_dev->hw_device) { switch (accel_pci_dev->pci_dev->device) { - case ADF_DH895XCC_PCI_DEVICE_ID: + case PCI_DEVICE_ID_INTEL_QAT_DH895XCC: adf_clean_hw_data_dh895xcc(accel_dev->hw_device); break; default: @@ -83,7 +80,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) int ret; switch (ent->device) { - case ADF_DH895XCC_PCI_DEVICE_ID: + case PCI_DEVICE_ID_INTEL_QAT_DH895XCC: break; default: dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device); @@ -143,10 +140,8 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) } /* Create dev top level debugfs entry */ - snprintf(name, sizeof(name), "%s%s_%02x:%02d.%d", - ADF_DEVICE_NAME_PREFIX, hw_data->dev_class->name, - pdev->bus->number, PCI_SLOT(pdev->devfn), - PCI_FUNC(pdev->devfn)); + snprintf(name, sizeof(name), "%s%s_%s", ADF_DEVICE_NAME_PREFIX, + hw_data->dev_class->name, pci_name(pdev)); accel_dev->debugfs_dir = debugfs_create_dir(name, NULL); @@ -205,7 +200,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) } pci_set_master(pdev); - if (adf_enable_aer(accel_dev, &adf_driver)) { + if (adf_enable_aer(accel_dev)) { dev_err(&pdev->dev, "Failed to enable aer\n"); ret = -EFAULT; goto out_err_free_reg; diff --git a/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c b/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c index 7d6e1db272c2..404cf9df6922 100644 --- a/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c +++ b/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c @@ -18,12 +18,9 @@ #include <adf_cfg.h> #include "adf_dh895xccvf_hw_data.h" -#define ADF_SYSTEM_DEVICE(device_id) \ - {PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)} - static const struct pci_device_id adf_pci_tbl[] = { - ADF_SYSTEM_DEVICE(ADF_DH895XCCIOV_PCI_DEVICE_ID), - {0,} + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_DH895XCC_VF), }, + { } }; MODULE_DEVICE_TABLE(pci, adf_pci_tbl); @@ -58,7 +55,7 @@ static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) if (accel_dev->hw_device) { switch (accel_pci_dev->pci_dev->device) { - case ADF_DH895XCCIOV_PCI_DEVICE_ID: + case PCI_DEVICE_ID_INTEL_QAT_DH895XCC_VF: adf_clean_hw_data_dh895xcciov(accel_dev->hw_device); break; default: @@ -85,7 +82,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) int ret; switch (ent->device) { - case ADF_DH895XCCIOV_PCI_DEVICE_ID: + case PCI_DEVICE_ID_INTEL_QAT_DH895XCC_VF: break; default: dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device); @@ -127,10 +124,8 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) accel_pci_dev->sku = hw_data->get_sku(hw_data); /* Create dev top level debugfs entry */ - snprintf(name, sizeof(name), "%s%s_%02x:%02d.%d", - ADF_DEVICE_NAME_PREFIX, hw_data->dev_class->name, - pdev->bus->number, PCI_SLOT(pdev->devfn), - PCI_FUNC(pdev->devfn)); + snprintf(name, sizeof(name), "%s%s_%s", ADF_DEVICE_NAME_PREFIX, + hw_data->dev_class->name, pci_name(pdev)); accel_dev->debugfs_dir = debugfs_create_dir(name, NULL); diff --git a/drivers/crypto/qce/core.c b/drivers/crypto/qce/core.c index cb6d61eb7302..ea616b7259ae 100644 --- a/drivers/crypto/qce/core.c +++ b/drivers/crypto/qce/core.c @@ -4,6 +4,7 @@ */ #include <linux/clk.h> +#include <linux/dma-mapping.h> #include <linux/interrupt.h> #include <linux/module.h> #include <linux/mod_devicetable.h> diff --git a/drivers/crypto/qce/sha.c b/drivers/crypto/qce/sha.c index c230843e2ffb..87be96a0b0bb 100644 --- a/drivers/crypto/qce/sha.c +++ b/drivers/crypto/qce/sha.c @@ -4,6 +4,7 @@ */ #include <linux/device.h> +#include <linux/dma-mapping.h> #include <linux/interrupt.h> #include <crypto/internal/hash.h> diff --git a/drivers/crypto/qce/skcipher.c b/drivers/crypto/qce/skcipher.c index 5630c5addd28..a2d3da0ad95f 100644 --- a/drivers/crypto/qce/skcipher.c +++ b/drivers/crypto/qce/skcipher.c @@ -4,6 +4,7 @@ */ #include <linux/device.h> +#include <linux/dma-mapping.h> #include <linux/interrupt.h> #include <linux/moduleparam.h> #include <linux/types.h> diff --git a/drivers/crypto/qcom-rng.c b/drivers/crypto/qcom-rng.c index 4730f84b646d..99ba8d51d102 100644 --- a/drivers/crypto/qcom-rng.c +++ b/drivers/crypto/qcom-rng.c @@ -7,6 +7,7 @@ #include <linux/acpi.h> #include <linux/clk.h> #include <linux/crypto.h> +#include <linux/io.h> #include <linux/module.h> #include <linux/of.h> #include <linux/platform_device.h> diff --git a/drivers/crypto/rockchip/rk3288_crypto.c b/drivers/crypto/rockchip/rk3288_crypto.c index f385587f99af..35d73061d156 100644 --- a/drivers/crypto/rockchip/rk3288_crypto.c +++ b/drivers/crypto/rockchip/rk3288_crypto.c @@ -10,6 +10,7 @@ */ #include "rk3288_crypto.h" +#include <linux/dma-mapping.h> #include <linux/module.h> #include <linux/platform_device.h> #include <linux/of.h> diff --git a/drivers/crypto/rockchip/rk3288_crypto.h b/drivers/crypto/rockchip/rk3288_crypto.h index 2b49c677afdb..3db595570c9c 100644 --- a/drivers/crypto/rockchip/rk3288_crypto.h +++ b/drivers/crypto/rockchip/rk3288_crypto.h @@ -7,6 +7,7 @@ #include <crypto/algapi.h> #include <linux/interrupt.h> #include <linux/delay.h> +#include <linux/scatterlist.h> #include <crypto/internal/hash.h> #include <crypto/internal/skcipher.h> diff --git a/drivers/crypto/rockchip/rk3288_crypto_ahash.c b/drivers/crypto/rockchip/rk3288_crypto_ahash.c index 6b7ecbec092e..81befe7febaa 100644 --- a/drivers/crypto/rockchip/rk3288_crypto_ahash.c +++ b/drivers/crypto/rockchip/rk3288_crypto_ahash.c @@ -8,6 +8,7 @@ * * Some ideas are from marvell/cesa.c and s5p-sss.c driver. */ +#include <linux/device.h> #include "rk3288_crypto.h" /* diff --git a/drivers/crypto/rockchip/rk3288_crypto_skcipher.c b/drivers/crypto/rockchip/rk3288_crypto_skcipher.c index 4a75c8e1fa6c..1cece1a7d3f0 100644 --- a/drivers/crypto/rockchip/rk3288_crypto_skcipher.c +++ b/drivers/crypto/rockchip/rk3288_crypto_skcipher.c @@ -8,6 +8,7 @@ * * Some ideas are from marvell-cesa.c and s5p-sss.c driver. */ +#include <linux/device.h> #include "rk3288_crypto.h" #define RK_CRYPTO_DEC BIT(0) diff --git a/drivers/crypto/s5p-sss.c b/drivers/crypto/s5p-sss.c index 341433fbcc4a..88a6c853ffd7 100644 --- a/drivers/crypto/s5p-sss.c +++ b/drivers/crypto/s5p-sss.c @@ -260,6 +260,7 @@ struct s5p_aes_ctx { * struct s5p_aes_dev - Crypto device state container * @dev: Associated device * @clk: Clock for accessing hardware + * @pclk: APB bus clock necessary to access the hardware * @ioaddr: Mapped IO memory region * @aes_ioaddr: Per-varian offset for AES block IO memory * @irq_fc: Feed control interrupt line @@ -342,13 +343,13 @@ struct s5p_aes_dev { * @engine: Bits for selecting type of HASH in SSS block * @sg: sg for DMA transfer * @sg_len: Length of sg for DMA transfer - * @sgl[]: sg for joining buffer and req->src scatterlist + * @sgl: sg for joining buffer and req->src scatterlist * @skip: Skip offset in req->src for current op * @total: Total number of bytes for current request * @finup: Keep state for finup or final. * @error: Keep track of error. * @bufcnt: Number of bytes holded in buffer[] - * @buffer[]: For byte(s) from end of req->src in UPDATE op + * @buffer: For byte(s) from end of req->src in UPDATE op */ struct s5p_hash_reqctx { struct s5p_aes_dev *dd; @@ -1125,7 +1126,7 @@ static int s5p_hash_copy_sg_lists(struct s5p_hash_reqctx *ctx, * s5p_hash_prepare_sgs() - prepare sg for processing * @ctx: request context * @sg: source scatterlist request - * @nbytes: number of bytes to process from sg + * @new_len: number of bytes to process from sg * @final: final flag * * Check two conditions: (1) if buffers in sg have len aligned data, and (2) @@ -2200,11 +2201,10 @@ static int s5p_aes_probe(struct platform_device *pdev) } pdata->clk = devm_clk_get(dev, variant->clk_names[0]); - if (IS_ERR(pdata->clk)) { - dev_err(dev, "failed to find secss clock %s\n", - variant->clk_names[0]); - return -ENOENT; - } + if (IS_ERR(pdata->clk)) + return dev_err_probe(dev, PTR_ERR(pdata->clk), + "failed to find secss clock %s\n", + variant->clk_names[0]); err = clk_prepare_enable(pdata->clk); if (err < 0) { @@ -2216,9 +2216,9 @@ static int s5p_aes_probe(struct platform_device *pdev) if (variant->clk_names[1]) { pdata->pclk = devm_clk_get(dev, variant->clk_names[1]); if (IS_ERR(pdata->pclk)) { - dev_err(dev, "failed to find clock %s\n", - variant->clk_names[1]); - err = -ENOENT; + err = dev_err_probe(dev, PTR_ERR(pdata->pclk), + "failed to find clock %s\n", + variant->clk_names[1]); goto err_clk; } @@ -2307,8 +2307,7 @@ err_algs: tasklet_kill(&pdata->tasklet); err_irq: - if (pdata->pclk) - clk_disable_unprepare(pdata->pclk); + clk_disable_unprepare(pdata->pclk); err_clk: clk_disable_unprepare(pdata->clk); @@ -2338,8 +2337,7 @@ static int s5p_aes_remove(struct platform_device *pdev) pdata->use_hash = false; } - if (pdata->pclk) - clk_disable_unprepare(pdata->pclk); + clk_disable_unprepare(pdata->pclk); clk_disable_unprepare(pdata->clk); s5p_dev = NULL; diff --git a/drivers/crypto/sa2ul.c b/drivers/crypto/sa2ul.c index 5bc099052bd2..eda93fab95fe 100644 --- a/drivers/crypto/sa2ul.c +++ b/drivers/crypto/sa2ul.c @@ -143,33 +143,38 @@ struct sa_alg_tmpl { }; /** + * struct sa_mapped_sg: scatterlist information for tx and rx + * @mapped: Set to true if the @sgt is mapped + * @dir: mapping direction used for @sgt + * @split_sg: Set if the sg is split and needs to be freed up + * @static_sg: Static scatterlist entry for overriding data + * @sgt: scatterlist table for DMA API use + */ +struct sa_mapped_sg { + bool mapped; + enum dma_data_direction dir; + struct scatterlist static_sg; + struct scatterlist *split_sg; + struct sg_table sgt; +}; +/** * struct sa_rx_data: RX Packet miscellaneous data place holder * @req: crypto request data pointer * @ddev: pointer to the DMA device * @tx_in: dma_async_tx_descriptor pointer for rx channel - * @split_src_sg: Set if the src sg is split and needs to be freed up - * @split_dst_sg: Set if the dst sg is split and needs to be freed up + * @mapped_sg: Information on tx (0) and rx (1) scatterlist DMA mapping * @enc: Flag indicating either encryption or decryption * @enc_iv_size: Initialisation vector size * @iv_idx: Initialisation vector index - * @rx_sg: Static scatterlist entry for overriding RX data - * @tx_sg: Static scatterlist entry for overriding TX data - * @src: Source data pointer - * @dst: Destination data pointer */ struct sa_rx_data { void *req; struct device *ddev; struct dma_async_tx_descriptor *tx_in; - struct scatterlist *split_src_sg; - struct scatterlist *split_dst_sg; + struct sa_mapped_sg mapped_sg[2]; u8 enc; u8 enc_iv_size; u8 iv_idx; - struct scatterlist rx_sg; - struct scatterlist tx_sg; - struct scatterlist *src; - struct scatterlist *dst; }; /** @@ -976,23 +981,46 @@ static int sa_3des_ecb_setkey(struct crypto_skcipher *tfm, const u8 *key, return sa_cipher_setkey(tfm, key, keylen, &ad); } +static void sa_sync_from_device(struct sa_rx_data *rxd) +{ + struct sg_table *sgt; + + if (rxd->mapped_sg[0].dir == DMA_BIDIRECTIONAL) + sgt = &rxd->mapped_sg[0].sgt; + else + sgt = &rxd->mapped_sg[1].sgt; + + dma_sync_sgtable_for_cpu(rxd->ddev, sgt, DMA_FROM_DEVICE); +} + +static void sa_free_sa_rx_data(struct sa_rx_data *rxd) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(rxd->mapped_sg); i++) { + struct sa_mapped_sg *mapped_sg = &rxd->mapped_sg[i]; + + if (mapped_sg->mapped) { + dma_unmap_sgtable(rxd->ddev, &mapped_sg->sgt, + mapped_sg->dir, 0); + kfree(mapped_sg->split_sg); + } + } + + kfree(rxd); +} + static void sa_aes_dma_in_callback(void *data) { struct sa_rx_data *rxd = (struct sa_rx_data *)data; struct skcipher_request *req; - int sglen; u32 *result; __be32 *mdptr; size_t ml, pl; int i; - enum dma_data_direction dir_src; - bool diff_dst; + sa_sync_from_device(rxd); req = container_of(rxd->req, struct skcipher_request, base); - sglen = sg_nents_for_len(req->src, req->cryptlen); - - diff_dst = (req->src != req->dst) ? true : false; - dir_src = diff_dst ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL; if (req->iv) { mdptr = (__be32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl, @@ -1003,18 +1031,7 @@ static void sa_aes_dma_in_callback(void *data) result[i] = be32_to_cpu(mdptr[i + rxd->iv_idx]); } - dma_unmap_sg(rxd->ddev, req->src, sglen, dir_src); - kfree(rxd->split_src_sg); - - if (diff_dst) { - sglen = sg_nents_for_len(req->dst, req->cryptlen); - - dma_unmap_sg(rxd->ddev, req->dst, sglen, - DMA_FROM_DEVICE); - kfree(rxd->split_dst_sg); - } - - kfree(rxd); + sa_free_sa_rx_data(rxd); skcipher_request_complete(req, 0); } @@ -1043,7 +1060,6 @@ static int sa_run(struct sa_req *req) struct device *ddev; struct dma_chan *dma_rx; int sg_nents, src_nents, dst_nents; - int mapped_src_nents, mapped_dst_nents; struct scatterlist *src, *dst; size_t pl, ml, split_size; struct sa_ctx_info *sa_ctx = req->enc ? &req->ctx->enc : &req->ctx->dec; @@ -1052,6 +1068,7 @@ static int sa_run(struct sa_req *req) u32 *mdptr; bool diff_dst; enum dma_data_direction dir_src; + struct sa_mapped_sg *mapped_sg; gfp_flags = req->base->flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; @@ -1082,6 +1099,7 @@ static int sa_run(struct sa_req *req) dma_rx = pdata->dma_rx1; ddev = dma_rx->device->dev; + rxd->ddev = ddev; memcpy(cmdl, sa_ctx->cmdl, sa_ctx->cmdl_size); @@ -1109,60 +1127,90 @@ static int sa_run(struct sa_req *req) split_size = req->size; + mapped_sg = &rxd->mapped_sg[0]; if (sg_nents == 1 && split_size <= req->src->length) { - src = &rxd->rx_sg; + src = &mapped_sg->static_sg; + src_nents = 1; sg_init_table(src, 1); sg_set_page(src, sg_page(req->src), split_size, req->src->offset); - src_nents = 1; - dma_map_sg(ddev, src, sg_nents, dir_src); + + mapped_sg->sgt.sgl = src; + mapped_sg->sgt.orig_nents = src_nents; + ret = dma_map_sgtable(ddev, &mapped_sg->sgt, dir_src, 0); + if (ret) + return ret; + + mapped_sg->dir = dir_src; + mapped_sg->mapped = true; } else { - mapped_src_nents = dma_map_sg(ddev, req->src, sg_nents, - dir_src); - ret = sg_split(req->src, mapped_src_nents, 0, 1, &split_size, - &src, &src_nents, gfp_flags); + mapped_sg->sgt.sgl = req->src; + mapped_sg->sgt.orig_nents = sg_nents; + ret = dma_map_sgtable(ddev, &mapped_sg->sgt, dir_src, 0); + if (ret) + return ret; + + mapped_sg->dir = dir_src; + mapped_sg->mapped = true; + + ret = sg_split(mapped_sg->sgt.sgl, mapped_sg->sgt.nents, 0, 1, + &split_size, &src, &src_nents, gfp_flags); if (ret) { - src_nents = sg_nents; - src = req->src; + src_nents = mapped_sg->sgt.nents; + src = mapped_sg->sgt.sgl; } else { - rxd->split_src_sg = src; + mapped_sg->split_sg = src; } } + dma_sync_sgtable_for_device(ddev, &mapped_sg->sgt, DMA_TO_DEVICE); + if (!diff_dst) { dst_nents = src_nents; dst = src; } else { dst_nents = sg_nents_for_len(req->dst, req->size); + mapped_sg = &rxd->mapped_sg[1]; if (dst_nents == 1 && split_size <= req->dst->length) { - dst = &rxd->tx_sg; + dst = &mapped_sg->static_sg; + dst_nents = 1; sg_init_table(dst, 1); sg_set_page(dst, sg_page(req->dst), split_size, req->dst->offset); - dst_nents = 1; - dma_map_sg(ddev, dst, dst_nents, DMA_FROM_DEVICE); + + mapped_sg->sgt.sgl = dst; + mapped_sg->sgt.orig_nents = dst_nents; + ret = dma_map_sgtable(ddev, &mapped_sg->sgt, + DMA_FROM_DEVICE, 0); + if (ret) + goto err_cleanup; + + mapped_sg->dir = DMA_FROM_DEVICE; + mapped_sg->mapped = true; } else { - mapped_dst_nents = dma_map_sg(ddev, req->dst, dst_nents, - DMA_FROM_DEVICE); - ret = sg_split(req->dst, mapped_dst_nents, 0, 1, - &split_size, &dst, &dst_nents, + mapped_sg->sgt.sgl = req->dst; + mapped_sg->sgt.orig_nents = dst_nents; + ret = dma_map_sgtable(ddev, &mapped_sg->sgt, + DMA_FROM_DEVICE, 0); + if (ret) + goto err_cleanup; + + mapped_sg->dir = DMA_FROM_DEVICE; + mapped_sg->mapped = true; + + ret = sg_split(mapped_sg->sgt.sgl, mapped_sg->sgt.nents, + 0, 1, &split_size, &dst, &dst_nents, gfp_flags); if (ret) { - dst_nents = dst_nents; - dst = req->dst; + dst_nents = mapped_sg->sgt.nents; + dst = mapped_sg->sgt.sgl; } else { - rxd->split_dst_sg = dst; + mapped_sg->split_sg = dst; } } } - if (unlikely(src_nents != sg_nents)) { - dev_warn_ratelimited(sa_k3_dev, "failed to map tx pkt\n"); - ret = -EIO; - goto err_cleanup; - } - rxd->tx_in = dmaengine_prep_slave_sg(dma_rx, dst, dst_nents, DMA_DEV_TO_MEM, DMA_PREP_INTERRUPT | DMA_CTRL_ACK); @@ -1174,9 +1222,6 @@ static int sa_run(struct sa_req *req) rxd->req = (void *)req->base; rxd->enc = req->enc; - rxd->ddev = ddev; - rxd->src = src; - rxd->dst = dst; rxd->iv_idx = req->ctx->iv_idx; rxd->enc_iv_size = sa_ctx->cmdl_upd_info.enc_iv.size; rxd->tx_in->callback = req->callback; @@ -1214,16 +1259,7 @@ static int sa_run(struct sa_req *req) return -EINPROGRESS; err_cleanup: - dma_unmap_sg(ddev, req->src, sg_nents, DMA_TO_DEVICE); - kfree(rxd->split_src_sg); - - if (req->src != req->dst) { - dst_nents = sg_nents_for_len(req->dst, req->size); - dma_unmap_sg(ddev, req->dst, dst_nents, DMA_FROM_DEVICE); - kfree(rxd->split_dst_sg); - } - - kfree(rxd); + sa_free_sa_rx_data(rxd); return ret; } @@ -1293,11 +1329,12 @@ static void sa_sha_dma_in_callback(void *data) struct ahash_request *req; struct crypto_ahash *tfm; unsigned int authsize; - int i, sg_nents; + int i; size_t ml, pl; u32 *result; __be32 *mdptr; + sa_sync_from_device(rxd); req = container_of(rxd->req, struct ahash_request, base); tfm = crypto_ahash_reqtfm(req); authsize = crypto_ahash_digestsize(tfm); @@ -1308,12 +1345,7 @@ static void sa_sha_dma_in_callback(void *data) for (i = 0; i < (authsize / 4); i++) result[i] = be32_to_cpu(mdptr[i + 4]); - sg_nents = sg_nents_for_len(req->src, req->nbytes); - dma_unmap_sg(rxd->ddev, req->src, sg_nents, DMA_FROM_DEVICE); - - kfree(rxd->split_src_sg); - - kfree(rxd); + sa_free_sa_rx_data(rxd); ahash_request_complete(req, 0); } @@ -1482,8 +1514,8 @@ static int sa_sha_init(struct ahash_request *req) struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm); - dev_dbg(sa_k3_dev, "init: digest size: %d, rctx=%llx\n", - crypto_ahash_digestsize(tfm), (u64)rctx); + dev_dbg(sa_k3_dev, "init: digest size: %u, rctx=%p\n", + crypto_ahash_digestsize(tfm), rctx); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash); rctx->fallback_req.base.flags = @@ -1637,43 +1669,28 @@ static void sa_aead_dma_in_callback(void *data) unsigned int authsize; u8 auth_tag[SA_MAX_AUTH_TAG_SZ]; size_t pl, ml; - int i, sglen; + int i; int err = 0; u16 auth_len; u32 *mdptr; - bool diff_dst; - enum dma_data_direction dir_src; + sa_sync_from_device(rxd); req = container_of(rxd->req, struct aead_request, base); tfm = crypto_aead_reqtfm(req); start = req->assoclen + req->cryptlen; authsize = crypto_aead_authsize(tfm); - diff_dst = (req->src != req->dst) ? true : false; - dir_src = diff_dst ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL; - mdptr = (u32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl, &ml); for (i = 0; i < (authsize / 4); i++) mdptr[i + 4] = swab32(mdptr[i + 4]); auth_len = req->assoclen + req->cryptlen; - if (!rxd->enc) - auth_len -= authsize; - - sglen = sg_nents_for_len(rxd->src, auth_len); - dma_unmap_sg(rxd->ddev, rxd->src, sglen, dir_src); - kfree(rxd->split_src_sg); - - if (diff_dst) { - sglen = sg_nents_for_len(rxd->dst, auth_len); - dma_unmap_sg(rxd->ddev, rxd->dst, sglen, DMA_FROM_DEVICE); - kfree(rxd->split_dst_sg); - } if (rxd->enc) { scatterwalk_map_and_copy(&mdptr[4], req->dst, start, authsize, 1); } else { + auth_len -= authsize; start -= authsize; scatterwalk_map_and_copy(auth_tag, req->src, start, authsize, 0); @@ -1681,7 +1698,7 @@ static void sa_aead_dma_in_callback(void *data) err = memcmp(&mdptr[4], auth_tag, authsize) ? -EBADMSG : 0; } - kfree(rxd); + sa_free_sa_rx_data(rxd); aead_request_complete(req, err); } @@ -2243,25 +2260,21 @@ static int sa_dma_init(struct sa_crypto_data *dd) return ret; dd->dma_rx1 = dma_request_chan(dd->dev, "rx1"); - if (IS_ERR(dd->dma_rx1)) { - if (PTR_ERR(dd->dma_rx1) != -EPROBE_DEFER) - dev_err(dd->dev, "Unable to request rx1 DMA channel\n"); - return PTR_ERR(dd->dma_rx1); - } + if (IS_ERR(dd->dma_rx1)) + return dev_err_probe(dd->dev, PTR_ERR(dd->dma_rx1), + "Unable to request rx1 DMA channel\n"); dd->dma_rx2 = dma_request_chan(dd->dev, "rx2"); if (IS_ERR(dd->dma_rx2)) { dma_release_channel(dd->dma_rx1); - if (PTR_ERR(dd->dma_rx2) != -EPROBE_DEFER) - dev_err(dd->dev, "Unable to request rx2 DMA channel\n"); - return PTR_ERR(dd->dma_rx2); + return dev_err_probe(dd->dev, PTR_ERR(dd->dma_rx2), + "Unable to request rx2 DMA channel\n"); } dd->dma_tx = dma_request_chan(dd->dev, "tx"); if (IS_ERR(dd->dma_tx)) { - if (PTR_ERR(dd->dma_tx) != -EPROBE_DEFER) - dev_err(dd->dev, "Unable to request tx DMA channel\n"); - ret = PTR_ERR(dd->dma_tx); + ret = dev_err_probe(dd->dev, PTR_ERR(dd->dma_tx), + "Unable to request tx DMA channel\n"); goto err_dma_tx; } @@ -2333,7 +2346,7 @@ static int sa_ul_probe(struct platform_device *pdev) pm_runtime_enable(dev); ret = pm_runtime_get_sync(dev); - if (ret) { + if (ret < 0) { dev_err(&pdev->dev, "%s: failed to get sync: %d\n", __func__, ret); return ret; diff --git a/drivers/crypto/sahara.c b/drivers/crypto/sahara.c index 0c8cb23ae708..d60679c79822 100644 --- a/drivers/crypto/sahara.c +++ b/drivers/crypto/sahara.c @@ -18,7 +18,7 @@ #include <crypto/sha.h> #include <linux/clk.h> -#include <linux/crypto.h> +#include <linux/dma-mapping.h> #include <linux/interrupt.h> #include <linux/io.h> #include <linux/irq.h> diff --git a/drivers/crypto/stm32/Kconfig b/drivers/crypto/stm32/Kconfig index 4ef3eb11361c..4a4c3284ae1f 100644 --- a/drivers/crypto/stm32/Kconfig +++ b/drivers/crypto/stm32/Kconfig @@ -3,6 +3,7 @@ config CRYPTO_DEV_STM32_CRC tristate "Support for STM32 crc accelerators" depends on ARCH_STM32 select CRYPTO_HASH + select CRC32 help This enables support for the CRC32 hw accelerator which can be found on STMicroelectronics STM32 SOC. diff --git a/drivers/crypto/stm32/stm32-crc32.c b/drivers/crypto/stm32/stm32-crc32.c index 3ba41148c2a4..75867c0b0017 100644 --- a/drivers/crypto/stm32/stm32-crc32.c +++ b/drivers/crypto/stm32/stm32-crc32.c @@ -6,7 +6,10 @@ #include <linux/bitrev.h> #include <linux/clk.h> +#include <linux/crc32.h> #include <linux/crc32poly.h> +#include <linux/io.h> +#include <linux/kernel.h> #include <linux/module.h> #include <linux/mod_devicetable.h> #include <linux/platform_device.h> @@ -147,7 +150,6 @@ static int burst_update(struct shash_desc *desc, const u8 *d8, struct stm32_crc_desc_ctx *ctx = shash_desc_ctx(desc); struct stm32_crc_ctx *mctx = crypto_shash_ctx(desc->tfm); struct stm32_crc *crc; - unsigned long flags; crc = stm32_crc_get_next_crc(); if (!crc) @@ -155,7 +157,15 @@ static int burst_update(struct shash_desc *desc, const u8 *d8, pm_runtime_get_sync(crc->dev); - spin_lock_irqsave(&crc->lock, flags); + if (!spin_trylock(&crc->lock)) { + /* Hardware is busy, calculate crc32 by software */ + if (mctx->poly == CRC32_POLY_LE) + ctx->partial = crc32_le(ctx->partial, d8, length); + else + ctx->partial = __crc32c_le(ctx->partial, d8, length); + + goto pm_out; + } /* * Restore previously calculated CRC for this context as init value @@ -195,8 +205,9 @@ static int burst_update(struct shash_desc *desc, const u8 *d8, /* Store partial result */ ctx->partial = readl_relaxed(crc->regs + CRC_DR); - spin_unlock_irqrestore(&crc->lock, flags); + spin_unlock(&crc->lock); +pm_out: pm_runtime_mark_last_busy(crc->dev); pm_runtime_put_autosuspend(crc->dev); @@ -216,9 +227,8 @@ static int stm32_crc_update(struct shash_desc *desc, const u8 *d8, return burst_update(desc, d8, length); /* Digest first bytes not 32bit aligned at first pass in the loop */ - size = min(length, - burst_sz + (unsigned int)d8 - ALIGN_DOWN((unsigned int)d8, - sizeof(u32))); + size = min_t(size_t, length, burst_sz + (size_t)d8 - + ALIGN_DOWN((size_t)d8, sizeof(u32))); for (rem_sz = length, cur = d8; rem_sz; rem_sz -= size, cur += size, size = min(rem_sz, burst_sz)) { ret = burst_update(desc, cur, size); diff --git a/drivers/crypto/stm32/stm32-cryp.c b/drivers/crypto/stm32/stm32-cryp.c index d347a1d6e351..2670c30332fa 100644 --- a/drivers/crypto/stm32/stm32-cryp.c +++ b/drivers/crypto/stm32/stm32-cryp.c @@ -118,7 +118,7 @@ struct stm32_cryp_ctx { struct crypto_engine_ctx enginectx; struct stm32_cryp *cryp; int keylen; - u32 key[AES_KEYSIZE_256 / sizeof(u32)]; + __be32 key[AES_KEYSIZE_256 / sizeof(u32)]; unsigned long flags; }; @@ -380,24 +380,24 @@ static int stm32_cryp_copy_sgs(struct stm32_cryp *cryp) return 0; } -static void stm32_cryp_hw_write_iv(struct stm32_cryp *cryp, u32 *iv) +static void stm32_cryp_hw_write_iv(struct stm32_cryp *cryp, __be32 *iv) { if (!iv) return; - stm32_cryp_write(cryp, CRYP_IV0LR, cpu_to_be32(*iv++)); - stm32_cryp_write(cryp, CRYP_IV0RR, cpu_to_be32(*iv++)); + stm32_cryp_write(cryp, CRYP_IV0LR, be32_to_cpu(*iv++)); + stm32_cryp_write(cryp, CRYP_IV0RR, be32_to_cpu(*iv++)); if (is_aes(cryp)) { - stm32_cryp_write(cryp, CRYP_IV1LR, cpu_to_be32(*iv++)); - stm32_cryp_write(cryp, CRYP_IV1RR, cpu_to_be32(*iv++)); + stm32_cryp_write(cryp, CRYP_IV1LR, be32_to_cpu(*iv++)); + stm32_cryp_write(cryp, CRYP_IV1RR, be32_to_cpu(*iv++)); } } static void stm32_cryp_get_iv(struct stm32_cryp *cryp) { struct skcipher_request *req = cryp->req; - u32 *tmp = (void *)req->iv; + __be32 *tmp = (void *)req->iv; if (!tmp) return; @@ -417,13 +417,13 @@ static void stm32_cryp_hw_write_key(struct stm32_cryp *c) int r_id; if (is_des(c)) { - stm32_cryp_write(c, CRYP_K1LR, cpu_to_be32(c->ctx->key[0])); - stm32_cryp_write(c, CRYP_K1RR, cpu_to_be32(c->ctx->key[1])); + stm32_cryp_write(c, CRYP_K1LR, be32_to_cpu(c->ctx->key[0])); + stm32_cryp_write(c, CRYP_K1RR, be32_to_cpu(c->ctx->key[1])); } else { r_id = CRYP_K3RR; for (i = c->ctx->keylen / sizeof(u32); i > 0; i--, r_id -= 4) stm32_cryp_write(c, r_id, - cpu_to_be32(c->ctx->key[i - 1])); + be32_to_cpu(c->ctx->key[i - 1])); } } @@ -469,7 +469,7 @@ static unsigned int stm32_cryp_get_input_text_len(struct stm32_cryp *cryp) static int stm32_cryp_gcm_init(struct stm32_cryp *cryp, u32 cfg) { int ret; - u32 iv[4]; + __be32 iv[4]; /* Phase 1 : init */ memcpy(iv, cryp->areq->iv, 12); @@ -491,6 +491,7 @@ static int stm32_cryp_ccm_init(struct stm32_cryp *cryp, u32 cfg) { int ret; u8 iv[AES_BLOCK_SIZE], b0[AES_BLOCK_SIZE]; + __be32 *bd; u32 *d; unsigned int i, textlen; @@ -498,7 +499,7 @@ static int stm32_cryp_ccm_init(struct stm32_cryp *cryp, u32 cfg) memcpy(iv, cryp->areq->iv, AES_BLOCK_SIZE); memset(iv + AES_BLOCK_SIZE - 1 - iv[0], 0, iv[0] + 1); iv[AES_BLOCK_SIZE - 1] = 1; - stm32_cryp_hw_write_iv(cryp, (u32 *)iv); + stm32_cryp_hw_write_iv(cryp, (__be32 *)iv); /* Build B0 */ memcpy(b0, iv, AES_BLOCK_SIZE); @@ -518,11 +519,14 @@ static int stm32_cryp_ccm_init(struct stm32_cryp *cryp, u32 cfg) /* Write B0 */ d = (u32 *)b0; + bd = (__be32 *)b0; for (i = 0; i < AES_BLOCK_32; i++) { + u32 xd = d[i]; + if (!cryp->caps->padding_wa) - *d = cpu_to_be32(*d); - stm32_cryp_write(cryp, CRYP_DIN, *d++); + xd = be32_to_cpu(bd[i]); + stm32_cryp_write(cryp, CRYP_DIN, xd); } /* Wait for end of processing */ @@ -617,7 +621,7 @@ static int stm32_cryp_hw_init(struct stm32_cryp *cryp) case CR_TDES_CBC: case CR_AES_CBC: case CR_AES_CTR: - stm32_cryp_hw_write_iv(cryp, (u32 *)cryp->req->iv); + stm32_cryp_hw_write_iv(cryp, (__be32 *)cryp->req->iv); break; default: @@ -1120,7 +1124,7 @@ static int stm32_cryp_read_auth_tag(struct stm32_cryp *cryp) /* GCM: write aad and payload size (in bits) */ size_bit = cryp->areq->assoclen * 8; if (cryp->caps->swap_final) - size_bit = cpu_to_be32(size_bit); + size_bit = (__force u32)cpu_to_be32(size_bit); stm32_cryp_write(cryp, CRYP_DIN, 0); stm32_cryp_write(cryp, CRYP_DIN, size_bit); @@ -1129,7 +1133,7 @@ static int stm32_cryp_read_auth_tag(struct stm32_cryp *cryp) cryp->areq->cryptlen - AES_BLOCK_SIZE; size_bit *= 8; if (cryp->caps->swap_final) - size_bit = cpu_to_be32(size_bit); + size_bit = (__force u32)cpu_to_be32(size_bit); stm32_cryp_write(cryp, CRYP_DIN, 0); stm32_cryp_write(cryp, CRYP_DIN, size_bit); @@ -1137,14 +1141,19 @@ static int stm32_cryp_read_auth_tag(struct stm32_cryp *cryp) /* CCM: write CTR0 */ u8 iv[AES_BLOCK_SIZE]; u32 *iv32 = (u32 *)iv; + __be32 *biv; + + biv = (void *)iv; memcpy(iv, cryp->areq->iv, AES_BLOCK_SIZE); memset(iv + AES_BLOCK_SIZE - 1 - iv[0], 0, iv[0] + 1); for (i = 0; i < AES_BLOCK_32; i++) { + u32 xiv = iv32[i]; + if (!cryp->caps->padding_wa) - *iv32 = cpu_to_be32(*iv32); - stm32_cryp_write(cryp, CRYP_DIN, *iv32++); + xiv = be32_to_cpu(biv[i]); + stm32_cryp_write(cryp, CRYP_DIN, xiv); } } diff --git a/drivers/crypto/stm32/stm32-hash.c b/drivers/crypto/stm32/stm32-hash.c index 03c5e6683805..e3e25278a970 100644 --- a/drivers/crypto/stm32/stm32-hash.c +++ b/drivers/crypto/stm32/stm32-hash.c @@ -9,6 +9,7 @@ #include <linux/clk.h> #include <linux/crypto.h> #include <linux/delay.h> +#include <linux/dma-mapping.h> #include <linux/dmaengine.h> #include <linux/interrupt.h> #include <linux/io.h> @@ -748,7 +749,7 @@ static int stm32_hash_final_req(struct stm32_hash_dev *hdev) static void stm32_hash_copy_hash(struct ahash_request *req) { struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req); - u32 *hash = (u32 *)rctx->digest; + __be32 *hash = (void *)rctx->digest; unsigned int i, hashsize; switch (rctx->flags & HASH_FLAGS_ALGO_MASK) { @@ -769,7 +770,7 @@ static void stm32_hash_copy_hash(struct ahash_request *req) } for (i = 0; i < hashsize / sizeof(u32); i++) - hash[i] = be32_to_cpu(stm32_hash_read(rctx->hdev, + hash[i] = cpu_to_be32(stm32_hash_read(rctx->hdev, HASH_HREG(i))); } @@ -1463,14 +1464,9 @@ static int stm32_hash_probe(struct platform_device *pdev) } hdev->clk = devm_clk_get(&pdev->dev, NULL); - if (IS_ERR(hdev->clk)) { - if (PTR_ERR(hdev->clk) != -EPROBE_DEFER) { - dev_err(dev, "failed to get clock for hash (%lu)\n", - PTR_ERR(hdev->clk)); - } - - return PTR_ERR(hdev->clk); - } + if (IS_ERR(hdev->clk)) + return dev_err_probe(dev, PTR_ERR(hdev->clk), + "failed to get clock for hash\n"); ret = clk_prepare_enable(hdev->clk); if (ret) { diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index 7c547352a862..66773892f665 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -806,10 +806,10 @@ static int talitos_register_rng(struct device *dev) struct talitos_private *priv = dev_get_drvdata(dev); int err; - priv->rng.name = dev_driver_string(dev), - priv->rng.init = talitos_rng_init, - priv->rng.data_present = talitos_rng_data_present, - priv->rng.data_read = talitos_rng_data_read, + priv->rng.name = dev_driver_string(dev); + priv->rng.init = talitos_rng_init; + priv->rng.data_present = talitos_rng_data_present; + priv->rng.data_read = talitos_rng_data_read; priv->rng.priv = (unsigned long)dev; err = hwrng_register(&priv->rng); diff --git a/drivers/crypto/ux500/cryp/cryp_core.c b/drivers/crypto/ux500/cryp/cryp_core.c index 800dfc4d16c4..c3adeb2e5823 100644 --- a/drivers/crypto/ux500/cryp/cryp_core.c +++ b/drivers/crypto/ux500/cryp/cryp_core.c @@ -11,13 +11,15 @@ #include <linux/clk.h> #include <linux/completion.h> -#include <linux/crypto.h> +#include <linux/device.h> +#include <linux/dma-mapping.h> #include <linux/dmaengine.h> #include <linux/err.h> #include <linux/errno.h> #include <linux/interrupt.h> #include <linux/io.h> #include <linux/irqreturn.h> +#include <linux/kernel.h> #include <linux/klist.h> #include <linux/module.h> #include <linux/mod_devicetable.h> @@ -27,7 +29,6 @@ #include <linux/platform_data/dma-ste-dma40.h> #include <crypto/aes.h> -#include <crypto/algapi.h> #include <crypto/ctr.h> #include <crypto/internal/des.h> #include <crypto/internal/skcipher.h> @@ -92,17 +93,6 @@ struct cryp_ctx { static struct cryp_driver_data driver_data; /** - * uint8p_to_uint32_be - 4*uint8 to uint32 big endian - * @in: Data to convert. - */ -static inline u32 uint8p_to_uint32_be(u8 *in) -{ - u32 *data = (u32 *)in; - - return cpu_to_be32p(data); -} - -/** * swap_bits_in_byte - mirror the bits in a byte * @b: the byte to be mirrored * @@ -284,6 +274,7 @@ static int cfg_ivs(struct cryp_device_data *device_data, struct cryp_ctx *ctx) int i; int status = 0; int num_of_regs = ctx->blocksize / 8; + __be32 *civ = (__be32 *)ctx->iv; u32 iv[AES_BLOCK_SIZE / 4]; dev_dbg(device_data->dev, "[%s]", __func__); @@ -300,7 +291,7 @@ static int cfg_ivs(struct cryp_device_data *device_data, struct cryp_ctx *ctx) } for (i = 0; i < ctx->blocksize / 4; i++) - iv[i] = uint8p_to_uint32_be(ctx->iv + i*4); + iv[i] = be32_to_cpup(civ + i); for (i = 0; i < num_of_regs; i++) { status = cfg_iv(device_data, iv[i*2], iv[i*2+1], @@ -339,23 +330,24 @@ static int cfg_keys(struct cryp_ctx *ctx) int i; int num_of_regs = ctx->keylen / 8; u32 swapped_key[CRYP_MAX_KEY_SIZE / 4]; + __be32 *ckey = (__be32 *)ctx->key; int cryp_error = 0; dev_dbg(ctx->device->dev, "[%s]", __func__); if (mode_is_aes(ctx->config.algomode)) { - swap_words_in_key_and_bits_in_byte((u8 *)ctx->key, + swap_words_in_key_and_bits_in_byte((u8 *)ckey, (u8 *)swapped_key, ctx->keylen); } else { for (i = 0; i < ctx->keylen / 4; i++) - swapped_key[i] = uint8p_to_uint32_be(ctx->key + i*4); + swapped_key[i] = be32_to_cpup(ckey + i); } for (i = 0; i < num_of_regs; i++) { cryp_error = set_key(ctx->device, - *(((u32 *)swapped_key)+i*2), - *(((u32 *)swapped_key)+i*2+1), + swapped_key[i * 2], + swapped_key[i * 2 + 1], (enum cryp_key_reg_index) i); if (cryp_error != 0) { diff --git a/drivers/crypto/ux500/hash/hash_core.c b/drivers/crypto/ux500/hash/hash_core.c index a5ee8c2fb4e0..3d407eebb2ba 100644 --- a/drivers/crypto/ux500/hash/hash_core.c +++ b/drivers/crypto/ux500/hash/hash_core.c @@ -15,6 +15,7 @@ #include <linux/clk.h> #include <linux/device.h> +#include <linux/dma-mapping.h> #include <linux/err.h> #include <linux/init.h> #include <linux/io.h> @@ -1071,27 +1072,32 @@ int hash_hw_update(struct ahash_request *req) struct hash_ctx *ctx = crypto_ahash_ctx(tfm); struct hash_req_ctx *req_ctx = ahash_request_ctx(req); struct crypto_hash_walk walk; - int msg_length = crypto_hash_walk_first(req, &walk); - - /* Empty message ("") is correct indata */ - if (msg_length == 0) - return ret; + int msg_length; index = req_ctx->state.index; buffer = (u8 *)req_ctx->state.buffer; + ret = hash_get_device_data(ctx, &device_data); + if (ret) + return ret; + + msg_length = crypto_hash_walk_first(req, &walk); + + /* Empty message ("") is correct indata */ + if (msg_length == 0) { + ret = 0; + goto release_dev; + } + /* Check if ctx->state.length + msg_length overflows */ if (msg_length > (req_ctx->state.length.low_word + msg_length) && HASH_HIGH_WORD_MAX_VAL == req_ctx->state.length.high_word) { pr_err("%s: HASH_MSG_LENGTH_OVERFLOW!\n", __func__); - return -EPERM; + ret = crypto_hash_walk_done(&walk, -EPERM); + goto release_dev; } - ret = hash_get_device_data(ctx, &device_data); - if (ret) - return ret; - /* Main loop */ while (0 != msg_length) { data_buffer = walk.data; @@ -1101,7 +1107,8 @@ int hash_hw_update(struct ahash_request *req) if (ret) { dev_err(device_data->dev, "%s: hash_internal_hw_update() failed!\n", __func__); - goto out; + crypto_hash_walk_done(&walk, ret); + goto release_dev; } msg_length = crypto_hash_walk_done(&walk, 0); @@ -1111,7 +1118,7 @@ int hash_hw_update(struct ahash_request *req) dev_dbg(device_data->dev, "%s: indata length=%d, bin=%d\n", __func__, req_ctx->state.index, req_ctx->state.bit_index); -out: +release_dev: release_hash_device(device_data); return ret; diff --git a/drivers/crypto/virtio/Kconfig b/drivers/crypto/virtio/Kconfig index fb294174e408..b894e3a8be4f 100644 --- a/drivers/crypto/virtio/Kconfig +++ b/drivers/crypto/virtio/Kconfig @@ -5,7 +5,6 @@ config CRYPTO_DEV_VIRTIO select CRYPTO_AEAD select CRYPTO_SKCIPHER select CRYPTO_ENGINE - default m help This driver provides support for virtio crypto device. If you choose 'M' here, this module will be called virtio_crypto. diff --git a/drivers/crypto/xilinx/zynqmp-aes-gcm.c b/drivers/crypto/xilinx/zynqmp-aes-gcm.c index 27079354dbe9..bf1f421e05f2 100644 --- a/drivers/crypto/xilinx/zynqmp-aes-gcm.c +++ b/drivers/crypto/xilinx/zynqmp-aes-gcm.c @@ -10,6 +10,7 @@ #include <crypto/internal/aead.h> #include <crypto/scatterwalk.h> +#include <linux/dma-mapping.h> #include <linux/module.h> #include <linux/of_device.h> #include <linux/platform_device.h> diff --git a/drivers/net/wireless/intel/ipw2x00/Kconfig b/drivers/net/wireless/intel/ipw2x00/Kconfig index b1e7b4470842..1650d5865aa0 100644 --- a/drivers/net/wireless/intel/ipw2x00/Kconfig +++ b/drivers/net/wireless/intel/ipw2x00/Kconfig @@ -160,11 +160,7 @@ config LIBIPW select WIRELESS_EXT select WEXT_SPY select CRYPTO - select CRYPTO_ARC4 - select CRYPTO_ECB - select CRYPTO_AES select CRYPTO_MICHAEL_MIC - select CRYPTO_ECB select CRC32 select LIB80211 select LIB80211_CRYPT_WEP diff --git a/drivers/net/wireless/intersil/hostap/Kconfig b/drivers/net/wireless/intersil/hostap/Kconfig index 6ad88299432f..c865d3156cea 100644 --- a/drivers/net/wireless/intersil/hostap/Kconfig +++ b/drivers/net/wireless/intersil/hostap/Kconfig @@ -5,11 +5,7 @@ config HOSTAP select WEXT_SPY select WEXT_PRIV select CRYPTO - select CRYPTO_ARC4 - select CRYPTO_ECB - select CRYPTO_AES select CRYPTO_MICHAEL_MIC - select CRYPTO_ECB select CRC32 select LIB80211 select LIB80211_CRYPT_WEP diff --git a/drivers/staging/rtl8192e/Kconfig b/drivers/staging/rtl8192e/Kconfig index 1007eea6c8fc..4c440bdaaf6e 100644 --- a/drivers/staging/rtl8192e/Kconfig +++ b/drivers/staging/rtl8192e/Kconfig @@ -25,7 +25,7 @@ config RTLLIB_CRYPTO_CCMP config RTLLIB_CRYPTO_TKIP tristate "Support for rtllib TKIP crypto" depends on RTLLIB - select CRYPTO_ARC4 + select CRYPTO_LIB_ARC4 select CRYPTO_MICHAEL_MIC default y help @@ -35,7 +35,7 @@ config RTLLIB_CRYPTO_TKIP config RTLLIB_CRYPTO_WEP tristate "Support for rtllib WEP crypto" - select CRYPTO_ARC4 + select CRYPTO_LIB_ARC4 depends on RTLLIB default y help diff --git a/drivers/staging/rtl8192e/rtllib_crypt_tkip.c b/drivers/staging/rtl8192e/rtllib_crypt_tkip.c index 8d2a58e706d5..8c2ff37b2d3a 100644 --- a/drivers/staging/rtl8192e/rtllib_crypt_tkip.c +++ b/drivers/staging/rtl8192e/rtllib_crypt_tkip.c @@ -5,8 +5,9 @@ * Copyright (c) 2003-2004, Jouni Malinen <jkmaline@cc.hut.fi> */ +#include <crypto/arc4.h> #include <crypto/hash.h> -#include <crypto/skcipher.h> +#include <linux/fips.h> #include <linux/module.h> #include <linux/init.h> #include <linux/slab.h> @@ -16,7 +17,6 @@ #include <linux/if_ether.h> #include <linux/if_arp.h> #include <linux/string.h> -#include <linux/scatterlist.h> #include <linux/crc32.h> #include <linux/etherdevice.h> @@ -45,9 +45,9 @@ struct rtllib_tkip_data { u32 dot11RSNAStatsTKIPLocalMICFailures; int key_idx; - struct crypto_sync_skcipher *rx_tfm_arc4; + struct arc4_ctx rx_ctx_arc4; + struct arc4_ctx tx_ctx_arc4; struct crypto_shash *rx_tfm_michael; - struct crypto_sync_skcipher *tx_tfm_arc4; struct crypto_shash *tx_tfm_michael; /* scratch buffers for virt_to_page() (crypto API) */ u8 rx_hdr[16]; @@ -58,16 +58,13 @@ static void *rtllib_tkip_init(int key_idx) { struct rtllib_tkip_data *priv; + if (fips_enabled) + return NULL; + priv = kzalloc(sizeof(*priv), GFP_ATOMIC); if (priv == NULL) goto fail; priv->key_idx = key_idx; - priv->tx_tfm_arc4 = crypto_alloc_sync_skcipher("ecb(arc4)", 0, 0); - if (IS_ERR(priv->tx_tfm_arc4)) { - pr_debug("Could not allocate crypto API arc4\n"); - priv->tx_tfm_arc4 = NULL; - goto fail; - } priv->tx_tfm_michael = crypto_alloc_shash("michael_mic", 0, 0); if (IS_ERR(priv->tx_tfm_michael)) { @@ -76,13 +73,6 @@ static void *rtllib_tkip_init(int key_idx) goto fail; } - priv->rx_tfm_arc4 = crypto_alloc_sync_skcipher("ecb(arc4)", 0, 0); - if (IS_ERR(priv->rx_tfm_arc4)) { - pr_debug("Could not allocate crypto API arc4\n"); - priv->rx_tfm_arc4 = NULL; - goto fail; - } - priv->rx_tfm_michael = crypto_alloc_shash("michael_mic", 0, 0); if (IS_ERR(priv->rx_tfm_michael)) { pr_debug("Could not allocate crypto API michael_mic\n"); @@ -94,9 +84,7 @@ static void *rtllib_tkip_init(int key_idx) fail: if (priv) { crypto_free_shash(priv->tx_tfm_michael); - crypto_free_sync_skcipher(priv->tx_tfm_arc4); crypto_free_shash(priv->rx_tfm_michael); - crypto_free_sync_skcipher(priv->rx_tfm_arc4); kfree(priv); } @@ -110,11 +98,9 @@ static void rtllib_tkip_deinit(void *priv) if (_priv) { crypto_free_shash(_priv->tx_tfm_michael); - crypto_free_sync_skcipher(_priv->tx_tfm_arc4); crypto_free_shash(_priv->rx_tfm_michael); - crypto_free_sync_skcipher(_priv->rx_tfm_arc4); } - kfree(priv); + kzfree(priv); } @@ -289,7 +275,6 @@ static int rtllib_tkip_encrypt(struct sk_buff *skb, int hdr_len, void *priv) int ret = 0; u8 rc4key[16], *icv; u32 crc; - struct scatterlist sg; if (skb_headroom(skb) < 8 || skb_tailroom(skb) < 4 || skb->len < hdr_len) @@ -331,8 +316,6 @@ static int rtllib_tkip_encrypt(struct sk_buff *skb, int hdr_len, void *priv) *pos++ = (tkey->tx_iv32 >> 24) & 0xff; if (!tcb_desc->bHwSec) { - SYNC_SKCIPHER_REQUEST_ON_STACK(req, tkey->tx_tfm_arc4); - icv = skb_put(skb, 4); crc = ~crc32_le(~0, pos, len); icv[0] = crc; @@ -340,15 +323,8 @@ static int rtllib_tkip_encrypt(struct sk_buff *skb, int hdr_len, void *priv) icv[2] = crc >> 16; icv[3] = crc >> 24; - sg_init_one(&sg, pos, len+4); - - - crypto_sync_skcipher_setkey(tkey->tx_tfm_arc4, rc4key, 16); - skcipher_request_set_sync_tfm(req, tkey->tx_tfm_arc4); - skcipher_request_set_callback(req, 0, NULL, NULL); - skcipher_request_set_crypt(req, &sg, &sg, len + 4, NULL); - ret = crypto_skcipher_encrypt(req); - skcipher_request_zero(req); + arc4_setkey(&tkey->tx_ctx_arc4, rc4key, 16); + arc4_crypt(&tkey->tx_ctx_arc4, pos, pos, len + 4); } tkey->tx_iv16++; @@ -376,9 +352,7 @@ static int rtllib_tkip_decrypt(struct sk_buff *skb, int hdr_len, void *priv) u8 rc4key[16]; u8 icv[4]; u32 crc; - struct scatterlist sg; int plen; - int err; if (skb->len < hdr_len + 8 + 4) return -1; @@ -414,8 +388,6 @@ static int rtllib_tkip_decrypt(struct sk_buff *skb, int hdr_len, void *priv) pos += 8; if (!tcb_desc->bHwSec || (skb->cb[0] == 1)) { - SYNC_SKCIPHER_REQUEST_ON_STACK(req, tkey->rx_tfm_arc4); - if ((iv32 < tkey->rx_iv32 || (iv32 == tkey->rx_iv32 && iv16 <= tkey->rx_iv16)) && tkey->initialized) { @@ -439,22 +411,8 @@ static int rtllib_tkip_decrypt(struct sk_buff *skb, int hdr_len, void *priv) plen = skb->len - hdr_len - 12; - sg_init_one(&sg, pos, plen+4); - - crypto_sync_skcipher_setkey(tkey->rx_tfm_arc4, rc4key, 16); - skcipher_request_set_sync_tfm(req, tkey->rx_tfm_arc4); - skcipher_request_set_callback(req, 0, NULL, NULL); - skcipher_request_set_crypt(req, &sg, &sg, plen + 4, NULL); - err = crypto_skcipher_decrypt(req); - skcipher_request_zero(req); - if (err) { - if (net_ratelimit()) { - netdev_dbg(skb->dev, - "Failed to decrypt received packet from %pM\n", - hdr->addr2); - } - return -7; - } + arc4_setkey(&tkey->rx_ctx_arc4, rc4key, 16); + arc4_crypt(&tkey->rx_ctx_arc4, pos, pos, plen + 4); crc = ~crc32_le(~0, pos, plen); icv[0] = crc; @@ -657,17 +615,13 @@ static int rtllib_tkip_set_key(void *key, int len, u8 *seq, void *priv) struct rtllib_tkip_data *tkey = priv; int keyidx; struct crypto_shash *tfm = tkey->tx_tfm_michael; - struct crypto_sync_skcipher *tfm2 = tkey->tx_tfm_arc4; struct crypto_shash *tfm3 = tkey->rx_tfm_michael; - struct crypto_sync_skcipher *tfm4 = tkey->rx_tfm_arc4; keyidx = tkey->key_idx; memset(tkey, 0, sizeof(*tkey)); tkey->key_idx = keyidx; tkey->tx_tfm_michael = tfm; - tkey->tx_tfm_arc4 = tfm2; tkey->rx_tfm_michael = tfm3; - tkey->rx_tfm_arc4 = tfm4; if (len == TKIP_KEY_LEN) { memcpy(tkey->key, key, TKIP_KEY_LEN); diff --git a/drivers/staging/rtl8192e/rtllib_crypt_wep.c b/drivers/staging/rtl8192e/rtllib_crypt_wep.c index b1ea650036d2..7cdd17f907fa 100644 --- a/drivers/staging/rtl8192e/rtllib_crypt_wep.c +++ b/drivers/staging/rtl8192e/rtllib_crypt_wep.c @@ -5,7 +5,8 @@ * Copyright (c) 2002-2004, Jouni Malinen <jkmaline@cc.hut.fi> */ -#include <crypto/skcipher.h> +#include <crypto/arc4.h> +#include <linux/fips.h> #include <linux/module.h> #include <linux/init.h> #include <linux/slab.h> @@ -14,7 +15,6 @@ #include <linux/string.h> #include "rtllib.h" -#include <linux/scatterlist.h> #include <linux/crc32.h> struct prism2_wep_data { @@ -23,8 +23,8 @@ struct prism2_wep_data { u8 key[WEP_KEY_LEN + 1]; u8 key_len; u8 key_idx; - struct crypto_sync_skcipher *tx_tfm; - struct crypto_sync_skcipher *rx_tfm; + struct arc4_ctx rx_ctx_arc4; + struct arc4_ctx tx_ctx_arc4; }; @@ -32,48 +32,24 @@ static void *prism2_wep_init(int keyidx) { struct prism2_wep_data *priv; + if (fips_enabled) + return NULL; + priv = kzalloc(sizeof(*priv), GFP_ATOMIC); if (priv == NULL) - goto fail; + return NULL; priv->key_idx = keyidx; - priv->tx_tfm = crypto_alloc_sync_skcipher("ecb(arc4)", 0, 0); - if (IS_ERR(priv->tx_tfm)) { - pr_debug("rtllib_crypt_wep: could not allocate crypto API arc4\n"); - priv->tx_tfm = NULL; - goto fail; - } - priv->rx_tfm = crypto_alloc_sync_skcipher("ecb(arc4)", 0, 0); - if (IS_ERR(priv->rx_tfm)) { - pr_debug("rtllib_crypt_wep: could not allocate crypto API arc4\n"); - priv->rx_tfm = NULL; - goto fail; - } - /* start WEP IV from a random value */ get_random_bytes(&priv->iv, 4); return priv; - -fail: - if (priv) { - crypto_free_sync_skcipher(priv->tx_tfm); - crypto_free_sync_skcipher(priv->rx_tfm); - kfree(priv); - } - return NULL; } static void prism2_wep_deinit(void *priv) { - struct prism2_wep_data *_priv = priv; - - if (_priv) { - crypto_free_sync_skcipher(_priv->tx_tfm); - crypto_free_sync_skcipher(_priv->rx_tfm); - } - kfree(priv); + kzfree(priv); } /* Perform WEP encryption on given skb that has at least 4 bytes of headroom @@ -92,8 +68,6 @@ static int prism2_wep_encrypt(struct sk_buff *skb, int hdr_len, void *priv) MAX_DEV_ADDR_SIZE); u32 crc; u8 *icv; - struct scatterlist sg; - int err; if (skb_headroom(skb) < 4 || skb_tailroom(skb) < 4 || skb->len < hdr_len){ @@ -131,8 +105,6 @@ static int prism2_wep_encrypt(struct sk_buff *skb, int hdr_len, void *priv) memcpy(key + 3, wep->key, wep->key_len); if (!tcb_desc->bHwSec) { - SYNC_SKCIPHER_REQUEST_ON_STACK(req, wep->tx_tfm); - /* Append little-endian CRC32 and encrypt it to produce ICV */ crc = ~crc32_le(~0, pos, len); icv = skb_put(skb, 4); @@ -141,14 +113,8 @@ static int prism2_wep_encrypt(struct sk_buff *skb, int hdr_len, void *priv) icv[2] = crc >> 16; icv[3] = crc >> 24; - sg_init_one(&sg, pos, len+4); - crypto_sync_skcipher_setkey(wep->tx_tfm, key, klen); - skcipher_request_set_sync_tfm(req, wep->tx_tfm); - skcipher_request_set_callback(req, 0, NULL, NULL); - skcipher_request_set_crypt(req, &sg, &sg, len + 4, NULL); - err = crypto_skcipher_encrypt(req); - skcipher_request_zero(req); - return err; + arc4_setkey(&wep->tx_ctx_arc4, key, klen); + arc4_crypt(&wep->tx_ctx_arc4, pos, pos, len + 4); } return 0; @@ -172,8 +138,6 @@ static int prism2_wep_decrypt(struct sk_buff *skb, int hdr_len, void *priv) MAX_DEV_ADDR_SIZE); u32 crc; u8 icv[4]; - struct scatterlist sg; - int err; if (skb->len < hdr_len + 8) return -1; @@ -195,17 +159,9 @@ static int prism2_wep_decrypt(struct sk_buff *skb, int hdr_len, void *priv) plen = skb->len - hdr_len - 8; if (!tcb_desc->bHwSec) { - SYNC_SKCIPHER_REQUEST_ON_STACK(req, wep->rx_tfm); - - sg_init_one(&sg, pos, plen+4); - crypto_sync_skcipher_setkey(wep->rx_tfm, key, klen); - skcipher_request_set_sync_tfm(req, wep->rx_tfm); - skcipher_request_set_callback(req, 0, NULL, NULL); - skcipher_request_set_crypt(req, &sg, &sg, plen + 4, NULL); - err = crypto_skcipher_decrypt(req); - skcipher_request_zero(req); - if (err) - return -7; + arc4_setkey(&wep->rx_ctx_arc4, key, klen); + arc4_crypt(&wep->rx_ctx_arc4, pos, pos, plen + 4); + crc = ~crc32_le(~0, pos, plen); icv[0] = crc; icv[1] = crc >> 8; diff --git a/drivers/staging/rtl8192u/Kconfig b/drivers/staging/rtl8192u/Kconfig index 1edca5c304fb..ef883d462d3d 100644 --- a/drivers/staging/rtl8192u/Kconfig +++ b/drivers/staging/rtl8192u/Kconfig @@ -8,3 +8,4 @@ config RTL8192U select CRYPTO select CRYPTO_AES select CRYPTO_CCM + select CRYPTO_LIB_ARC4 diff --git a/drivers/staging/rtl8192u/ieee80211/ieee80211_crypt_tkip.c b/drivers/staging/rtl8192u/ieee80211/ieee80211_crypt_tkip.c index ffe624ed0c0c..4b415cc76715 100644 --- a/drivers/staging/rtl8192u/ieee80211/ieee80211_crypt_tkip.c +++ b/drivers/staging/rtl8192u/ieee80211/ieee80211_crypt_tkip.c @@ -5,6 +5,7 @@ * Copyright (c) 2003-2004, Jouni Malinen <jkmaline@cc.hut.fi> */ +#include <linux/fips.h> #include <linux/module.h> #include <linux/init.h> #include <linux/slab.h> @@ -17,9 +18,8 @@ #include "ieee80211.h" +#include <crypto/arc4.h> #include <crypto/hash.h> -#include <crypto/skcipher.h> - #include <linux/scatterlist.h> #include <linux/crc32.h> MODULE_AUTHOR("Jouni Malinen"); @@ -49,9 +49,9 @@ struct ieee80211_tkip_data { int key_idx; - struct crypto_sync_skcipher *rx_tfm_arc4; + struct arc4_ctx rx_ctx_arc4; + struct arc4_ctx tx_ctx_arc4; struct crypto_shash *rx_tfm_michael; - struct crypto_sync_skcipher *tx_tfm_arc4; struct crypto_shash *tx_tfm_michael; /* scratch buffers for virt_to_page() (crypto API) */ @@ -62,19 +62,14 @@ static void *ieee80211_tkip_init(int key_idx) { struct ieee80211_tkip_data *priv; + if (fips_enabled) + return NULL; + priv = kzalloc(sizeof(*priv), GFP_KERNEL); if (!priv) goto fail; priv->key_idx = key_idx; - priv->tx_tfm_arc4 = crypto_alloc_sync_skcipher("ecb(arc4)", 0, 0); - if (IS_ERR(priv->tx_tfm_arc4)) { - printk(KERN_DEBUG "ieee80211_crypt_tkip: could not allocate " - "crypto API arc4\n"); - priv->tx_tfm_arc4 = NULL; - goto fail; - } - priv->tx_tfm_michael = crypto_alloc_shash("michael_mic", 0, 0); if (IS_ERR(priv->tx_tfm_michael)) { printk(KERN_DEBUG "ieee80211_crypt_tkip: could not allocate " @@ -83,14 +78,6 @@ static void *ieee80211_tkip_init(int key_idx) goto fail; } - priv->rx_tfm_arc4 = crypto_alloc_sync_skcipher("ecb(arc4)", 0, 0); - if (IS_ERR(priv->rx_tfm_arc4)) { - printk(KERN_DEBUG "ieee80211_crypt_tkip: could not allocate " - "crypto API arc4\n"); - priv->rx_tfm_arc4 = NULL; - goto fail; - } - priv->rx_tfm_michael = crypto_alloc_shash("michael_mic", 0, 0); if (IS_ERR(priv->rx_tfm_michael)) { printk(KERN_DEBUG "ieee80211_crypt_tkip: could not allocate " @@ -104,9 +91,7 @@ static void *ieee80211_tkip_init(int key_idx) fail: if (priv) { crypto_free_shash(priv->tx_tfm_michael); - crypto_free_sync_skcipher(priv->tx_tfm_arc4); crypto_free_shash(priv->rx_tfm_michael); - crypto_free_sync_skcipher(priv->rx_tfm_arc4); kfree(priv); } @@ -120,11 +105,9 @@ static void ieee80211_tkip_deinit(void *priv) if (_priv) { crypto_free_shash(_priv->tx_tfm_michael); - crypto_free_sync_skcipher(_priv->tx_tfm_arc4); crypto_free_shash(_priv->rx_tfm_michael); - crypto_free_sync_skcipher(_priv->rx_tfm_arc4); } - kfree(priv); + kzfree(priv); } @@ -290,10 +273,8 @@ static int ieee80211_tkip_encrypt(struct sk_buff *skb, int hdr_len, void *priv) u8 *pos; struct rtl_80211_hdr_4addr *hdr; struct cb_desc *tcb_desc = (struct cb_desc *)(skb->cb + MAX_DEV_ADDR_SIZE); - int ret = 0; u8 rc4key[16], *icv; u32 crc; - struct scatterlist sg; if (skb_headroom(skb) < 8 || skb_tailroom(skb) < 4 || skb->len < hdr_len) @@ -334,21 +315,15 @@ static int ieee80211_tkip_encrypt(struct sk_buff *skb, int hdr_len, void *priv) *pos++ = (tkey->tx_iv32 >> 24) & 0xff; if (!tcb_desc->bHwSec) { - SYNC_SKCIPHER_REQUEST_ON_STACK(req, tkey->tx_tfm_arc4); - icv = skb_put(skb, 4); crc = ~crc32_le(~0, pos, len); icv[0] = crc; icv[1] = crc >> 8; icv[2] = crc >> 16; icv[3] = crc >> 24; - crypto_sync_skcipher_setkey(tkey->tx_tfm_arc4, rc4key, 16); - sg_init_one(&sg, pos, len + 4); - skcipher_request_set_sync_tfm(req, tkey->tx_tfm_arc4); - skcipher_request_set_callback(req, 0, NULL, NULL); - skcipher_request_set_crypt(req, &sg, &sg, len + 4, NULL); - ret = crypto_skcipher_encrypt(req); - skcipher_request_zero(req); + + arc4_setkey(&tkey->tx_ctx_arc4, rc4key, 16); + arc4_crypt(&tkey->tx_ctx_arc4, pos, pos, len + 4); } tkey->tx_iv16++; @@ -357,12 +332,7 @@ static int ieee80211_tkip_encrypt(struct sk_buff *skb, int hdr_len, void *priv) tkey->tx_iv32++; } - if (!tcb_desc->bHwSec) - return ret; - else - return 0; - - + return 0; } static int ieee80211_tkip_decrypt(struct sk_buff *skb, int hdr_len, void *priv) @@ -376,9 +346,7 @@ static int ieee80211_tkip_decrypt(struct sk_buff *skb, int hdr_len, void *priv) u8 rc4key[16]; u8 icv[4]; u32 crc; - struct scatterlist sg; int plen; - int err; if (skb->len < hdr_len + 8 + 4) return -1; @@ -412,8 +380,6 @@ static int ieee80211_tkip_decrypt(struct sk_buff *skb, int hdr_len, void *priv) pos += 8; if (!tcb_desc->bHwSec) { - SYNC_SKCIPHER_REQUEST_ON_STACK(req, tkey->rx_tfm_arc4); - if (iv32 < tkey->rx_iv32 || (iv32 == tkey->rx_iv32 && iv16 <= tkey->rx_iv16)) { if (net_ratelimit()) { @@ -434,23 +400,8 @@ static int ieee80211_tkip_decrypt(struct sk_buff *skb, int hdr_len, void *priv) plen = skb->len - hdr_len - 12; - crypto_sync_skcipher_setkey(tkey->rx_tfm_arc4, rc4key, 16); - sg_init_one(&sg, pos, plen + 4); - - skcipher_request_set_sync_tfm(req, tkey->rx_tfm_arc4); - skcipher_request_set_callback(req, 0, NULL, NULL); - skcipher_request_set_crypt(req, &sg, &sg, plen + 4, NULL); - - err = crypto_skcipher_decrypt(req); - skcipher_request_zero(req); - if (err) { - if (net_ratelimit()) { - netdev_dbg(skb->dev, "TKIP: failed to decrypt " - "received packet from %pM\n", - hdr->addr2); - } - return -7; - } + arc4_setkey(&tkey->rx_ctx_arc4, rc4key, 16); + arc4_crypt(&tkey->rx_ctx_arc4, pos, pos, plen + 4); crc = ~crc32_le(~0, pos, plen); icv[0] = crc; @@ -655,17 +606,13 @@ static int ieee80211_tkip_set_key(void *key, int len, u8 *seq, void *priv) struct ieee80211_tkip_data *tkey = priv; int keyidx; struct crypto_shash *tfm = tkey->tx_tfm_michael; - struct crypto_sync_skcipher *tfm2 = tkey->tx_tfm_arc4; struct crypto_shash *tfm3 = tkey->rx_tfm_michael; - struct crypto_sync_skcipher *tfm4 = tkey->rx_tfm_arc4; keyidx = tkey->key_idx; memset(tkey, 0, sizeof(*tkey)); tkey->key_idx = keyidx; tkey->tx_tfm_michael = tfm; - tkey->tx_tfm_arc4 = tfm2; tkey->rx_tfm_michael = tfm3; - tkey->rx_tfm_arc4 = tfm4; if (len == TKIP_KEY_LEN) { memcpy(tkey->key, key, TKIP_KEY_LEN); diff --git a/drivers/staging/rtl8192u/ieee80211/ieee80211_crypt_wep.c b/drivers/staging/rtl8192u/ieee80211/ieee80211_crypt_wep.c index 26482c3dcd1c..1c56e2d03aae 100644 --- a/drivers/staging/rtl8192u/ieee80211/ieee80211_crypt_wep.c +++ b/drivers/staging/rtl8192u/ieee80211/ieee80211_crypt_wep.c @@ -5,6 +5,7 @@ * Copyright (c) 2002-2004, Jouni Malinen <jkmaline@cc.hut.fi> */ +#include <linux/fips.h> #include <linux/module.h> #include <linux/init.h> #include <linux/slab.h> @@ -14,8 +15,7 @@ #include "ieee80211.h" -#include <crypto/skcipher.h> -#include <linux/scatterlist.h> +#include <crypto/arc4.h> #include <linux/crc32.h> MODULE_AUTHOR("Jouni Malinen"); @@ -28,8 +28,8 @@ struct prism2_wep_data { u8 key[WEP_KEY_LEN + 1]; u8 key_len; u8 key_idx; - struct crypto_sync_skcipher *tx_tfm; - struct crypto_sync_skcipher *rx_tfm; + struct arc4_ctx rx_ctx_arc4; + struct arc4_ctx tx_ctx_arc4; }; @@ -37,39 +37,24 @@ static void *prism2_wep_init(int keyidx) { struct prism2_wep_data *priv; + if (fips_enabled) + return NULL; + priv = kzalloc(sizeof(*priv), GFP_KERNEL); if (!priv) return NULL; priv->key_idx = keyidx; - priv->tx_tfm = crypto_alloc_sync_skcipher("ecb(arc4)", 0, 0); - if (IS_ERR(priv->tx_tfm)) - goto free_priv; - priv->rx_tfm = crypto_alloc_sync_skcipher("ecb(arc4)", 0, 0); - if (IS_ERR(priv->rx_tfm)) - goto free_tx; - /* start WEP IV from a random value */ get_random_bytes(&priv->iv, 4); return priv; -free_tx: - crypto_free_sync_skcipher(priv->tx_tfm); -free_priv: - kfree(priv); - return NULL; } static void prism2_wep_deinit(void *priv) { - struct prism2_wep_data *_priv = priv; - - if (_priv) { - crypto_free_sync_skcipher(_priv->tx_tfm); - crypto_free_sync_skcipher(_priv->rx_tfm); - } - kfree(priv); + kzfree(priv); } /* Perform WEP encryption on given skb that has at least 4 bytes of headroom @@ -87,8 +72,6 @@ static int prism2_wep_encrypt(struct sk_buff *skb, int hdr_len, void *priv) struct cb_desc *tcb_desc = (struct cb_desc *)(skb->cb + MAX_DEV_ADDR_SIZE); u32 crc; u8 *icv; - struct scatterlist sg; - int err; if (skb_headroom(skb) < 4 || skb_tailroom(skb) < 4 || skb->len < hdr_len) @@ -124,8 +107,6 @@ static int prism2_wep_encrypt(struct sk_buff *skb, int hdr_len, void *priv) memcpy(key + 3, wep->key, wep->key_len); if (!tcb_desc->bHwSec) { - SYNC_SKCIPHER_REQUEST_ON_STACK(req, wep->tx_tfm); - /* Append little-endian CRC32 and encrypt it to produce ICV */ crc = ~crc32_le(~0, pos, len); icv = skb_put(skb, 4); @@ -134,16 +115,8 @@ static int prism2_wep_encrypt(struct sk_buff *skb, int hdr_len, void *priv) icv[2] = crc >> 16; icv[3] = crc >> 24; - crypto_sync_skcipher_setkey(wep->tx_tfm, key, klen); - sg_init_one(&sg, pos, len + 4); - - skcipher_request_set_sync_tfm(req, wep->tx_tfm); - skcipher_request_set_callback(req, 0, NULL, NULL); - skcipher_request_set_crypt(req, &sg, &sg, len + 4, NULL); - - err = crypto_skcipher_encrypt(req); - skcipher_request_zero(req); - return err; + arc4_setkey(&wep->tx_ctx_arc4, key, klen); + arc4_crypt(&wep->tx_ctx_arc4, pos, pos, len + 4); } return 0; @@ -166,8 +139,6 @@ static int prism2_wep_decrypt(struct sk_buff *skb, int hdr_len, void *priv) struct cb_desc *tcb_desc = (struct cb_desc *)(skb->cb + MAX_DEV_ADDR_SIZE); u32 crc; u8 icv[4]; - struct scatterlist sg; - int err; if (skb->len < hdr_len + 8) return -1; @@ -189,19 +160,8 @@ static int prism2_wep_decrypt(struct sk_buff *skb, int hdr_len, void *priv) plen = skb->len - hdr_len - 8; if (!tcb_desc->bHwSec) { - SYNC_SKCIPHER_REQUEST_ON_STACK(req, wep->rx_tfm); - - crypto_sync_skcipher_setkey(wep->rx_tfm, key, klen); - sg_init_one(&sg, pos, plen + 4); - - skcipher_request_set_sync_tfm(req, wep->rx_tfm); - skcipher_request_set_callback(req, 0, NULL, NULL); - skcipher_request_set_crypt(req, &sg, &sg, plen + 4, NULL); - - err = crypto_skcipher_decrypt(req); - skcipher_request_zero(req); - if (err) - return -7; + arc4_setkey(&wep->rx_ctx_arc4, key, klen); + arc4_crypt(&wep->rx_ctx_arc4, pos, pos, plen + 4); crc = ~crc32_le(~0, pos, plen); icv[0] = crc; diff --git a/include/crypto/algapi.h b/include/crypto/algapi.h index 143d884d65c7..18dd7a4aaf7d 100644 --- a/include/crypto/algapi.h +++ b/include/crypto/algapi.h @@ -10,7 +10,6 @@ #include <linux/crypto.h> #include <linux/list.h> #include <linux/kernel.h> -#include <linux/skbuff.h> /* * Maximum values for blocksize and alignmask, used to allocate @@ -27,6 +26,7 @@ struct crypto_instance; struct module; struct rtattr; struct seq_file; +struct sk_buff; struct crypto_type { unsigned int (*ctxsize)(struct crypto_alg *alg, u32 type, u32 mask); @@ -277,12 +277,6 @@ static inline int crypto_memneq(const void *a, const void *b, size_t size) return __crypto_memneq(a, b, size) != 0UL ? 1 : 0; } -static inline void crypto_yield(u32 flags) -{ - if (flags & CRYPTO_TFM_REQ_MAY_SLEEP) - cond_resched(); -} - int crypto_register_notifier(struct notifier_block *nb); int crypto_unregister_notifier(struct notifier_block *nb); diff --git a/include/crypto/cbc.h b/include/crypto/cbc.h deleted file mode 100644 index 2b6422db42e2..000000000000 --- a/include/crypto/cbc.h +++ /dev/null @@ -1,141 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * CBC: Cipher Block Chaining mode - * - * Copyright (c) 2016 Herbert Xu <herbert@gondor.apana.org.au> - */ - -#ifndef _CRYPTO_CBC_H -#define _CRYPTO_CBC_H - -#include <crypto/internal/skcipher.h> -#include <linux/string.h> -#include <linux/types.h> - -static inline int crypto_cbc_encrypt_segment( - struct skcipher_walk *walk, struct crypto_skcipher *tfm, - void (*fn)(struct crypto_skcipher *, const u8 *, u8 *)) -{ - unsigned int bsize = crypto_skcipher_blocksize(tfm); - unsigned int nbytes = walk->nbytes; - u8 *src = walk->src.virt.addr; - u8 *dst = walk->dst.virt.addr; - u8 *iv = walk->iv; - - do { - crypto_xor(iv, src, bsize); - fn(tfm, iv, dst); - memcpy(iv, dst, bsize); - - src += bsize; - dst += bsize; - } while ((nbytes -= bsize) >= bsize); - - return nbytes; -} - -static inline int crypto_cbc_encrypt_inplace( - struct skcipher_walk *walk, struct crypto_skcipher *tfm, - void (*fn)(struct crypto_skcipher *, const u8 *, u8 *)) -{ - unsigned int bsize = crypto_skcipher_blocksize(tfm); - unsigned int nbytes = walk->nbytes; - u8 *src = walk->src.virt.addr; - u8 *iv = walk->iv; - - do { - crypto_xor(src, iv, bsize); - fn(tfm, src, src); - iv = src; - - src += bsize; - } while ((nbytes -= bsize) >= bsize); - - memcpy(walk->iv, iv, bsize); - - return nbytes; -} - -static inline int crypto_cbc_encrypt_walk(struct skcipher_request *req, - void (*fn)(struct crypto_skcipher *, - const u8 *, u8 *)) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct skcipher_walk walk; - int err; - - err = skcipher_walk_virt(&walk, req, false); - - while (walk.nbytes) { - if (walk.src.virt.addr == walk.dst.virt.addr) - err = crypto_cbc_encrypt_inplace(&walk, tfm, fn); - else - err = crypto_cbc_encrypt_segment(&walk, tfm, fn); - err = skcipher_walk_done(&walk, err); - } - - return err; -} - -static inline int crypto_cbc_decrypt_segment( - struct skcipher_walk *walk, struct crypto_skcipher *tfm, - void (*fn)(struct crypto_skcipher *, const u8 *, u8 *)) -{ - unsigned int bsize = crypto_skcipher_blocksize(tfm); - unsigned int nbytes = walk->nbytes; - u8 *src = walk->src.virt.addr; - u8 *dst = walk->dst.virt.addr; - u8 *iv = walk->iv; - - do { - fn(tfm, src, dst); - crypto_xor(dst, iv, bsize); - iv = src; - - src += bsize; - dst += bsize; - } while ((nbytes -= bsize) >= bsize); - - memcpy(walk->iv, iv, bsize); - - return nbytes; -} - -static inline int crypto_cbc_decrypt_inplace( - struct skcipher_walk *walk, struct crypto_skcipher *tfm, - void (*fn)(struct crypto_skcipher *, const u8 *, u8 *)) -{ - unsigned int bsize = crypto_skcipher_blocksize(tfm); - unsigned int nbytes = walk->nbytes; - u8 *src = walk->src.virt.addr; - u8 last_iv[MAX_CIPHER_BLOCKSIZE]; - - /* Start of the last block. */ - src += nbytes - (nbytes & (bsize - 1)) - bsize; - memcpy(last_iv, src, bsize); - - for (;;) { - fn(tfm, src, src); - if ((nbytes -= bsize) < bsize) - break; - crypto_xor(src, src - bsize, bsize); - src -= bsize; - } - - crypto_xor(src, walk->iv, bsize); - memcpy(walk->iv, last_iv, bsize); - - return nbytes; -} - -static inline int crypto_cbc_decrypt_blocks( - struct skcipher_walk *walk, struct crypto_skcipher *tfm, - void (*fn)(struct crypto_skcipher *, const u8 *, u8 *)) -{ - if (walk->src.virt.addr == walk->dst.virt.addr) - return crypto_cbc_decrypt_inplace(walk, tfm, fn); - else - return crypto_cbc_decrypt_segment(walk, tfm, fn); -} - -#endif /* _CRYPTO_CBC_H */ diff --git a/include/crypto/hash.h b/include/crypto/hash.h index 0d1b403888c9..af2ff31ff619 100644 --- a/include/crypto/hash.h +++ b/include/crypto/hash.h @@ -59,11 +59,6 @@ struct ahash_request { void *__ctx[] CRYPTO_MINALIGN_ATTR; }; -#define AHASH_REQUEST_ON_STACK(name, ahash) \ - char __##name##_desc[sizeof(struct ahash_request) + \ - crypto_ahash_reqsize(ahash)] CRYPTO_MINALIGN_ATTR; \ - struct ahash_request *name = (void *)__##name##_desc - /** * struct ahash_alg - asynchronous message digest definition * @init: **[mandatory]** Initialize the transformation context. Intended only to initialize the @@ -123,6 +118,17 @@ struct ahash_request { * data so the transformation can continue from this point onward. No * data processing happens at this point. Driver must not use * req->result. + * @init_tfm: Initialize the cryptographic transformation object. + * This function is called only once at the instantiation + * time, right after the transformation context was + * allocated. In case the cryptographic hardware has + * some special requirements which need to be handled + * by software, this function shall check for the precise + * requirement of the transformation and put any software + * fallbacks in place. + * @exit_tfm: Deinitialize the cryptographic transformation object. + * This is a counterpart to @init_tfm, used to remove + * various changes set in @init_tfm. * @halg: see struct hash_alg_common */ struct ahash_alg { @@ -135,6 +141,8 @@ struct ahash_alg { int (*import)(struct ahash_request *req, const void *in); int (*setkey)(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen); + int (*init_tfm)(struct crypto_ahash *tfm); + void (*exit_tfm)(struct crypto_ahash *tfm); struct hash_alg_common halg; }; diff --git a/include/crypto/if_alg.h b/include/crypto/if_alg.h index ee6412314f8f..a5db86670bdf 100644 --- a/include/crypto/if_alg.h +++ b/include/crypto/if_alg.h @@ -46,6 +46,7 @@ struct af_alg_type { void *(*bind)(const char *name, u32 type, u32 mask); void (*release)(void *private); int (*setkey)(void *private, const u8 *key, unsigned int keylen); + int (*setentropy)(void *private, sockptr_t entropy, unsigned int len); int (*accept)(void *private, struct sock *sk); int (*accept_nokey)(void *private, struct sock *sk); int (*setauthsize)(void *private, unsigned int authsize); diff --git a/include/crypto/internal/hash.h b/include/crypto/internal/hash.h index 89f6f46ab2b8..0a288dddcf5b 100644 --- a/include/crypto/internal/hash.h +++ b/include/crypto/internal/hash.h @@ -62,25 +62,12 @@ struct crypto_shash_spawn { int crypto_hash_walk_done(struct crypto_hash_walk *walk, int err); int crypto_hash_walk_first(struct ahash_request *req, struct crypto_hash_walk *walk); -int crypto_ahash_walk_first(struct ahash_request *req, - struct crypto_hash_walk *walk); - -static inline int crypto_ahash_walk_done(struct crypto_hash_walk *walk, - int err) -{ - return crypto_hash_walk_done(walk, err); -} static inline int crypto_hash_walk_last(struct crypto_hash_walk *walk) { return !(walk->entrylen | walk->total); } -static inline int crypto_ahash_walk_last(struct crypto_hash_walk *walk) -{ - return crypto_hash_walk_last(walk); -} - int crypto_register_ahash(struct ahash_alg *alg); void crypto_unregister_ahash(struct ahash_alg *alg); int crypto_register_ahashes(struct ahash_alg *algs, int count); @@ -177,6 +164,12 @@ static inline struct ahash_instance *ahash_instance( return container_of(inst, struct ahash_instance, s.base); } +static inline struct ahash_instance *ahash_alg_instance( + struct crypto_ahash *ahash) +{ + return ahash_instance(crypto_tfm_alg_instance(&ahash->base)); +} + static inline void *ahash_instance_ctx(struct ahash_instance *inst) { return crypto_instance_ctx(ahash_crypto_instance(inst)); diff --git a/include/crypto/public_key.h b/include/crypto/public_key.h index 11f535cfb810..948c5203ca9c 100644 --- a/include/crypto/public_key.h +++ b/include/crypto/public_key.h @@ -12,6 +12,7 @@ #include <linux/keyctl.h> #include <linux/oid_registry.h> +#include <crypto/akcipher.h> /* * Cryptographic data for the public-key subtype of the asymmetric key type. @@ -44,6 +45,8 @@ struct public_key_signature { const char *pkey_algo; const char *hash_algo; const char *encoding; + const void *data; + unsigned int data_size; }; extern void public_key_signature_free(struct public_key_signature *sig); diff --git a/include/crypto/sm2.h b/include/crypto/sm2.h new file mode 100644 index 000000000000..af452556dcd4 --- /dev/null +++ b/include/crypto/sm2.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * sm2.h - SM2 asymmetric public-key algorithm + * as specified by OSCCA GM/T 0003.1-2012 -- 0003.5-2012 SM2 and + * described at https://tools.ietf.org/html/draft-shen-sm2-ecdsa-02 + * + * Copyright (c) 2020, Alibaba Group. + * Written by Tianjia Zhang <tianjia.zhang@linux.alibaba.com> + */ + +#ifndef _CRYPTO_SM2_H +#define _CRYPTO_SM2_H + +#include <crypto/sm3.h> +#include <crypto/akcipher.h> + +/* The default user id as specified in GM/T 0009-2012 */ +#define SM2_DEFAULT_USERID "1234567812345678" +#define SM2_DEFAULT_USERID_LEN 16 + +extern int sm2_compute_z_digest(struct crypto_akcipher *tfm, + const unsigned char *id, size_t id_len, + unsigned char dgst[SM3_DIGEST_SIZE]); + +#endif /* _CRYPTO_SM2_H */ diff --git a/include/crypto/sm3.h b/include/crypto/sm3.h index 1438942dc773..42ea21289ba9 100644 --- a/include/crypto/sm3.h +++ b/include/crypto/sm3.h @@ -35,6 +35,8 @@ struct shash_desc; extern int crypto_sm3_update(struct shash_desc *desc, const u8 *data, unsigned int len); +extern int crypto_sm3_final(struct shash_desc *desc, u8 *out); + extern int crypto_sm3_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *hash); #endif diff --git a/include/linux/mpi.h b/include/linux/mpi.h index 5d906dfbf3ed..3e5358f4de2f 100644 --- a/include/linux/mpi.h +++ b/include/linux/mpi.h @@ -40,21 +40,79 @@ struct gcry_mpi { typedef struct gcry_mpi *MPI; #define mpi_get_nlimbs(a) ((a)->nlimbs) +#define mpi_has_sign(a) ((a)->sign) /*-- mpiutil.c --*/ MPI mpi_alloc(unsigned nlimbs); +void mpi_clear(MPI a); void mpi_free(MPI a); int mpi_resize(MPI a, unsigned nlimbs); +static inline MPI mpi_new(unsigned int nbits) +{ + return mpi_alloc((nbits + BITS_PER_MPI_LIMB - 1) / BITS_PER_MPI_LIMB); +} + +MPI mpi_copy(MPI a); +MPI mpi_alloc_like(MPI a); +void mpi_snatch(MPI w, MPI u); +MPI mpi_set(MPI w, MPI u); +MPI mpi_set_ui(MPI w, unsigned long u); +MPI mpi_alloc_set_ui(unsigned long u); +void mpi_swap_cond(MPI a, MPI b, unsigned long swap); + +/* Constants used to return constant MPIs. See mpi_init if you + * want to add more constants. + */ +#define MPI_NUMBER_OF_CONSTANTS 6 +enum gcry_mpi_constants { + MPI_C_ZERO, + MPI_C_ONE, + MPI_C_TWO, + MPI_C_THREE, + MPI_C_FOUR, + MPI_C_EIGHT +}; + +MPI mpi_const(enum gcry_mpi_constants no); + /*-- mpicoder.c --*/ + +/* Different formats of external big integer representation. */ +enum gcry_mpi_format { + GCRYMPI_FMT_NONE = 0, + GCRYMPI_FMT_STD = 1, /* Twos complement stored without length. */ + GCRYMPI_FMT_PGP = 2, /* As used by OpenPGP (unsigned only). */ + GCRYMPI_FMT_SSH = 3, /* As used by SSH (like STD but with length). */ + GCRYMPI_FMT_HEX = 4, /* Hex format. */ + GCRYMPI_FMT_USG = 5, /* Like STD but unsigned. */ + GCRYMPI_FMT_OPAQUE = 8 /* Opaque format (some functions only). */ +}; + MPI mpi_read_raw_data(const void *xbuffer, size_t nbytes); MPI mpi_read_from_buffer(const void *buffer, unsigned *ret_nread); +int mpi_fromstr(MPI val, const char *str); +MPI mpi_scanval(const char *string); MPI mpi_read_raw_from_sgl(struct scatterlist *sgl, unsigned int len); void *mpi_get_buffer(MPI a, unsigned *nbytes, int *sign); int mpi_read_buffer(MPI a, uint8_t *buf, unsigned buf_len, unsigned *nbytes, int *sign); int mpi_write_to_sgl(MPI a, struct scatterlist *sg, unsigned nbytes, int *sign); +int mpi_print(enum gcry_mpi_format format, unsigned char *buffer, + size_t buflen, size_t *nwritten, MPI a); + +/*-- mpi-mod.c --*/ +void mpi_mod(MPI rem, MPI dividend, MPI divisor); + +/* Context used with Barrett reduction. */ +struct barrett_ctx_s; +typedef struct barrett_ctx_s *mpi_barrett_t; + +mpi_barrett_t mpi_barrett_init(MPI m, int copy); +void mpi_barrett_free(mpi_barrett_t ctx); +void mpi_mod_barrett(MPI r, MPI x, mpi_barrett_t ctx); +void mpi_mul_barrett(MPI w, MPI u, MPI v, mpi_barrett_t ctx); /*-- mpi-pow.c --*/ int mpi_powm(MPI res, MPI base, MPI exp, MPI mod); @@ -62,6 +120,7 @@ int mpi_powm(MPI res, MPI base, MPI exp, MPI mod); /*-- mpi-cmp.c --*/ int mpi_cmp_ui(MPI u, ulong v); int mpi_cmp(MPI u, MPI v); +int mpi_cmpabs(MPI u, MPI v); /*-- mpi-sub-ui.c --*/ int mpi_sub_ui(MPI w, MPI u, unsigned long vval); @@ -69,6 +128,139 @@ int mpi_sub_ui(MPI w, MPI u, unsigned long vval); /*-- mpi-bit.c --*/ void mpi_normalize(MPI a); unsigned mpi_get_nbits(MPI a); +int mpi_test_bit(MPI a, unsigned int n); +void mpi_set_bit(MPI a, unsigned int n); +void mpi_set_highbit(MPI a, unsigned int n); +void mpi_clear_highbit(MPI a, unsigned int n); +void mpi_clear_bit(MPI a, unsigned int n); +void mpi_rshift_limbs(MPI a, unsigned int count); +void mpi_rshift(MPI x, MPI a, unsigned int n); +void mpi_lshift_limbs(MPI a, unsigned int count); +void mpi_lshift(MPI x, MPI a, unsigned int n); + +/*-- mpi-add.c --*/ +void mpi_add_ui(MPI w, MPI u, unsigned long v); +void mpi_add(MPI w, MPI u, MPI v); +void mpi_sub(MPI w, MPI u, MPI v); +void mpi_addm(MPI w, MPI u, MPI v, MPI m); +void mpi_subm(MPI w, MPI u, MPI v, MPI m); + +/*-- mpi-mul.c --*/ +void mpi_mul(MPI w, MPI u, MPI v); +void mpi_mulm(MPI w, MPI u, MPI v, MPI m); + +/*-- mpi-div.c --*/ +void mpi_tdiv_r(MPI rem, MPI num, MPI den); +void mpi_fdiv_r(MPI rem, MPI dividend, MPI divisor); +void mpi_fdiv_q(MPI quot, MPI dividend, MPI divisor); + +/*-- mpi-inv.c --*/ +int mpi_invm(MPI x, MPI a, MPI n); + +/*-- ec.c --*/ + +/* Object to represent a point in projective coordinates */ +struct gcry_mpi_point { + MPI x; + MPI y; + MPI z; +}; + +typedef struct gcry_mpi_point *MPI_POINT; + +/* Models describing an elliptic curve */ +enum gcry_mpi_ec_models { + /* The Short Weierstrass equation is + * y^2 = x^3 + ax + b + */ + MPI_EC_WEIERSTRASS = 0, + /* The Montgomery equation is + * by^2 = x^3 + ax^2 + x + */ + MPI_EC_MONTGOMERY, + /* The Twisted Edwards equation is + * ax^2 + y^2 = 1 + bx^2y^2 + * Note that we use 'b' instead of the commonly used 'd'. + */ + MPI_EC_EDWARDS +}; + +/* Dialects used with elliptic curves */ +enum ecc_dialects { + ECC_DIALECT_STANDARD = 0, + ECC_DIALECT_ED25519, + ECC_DIALECT_SAFECURVE +}; + +/* This context is used with all our EC functions. */ +struct mpi_ec_ctx { + enum gcry_mpi_ec_models model; /* The model describing this curve. */ + enum ecc_dialects dialect; /* The ECC dialect used with the curve. */ + int flags; /* Public key flags (not always used). */ + unsigned int nbits; /* Number of bits. */ + + /* Domain parameters. Note that they may not all be set and if set + * the MPIs may be flaged as constant. + */ + MPI p; /* Prime specifying the field GF(p). */ + MPI a; /* First coefficient of the Weierstrass equation. */ + MPI b; /* Second coefficient of the Weierstrass equation. */ + MPI_POINT G; /* Base point (generator). */ + MPI n; /* Order of G. */ + unsigned int h; /* Cofactor. */ + + /* The actual key. May not be set. */ + MPI_POINT Q; /* Public key. */ + MPI d; /* Private key. */ + + const char *name; /* Name of the curve. */ + + /* This structure is private to mpi/ec.c! */ + struct { + struct { + unsigned int a_is_pminus3:1; + unsigned int two_inv_p:1; + } valid; /* Flags to help setting the helper vars below. */ + + int a_is_pminus3; /* True if A = P - 3. */ + + MPI two_inv_p; + + mpi_barrett_t p_barrett; + + /* Scratch variables. */ + MPI scratch[11]; + + /* Helper for fast reduction. */ + /* int nist_nbits; /\* If this is a NIST curve, the # of bits. *\/ */ + /* MPI s[10]; */ + /* MPI c; */ + } t; + + /* Curve specific computation routines for the field. */ + void (*addm)(MPI w, MPI u, MPI v, struct mpi_ec_ctx *ctx); + void (*subm)(MPI w, MPI u, MPI v, struct mpi_ec_ctx *ec); + void (*mulm)(MPI w, MPI u, MPI v, struct mpi_ec_ctx *ctx); + void (*pow2)(MPI w, const MPI b, struct mpi_ec_ctx *ctx); + void (*mul2)(MPI w, MPI u, struct mpi_ec_ctx *ctx); +}; + +void mpi_ec_init(struct mpi_ec_ctx *ctx, enum gcry_mpi_ec_models model, + enum ecc_dialects dialect, + int flags, MPI p, MPI a, MPI b); +void mpi_ec_deinit(struct mpi_ec_ctx *ctx); +MPI_POINT mpi_point_new(unsigned int nbits); +void mpi_point_release(MPI_POINT p); +void mpi_point_init(MPI_POINT p); +void mpi_point_free_parts(MPI_POINT p); +int mpi_ec_get_affine(MPI x, MPI y, MPI_POINT point, struct mpi_ec_ctx *ctx); +void mpi_ec_add_points(MPI_POINT result, + MPI_POINT p1, MPI_POINT p2, + struct mpi_ec_ctx *ctx); +void mpi_ec_mul_point(MPI_POINT result, + MPI scalar, MPI_POINT point, + struct mpi_ec_ctx *ctx); +int mpi_ec_curve_point(MPI_POINT point, struct mpi_ec_ctx *ctx); /* inline functions */ diff --git a/include/linux/oid_registry.h b/include/linux/oid_registry.h index 657d6bf2c064..4462ed2c18cd 100644 --- a/include/linux/oid_registry.h +++ b/include/linux/oid_registry.h @@ -107,6 +107,12 @@ enum OID { OID_gostTC26Sign512B, /* 1.2.643.7.1.2.1.2.2 */ OID_gostTC26Sign512C, /* 1.2.643.7.1.2.1.2.3 */ + /* OSCCA */ + OID_sm2, /* 1.2.156.10197.1.301 */ + OID_sm3, /* 1.2.156.10197.1.401 */ + OID_SM2_with_SM3, /* 1.2.156.10197.1.501 */ + OID_sm3WithRSAEncryption, /* 1.2.156.10197.1.504 */ + OID__NR }; diff --git a/include/linux/sunrpc/gss_krb5.h b/include/linux/sunrpc/gss_krb5.h index e8f8ffe7448b..91f43d86879d 100644 --- a/include/linux/sunrpc/gss_krb5.h +++ b/include/linux/sunrpc/gss_krb5.h @@ -141,14 +141,12 @@ enum sgn_alg { SGN_ALG_MD2_5 = 0x0001, SGN_ALG_DES_MAC = 0x0002, SGN_ALG_3 = 0x0003, /* not published */ - SGN_ALG_HMAC_MD5 = 0x0011, /* microsoft w2k; no support */ SGN_ALG_HMAC_SHA1_DES3_KD = 0x0004 }; enum seal_alg { SEAL_ALG_NONE = 0xffff, SEAL_ALG_DES = 0x0000, SEAL_ALG_1 = 0x0001, /* not published */ - SEAL_ALG_MICROSOFT_RC4 = 0x0010,/* microsoft w2k; no support */ SEAL_ALG_DES3KD = 0x0002 }; @@ -316,14 +314,5 @@ gss_krb5_aes_decrypt(struct krb5_ctx *kctx, u32 offset, u32 len, struct xdr_buf *buf, u32 *plainoffset, u32 *plainlen); -int -krb5_rc4_setup_seq_key(struct krb5_ctx *kctx, - struct crypto_sync_skcipher *cipher, - unsigned char *cksum); - -int -krb5_rc4_setup_enc_key(struct krb5_ctx *kctx, - struct crypto_sync_skcipher *cipher, - s32 seqnum); void gss_krb5_make_confounder(char *p, u32 conflen); diff --git a/include/linux/sunrpc/gss_krb5_enctypes.h b/include/linux/sunrpc/gss_krb5_enctypes.h index 981c89cef19d..87eea679d750 100644 --- a/include/linux/sunrpc/gss_krb5_enctypes.h +++ b/include/linux/sunrpc/gss_krb5_enctypes.h @@ -13,15 +13,13 @@ #ifdef CONFIG_SUNRPC_DISABLE_INSECURE_ENCTYPES /* - * NB: This list includes encryption types that were deprecated - * by RFC 8429 (DES3_CBC_SHA1 and ARCFOUR_HMAC). + * NB: This list includes DES3_CBC_SHA1, which was deprecated by RFC 8429. * * ENCTYPE_AES256_CTS_HMAC_SHA1_96 * ENCTYPE_AES128_CTS_HMAC_SHA1_96 * ENCTYPE_DES3_CBC_SHA1 - * ENCTYPE_ARCFOUR_HMAC */ -#define KRB5_SUPPORTED_ENCTYPES "18,17,16,23" +#define KRB5_SUPPORTED_ENCTYPES "18,17,16" #else /* CONFIG_SUNRPC_DISABLE_INSECURE_ENCTYPES */ @@ -32,12 +30,11 @@ * ENCTYPE_AES256_CTS_HMAC_SHA1_96 * ENCTYPE_AES128_CTS_HMAC_SHA1_96 * ENCTYPE_DES3_CBC_SHA1 - * ENCTYPE_ARCFOUR_HMAC * ENCTYPE_DES_CBC_MD5 * ENCTYPE_DES_CBC_CRC * ENCTYPE_DES_CBC_MD4 */ -#define KRB5_SUPPORTED_ENCTYPES "18,17,16,23,3,1,2" +#define KRB5_SUPPORTED_ENCTYPES "18,17,16,3,1,2" #endif /* CONFIG_SUNRPC_DISABLE_INSECURE_ENCTYPES */ diff --git a/include/uapi/linux/if_alg.h b/include/uapi/linux/if_alg.h index bc2bcdec377b..60b7c2efd921 100644 --- a/include/uapi/linux/if_alg.h +++ b/include/uapi/linux/if_alg.h @@ -35,6 +35,7 @@ struct af_alg_iv { #define ALG_SET_OP 3 #define ALG_SET_AEAD_ASSOCLEN 4 #define ALG_SET_AEAD_AUTHSIZE 5 +#define ALG_SET_DRBG_ENTROPY 6 /* Operations */ #define ALG_OP_DECRYPT 0 diff --git a/lib/crypto/chacha20poly1305.c b/lib/crypto/chacha20poly1305.c index 431e04280332..5850f3b87359 100644 --- a/lib/crypto/chacha20poly1305.c +++ b/lib/crypto/chacha20poly1305.c @@ -251,9 +251,7 @@ bool chacha20poly1305_crypt_sg_inplace(struct scatterlist *src, poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf)); } - flags = SG_MITER_TO_SG; - if (!preemptible()) - flags |= SG_MITER_ATOMIC; + flags = SG_MITER_TO_SG | SG_MITER_ATOMIC; sg_miter_start(&miter, src, sg_nents(src), flags); diff --git a/lib/mpi/Makefile b/lib/mpi/Makefile index 43b8fce14079..6e6ef9a34fe1 100644 --- a/lib/mpi/Makefile +++ b/lib/mpi/Makefile @@ -13,10 +13,16 @@ mpi-y = \ generic_mpih-rshift.o \ generic_mpih-sub1.o \ generic_mpih-add1.o \ + ec.o \ mpicoder.o \ + mpi-add.o \ mpi-bit.o \ mpi-cmp.o \ mpi-sub-ui.o \ + mpi-div.o \ + mpi-inv.o \ + mpi-mod.o \ + mpi-mul.o \ mpih-cmp.o \ mpih-div.o \ mpih-mul.o \ diff --git a/lib/mpi/ec.c b/lib/mpi/ec.c new file mode 100644 index 000000000000..c21470122dfc --- /dev/null +++ b/lib/mpi/ec.c @@ -0,0 +1,1509 @@ +/* ec.c - Elliptic Curve functions + * Copyright (C) 2007 Free Software Foundation, Inc. + * Copyright (C) 2013 g10 Code GmbH + * + * This file is part of Libgcrypt. + * + * Libgcrypt is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation; either version 2.1 of + * the License, or (at your option) any later version. + * + * Libgcrypt is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this program; if not, see <http://www.gnu.org/licenses/>. + */ + +#include "mpi-internal.h" +#include "longlong.h" + +#define point_init(a) mpi_point_init((a)) +#define point_free(a) mpi_point_free_parts((a)) + +#define log_error(fmt, ...) pr_err(fmt, ##__VA_ARGS__) +#define log_fatal(fmt, ...) pr_err(fmt, ##__VA_ARGS__) + +#define DIM(v) (sizeof(v)/sizeof((v)[0])) + + +/* Create a new point option. NBITS gives the size in bits of one + * coordinate; it is only used to pre-allocate some resources and + * might also be passed as 0 to use a default value. + */ +MPI_POINT mpi_point_new(unsigned int nbits) +{ + MPI_POINT p; + + (void)nbits; /* Currently not used. */ + + p = kmalloc(sizeof(*p), GFP_KERNEL); + if (p) + mpi_point_init(p); + return p; +} +EXPORT_SYMBOL_GPL(mpi_point_new); + +/* Release the point object P. P may be NULL. */ +void mpi_point_release(MPI_POINT p) +{ + if (p) { + mpi_point_free_parts(p); + kfree(p); + } +} +EXPORT_SYMBOL_GPL(mpi_point_release); + +/* Initialize the fields of a point object. gcry_mpi_point_free_parts + * may be used to release the fields. + */ +void mpi_point_init(MPI_POINT p) +{ + p->x = mpi_new(0); + p->y = mpi_new(0); + p->z = mpi_new(0); +} +EXPORT_SYMBOL_GPL(mpi_point_init); + +/* Release the parts of a point object. */ +void mpi_point_free_parts(MPI_POINT p) +{ + mpi_free(p->x); p->x = NULL; + mpi_free(p->y); p->y = NULL; + mpi_free(p->z); p->z = NULL; +} +EXPORT_SYMBOL_GPL(mpi_point_free_parts); + +/* Set the value from S into D. */ +static void point_set(MPI_POINT d, MPI_POINT s) +{ + mpi_set(d->x, s->x); + mpi_set(d->y, s->y); + mpi_set(d->z, s->z); +} + +static void point_resize(MPI_POINT p, struct mpi_ec_ctx *ctx) +{ + size_t nlimbs = ctx->p->nlimbs; + + mpi_resize(p->x, nlimbs); + p->x->nlimbs = nlimbs; + mpi_resize(p->z, nlimbs); + p->z->nlimbs = nlimbs; + + if (ctx->model != MPI_EC_MONTGOMERY) { + mpi_resize(p->y, nlimbs); + p->y->nlimbs = nlimbs; + } +} + +static void point_swap_cond(MPI_POINT d, MPI_POINT s, unsigned long swap, + struct mpi_ec_ctx *ctx) +{ + mpi_swap_cond(d->x, s->x, swap); + if (ctx->model != MPI_EC_MONTGOMERY) + mpi_swap_cond(d->y, s->y, swap); + mpi_swap_cond(d->z, s->z, swap); +} + + +/* W = W mod P. */ +static void ec_mod(MPI w, struct mpi_ec_ctx *ec) +{ + if (ec->t.p_barrett) + mpi_mod_barrett(w, w, ec->t.p_barrett); + else + mpi_mod(w, w, ec->p); +} + +static void ec_addm(MPI w, MPI u, MPI v, struct mpi_ec_ctx *ctx) +{ + mpi_add(w, u, v); + ec_mod(w, ctx); +} + +static void ec_subm(MPI w, MPI u, MPI v, struct mpi_ec_ctx *ec) +{ + mpi_sub(w, u, v); + while (w->sign) + mpi_add(w, w, ec->p); + /*ec_mod(w, ec);*/ +} + +static void ec_mulm(MPI w, MPI u, MPI v, struct mpi_ec_ctx *ctx) +{ + mpi_mul(w, u, v); + ec_mod(w, ctx); +} + +/* W = 2 * U mod P. */ +static void ec_mul2(MPI w, MPI u, struct mpi_ec_ctx *ctx) +{ + mpi_lshift(w, u, 1); + ec_mod(w, ctx); +} + +static void ec_powm(MPI w, const MPI b, const MPI e, + struct mpi_ec_ctx *ctx) +{ + mpi_powm(w, b, e, ctx->p); + /* mpi_abs(w); */ +} + +/* Shortcut for + * ec_powm(B, B, mpi_const(MPI_C_TWO), ctx); + * for easier optimization. + */ +static void ec_pow2(MPI w, const MPI b, struct mpi_ec_ctx *ctx) +{ + /* Using mpi_mul is slightly faster (at least on amd64). */ + /* mpi_powm(w, b, mpi_const(MPI_C_TWO), ctx->p); */ + ec_mulm(w, b, b, ctx); +} + +/* Shortcut for + * ec_powm(B, B, mpi_const(MPI_C_THREE), ctx); + * for easier optimization. + */ +static void ec_pow3(MPI w, const MPI b, struct mpi_ec_ctx *ctx) +{ + mpi_powm(w, b, mpi_const(MPI_C_THREE), ctx->p); +} + +static void ec_invm(MPI x, MPI a, struct mpi_ec_ctx *ctx) +{ + if (!mpi_invm(x, a, ctx->p)) + log_error("ec_invm: inverse does not exist:\n"); +} + +static void mpih_set_cond(mpi_ptr_t wp, mpi_ptr_t up, + mpi_size_t usize, unsigned long set) +{ + mpi_size_t i; + mpi_limb_t mask = ((mpi_limb_t)0) - set; + mpi_limb_t x; + + for (i = 0; i < usize; i++) { + x = mask & (wp[i] ^ up[i]); + wp[i] = wp[i] ^ x; + } +} + +/* Routines for 2^255 - 19. */ + +#define LIMB_SIZE_25519 ((256+BITS_PER_MPI_LIMB-1)/BITS_PER_MPI_LIMB) + +static void ec_addm_25519(MPI w, MPI u, MPI v, struct mpi_ec_ctx *ctx) +{ + mpi_ptr_t wp, up, vp; + mpi_size_t wsize = LIMB_SIZE_25519; + mpi_limb_t n[LIMB_SIZE_25519]; + mpi_limb_t borrow; + + if (w->nlimbs != wsize || u->nlimbs != wsize || v->nlimbs != wsize) + log_bug("addm_25519: different sizes\n"); + + memset(n, 0, sizeof(n)); + up = u->d; + vp = v->d; + wp = w->d; + + mpihelp_add_n(wp, up, vp, wsize); + borrow = mpihelp_sub_n(wp, wp, ctx->p->d, wsize); + mpih_set_cond(n, ctx->p->d, wsize, (borrow != 0UL)); + mpihelp_add_n(wp, wp, n, wsize); + wp[LIMB_SIZE_25519-1] &= ~((mpi_limb_t)1 << (255 % BITS_PER_MPI_LIMB)); +} + +static void ec_subm_25519(MPI w, MPI u, MPI v, struct mpi_ec_ctx *ctx) +{ + mpi_ptr_t wp, up, vp; + mpi_size_t wsize = LIMB_SIZE_25519; + mpi_limb_t n[LIMB_SIZE_25519]; + mpi_limb_t borrow; + + if (w->nlimbs != wsize || u->nlimbs != wsize || v->nlimbs != wsize) + log_bug("subm_25519: different sizes\n"); + + memset(n, 0, sizeof(n)); + up = u->d; + vp = v->d; + wp = w->d; + + borrow = mpihelp_sub_n(wp, up, vp, wsize); + mpih_set_cond(n, ctx->p->d, wsize, (borrow != 0UL)); + mpihelp_add_n(wp, wp, n, wsize); + wp[LIMB_SIZE_25519-1] &= ~((mpi_limb_t)1 << (255 % BITS_PER_MPI_LIMB)); +} + +static void ec_mulm_25519(MPI w, MPI u, MPI v, struct mpi_ec_ctx *ctx) +{ + mpi_ptr_t wp, up, vp; + mpi_size_t wsize = LIMB_SIZE_25519; + mpi_limb_t n[LIMB_SIZE_25519*2]; + mpi_limb_t m[LIMB_SIZE_25519+1]; + mpi_limb_t cy; + int msb; + + (void)ctx; + if (w->nlimbs != wsize || u->nlimbs != wsize || v->nlimbs != wsize) + log_bug("mulm_25519: different sizes\n"); + + up = u->d; + vp = v->d; + wp = w->d; + + mpihelp_mul_n(n, up, vp, wsize); + memcpy(wp, n, wsize * BYTES_PER_MPI_LIMB); + wp[LIMB_SIZE_25519-1] &= ~((mpi_limb_t)1 << (255 % BITS_PER_MPI_LIMB)); + + memcpy(m, n+LIMB_SIZE_25519-1, (wsize+1) * BYTES_PER_MPI_LIMB); + mpihelp_rshift(m, m, LIMB_SIZE_25519+1, (255 % BITS_PER_MPI_LIMB)); + + memcpy(n, m, wsize * BYTES_PER_MPI_LIMB); + cy = mpihelp_lshift(m, m, LIMB_SIZE_25519, 4); + m[LIMB_SIZE_25519] = cy; + cy = mpihelp_add_n(m, m, n, wsize); + m[LIMB_SIZE_25519] += cy; + cy = mpihelp_add_n(m, m, n, wsize); + m[LIMB_SIZE_25519] += cy; + cy = mpihelp_add_n(m, m, n, wsize); + m[LIMB_SIZE_25519] += cy; + + cy = mpihelp_add_n(wp, wp, m, wsize); + m[LIMB_SIZE_25519] += cy; + + memset(m, 0, wsize * BYTES_PER_MPI_LIMB); + msb = (wp[LIMB_SIZE_25519-1] >> (255 % BITS_PER_MPI_LIMB)); + m[0] = (m[LIMB_SIZE_25519] * 2 + msb) * 19; + wp[LIMB_SIZE_25519-1] &= ~((mpi_limb_t)1 << (255 % BITS_PER_MPI_LIMB)); + mpihelp_add_n(wp, wp, m, wsize); + + m[0] = 0; + cy = mpihelp_sub_n(wp, wp, ctx->p->d, wsize); + mpih_set_cond(m, ctx->p->d, wsize, (cy != 0UL)); + mpihelp_add_n(wp, wp, m, wsize); +} + +static void ec_mul2_25519(MPI w, MPI u, struct mpi_ec_ctx *ctx) +{ + ec_addm_25519(w, u, u, ctx); +} + +static void ec_pow2_25519(MPI w, const MPI b, struct mpi_ec_ctx *ctx) +{ + ec_mulm_25519(w, b, b, ctx); +} + +/* Routines for 2^448 - 2^224 - 1. */ + +#define LIMB_SIZE_448 ((448+BITS_PER_MPI_LIMB-1)/BITS_PER_MPI_LIMB) +#define LIMB_SIZE_HALF_448 ((LIMB_SIZE_448+1)/2) + +static void ec_addm_448(MPI w, MPI u, MPI v, struct mpi_ec_ctx *ctx) +{ + mpi_ptr_t wp, up, vp; + mpi_size_t wsize = LIMB_SIZE_448; + mpi_limb_t n[LIMB_SIZE_448]; + mpi_limb_t cy; + + if (w->nlimbs != wsize || u->nlimbs != wsize || v->nlimbs != wsize) + log_bug("addm_448: different sizes\n"); + + memset(n, 0, sizeof(n)); + up = u->d; + vp = v->d; + wp = w->d; + + cy = mpihelp_add_n(wp, up, vp, wsize); + mpih_set_cond(n, ctx->p->d, wsize, (cy != 0UL)); + mpihelp_sub_n(wp, wp, n, wsize); +} + +static void ec_subm_448(MPI w, MPI u, MPI v, struct mpi_ec_ctx *ctx) +{ + mpi_ptr_t wp, up, vp; + mpi_size_t wsize = LIMB_SIZE_448; + mpi_limb_t n[LIMB_SIZE_448]; + mpi_limb_t borrow; + + if (w->nlimbs != wsize || u->nlimbs != wsize || v->nlimbs != wsize) + log_bug("subm_448: different sizes\n"); + + memset(n, 0, sizeof(n)); + up = u->d; + vp = v->d; + wp = w->d; + + borrow = mpihelp_sub_n(wp, up, vp, wsize); + mpih_set_cond(n, ctx->p->d, wsize, (borrow != 0UL)); + mpihelp_add_n(wp, wp, n, wsize); +} + +static void ec_mulm_448(MPI w, MPI u, MPI v, struct mpi_ec_ctx *ctx) +{ + mpi_ptr_t wp, up, vp; + mpi_size_t wsize = LIMB_SIZE_448; + mpi_limb_t n[LIMB_SIZE_448*2]; + mpi_limb_t a2[LIMB_SIZE_HALF_448]; + mpi_limb_t a3[LIMB_SIZE_HALF_448]; + mpi_limb_t b0[LIMB_SIZE_HALF_448]; + mpi_limb_t b1[LIMB_SIZE_HALF_448]; + mpi_limb_t cy; + int i; +#if (LIMB_SIZE_HALF_448 > LIMB_SIZE_448/2) + mpi_limb_t b1_rest, a3_rest; +#endif + + if (w->nlimbs != wsize || u->nlimbs != wsize || v->nlimbs != wsize) + log_bug("mulm_448: different sizes\n"); + + up = u->d; + vp = v->d; + wp = w->d; + + mpihelp_mul_n(n, up, vp, wsize); + + for (i = 0; i < (wsize + 1) / 2; i++) { + b0[i] = n[i]; + b1[i] = n[i+wsize/2]; + a2[i] = n[i+wsize]; + a3[i] = n[i+wsize+wsize/2]; + } + +#if (LIMB_SIZE_HALF_448 > LIMB_SIZE_448/2) + b0[LIMB_SIZE_HALF_448-1] &= ((mpi_limb_t)1UL << 32)-1; + a2[LIMB_SIZE_HALF_448-1] &= ((mpi_limb_t)1UL << 32)-1; + + b1_rest = 0; + a3_rest = 0; + + for (i = (wsize + 1) / 2 - 1; i >= 0; i--) { + mpi_limb_t b1v, a3v; + b1v = b1[i]; + a3v = a3[i]; + b1[i] = (b1_rest << 32) | (b1v >> 32); + a3[i] = (a3_rest << 32) | (a3v >> 32); + b1_rest = b1v & (((mpi_limb_t)1UL << 32)-1); + a3_rest = a3v & (((mpi_limb_t)1UL << 32)-1); + } +#endif + + cy = mpihelp_add_n(b0, b0, a2, LIMB_SIZE_HALF_448); + cy += mpihelp_add_n(b0, b0, a3, LIMB_SIZE_HALF_448); + for (i = 0; i < (wsize + 1) / 2; i++) + wp[i] = b0[i]; +#if (LIMB_SIZE_HALF_448 > LIMB_SIZE_448/2) + wp[LIMB_SIZE_HALF_448-1] &= (((mpi_limb_t)1UL << 32)-1); +#endif + +#if (LIMB_SIZE_HALF_448 > LIMB_SIZE_448/2) + cy = b0[LIMB_SIZE_HALF_448-1] >> 32; +#endif + + cy = mpihelp_add_1(b1, b1, LIMB_SIZE_HALF_448, cy); + cy += mpihelp_add_n(b1, b1, a2, LIMB_SIZE_HALF_448); + cy += mpihelp_add_n(b1, b1, a3, LIMB_SIZE_HALF_448); + cy += mpihelp_add_n(b1, b1, a3, LIMB_SIZE_HALF_448); +#if (LIMB_SIZE_HALF_448 > LIMB_SIZE_448/2) + b1_rest = 0; + for (i = (wsize + 1) / 2 - 1; i >= 0; i--) { + mpi_limb_t b1v = b1[i]; + b1[i] = (b1_rest << 32) | (b1v >> 32); + b1_rest = b1v & (((mpi_limb_t)1UL << 32)-1); + } + wp[LIMB_SIZE_HALF_448-1] |= (b1_rest << 32); +#endif + for (i = 0; i < wsize / 2; i++) + wp[i+(wsize + 1) / 2] = b1[i]; + +#if (LIMB_SIZE_HALF_448 > LIMB_SIZE_448/2) + cy = b1[LIMB_SIZE_HALF_448-1]; +#endif + + memset(n, 0, wsize * BYTES_PER_MPI_LIMB); + +#if (LIMB_SIZE_HALF_448 > LIMB_SIZE_448/2) + n[LIMB_SIZE_HALF_448-1] = cy << 32; +#else + n[LIMB_SIZE_HALF_448] = cy; +#endif + n[0] = cy; + mpihelp_add_n(wp, wp, n, wsize); + + memset(n, 0, wsize * BYTES_PER_MPI_LIMB); + cy = mpihelp_sub_n(wp, wp, ctx->p->d, wsize); + mpih_set_cond(n, ctx->p->d, wsize, (cy != 0UL)); + mpihelp_add_n(wp, wp, n, wsize); +} + +static void ec_mul2_448(MPI w, MPI u, struct mpi_ec_ctx *ctx) +{ + ec_addm_448(w, u, u, ctx); +} + +static void ec_pow2_448(MPI w, const MPI b, struct mpi_ec_ctx *ctx) +{ + ec_mulm_448(w, b, b, ctx); +} + +struct field_table { + const char *p; + + /* computation routines for the field. */ + void (*addm)(MPI w, MPI u, MPI v, struct mpi_ec_ctx *ctx); + void (*subm)(MPI w, MPI u, MPI v, struct mpi_ec_ctx *ctx); + void (*mulm)(MPI w, MPI u, MPI v, struct mpi_ec_ctx *ctx); + void (*mul2)(MPI w, MPI u, struct mpi_ec_ctx *ctx); + void (*pow2)(MPI w, const MPI b, struct mpi_ec_ctx *ctx); +}; + +static const struct field_table field_table[] = { + { + "0x7FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFED", + ec_addm_25519, + ec_subm_25519, + ec_mulm_25519, + ec_mul2_25519, + ec_pow2_25519 + }, + { + "0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE" + "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF", + ec_addm_448, + ec_subm_448, + ec_mulm_448, + ec_mul2_448, + ec_pow2_448 + }, + { NULL, NULL, NULL, NULL, NULL, NULL }, +}; + +/* Force recomputation of all helper variables. */ +static void mpi_ec_get_reset(struct mpi_ec_ctx *ec) +{ + ec->t.valid.a_is_pminus3 = 0; + ec->t.valid.two_inv_p = 0; +} + +/* Accessor for helper variable. */ +static int ec_get_a_is_pminus3(struct mpi_ec_ctx *ec) +{ + MPI tmp; + + if (!ec->t.valid.a_is_pminus3) { + ec->t.valid.a_is_pminus3 = 1; + tmp = mpi_alloc_like(ec->p); + mpi_sub_ui(tmp, ec->p, 3); + ec->t.a_is_pminus3 = !mpi_cmp(ec->a, tmp); + mpi_free(tmp); + } + + return ec->t.a_is_pminus3; +} + +/* Accessor for helper variable. */ +static MPI ec_get_two_inv_p(struct mpi_ec_ctx *ec) +{ + if (!ec->t.valid.two_inv_p) { + ec->t.valid.two_inv_p = 1; + if (!ec->t.two_inv_p) + ec->t.two_inv_p = mpi_alloc(0); + ec_invm(ec->t.two_inv_p, mpi_const(MPI_C_TWO), ec); + } + return ec->t.two_inv_p; +} + +static const char *const curve25519_bad_points[] = { + "0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffed", + "0x0000000000000000000000000000000000000000000000000000000000000000", + "0x0000000000000000000000000000000000000000000000000000000000000001", + "0x00b8495f16056286fdb1329ceb8d09da6ac49ff1fae35616aeb8413b7c7aebe0", + "0x57119fd0dd4e22d8868e1c58c45c44045bef839c55b1d0b1248c50a3bc959c5f", + "0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffec", + "0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffee", + NULL +}; + +static const char *const curve448_bad_points[] = { + "0xfffffffffffffffffffffffffffffffffffffffffffffffffffffffe" + "ffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "0x00000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000", + "0x00000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000001", + "0xfffffffffffffffffffffffffffffffffffffffffffffffffffffffe" + "fffffffffffffffffffffffffffffffffffffffffffffffffffffffe", + "0xffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + "00000000000000000000000000000000000000000000000000000000", + NULL +}; + +static const char *const *bad_points_table[] = { + curve25519_bad_points, + curve448_bad_points, +}; + +static void mpi_ec_coefficient_normalize(MPI a, MPI p) +{ + if (a->sign) { + mpi_resize(a, p->nlimbs); + mpihelp_sub_n(a->d, p->d, a->d, p->nlimbs); + a->nlimbs = p->nlimbs; + a->sign = 0; + } +} + +/* This function initialized a context for elliptic curve based on the + * field GF(p). P is the prime specifying this field, A is the first + * coefficient. CTX is expected to be zeroized. + */ +void mpi_ec_init(struct mpi_ec_ctx *ctx, enum gcry_mpi_ec_models model, + enum ecc_dialects dialect, + int flags, MPI p, MPI a, MPI b) +{ + int i; + static int use_barrett = -1 /* TODO: 1 or -1 */; + + mpi_ec_coefficient_normalize(a, p); + mpi_ec_coefficient_normalize(b, p); + + /* Fixme: Do we want to check some constraints? e.g. a < p */ + + ctx->model = model; + ctx->dialect = dialect; + ctx->flags = flags; + if (dialect == ECC_DIALECT_ED25519) + ctx->nbits = 256; + else + ctx->nbits = mpi_get_nbits(p); + ctx->p = mpi_copy(p); + ctx->a = mpi_copy(a); + ctx->b = mpi_copy(b); + + ctx->t.p_barrett = use_barrett > 0 ? mpi_barrett_init(ctx->p, 0) : NULL; + + mpi_ec_get_reset(ctx); + + if (model == MPI_EC_MONTGOMERY) { + for (i = 0; i < DIM(bad_points_table); i++) { + MPI p_candidate = mpi_scanval(bad_points_table[i][0]); + int match_p = !mpi_cmp(ctx->p, p_candidate); + int j; + + mpi_free(p_candidate); + if (!match_p) + continue; + + for (j = 0; i < DIM(ctx->t.scratch) && bad_points_table[i][j]; j++) + ctx->t.scratch[j] = mpi_scanval(bad_points_table[i][j]); + } + } else { + /* Allocate scratch variables. */ + for (i = 0; i < DIM(ctx->t.scratch); i++) + ctx->t.scratch[i] = mpi_alloc_like(ctx->p); + } + + ctx->addm = ec_addm; + ctx->subm = ec_subm; + ctx->mulm = ec_mulm; + ctx->mul2 = ec_mul2; + ctx->pow2 = ec_pow2; + + for (i = 0; field_table[i].p; i++) { + MPI f_p; + + f_p = mpi_scanval(field_table[i].p); + if (!f_p) + break; + + if (!mpi_cmp(p, f_p)) { + ctx->addm = field_table[i].addm; + ctx->subm = field_table[i].subm; + ctx->mulm = field_table[i].mulm; + ctx->mul2 = field_table[i].mul2; + ctx->pow2 = field_table[i].pow2; + mpi_free(f_p); + + mpi_resize(ctx->a, ctx->p->nlimbs); + ctx->a->nlimbs = ctx->p->nlimbs; + + mpi_resize(ctx->b, ctx->p->nlimbs); + ctx->b->nlimbs = ctx->p->nlimbs; + + for (i = 0; i < DIM(ctx->t.scratch) && ctx->t.scratch[i]; i++) + ctx->t.scratch[i]->nlimbs = ctx->p->nlimbs; + + break; + } + + mpi_free(f_p); + } +} +EXPORT_SYMBOL_GPL(mpi_ec_init); + +void mpi_ec_deinit(struct mpi_ec_ctx *ctx) +{ + int i; + + mpi_barrett_free(ctx->t.p_barrett); + + /* Domain parameter. */ + mpi_free(ctx->p); + mpi_free(ctx->a); + mpi_free(ctx->b); + mpi_point_release(ctx->G); + mpi_free(ctx->n); + + /* The key. */ + mpi_point_release(ctx->Q); + mpi_free(ctx->d); + + /* Private data of ec.c. */ + mpi_free(ctx->t.two_inv_p); + + for (i = 0; i < DIM(ctx->t.scratch); i++) + mpi_free(ctx->t.scratch[i]); +} +EXPORT_SYMBOL_GPL(mpi_ec_deinit); + +/* Compute the affine coordinates from the projective coordinates in + * POINT. Set them into X and Y. If one coordinate is not required, + * X or Y may be passed as NULL. CTX is the usual context. Returns: 0 + * on success or !0 if POINT is at infinity. + */ +int mpi_ec_get_affine(MPI x, MPI y, MPI_POINT point, struct mpi_ec_ctx *ctx) +{ + if (!mpi_cmp_ui(point->z, 0)) + return -1; + + switch (ctx->model) { + case MPI_EC_WEIERSTRASS: /* Using Jacobian coordinates. */ + { + MPI z1, z2, z3; + + z1 = mpi_new(0); + z2 = mpi_new(0); + ec_invm(z1, point->z, ctx); /* z1 = z^(-1) mod p */ + ec_mulm(z2, z1, z1, ctx); /* z2 = z^(-2) mod p */ + + if (x) + ec_mulm(x, point->x, z2, ctx); + + if (y) { + z3 = mpi_new(0); + ec_mulm(z3, z2, z1, ctx); /* z3 = z^(-3) mod p */ + ec_mulm(y, point->y, z3, ctx); + mpi_free(z3); + } + + mpi_free(z2); + mpi_free(z1); + } + return 0; + + case MPI_EC_MONTGOMERY: + { + if (x) + mpi_set(x, point->x); + + if (y) { + log_fatal("%s: Getting Y-coordinate on %s is not supported\n", + "mpi_ec_get_affine", "Montgomery"); + return -1; + } + } + return 0; + + case MPI_EC_EDWARDS: + { + MPI z; + + z = mpi_new(0); + ec_invm(z, point->z, ctx); + + mpi_resize(z, ctx->p->nlimbs); + z->nlimbs = ctx->p->nlimbs; + + if (x) { + mpi_resize(x, ctx->p->nlimbs); + x->nlimbs = ctx->p->nlimbs; + ctx->mulm(x, point->x, z, ctx); + } + if (y) { + mpi_resize(y, ctx->p->nlimbs); + y->nlimbs = ctx->p->nlimbs; + ctx->mulm(y, point->y, z, ctx); + } + + mpi_free(z); + } + return 0; + + default: + return -1; + } +} +EXPORT_SYMBOL_GPL(mpi_ec_get_affine); + +/* RESULT = 2 * POINT (Weierstrass version). */ +static void dup_point_weierstrass(MPI_POINT result, + MPI_POINT point, struct mpi_ec_ctx *ctx) +{ +#define x3 (result->x) +#define y3 (result->y) +#define z3 (result->z) +#define t1 (ctx->t.scratch[0]) +#define t2 (ctx->t.scratch[1]) +#define t3 (ctx->t.scratch[2]) +#define l1 (ctx->t.scratch[3]) +#define l2 (ctx->t.scratch[4]) +#define l3 (ctx->t.scratch[5]) + + if (!mpi_cmp_ui(point->y, 0) || !mpi_cmp_ui(point->z, 0)) { + /* P_y == 0 || P_z == 0 => [1:1:0] */ + mpi_set_ui(x3, 1); + mpi_set_ui(y3, 1); + mpi_set_ui(z3, 0); + } else { + if (ec_get_a_is_pminus3(ctx)) { + /* Use the faster case. */ + /* L1 = 3(X - Z^2)(X + Z^2) */ + /* T1: used for Z^2. */ + /* T2: used for the right term. */ + ec_pow2(t1, point->z, ctx); + ec_subm(l1, point->x, t1, ctx); + ec_mulm(l1, l1, mpi_const(MPI_C_THREE), ctx); + ec_addm(t2, point->x, t1, ctx); + ec_mulm(l1, l1, t2, ctx); + } else { + /* Standard case. */ + /* L1 = 3X^2 + aZ^4 */ + /* T1: used for aZ^4. */ + ec_pow2(l1, point->x, ctx); + ec_mulm(l1, l1, mpi_const(MPI_C_THREE), ctx); + ec_powm(t1, point->z, mpi_const(MPI_C_FOUR), ctx); + ec_mulm(t1, t1, ctx->a, ctx); + ec_addm(l1, l1, t1, ctx); + } + /* Z3 = 2YZ */ + ec_mulm(z3, point->y, point->z, ctx); + ec_mul2(z3, z3, ctx); + + /* L2 = 4XY^2 */ + /* T2: used for Y2; required later. */ + ec_pow2(t2, point->y, ctx); + ec_mulm(l2, t2, point->x, ctx); + ec_mulm(l2, l2, mpi_const(MPI_C_FOUR), ctx); + + /* X3 = L1^2 - 2L2 */ + /* T1: used for L2^2. */ + ec_pow2(x3, l1, ctx); + ec_mul2(t1, l2, ctx); + ec_subm(x3, x3, t1, ctx); + + /* L3 = 8Y^4 */ + /* T2: taken from above. */ + ec_pow2(t2, t2, ctx); + ec_mulm(l3, t2, mpi_const(MPI_C_EIGHT), ctx); + + /* Y3 = L1(L2 - X3) - L3 */ + ec_subm(y3, l2, x3, ctx); + ec_mulm(y3, y3, l1, ctx); + ec_subm(y3, y3, l3, ctx); + } + +#undef x3 +#undef y3 +#undef z3 +#undef t1 +#undef t2 +#undef t3 +#undef l1 +#undef l2 +#undef l3 +} + +/* RESULT = 2 * POINT (Montgomery version). */ +static void dup_point_montgomery(MPI_POINT result, + MPI_POINT point, struct mpi_ec_ctx *ctx) +{ + (void)result; + (void)point; + (void)ctx; + log_fatal("%s: %s not yet supported\n", + "mpi_ec_dup_point", "Montgomery"); +} + +/* RESULT = 2 * POINT (Twisted Edwards version). */ +static void dup_point_edwards(MPI_POINT result, + MPI_POINT point, struct mpi_ec_ctx *ctx) +{ +#define X1 (point->x) +#define Y1 (point->y) +#define Z1 (point->z) +#define X3 (result->x) +#define Y3 (result->y) +#define Z3 (result->z) +#define B (ctx->t.scratch[0]) +#define C (ctx->t.scratch[1]) +#define D (ctx->t.scratch[2]) +#define E (ctx->t.scratch[3]) +#define F (ctx->t.scratch[4]) +#define H (ctx->t.scratch[5]) +#define J (ctx->t.scratch[6]) + + /* Compute: (X_3 : Y_3 : Z_3) = 2( X_1 : Y_1 : Z_1 ) */ + + /* B = (X_1 + Y_1)^2 */ + ctx->addm(B, X1, Y1, ctx); + ctx->pow2(B, B, ctx); + + /* C = X_1^2 */ + /* D = Y_1^2 */ + ctx->pow2(C, X1, ctx); + ctx->pow2(D, Y1, ctx); + + /* E = aC */ + if (ctx->dialect == ECC_DIALECT_ED25519) + ctx->subm(E, ctx->p, C, ctx); + else + ctx->mulm(E, ctx->a, C, ctx); + + /* F = E + D */ + ctx->addm(F, E, D, ctx); + + /* H = Z_1^2 */ + ctx->pow2(H, Z1, ctx); + + /* J = F - 2H */ + ctx->mul2(J, H, ctx); + ctx->subm(J, F, J, ctx); + + /* X_3 = (B - C - D) · J */ + ctx->subm(X3, B, C, ctx); + ctx->subm(X3, X3, D, ctx); + ctx->mulm(X3, X3, J, ctx); + + /* Y_3 = F · (E - D) */ + ctx->subm(Y3, E, D, ctx); + ctx->mulm(Y3, Y3, F, ctx); + + /* Z_3 = F · J */ + ctx->mulm(Z3, F, J, ctx); + +#undef X1 +#undef Y1 +#undef Z1 +#undef X3 +#undef Y3 +#undef Z3 +#undef B +#undef C +#undef D +#undef E +#undef F +#undef H +#undef J +} + +/* RESULT = 2 * POINT */ +static void +mpi_ec_dup_point(MPI_POINT result, MPI_POINT point, struct mpi_ec_ctx *ctx) +{ + switch (ctx->model) { + case MPI_EC_WEIERSTRASS: + dup_point_weierstrass(result, point, ctx); + break; + case MPI_EC_MONTGOMERY: + dup_point_montgomery(result, point, ctx); + break; + case MPI_EC_EDWARDS: + dup_point_edwards(result, point, ctx); + break; + } +} + +/* RESULT = P1 + P2 (Weierstrass version).*/ +static void add_points_weierstrass(MPI_POINT result, + MPI_POINT p1, MPI_POINT p2, + struct mpi_ec_ctx *ctx) +{ +#define x1 (p1->x) +#define y1 (p1->y) +#define z1 (p1->z) +#define x2 (p2->x) +#define y2 (p2->y) +#define z2 (p2->z) +#define x3 (result->x) +#define y3 (result->y) +#define z3 (result->z) +#define l1 (ctx->t.scratch[0]) +#define l2 (ctx->t.scratch[1]) +#define l3 (ctx->t.scratch[2]) +#define l4 (ctx->t.scratch[3]) +#define l5 (ctx->t.scratch[4]) +#define l6 (ctx->t.scratch[5]) +#define l7 (ctx->t.scratch[6]) +#define l8 (ctx->t.scratch[7]) +#define l9 (ctx->t.scratch[8]) +#define t1 (ctx->t.scratch[9]) +#define t2 (ctx->t.scratch[10]) + + if ((!mpi_cmp(x1, x2)) && (!mpi_cmp(y1, y2)) && (!mpi_cmp(z1, z2))) { + /* Same point; need to call the duplicate function. */ + mpi_ec_dup_point(result, p1, ctx); + } else if (!mpi_cmp_ui(z1, 0)) { + /* P1 is at infinity. */ + mpi_set(x3, p2->x); + mpi_set(y3, p2->y); + mpi_set(z3, p2->z); + } else if (!mpi_cmp_ui(z2, 0)) { + /* P2 is at infinity. */ + mpi_set(x3, p1->x); + mpi_set(y3, p1->y); + mpi_set(z3, p1->z); + } else { + int z1_is_one = !mpi_cmp_ui(z1, 1); + int z2_is_one = !mpi_cmp_ui(z2, 1); + + /* l1 = x1 z2^2 */ + /* l2 = x2 z1^2 */ + if (z2_is_one) + mpi_set(l1, x1); + else { + ec_pow2(l1, z2, ctx); + ec_mulm(l1, l1, x1, ctx); + } + if (z1_is_one) + mpi_set(l2, x2); + else { + ec_pow2(l2, z1, ctx); + ec_mulm(l2, l2, x2, ctx); + } + /* l3 = l1 - l2 */ + ec_subm(l3, l1, l2, ctx); + /* l4 = y1 z2^3 */ + ec_powm(l4, z2, mpi_const(MPI_C_THREE), ctx); + ec_mulm(l4, l4, y1, ctx); + /* l5 = y2 z1^3 */ + ec_powm(l5, z1, mpi_const(MPI_C_THREE), ctx); + ec_mulm(l5, l5, y2, ctx); + /* l6 = l4 - l5 */ + ec_subm(l6, l4, l5, ctx); + + if (!mpi_cmp_ui(l3, 0)) { + if (!mpi_cmp_ui(l6, 0)) { + /* P1 and P2 are the same - use duplicate function. */ + mpi_ec_dup_point(result, p1, ctx); + } else { + /* P1 is the inverse of P2. */ + mpi_set_ui(x3, 1); + mpi_set_ui(y3, 1); + mpi_set_ui(z3, 0); + } + } else { + /* l7 = l1 + l2 */ + ec_addm(l7, l1, l2, ctx); + /* l8 = l4 + l5 */ + ec_addm(l8, l4, l5, ctx); + /* z3 = z1 z2 l3 */ + ec_mulm(z3, z1, z2, ctx); + ec_mulm(z3, z3, l3, ctx); + /* x3 = l6^2 - l7 l3^2 */ + ec_pow2(t1, l6, ctx); + ec_pow2(t2, l3, ctx); + ec_mulm(t2, t2, l7, ctx); + ec_subm(x3, t1, t2, ctx); + /* l9 = l7 l3^2 - 2 x3 */ + ec_mul2(t1, x3, ctx); + ec_subm(l9, t2, t1, ctx); + /* y3 = (l9 l6 - l8 l3^3)/2 */ + ec_mulm(l9, l9, l6, ctx); + ec_powm(t1, l3, mpi_const(MPI_C_THREE), ctx); /* fixme: Use saved value*/ + ec_mulm(t1, t1, l8, ctx); + ec_subm(y3, l9, t1, ctx); + ec_mulm(y3, y3, ec_get_two_inv_p(ctx), ctx); + } + } + +#undef x1 +#undef y1 +#undef z1 +#undef x2 +#undef y2 +#undef z2 +#undef x3 +#undef y3 +#undef z3 +#undef l1 +#undef l2 +#undef l3 +#undef l4 +#undef l5 +#undef l6 +#undef l7 +#undef l8 +#undef l9 +#undef t1 +#undef t2 +} + +/* RESULT = P1 + P2 (Montgomery version).*/ +static void add_points_montgomery(MPI_POINT result, + MPI_POINT p1, MPI_POINT p2, + struct mpi_ec_ctx *ctx) +{ + (void)result; + (void)p1; + (void)p2; + (void)ctx; + log_fatal("%s: %s not yet supported\n", + "mpi_ec_add_points", "Montgomery"); +} + +/* RESULT = P1 + P2 (Twisted Edwards version).*/ +static void add_points_edwards(MPI_POINT result, + MPI_POINT p1, MPI_POINT p2, + struct mpi_ec_ctx *ctx) +{ +#define X1 (p1->x) +#define Y1 (p1->y) +#define Z1 (p1->z) +#define X2 (p2->x) +#define Y2 (p2->y) +#define Z2 (p2->z) +#define X3 (result->x) +#define Y3 (result->y) +#define Z3 (result->z) +#define A (ctx->t.scratch[0]) +#define B (ctx->t.scratch[1]) +#define C (ctx->t.scratch[2]) +#define D (ctx->t.scratch[3]) +#define E (ctx->t.scratch[4]) +#define F (ctx->t.scratch[5]) +#define G (ctx->t.scratch[6]) +#define tmp (ctx->t.scratch[7]) + + point_resize(result, ctx); + + /* Compute: (X_3 : Y_3 : Z_3) = (X_1 : Y_1 : Z_1) + (X_2 : Y_2 : Z_3) */ + + /* A = Z1 · Z2 */ + ctx->mulm(A, Z1, Z2, ctx); + + /* B = A^2 */ + ctx->pow2(B, A, ctx); + + /* C = X1 · X2 */ + ctx->mulm(C, X1, X2, ctx); + + /* D = Y1 · Y2 */ + ctx->mulm(D, Y1, Y2, ctx); + + /* E = d · C · D */ + ctx->mulm(E, ctx->b, C, ctx); + ctx->mulm(E, E, D, ctx); + + /* F = B - E */ + ctx->subm(F, B, E, ctx); + + /* G = B + E */ + ctx->addm(G, B, E, ctx); + + /* X_3 = A · F · ((X_1 + Y_1) · (X_2 + Y_2) - C - D) */ + ctx->addm(tmp, X1, Y1, ctx); + ctx->addm(X3, X2, Y2, ctx); + ctx->mulm(X3, X3, tmp, ctx); + ctx->subm(X3, X3, C, ctx); + ctx->subm(X3, X3, D, ctx); + ctx->mulm(X3, X3, F, ctx); + ctx->mulm(X3, X3, A, ctx); + + /* Y_3 = A · G · (D - aC) */ + if (ctx->dialect == ECC_DIALECT_ED25519) { + ctx->addm(Y3, D, C, ctx); + } else { + ctx->mulm(Y3, ctx->a, C, ctx); + ctx->subm(Y3, D, Y3, ctx); + } + ctx->mulm(Y3, Y3, G, ctx); + ctx->mulm(Y3, Y3, A, ctx); + + /* Z_3 = F · G */ + ctx->mulm(Z3, F, G, ctx); + + +#undef X1 +#undef Y1 +#undef Z1 +#undef X2 +#undef Y2 +#undef Z2 +#undef X3 +#undef Y3 +#undef Z3 +#undef A +#undef B +#undef C +#undef D +#undef E +#undef F +#undef G +#undef tmp +} + +/* Compute a step of Montgomery Ladder (only use X and Z in the point). + * Inputs: P1, P2, and x-coordinate of DIF = P1 - P1. + * Outputs: PRD = 2 * P1 and SUM = P1 + P2. + */ +static void montgomery_ladder(MPI_POINT prd, MPI_POINT sum, + MPI_POINT p1, MPI_POINT p2, MPI dif_x, + struct mpi_ec_ctx *ctx) +{ + ctx->addm(sum->x, p2->x, p2->z, ctx); + ctx->subm(p2->z, p2->x, p2->z, ctx); + ctx->addm(prd->x, p1->x, p1->z, ctx); + ctx->subm(p1->z, p1->x, p1->z, ctx); + ctx->mulm(p2->x, p1->z, sum->x, ctx); + ctx->mulm(p2->z, prd->x, p2->z, ctx); + ctx->pow2(p1->x, prd->x, ctx); + ctx->pow2(p1->z, p1->z, ctx); + ctx->addm(sum->x, p2->x, p2->z, ctx); + ctx->subm(p2->z, p2->x, p2->z, ctx); + ctx->mulm(prd->x, p1->x, p1->z, ctx); + ctx->subm(p1->z, p1->x, p1->z, ctx); + ctx->pow2(sum->x, sum->x, ctx); + ctx->pow2(sum->z, p2->z, ctx); + ctx->mulm(prd->z, p1->z, ctx->a, ctx); /* CTX->A: (a-2)/4 */ + ctx->mulm(sum->z, sum->z, dif_x, ctx); + ctx->addm(prd->z, p1->x, prd->z, ctx); + ctx->mulm(prd->z, prd->z, p1->z, ctx); +} + +/* RESULT = P1 + P2 */ +void mpi_ec_add_points(MPI_POINT result, + MPI_POINT p1, MPI_POINT p2, + struct mpi_ec_ctx *ctx) +{ + switch (ctx->model) { + case MPI_EC_WEIERSTRASS: + add_points_weierstrass(result, p1, p2, ctx); + break; + case MPI_EC_MONTGOMERY: + add_points_montgomery(result, p1, p2, ctx); + break; + case MPI_EC_EDWARDS: + add_points_edwards(result, p1, p2, ctx); + break; + } +} +EXPORT_SYMBOL_GPL(mpi_ec_add_points); + +/* Scalar point multiplication - the main function for ECC. If takes + * an integer SCALAR and a POINT as well as the usual context CTX. + * RESULT will be set to the resulting point. + */ +void mpi_ec_mul_point(MPI_POINT result, + MPI scalar, MPI_POINT point, + struct mpi_ec_ctx *ctx) +{ + MPI x1, y1, z1, k, h, yy; + unsigned int i, loops; + struct gcry_mpi_point p1, p2, p1inv; + + if (ctx->model == MPI_EC_EDWARDS) { + /* Simple left to right binary method. Algorithm 3.27 from + * {author={Hankerson, Darrel and Menezes, Alfred J. and Vanstone, Scott}, + * title = {Guide to Elliptic Curve Cryptography}, + * year = {2003}, isbn = {038795273X}, + * url = {http://www.cacr.math.uwaterloo.ca/ecc/}, + * publisher = {Springer-Verlag New York, Inc.}} + */ + unsigned int nbits; + int j; + + if (mpi_cmp(scalar, ctx->p) >= 0) + nbits = mpi_get_nbits(scalar); + else + nbits = mpi_get_nbits(ctx->p); + + mpi_set_ui(result->x, 0); + mpi_set_ui(result->y, 1); + mpi_set_ui(result->z, 1); + point_resize(point, ctx); + + point_resize(result, ctx); + point_resize(point, ctx); + + for (j = nbits-1; j >= 0; j--) { + mpi_ec_dup_point(result, result, ctx); + if (mpi_test_bit(scalar, j)) + mpi_ec_add_points(result, result, point, ctx); + } + return; + } else if (ctx->model == MPI_EC_MONTGOMERY) { + unsigned int nbits; + int j; + struct gcry_mpi_point p1_, p2_; + MPI_POINT q1, q2, prd, sum; + unsigned long sw; + mpi_size_t rsize; + int scalar_copied = 0; + + /* Compute scalar point multiplication with Montgomery Ladder. + * Note that we don't use Y-coordinate in the points at all. + * RESULT->Y will be filled by zero. + */ + + nbits = mpi_get_nbits(scalar); + point_init(&p1); + point_init(&p2); + point_init(&p1_); + point_init(&p2_); + mpi_set_ui(p1.x, 1); + mpi_free(p2.x); + p2.x = mpi_copy(point->x); + mpi_set_ui(p2.z, 1); + + point_resize(&p1, ctx); + point_resize(&p2, ctx); + point_resize(&p1_, ctx); + point_resize(&p2_, ctx); + + mpi_resize(point->x, ctx->p->nlimbs); + point->x->nlimbs = ctx->p->nlimbs; + + q1 = &p1; + q2 = &p2; + prd = &p1_; + sum = &p2_; + + for (j = nbits-1; j >= 0; j--) { + MPI_POINT t; + + sw = mpi_test_bit(scalar, j); + point_swap_cond(q1, q2, sw, ctx); + montgomery_ladder(prd, sum, q1, q2, point->x, ctx); + point_swap_cond(prd, sum, sw, ctx); + t = q1; q1 = prd; prd = t; + t = q2; q2 = sum; sum = t; + } + + mpi_clear(result->y); + sw = (nbits & 1); + point_swap_cond(&p1, &p1_, sw, ctx); + + rsize = p1.z->nlimbs; + MPN_NORMALIZE(p1.z->d, rsize); + if (rsize == 0) { + mpi_set_ui(result->x, 1); + mpi_set_ui(result->z, 0); + } else { + z1 = mpi_new(0); + ec_invm(z1, p1.z, ctx); + ec_mulm(result->x, p1.x, z1, ctx); + mpi_set_ui(result->z, 1); + mpi_free(z1); + } + + point_free(&p1); + point_free(&p2); + point_free(&p1_); + point_free(&p2_); + if (scalar_copied) + mpi_free(scalar); + return; + } + + x1 = mpi_alloc_like(ctx->p); + y1 = mpi_alloc_like(ctx->p); + h = mpi_alloc_like(ctx->p); + k = mpi_copy(scalar); + yy = mpi_copy(point->y); + + if (mpi_has_sign(k)) { + k->sign = 0; + ec_invm(yy, yy, ctx); + } + + if (!mpi_cmp_ui(point->z, 1)) { + mpi_set(x1, point->x); + mpi_set(y1, yy); + } else { + MPI z2, z3; + + z2 = mpi_alloc_like(ctx->p); + z3 = mpi_alloc_like(ctx->p); + ec_mulm(z2, point->z, point->z, ctx); + ec_mulm(z3, point->z, z2, ctx); + ec_invm(z2, z2, ctx); + ec_mulm(x1, point->x, z2, ctx); + ec_invm(z3, z3, ctx); + ec_mulm(y1, yy, z3, ctx); + mpi_free(z2); + mpi_free(z3); + } + z1 = mpi_copy(mpi_const(MPI_C_ONE)); + + mpi_mul(h, k, mpi_const(MPI_C_THREE)); /* h = 3k */ + loops = mpi_get_nbits(h); + if (loops < 2) { + /* If SCALAR is zero, the above mpi_mul sets H to zero and thus + * LOOPs will be zero. To avoid an underflow of I in the main + * loop we set LOOP to 2 and the result to (0,0,0). + */ + loops = 2; + mpi_clear(result->x); + mpi_clear(result->y); + mpi_clear(result->z); + } else { + mpi_set(result->x, point->x); + mpi_set(result->y, yy); + mpi_set(result->z, point->z); + } + mpi_free(yy); yy = NULL; + + p1.x = x1; x1 = NULL; + p1.y = y1; y1 = NULL; + p1.z = z1; z1 = NULL; + point_init(&p2); + point_init(&p1inv); + + /* Invert point: y = p - y mod p */ + point_set(&p1inv, &p1); + ec_subm(p1inv.y, ctx->p, p1inv.y, ctx); + + for (i = loops-2; i > 0; i--) { + mpi_ec_dup_point(result, result, ctx); + if (mpi_test_bit(h, i) == 1 && mpi_test_bit(k, i) == 0) { + point_set(&p2, result); + mpi_ec_add_points(result, &p2, &p1, ctx); + } + if (mpi_test_bit(h, i) == 0 && mpi_test_bit(k, i) == 1) { + point_set(&p2, result); + mpi_ec_add_points(result, &p2, &p1inv, ctx); + } + } + + point_free(&p1); + point_free(&p2); + point_free(&p1inv); + mpi_free(h); + mpi_free(k); +} +EXPORT_SYMBOL_GPL(mpi_ec_mul_point); + +/* Return true if POINT is on the curve described by CTX. */ +int mpi_ec_curve_point(MPI_POINT point, struct mpi_ec_ctx *ctx) +{ + int res = 0; + MPI x, y, w; + + x = mpi_new(0); + y = mpi_new(0); + w = mpi_new(0); + + /* Check that the point is in range. This needs to be done here and + * not after conversion to affine coordinates. + */ + if (mpi_cmpabs(point->x, ctx->p) >= 0) + goto leave; + if (mpi_cmpabs(point->y, ctx->p) >= 0) + goto leave; + if (mpi_cmpabs(point->z, ctx->p) >= 0) + goto leave; + + switch (ctx->model) { + case MPI_EC_WEIERSTRASS: + { + MPI xxx; + + if (mpi_ec_get_affine(x, y, point, ctx)) + goto leave; + + xxx = mpi_new(0); + + /* y^2 == x^3 + a·x + b */ + ec_pow2(y, y, ctx); + + ec_pow3(xxx, x, ctx); + ec_mulm(w, ctx->a, x, ctx); + ec_addm(w, w, ctx->b, ctx); + ec_addm(w, w, xxx, ctx); + + if (!mpi_cmp(y, w)) + res = 1; + + mpi_free(xxx); + } + break; + + case MPI_EC_MONTGOMERY: + { +#define xx y + /* With Montgomery curve, only X-coordinate is valid. */ + if (mpi_ec_get_affine(x, NULL, point, ctx)) + goto leave; + + /* The equation is: b * y^2 == x^3 + a · x^2 + x */ + /* We check if right hand is quadratic residue or not by + * Euler's criterion. + */ + /* CTX->A has (a-2)/4 and CTX->B has b^-1 */ + ec_mulm(w, ctx->a, mpi_const(MPI_C_FOUR), ctx); + ec_addm(w, w, mpi_const(MPI_C_TWO), ctx); + ec_mulm(w, w, x, ctx); + ec_pow2(xx, x, ctx); + ec_addm(w, w, xx, ctx); + ec_addm(w, w, mpi_const(MPI_C_ONE), ctx); + ec_mulm(w, w, x, ctx); + ec_mulm(w, w, ctx->b, ctx); +#undef xx + /* Compute Euler's criterion: w^(p-1)/2 */ +#define p_minus1 y + ec_subm(p_minus1, ctx->p, mpi_const(MPI_C_ONE), ctx); + mpi_rshift(p_minus1, p_minus1, 1); + ec_powm(w, w, p_minus1, ctx); + + res = !mpi_cmp_ui(w, 1); +#undef p_minus1 + } + break; + + case MPI_EC_EDWARDS: + { + if (mpi_ec_get_affine(x, y, point, ctx)) + goto leave; + + mpi_resize(w, ctx->p->nlimbs); + w->nlimbs = ctx->p->nlimbs; + + /* a · x^2 + y^2 - 1 - b · x^2 · y^2 == 0 */ + ctx->pow2(x, x, ctx); + ctx->pow2(y, y, ctx); + if (ctx->dialect == ECC_DIALECT_ED25519) + ctx->subm(w, ctx->p, x, ctx); + else + ctx->mulm(w, ctx->a, x, ctx); + ctx->addm(w, w, y, ctx); + ctx->mulm(x, x, y, ctx); + ctx->mulm(x, x, ctx->b, ctx); + ctx->subm(w, w, x, ctx); + if (!mpi_cmp_ui(w, 1)) + res = 1; + } + break; + } + +leave: + mpi_free(w); + mpi_free(x); + mpi_free(y); + + return res; +} +EXPORT_SYMBOL_GPL(mpi_ec_curve_point); diff --git a/lib/mpi/mpi-add.c b/lib/mpi/mpi-add.c new file mode 100644 index 000000000000..2cdae54c1bd0 --- /dev/null +++ b/lib/mpi/mpi-add.c @@ -0,0 +1,155 @@ +/* mpi-add.c - MPI functions + * Copyright (C) 1994, 1996, 1998, 2001, 2002, + * 2003 Free Software Foundation, Inc. + * + * This file is part of Libgcrypt. + * + * Note: This code is heavily based on the GNU MP Library. + * Actually it's the same code with only minor changes in the + * way the data is stored; this is to support the abstraction + * of an optional secure memory allocation which may be used + * to avoid revealing of sensitive data due to paging etc. + */ + +#include "mpi-internal.h" + +/**************** + * Add the unsigned integer V to the mpi-integer U and store the + * result in W. U and V may be the same. + */ +void mpi_add_ui(MPI w, MPI u, unsigned long v) +{ + mpi_ptr_t wp, up; + mpi_size_t usize, wsize; + int usign, wsign; + + usize = u->nlimbs; + usign = u->sign; + wsign = 0; + + /* If not space for W (and possible carry), increase space. */ + wsize = usize + 1; + if (w->alloced < wsize) + mpi_resize(w, wsize); + + /* These must be after realloc (U may be the same as W). */ + up = u->d; + wp = w->d; + + if (!usize) { /* simple */ + wp[0] = v; + wsize = v ? 1:0; + } else if (!usign) { /* mpi is not negative */ + mpi_limb_t cy; + cy = mpihelp_add_1(wp, up, usize, v); + wp[usize] = cy; + wsize = usize + cy; + } else { + /* The signs are different. Need exact comparison to determine + * which operand to subtract from which. + */ + if (usize == 1 && up[0] < v) { + wp[0] = v - up[0]; + wsize = 1; + } else { + mpihelp_sub_1(wp, up, usize, v); + /* Size can decrease with at most one limb. */ + wsize = usize - (wp[usize-1] == 0); + wsign = 1; + } + } + + w->nlimbs = wsize; + w->sign = wsign; +} + + +void mpi_add(MPI w, MPI u, MPI v) +{ + mpi_ptr_t wp, up, vp; + mpi_size_t usize, vsize, wsize; + int usign, vsign, wsign; + + if (u->nlimbs < v->nlimbs) { /* Swap U and V. */ + usize = v->nlimbs; + usign = v->sign; + vsize = u->nlimbs; + vsign = u->sign; + wsize = usize + 1; + RESIZE_IF_NEEDED(w, wsize); + /* These must be after realloc (u or v may be the same as w). */ + up = v->d; + vp = u->d; + } else { + usize = u->nlimbs; + usign = u->sign; + vsize = v->nlimbs; + vsign = v->sign; + wsize = usize + 1; + RESIZE_IF_NEEDED(w, wsize); + /* These must be after realloc (u or v may be the same as w). */ + up = u->d; + vp = v->d; + } + wp = w->d; + wsign = 0; + + if (!vsize) { /* simple */ + MPN_COPY(wp, up, usize); + wsize = usize; + wsign = usign; + } else if (usign != vsign) { /* different sign */ + /* This test is right since USIZE >= VSIZE */ + if (usize != vsize) { + mpihelp_sub(wp, up, usize, vp, vsize); + wsize = usize; + MPN_NORMALIZE(wp, wsize); + wsign = usign; + } else if (mpihelp_cmp(up, vp, usize) < 0) { + mpihelp_sub_n(wp, vp, up, usize); + wsize = usize; + MPN_NORMALIZE(wp, wsize); + if (!usign) + wsign = 1; + } else { + mpihelp_sub_n(wp, up, vp, usize); + wsize = usize; + MPN_NORMALIZE(wp, wsize); + if (usign) + wsign = 1; + } + } else { /* U and V have same sign. Add them. */ + mpi_limb_t cy = mpihelp_add(wp, up, usize, vp, vsize); + wp[usize] = cy; + wsize = usize + cy; + if (usign) + wsign = 1; + } + + w->nlimbs = wsize; + w->sign = wsign; +} +EXPORT_SYMBOL_GPL(mpi_add); + +void mpi_sub(MPI w, MPI u, MPI v) +{ + MPI vv = mpi_copy(v); + vv->sign = !vv->sign; + mpi_add(w, u, vv); + mpi_free(vv); +} + + +void mpi_addm(MPI w, MPI u, MPI v, MPI m) +{ + mpi_add(w, u, v); + mpi_mod(w, w, m); +} +EXPORT_SYMBOL_GPL(mpi_addm); + +void mpi_subm(MPI w, MPI u, MPI v, MPI m) +{ + mpi_sub(w, u, v); + mpi_mod(w, w, m); +} +EXPORT_SYMBOL_GPL(mpi_subm); diff --git a/lib/mpi/mpi-bit.c b/lib/mpi/mpi-bit.c index 503537e08436..a5119a2bcdd4 100644 --- a/lib/mpi/mpi-bit.c +++ b/lib/mpi/mpi-bit.c @@ -32,6 +32,7 @@ void mpi_normalize(MPI a) for (; a->nlimbs && !a->d[a->nlimbs - 1]; a->nlimbs--) ; } +EXPORT_SYMBOL_GPL(mpi_normalize); /**************** * Return the number of bits in A. @@ -54,3 +55,253 @@ unsigned mpi_get_nbits(MPI a) return n; } EXPORT_SYMBOL_GPL(mpi_get_nbits); + +/**************** + * Test whether bit N is set. + */ +int mpi_test_bit(MPI a, unsigned int n) +{ + unsigned int limbno, bitno; + mpi_limb_t limb; + + limbno = n / BITS_PER_MPI_LIMB; + bitno = n % BITS_PER_MPI_LIMB; + + if (limbno >= a->nlimbs) + return 0; /* too far left: this is a 0 */ + limb = a->d[limbno]; + return (limb & (A_LIMB_1 << bitno)) ? 1 : 0; +} +EXPORT_SYMBOL_GPL(mpi_test_bit); + +/**************** + * Set bit N of A. + */ +void mpi_set_bit(MPI a, unsigned int n) +{ + unsigned int i, limbno, bitno; + + limbno = n / BITS_PER_MPI_LIMB; + bitno = n % BITS_PER_MPI_LIMB; + + if (limbno >= a->nlimbs) { + for (i = a->nlimbs; i < a->alloced; i++) + a->d[i] = 0; + mpi_resize(a, limbno+1); + a->nlimbs = limbno+1; + } + a->d[limbno] |= (A_LIMB_1<<bitno); +} + +/**************** + * Set bit N of A. and clear all bits above + */ +void mpi_set_highbit(MPI a, unsigned int n) +{ + unsigned int i, limbno, bitno; + + limbno = n / BITS_PER_MPI_LIMB; + bitno = n % BITS_PER_MPI_LIMB; + + if (limbno >= a->nlimbs) { + for (i = a->nlimbs; i < a->alloced; i++) + a->d[i] = 0; + mpi_resize(a, limbno+1); + a->nlimbs = limbno+1; + } + a->d[limbno] |= (A_LIMB_1<<bitno); + for (bitno++; bitno < BITS_PER_MPI_LIMB; bitno++) + a->d[limbno] &= ~(A_LIMB_1 << bitno); + a->nlimbs = limbno+1; +} +EXPORT_SYMBOL_GPL(mpi_set_highbit); + +/**************** + * clear bit N of A and all bits above + */ +void mpi_clear_highbit(MPI a, unsigned int n) +{ + unsigned int limbno, bitno; + + limbno = n / BITS_PER_MPI_LIMB; + bitno = n % BITS_PER_MPI_LIMB; + + if (limbno >= a->nlimbs) + return; /* not allocated, therefore no need to clear bits :-) */ + + for ( ; bitno < BITS_PER_MPI_LIMB; bitno++) + a->d[limbno] &= ~(A_LIMB_1 << bitno); + a->nlimbs = limbno+1; +} + +/**************** + * Clear bit N of A. + */ +void mpi_clear_bit(MPI a, unsigned int n) +{ + unsigned int limbno, bitno; + + limbno = n / BITS_PER_MPI_LIMB; + bitno = n % BITS_PER_MPI_LIMB; + + if (limbno >= a->nlimbs) + return; /* Don't need to clear this bit, it's far too left. */ + a->d[limbno] &= ~(A_LIMB_1 << bitno); +} +EXPORT_SYMBOL_GPL(mpi_clear_bit); + + +/**************** + * Shift A by COUNT limbs to the right + * This is used only within the MPI library + */ +void mpi_rshift_limbs(MPI a, unsigned int count) +{ + mpi_ptr_t ap = a->d; + mpi_size_t n = a->nlimbs; + unsigned int i; + + if (count >= n) { + a->nlimbs = 0; + return; + } + + for (i = 0; i < n - count; i++) + ap[i] = ap[i+count]; + ap[i] = 0; + a->nlimbs -= count; +} + +/* + * Shift A by N bits to the right. + */ +void mpi_rshift(MPI x, MPI a, unsigned int n) +{ + mpi_size_t xsize; + unsigned int i; + unsigned int nlimbs = (n/BITS_PER_MPI_LIMB); + unsigned int nbits = (n%BITS_PER_MPI_LIMB); + + if (x == a) { + /* In-place operation. */ + if (nlimbs >= x->nlimbs) { + x->nlimbs = 0; + return; + } + + if (nlimbs) { + for (i = 0; i < x->nlimbs - nlimbs; i++) + x->d[i] = x->d[i+nlimbs]; + x->d[i] = 0; + x->nlimbs -= nlimbs; + } + if (x->nlimbs && nbits) + mpihelp_rshift(x->d, x->d, x->nlimbs, nbits); + } else if (nlimbs) { + /* Copy and shift by more or equal bits than in a limb. */ + xsize = a->nlimbs; + x->sign = a->sign; + RESIZE_IF_NEEDED(x, xsize); + x->nlimbs = xsize; + for (i = 0; i < a->nlimbs; i++) + x->d[i] = a->d[i]; + x->nlimbs = i; + + if (nlimbs >= x->nlimbs) { + x->nlimbs = 0; + return; + } + + if (nlimbs) { + for (i = 0; i < x->nlimbs - nlimbs; i++) + x->d[i] = x->d[i+nlimbs]; + x->d[i] = 0; + x->nlimbs -= nlimbs; + } + + if (x->nlimbs && nbits) + mpihelp_rshift(x->d, x->d, x->nlimbs, nbits); + } else { + /* Copy and shift by less than bits in a limb. */ + xsize = a->nlimbs; + x->sign = a->sign; + RESIZE_IF_NEEDED(x, xsize); + x->nlimbs = xsize; + + if (xsize) { + if (nbits) + mpihelp_rshift(x->d, a->d, x->nlimbs, nbits); + else { + /* The rshift helper function is not specified for + * NBITS==0, thus we do a plain copy here. + */ + for (i = 0; i < x->nlimbs; i++) + x->d[i] = a->d[i]; + } + } + } + MPN_NORMALIZE(x->d, x->nlimbs); +} + +/**************** + * Shift A by COUNT limbs to the left + * This is used only within the MPI library + */ +void mpi_lshift_limbs(MPI a, unsigned int count) +{ + mpi_ptr_t ap; + int n = a->nlimbs; + int i; + + if (!count || !n) + return; + + RESIZE_IF_NEEDED(a, n+count); + + ap = a->d; + for (i = n-1; i >= 0; i--) + ap[i+count] = ap[i]; + for (i = 0; i < count; i++) + ap[i] = 0; + a->nlimbs += count; +} + +/* + * Shift A by N bits to the left. + */ +void mpi_lshift(MPI x, MPI a, unsigned int n) +{ + unsigned int nlimbs = (n/BITS_PER_MPI_LIMB); + unsigned int nbits = (n%BITS_PER_MPI_LIMB); + + if (x == a && !n) + return; /* In-place shift with an amount of zero. */ + + if (x != a) { + /* Copy A to X. */ + unsigned int alimbs = a->nlimbs; + int asign = a->sign; + mpi_ptr_t xp, ap; + + RESIZE_IF_NEEDED(x, alimbs+nlimbs+1); + xp = x->d; + ap = a->d; + MPN_COPY(xp, ap, alimbs); + x->nlimbs = alimbs; + x->flags = a->flags; + x->sign = asign; + } + + if (nlimbs && !nbits) { + /* Shift a full number of limbs. */ + mpi_lshift_limbs(x, nlimbs); + } else if (n) { + /* We use a very dump approach: Shift left by the number of + * limbs plus one and than fix it up by an rshift. + */ + mpi_lshift_limbs(x, nlimbs+1); + mpi_rshift(x, x, BITS_PER_MPI_LIMB - nbits); + } + + MPN_NORMALIZE(x->d, x->nlimbs); +} diff --git a/lib/mpi/mpi-cmp.c b/lib/mpi/mpi-cmp.c index d25e9e96c310..c4cfa3ff0581 100644 --- a/lib/mpi/mpi-cmp.c +++ b/lib/mpi/mpi-cmp.c @@ -41,28 +41,54 @@ int mpi_cmp_ui(MPI u, unsigned long v) } EXPORT_SYMBOL_GPL(mpi_cmp_ui); -int mpi_cmp(MPI u, MPI v) +static int do_mpi_cmp(MPI u, MPI v, int absmode) { - mpi_size_t usize, vsize; + mpi_size_t usize; + mpi_size_t vsize; + int usign; + int vsign; int cmp; mpi_normalize(u); mpi_normalize(v); + usize = u->nlimbs; vsize = v->nlimbs; - if (!u->sign && v->sign) + usign = absmode ? 0 : u->sign; + vsign = absmode ? 0 : v->sign; + + /* Compare sign bits. */ + + if (!usign && vsign) return 1; - if (u->sign && !v->sign) + if (usign && !vsign) return -1; - if (usize != vsize && !u->sign && !v->sign) + + /* U and V are either both positive or both negative. */ + + if (usize != vsize && !usign && !vsign) return usize - vsize; - if (usize != vsize && u->sign && v->sign) - return vsize - usize; + if (usize != vsize && usign && vsign) + return vsize + usize; if (!usize) return 0; cmp = mpihelp_cmp(u->d, v->d, usize); - if (u->sign) - return -cmp; - return cmp; + if (!cmp) + return 0; + if ((cmp < 0?1:0) == (usign?1:0)) + return 1; + + return -1; +} + +int mpi_cmp(MPI u, MPI v) +{ + return do_mpi_cmp(u, v, 0); } EXPORT_SYMBOL_GPL(mpi_cmp); + +int mpi_cmpabs(MPI u, MPI v) +{ + return do_mpi_cmp(u, v, 1); +} +EXPORT_SYMBOL_GPL(mpi_cmpabs); diff --git a/lib/mpi/mpi-div.c b/lib/mpi/mpi-div.c new file mode 100644 index 000000000000..45beab8b9e9e --- /dev/null +++ b/lib/mpi/mpi-div.c @@ -0,0 +1,234 @@ +/* mpi-div.c - MPI functions + * Copyright (C) 1994, 1996, 1998, 2001, 2002, + * 2003 Free Software Foundation, Inc. + * + * This file is part of Libgcrypt. + * + * Note: This code is heavily based on the GNU MP Library. + * Actually it's the same code with only minor changes in the + * way the data is stored; this is to support the abstraction + * of an optional secure memory allocation which may be used + * to avoid revealing of sensitive data due to paging etc. + */ + +#include "mpi-internal.h" +#include "longlong.h" + +void mpi_tdiv_qr(MPI quot, MPI rem, MPI num, MPI den); +void mpi_fdiv_qr(MPI quot, MPI rem, MPI dividend, MPI divisor); + +void mpi_fdiv_r(MPI rem, MPI dividend, MPI divisor) +{ + int divisor_sign = divisor->sign; + MPI temp_divisor = NULL; + + /* We need the original value of the divisor after the remainder has been + * preliminary calculated. We have to copy it to temporary space if it's + * the same variable as REM. + */ + if (rem == divisor) { + temp_divisor = mpi_copy(divisor); + divisor = temp_divisor; + } + + mpi_tdiv_r(rem, dividend, divisor); + + if (((divisor_sign?1:0) ^ (dividend->sign?1:0)) && rem->nlimbs) + mpi_add(rem, rem, divisor); + + if (temp_divisor) + mpi_free(temp_divisor); +} + +void mpi_fdiv_q(MPI quot, MPI dividend, MPI divisor) +{ + MPI tmp = mpi_alloc(mpi_get_nlimbs(quot)); + mpi_fdiv_qr(quot, tmp, dividend, divisor); + mpi_free(tmp); +} + +void mpi_fdiv_qr(MPI quot, MPI rem, MPI dividend, MPI divisor) +{ + int divisor_sign = divisor->sign; + MPI temp_divisor = NULL; + + if (quot == divisor || rem == divisor) { + temp_divisor = mpi_copy(divisor); + divisor = temp_divisor; + } + + mpi_tdiv_qr(quot, rem, dividend, divisor); + + if ((divisor_sign ^ dividend->sign) && rem->nlimbs) { + mpi_sub_ui(quot, quot, 1); + mpi_add(rem, rem, divisor); + } + + if (temp_divisor) + mpi_free(temp_divisor); +} + +/* If den == quot, den needs temporary storage. + * If den == rem, den needs temporary storage. + * If num == quot, num needs temporary storage. + * If den has temporary storage, it can be normalized while being copied, + * i.e no extra storage should be allocated. + */ + +void mpi_tdiv_r(MPI rem, MPI num, MPI den) +{ + mpi_tdiv_qr(NULL, rem, num, den); +} + +void mpi_tdiv_qr(MPI quot, MPI rem, MPI num, MPI den) +{ + mpi_ptr_t np, dp; + mpi_ptr_t qp, rp; + mpi_size_t nsize = num->nlimbs; + mpi_size_t dsize = den->nlimbs; + mpi_size_t qsize, rsize; + mpi_size_t sign_remainder = num->sign; + mpi_size_t sign_quotient = num->sign ^ den->sign; + unsigned int normalization_steps; + mpi_limb_t q_limb; + mpi_ptr_t marker[5]; + int markidx = 0; + + /* Ensure space is enough for quotient and remainder. + * We need space for an extra limb in the remainder, because it's + * up-shifted (normalized) below. + */ + rsize = nsize + 1; + mpi_resize(rem, rsize); + + qsize = rsize - dsize; /* qsize cannot be bigger than this. */ + if (qsize <= 0) { + if (num != rem) { + rem->nlimbs = num->nlimbs; + rem->sign = num->sign; + MPN_COPY(rem->d, num->d, nsize); + } + if (quot) { + /* This needs to follow the assignment to rem, in case the + * numerator and quotient are the same. + */ + quot->nlimbs = 0; + quot->sign = 0; + } + return; + } + + if (quot) + mpi_resize(quot, qsize); + + /* Read pointers here, when reallocation is finished. */ + np = num->d; + dp = den->d; + rp = rem->d; + + /* Optimize division by a single-limb divisor. */ + if (dsize == 1) { + mpi_limb_t rlimb; + if (quot) { + qp = quot->d; + rlimb = mpihelp_divmod_1(qp, np, nsize, dp[0]); + qsize -= qp[qsize - 1] == 0; + quot->nlimbs = qsize; + quot->sign = sign_quotient; + } else + rlimb = mpihelp_mod_1(np, nsize, dp[0]); + rp[0] = rlimb; + rsize = rlimb != 0?1:0; + rem->nlimbs = rsize; + rem->sign = sign_remainder; + return; + } + + + if (quot) { + qp = quot->d; + /* Make sure QP and NP point to different objects. Otherwise the + * numerator would be gradually overwritten by the quotient limbs. + */ + if (qp == np) { /* Copy NP object to temporary space. */ + np = marker[markidx++] = mpi_alloc_limb_space(nsize); + MPN_COPY(np, qp, nsize); + } + } else /* Put quotient at top of remainder. */ + qp = rp + dsize; + + normalization_steps = count_leading_zeros(dp[dsize - 1]); + + /* Normalize the denominator, i.e. make its most significant bit set by + * shifting it NORMALIZATION_STEPS bits to the left. Also shift the + * numerator the same number of steps (to keep the quotient the same!). + */ + if (normalization_steps) { + mpi_ptr_t tp; + mpi_limb_t nlimb; + + /* Shift up the denominator setting the most significant bit of + * the most significant word. Use temporary storage not to clobber + * the original contents of the denominator. + */ + tp = marker[markidx++] = mpi_alloc_limb_space(dsize); + mpihelp_lshift(tp, dp, dsize, normalization_steps); + dp = tp; + + /* Shift up the numerator, possibly introducing a new most + * significant word. Move the shifted numerator in the remainder + * meanwhile. + */ + nlimb = mpihelp_lshift(rp, np, nsize, normalization_steps); + if (nlimb) { + rp[nsize] = nlimb; + rsize = nsize + 1; + } else + rsize = nsize; + } else { + /* The denominator is already normalized, as required. Copy it to + * temporary space if it overlaps with the quotient or remainder. + */ + if (dp == rp || (quot && (dp == qp))) { + mpi_ptr_t tp; + + tp = marker[markidx++] = mpi_alloc_limb_space(dsize); + MPN_COPY(tp, dp, dsize); + dp = tp; + } + + /* Move the numerator to the remainder. */ + if (rp != np) + MPN_COPY(rp, np, nsize); + + rsize = nsize; + } + + q_limb = mpihelp_divrem(qp, 0, rp, rsize, dp, dsize); + + if (quot) { + qsize = rsize - dsize; + if (q_limb) { + qp[qsize] = q_limb; + qsize += 1; + } + + quot->nlimbs = qsize; + quot->sign = sign_quotient; + } + + rsize = dsize; + MPN_NORMALIZE(rp, rsize); + + if (normalization_steps && rsize) { + mpihelp_rshift(rp, rp, rsize, normalization_steps); + rsize -= rp[rsize - 1] == 0?1:0; + } + + rem->nlimbs = rsize; + rem->sign = sign_remainder; + while (markidx) { + markidx--; + mpi_free_limb_space(marker[markidx]); + } +} diff --git a/lib/mpi/mpi-internal.h b/lib/mpi/mpi-internal.h index 91df5f0b70f2..554002182db1 100644 --- a/lib/mpi/mpi-internal.h +++ b/lib/mpi/mpi-internal.h @@ -52,6 +52,12 @@ typedef mpi_limb_t *mpi_ptr_t; /* pointer to a limb */ typedef int mpi_size_t; /* (must be a signed type) */ +#define RESIZE_IF_NEEDED(a, b) \ + do { \ + if ((a)->alloced < (b)) \ + mpi_resize((a), (b)); \ + } while (0) + /* Copy N limbs from S to D. */ #define MPN_COPY(d, s, n) \ do { \ @@ -60,6 +66,14 @@ typedef int mpi_size_t; /* (must be a signed type) */ (d)[_i] = (s)[_i]; \ } while (0) +#define MPN_COPY_INCR(d, s, n) \ + do { \ + mpi_size_t _i; \ + for (_i = 0; _i < (n); _i++) \ + (d)[_i] = (s)[_i]; \ + } while (0) + + #define MPN_COPY_DECR(d, s, n) \ do { \ mpi_size_t _i; \ @@ -92,6 +106,38 @@ typedef int mpi_size_t; /* (must be a signed type) */ mul_n(prodp, up, vp, size, tspace); \ } while (0); +/* Divide the two-limb number in (NH,,NL) by D, with DI being the largest + * limb not larger than (2**(2*BITS_PER_MP_LIMB))/D - (2**BITS_PER_MP_LIMB). + * If this would yield overflow, DI should be the largest possible number + * (i.e., only ones). For correct operation, the most significant bit of D + * has to be set. Put the quotient in Q and the remainder in R. + */ +#define UDIV_QRNND_PREINV(q, r, nh, nl, d, di) \ + do { \ + mpi_limb_t _ql __maybe_unused; \ + mpi_limb_t _q, _r; \ + mpi_limb_t _xh, _xl; \ + umul_ppmm(_q, _ql, (nh), (di)); \ + _q += (nh); /* DI is 2**BITS_PER_MPI_LIMB too small */ \ + umul_ppmm(_xh, _xl, _q, (d)); \ + sub_ddmmss(_xh, _r, (nh), (nl), _xh, _xl); \ + if (_xh) { \ + sub_ddmmss(_xh, _r, _xh, _r, 0, (d)); \ + _q++; \ + if (_xh) { \ + sub_ddmmss(_xh, _r, _xh, _r, 0, (d)); \ + _q++; \ + } \ + } \ + if (_r >= (d)) { \ + _r -= (d); \ + _q++; \ + } \ + (r) = _r; \ + (q) = _q; \ + } while (0) + + /*-- mpiutil.c --*/ mpi_ptr_t mpi_alloc_limb_space(unsigned nlimbs); void mpi_free_limb_space(mpi_ptr_t a); @@ -135,6 +181,8 @@ int mpihelp_mul(mpi_ptr_t prodp, mpi_ptr_t up, mpi_size_t usize, void mpih_sqr_n_basecase(mpi_ptr_t prodp, mpi_ptr_t up, mpi_size_t size); void mpih_sqr_n(mpi_ptr_t prodp, mpi_ptr_t up, mpi_size_t size, mpi_ptr_t tspace); +void mpihelp_mul_n(mpi_ptr_t prodp, + mpi_ptr_t up, mpi_ptr_t vp, mpi_size_t size); int mpihelp_mul_karatsuba_case(mpi_ptr_t prodp, mpi_ptr_t up, mpi_size_t usize, @@ -146,9 +194,14 @@ mpi_limb_t mpihelp_mul_1(mpi_ptr_t res_ptr, mpi_ptr_t s1_ptr, mpi_size_t s1_size, mpi_limb_t s2_limb); /*-- mpih-div.c --*/ +mpi_limb_t mpihelp_mod_1(mpi_ptr_t dividend_ptr, mpi_size_t dividend_size, + mpi_limb_t divisor_limb); mpi_limb_t mpihelp_divrem(mpi_ptr_t qp, mpi_size_t qextra_limbs, mpi_ptr_t np, mpi_size_t nsize, mpi_ptr_t dp, mpi_size_t dsize); +mpi_limb_t mpihelp_divmod_1(mpi_ptr_t quot_ptr, + mpi_ptr_t dividend_ptr, mpi_size_t dividend_size, + mpi_limb_t divisor_limb); /*-- generic_mpih-[lr]shift.c --*/ mpi_limb_t mpihelp_lshift(mpi_ptr_t wp, mpi_ptr_t up, mpi_size_t usize, diff --git a/lib/mpi/mpi-inv.c b/lib/mpi/mpi-inv.c new file mode 100644 index 000000000000..61e37d18f793 --- /dev/null +++ b/lib/mpi/mpi-inv.c @@ -0,0 +1,143 @@ +/* mpi-inv.c - MPI functions + * Copyright (C) 1998, 2001, 2002, 2003 Free Software Foundation, Inc. + * + * This file is part of Libgcrypt. + * + * Libgcrypt is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation; either version 2.1 of + * the License, or (at your option) any later version. + * + * Libgcrypt is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this program; if not, see <http://www.gnu.org/licenses/>. + */ + +#include "mpi-internal.h" + +/**************** + * Calculate the multiplicative inverse X of A mod N + * That is: Find the solution x for + * 1 = (a*x) mod n + */ +int mpi_invm(MPI x, MPI a, MPI n) +{ + /* Extended Euclid's algorithm (See TAOCP Vol II, 4.5.2, Alg X) + * modified according to Michael Penk's solution for Exercise 35 + * with further enhancement + */ + MPI u, v, u1, u2 = NULL, u3, v1, v2 = NULL, v3, t1, t2 = NULL, t3; + unsigned int k; + int sign; + int odd; + + if (!mpi_cmp_ui(a, 0)) + return 0; /* Inverse does not exists. */ + if (!mpi_cmp_ui(n, 1)) + return 0; /* Inverse does not exists. */ + + u = mpi_copy(a); + v = mpi_copy(n); + + for (k = 0; !mpi_test_bit(u, 0) && !mpi_test_bit(v, 0); k++) { + mpi_rshift(u, u, 1); + mpi_rshift(v, v, 1); + } + odd = mpi_test_bit(v, 0); + + u1 = mpi_alloc_set_ui(1); + if (!odd) + u2 = mpi_alloc_set_ui(0); + u3 = mpi_copy(u); + v1 = mpi_copy(v); + if (!odd) { + v2 = mpi_alloc(mpi_get_nlimbs(u)); + mpi_sub(v2, u1, u); /* U is used as const 1 */ + } + v3 = mpi_copy(v); + if (mpi_test_bit(u, 0)) { /* u is odd */ + t1 = mpi_alloc_set_ui(0); + if (!odd) { + t2 = mpi_alloc_set_ui(1); + t2->sign = 1; + } + t3 = mpi_copy(v); + t3->sign = !t3->sign; + goto Y4; + } else { + t1 = mpi_alloc_set_ui(1); + if (!odd) + t2 = mpi_alloc_set_ui(0); + t3 = mpi_copy(u); + } + + do { + do { + if (!odd) { + if (mpi_test_bit(t1, 0) || mpi_test_bit(t2, 0)) { + /* one is odd */ + mpi_add(t1, t1, v); + mpi_sub(t2, t2, u); + } + mpi_rshift(t1, t1, 1); + mpi_rshift(t2, t2, 1); + mpi_rshift(t3, t3, 1); + } else { + if (mpi_test_bit(t1, 0)) + mpi_add(t1, t1, v); + mpi_rshift(t1, t1, 1); + mpi_rshift(t3, t3, 1); + } +Y4: + ; + } while (!mpi_test_bit(t3, 0)); /* while t3 is even */ + + if (!t3->sign) { + mpi_set(u1, t1); + if (!odd) + mpi_set(u2, t2); + mpi_set(u3, t3); + } else { + mpi_sub(v1, v, t1); + sign = u->sign; u->sign = !u->sign; + if (!odd) + mpi_sub(v2, u, t2); + u->sign = sign; + sign = t3->sign; t3->sign = !t3->sign; + mpi_set(v3, t3); + t3->sign = sign; + } + mpi_sub(t1, u1, v1); + if (!odd) + mpi_sub(t2, u2, v2); + mpi_sub(t3, u3, v3); + if (t1->sign) { + mpi_add(t1, t1, v); + if (!odd) + mpi_sub(t2, t2, u); + } + } while (mpi_cmp_ui(t3, 0)); /* while t3 != 0 */ + /* mpi_lshift( u3, k ); */ + mpi_set(x, u1); + + mpi_free(u1); + mpi_free(v1); + mpi_free(t1); + if (!odd) { + mpi_free(u2); + mpi_free(v2); + mpi_free(t2); + } + mpi_free(u3); + mpi_free(v3); + mpi_free(t3); + + mpi_free(u); + mpi_free(v); + return 1; +} +EXPORT_SYMBOL_GPL(mpi_invm); diff --git a/lib/mpi/mpi-mod.c b/lib/mpi/mpi-mod.c new file mode 100644 index 000000000000..47bc59edd4ff --- /dev/null +++ b/lib/mpi/mpi-mod.c @@ -0,0 +1,155 @@ +/* mpi-mod.c - Modular reduction + * Copyright (C) 1998, 1999, 2001, 2002, 2003, + * 2007 Free Software Foundation, Inc. + * + * This file is part of Libgcrypt. + */ + + +#include "mpi-internal.h" +#include "longlong.h" + +/* Context used with Barrett reduction. */ +struct barrett_ctx_s { + MPI m; /* The modulus - may not be modified. */ + int m_copied; /* If true, M needs to be released. */ + int k; + MPI y; + MPI r1; /* Helper MPI. */ + MPI r2; /* Helper MPI. */ + MPI r3; /* Helper MPI allocated on demand. */ +}; + + + +void mpi_mod(MPI rem, MPI dividend, MPI divisor) +{ + mpi_fdiv_r(rem, dividend, divisor); +} + +/* This function returns a new context for Barrett based operations on + * the modulus M. This context needs to be released using + * _gcry_mpi_barrett_free. If COPY is true M will be transferred to + * the context and the user may change M. If COPY is false, M may not + * be changed until gcry_mpi_barrett_free has been called. + */ +mpi_barrett_t mpi_barrett_init(MPI m, int copy) +{ + mpi_barrett_t ctx; + MPI tmp; + + mpi_normalize(m); + ctx = kcalloc(1, sizeof(*ctx), GFP_KERNEL); + + if (copy) { + ctx->m = mpi_copy(m); + ctx->m_copied = 1; + } else + ctx->m = m; + + ctx->k = mpi_get_nlimbs(m); + tmp = mpi_alloc(ctx->k + 1); + + /* Barrett precalculation: y = floor(b^(2k) / m). */ + mpi_set_ui(tmp, 1); + mpi_lshift_limbs(tmp, 2 * ctx->k); + mpi_fdiv_q(tmp, tmp, m); + + ctx->y = tmp; + ctx->r1 = mpi_alloc(2 * ctx->k + 1); + ctx->r2 = mpi_alloc(2 * ctx->k + 1); + + return ctx; +} + +void mpi_barrett_free(mpi_barrett_t ctx) +{ + if (ctx) { + mpi_free(ctx->y); + mpi_free(ctx->r1); + mpi_free(ctx->r2); + if (ctx->r3) + mpi_free(ctx->r3); + if (ctx->m_copied) + mpi_free(ctx->m); + kfree(ctx); + } +} + + +/* R = X mod M + * + * Using Barrett reduction. Before using this function + * _gcry_mpi_barrett_init must have been called to do the + * precalculations. CTX is the context created by this precalculation + * and also conveys M. If the Barret reduction could no be done a + * straightforward reduction method is used. + * + * We assume that these conditions are met: + * Input: x =(x_2k-1 ...x_0)_b + * m =(m_k-1 ....m_0)_b with m_k-1 != 0 + * Output: r = x mod m + */ +void mpi_mod_barrett(MPI r, MPI x, mpi_barrett_t ctx) +{ + MPI m = ctx->m; + int k = ctx->k; + MPI y = ctx->y; + MPI r1 = ctx->r1; + MPI r2 = ctx->r2; + int sign; + + mpi_normalize(x); + if (mpi_get_nlimbs(x) > 2*k) { + mpi_mod(r, x, m); + return; + } + + sign = x->sign; + x->sign = 0; + + /* 1. q1 = floor( x / b^k-1) + * q2 = q1 * y + * q3 = floor( q2 / b^k+1 ) + * Actually, we don't need qx, we can work direct on r2 + */ + mpi_set(r2, x); + mpi_rshift_limbs(r2, k-1); + mpi_mul(r2, r2, y); + mpi_rshift_limbs(r2, k+1); + + /* 2. r1 = x mod b^k+1 + * r2 = q3 * m mod b^k+1 + * r = r1 - r2 + * 3. if r < 0 then r = r + b^k+1 + */ + mpi_set(r1, x); + if (r1->nlimbs > k+1) /* Quick modulo operation. */ + r1->nlimbs = k+1; + mpi_mul(r2, r2, m); + if (r2->nlimbs > k+1) /* Quick modulo operation. */ + r2->nlimbs = k+1; + mpi_sub(r, r1, r2); + + if (mpi_has_sign(r)) { + if (!ctx->r3) { + ctx->r3 = mpi_alloc(k + 2); + mpi_set_ui(ctx->r3, 1); + mpi_lshift_limbs(ctx->r3, k + 1); + } + mpi_add(r, r, ctx->r3); + } + + /* 4. while r >= m do r = r - m */ + while (mpi_cmp(r, m) >= 0) + mpi_sub(r, r, m); + + x->sign = sign; +} + + +void mpi_mul_barrett(MPI w, MPI u, MPI v, mpi_barrett_t ctx) +{ + mpi_mul(w, u, v); + mpi_mod_barrett(w, w, ctx); +} diff --git a/lib/mpi/mpi-mul.c b/lib/mpi/mpi-mul.c new file mode 100644 index 000000000000..8f5fa200f297 --- /dev/null +++ b/lib/mpi/mpi-mul.c @@ -0,0 +1,91 @@ +/* mpi-mul.c - MPI functions + * Copyright (C) 1994, 1996, 1998, 2001, 2002, + * 2003 Free Software Foundation, Inc. + * + * This file is part of Libgcrypt. + * + * Note: This code is heavily based on the GNU MP Library. + * Actually it's the same code with only minor changes in the + * way the data is stored; this is to support the abstraction + * of an optional secure memory allocation which may be used + * to avoid revealing of sensitive data due to paging etc. + */ + +#include "mpi-internal.h" + +void mpi_mul(MPI w, MPI u, MPI v) +{ + mpi_size_t usize, vsize, wsize; + mpi_ptr_t up, vp, wp; + mpi_limb_t cy; + int usign, vsign, sign_product; + int assign_wp = 0; + mpi_ptr_t tmp_limb = NULL; + + if (u->nlimbs < v->nlimbs) { + /* Swap U and V. */ + usize = v->nlimbs; + usign = v->sign; + up = v->d; + vsize = u->nlimbs; + vsign = u->sign; + vp = u->d; + } else { + usize = u->nlimbs; + usign = u->sign; + up = u->d; + vsize = v->nlimbs; + vsign = v->sign; + vp = v->d; + } + sign_product = usign ^ vsign; + wp = w->d; + + /* Ensure W has space enough to store the result. */ + wsize = usize + vsize; + if (w->alloced < wsize) { + if (wp == up || wp == vp) { + wp = mpi_alloc_limb_space(wsize); + assign_wp = 1; + } else { + mpi_resize(w, wsize); + wp = w->d; + } + } else { /* Make U and V not overlap with W. */ + if (wp == up) { + /* W and U are identical. Allocate temporary space for U. */ + up = tmp_limb = mpi_alloc_limb_space(usize); + /* Is V identical too? Keep it identical with U. */ + if (wp == vp) + vp = up; + /* Copy to the temporary space. */ + MPN_COPY(up, wp, usize); + } else if (wp == vp) { + /* W and V are identical. Allocate temporary space for V. */ + vp = tmp_limb = mpi_alloc_limb_space(vsize); + /* Copy to the temporary space. */ + MPN_COPY(vp, wp, vsize); + } + } + + if (!vsize) + wsize = 0; + else { + mpihelp_mul(wp, up, usize, vp, vsize, &cy); + wsize -= cy ? 0:1; + } + + if (assign_wp) + mpi_assign_limb_space(w, wp, wsize); + w->nlimbs = wsize; + w->sign = sign_product; + if (tmp_limb) + mpi_free_limb_space(tmp_limb); +} + +void mpi_mulm(MPI w, MPI u, MPI v, MPI m) +{ + mpi_mul(w, u, v); + mpi_tdiv_r(w, w, m); +} +EXPORT_SYMBOL_GPL(mpi_mulm); diff --git a/lib/mpi/mpicoder.c b/lib/mpi/mpicoder.c index eead4b339466..7ea225b2204f 100644 --- a/lib/mpi/mpicoder.c +++ b/lib/mpi/mpicoder.c @@ -25,6 +25,7 @@ #include <linux/string.h> #include "mpi-internal.h" +#define MAX_EXTERN_SCAN_BYTES (16*1024*1024) #define MAX_EXTERN_MPI_BITS 16384 /** @@ -109,6 +110,112 @@ MPI mpi_read_from_buffer(const void *xbuffer, unsigned *ret_nread) } EXPORT_SYMBOL_GPL(mpi_read_from_buffer); +/**************** + * Fill the mpi VAL from the hex string in STR. + */ +int mpi_fromstr(MPI val, const char *str) +{ + int sign = 0; + int prepend_zero = 0; + int i, j, c, c1, c2; + unsigned int nbits, nbytes, nlimbs; + mpi_limb_t a; + + if (*str == '-') { + sign = 1; + str++; + } + + /* Skip optional hex prefix. */ + if (*str == '0' && str[1] == 'x') + str += 2; + + nbits = strlen(str); + if (nbits > MAX_EXTERN_SCAN_BYTES) { + mpi_clear(val); + return -EINVAL; + } + nbits *= 4; + if ((nbits % 8)) + prepend_zero = 1; + + nbytes = (nbits+7) / 8; + nlimbs = (nbytes+BYTES_PER_MPI_LIMB-1) / BYTES_PER_MPI_LIMB; + + if (val->alloced < nlimbs) + mpi_resize(val, nlimbs); + + i = BYTES_PER_MPI_LIMB - (nbytes % BYTES_PER_MPI_LIMB); + i %= BYTES_PER_MPI_LIMB; + j = val->nlimbs = nlimbs; + val->sign = sign; + for (; j > 0; j--) { + a = 0; + for (; i < BYTES_PER_MPI_LIMB; i++) { + if (prepend_zero) { + c1 = '0'; + prepend_zero = 0; + } else + c1 = *str++; + + if (!c1) { + mpi_clear(val); + return -EINVAL; + } + c2 = *str++; + if (!c2) { + mpi_clear(val); + return -EINVAL; + } + if (c1 >= '0' && c1 <= '9') + c = c1 - '0'; + else if (c1 >= 'a' && c1 <= 'f') + c = c1 - 'a' + 10; + else if (c1 >= 'A' && c1 <= 'F') + c = c1 - 'A' + 10; + else { + mpi_clear(val); + return -EINVAL; + } + c <<= 4; + if (c2 >= '0' && c2 <= '9') + c |= c2 - '0'; + else if (c2 >= 'a' && c2 <= 'f') + c |= c2 - 'a' + 10; + else if (c2 >= 'A' && c2 <= 'F') + c |= c2 - 'A' + 10; + else { + mpi_clear(val); + return -EINVAL; + } + a <<= 8; + a |= c; + } + i = 0; + val->d[j-1] = a; + } + + return 0; +} +EXPORT_SYMBOL_GPL(mpi_fromstr); + +MPI mpi_scanval(const char *string) +{ + MPI a; + + a = mpi_alloc(0); + if (!a) + return NULL; + + if (mpi_fromstr(a, string)) { + mpi_free(a); + return NULL; + } + mpi_normalize(a); + return a; +} +EXPORT_SYMBOL_GPL(mpi_scanval); + static int count_lzeros(MPI a) { mpi_limb_t alimb; @@ -413,3 +520,232 @@ MPI mpi_read_raw_from_sgl(struct scatterlist *sgl, unsigned int nbytes) return val; } EXPORT_SYMBOL_GPL(mpi_read_raw_from_sgl); + +/* Perform a two's complement operation on buffer P of size N bytes. */ +static void twocompl(unsigned char *p, unsigned int n) +{ + int i; + + for (i = n-1; i >= 0 && !p[i]; i--) + ; + if (i >= 0) { + if ((p[i] & 0x01)) + p[i] = (((p[i] ^ 0xfe) | 0x01) & 0xff); + else if ((p[i] & 0x02)) + p[i] = (((p[i] ^ 0xfc) | 0x02) & 0xfe); + else if ((p[i] & 0x04)) + p[i] = (((p[i] ^ 0xf8) | 0x04) & 0xfc); + else if ((p[i] & 0x08)) + p[i] = (((p[i] ^ 0xf0) | 0x08) & 0xf8); + else if ((p[i] & 0x10)) + p[i] = (((p[i] ^ 0xe0) | 0x10) & 0xf0); + else if ((p[i] & 0x20)) + p[i] = (((p[i] ^ 0xc0) | 0x20) & 0xe0); + else if ((p[i] & 0x40)) + p[i] = (((p[i] ^ 0x80) | 0x40) & 0xc0); + else + p[i] = 0x80; + + for (i--; i >= 0; i--) + p[i] ^= 0xff; + } +} + +int mpi_print(enum gcry_mpi_format format, unsigned char *buffer, + size_t buflen, size_t *nwritten, MPI a) +{ + unsigned int nbits = mpi_get_nbits(a); + size_t len; + size_t dummy_nwritten; + int negative; + + if (!nwritten) + nwritten = &dummy_nwritten; + + /* Libgcrypt does no always care to set clear the sign if the value + * is 0. For printing this is a bit of a surprise, in particular + * because if some of the formats don't support negative numbers but + * should be able to print a zero. Thus we need this extra test + * for a negative number. + */ + if (a->sign && mpi_cmp_ui(a, 0)) + negative = 1; + else + negative = 0; + + len = buflen; + *nwritten = 0; + if (format == GCRYMPI_FMT_STD) { + unsigned char *tmp; + int extra = 0; + unsigned int n; + + tmp = mpi_get_buffer(a, &n, NULL); + if (!tmp) + return -EINVAL; + + if (negative) { + twocompl(tmp, n); + if (!(*tmp & 0x80)) { + /* Need to extend the sign. */ + n++; + extra = 2; + } + } else if (n && (*tmp & 0x80)) { + /* Positive but the high bit of the returned buffer is set. + * Thus we need to print an extra leading 0x00 so that the + * output is interpreted as a positive number. + */ + n++; + extra = 1; + } + + if (buffer && n > len) { + /* The provided buffer is too short. */ + kfree(tmp); + return -E2BIG; + } + if (buffer) { + unsigned char *s = buffer; + + if (extra == 1) + *s++ = 0; + else if (extra) + *s++ = 0xff; + memcpy(s, tmp, n-!!extra); + } + kfree(tmp); + *nwritten = n; + return 0; + } else if (format == GCRYMPI_FMT_USG) { + unsigned int n = (nbits + 7)/8; + + /* Note: We ignore the sign for this format. */ + /* FIXME: for performance reasons we should put this into + * mpi_aprint because we can then use the buffer directly. + */ + + if (buffer && n > len) + return -E2BIG; + if (buffer) { + unsigned char *tmp; + + tmp = mpi_get_buffer(a, &n, NULL); + if (!tmp) + return -EINVAL; + memcpy(buffer, tmp, n); + kfree(tmp); + } + *nwritten = n; + return 0; + } else if (format == GCRYMPI_FMT_PGP) { + unsigned int n = (nbits + 7)/8; + + /* The PGP format can only handle unsigned integers. */ + if (negative) + return -EINVAL; + + if (buffer && n+2 > len) + return -E2BIG; + + if (buffer) { + unsigned char *tmp; + unsigned char *s = buffer; + + s[0] = nbits >> 8; + s[1] = nbits; + + tmp = mpi_get_buffer(a, &n, NULL); + if (!tmp) + return -EINVAL; + memcpy(s+2, tmp, n); + kfree(tmp); + } + *nwritten = n+2; + return 0; + } else if (format == GCRYMPI_FMT_SSH) { + unsigned char *tmp; + int extra = 0; + unsigned int n; + + tmp = mpi_get_buffer(a, &n, NULL); + if (!tmp) + return -EINVAL; + + if (negative) { + twocompl(tmp, n); + if (!(*tmp & 0x80)) { + /* Need to extend the sign. */ + n++; + extra = 2; + } + } else if (n && (*tmp & 0x80)) { + n++; + extra = 1; + } + + if (buffer && n+4 > len) { + kfree(tmp); + return -E2BIG; + } + + if (buffer) { + unsigned char *s = buffer; + + *s++ = n >> 24; + *s++ = n >> 16; + *s++ = n >> 8; + *s++ = n; + if (extra == 1) + *s++ = 0; + else if (extra) + *s++ = 0xff; + memcpy(s, tmp, n-!!extra); + } + kfree(tmp); + *nwritten = 4+n; + return 0; + } else if (format == GCRYMPI_FMT_HEX) { + unsigned char *tmp; + int i; + int extra = 0; + unsigned int n = 0; + + tmp = mpi_get_buffer(a, &n, NULL); + if (!tmp) + return -EINVAL; + if (!n || (*tmp & 0x80)) + extra = 2; + + if (buffer && 2*n + extra + negative + 1 > len) { + kfree(tmp); + return -E2BIG; + } + if (buffer) { + unsigned char *s = buffer; + + if (negative) + *s++ = '-'; + if (extra) { + *s++ = '0'; + *s++ = '0'; + } + + for (i = 0; i < n; i++) { + unsigned int c = tmp[i]; + + *s++ = (c >> 4) < 10 ? '0'+(c>>4) : 'A'+(c>>4)-10; + c &= 15; + *s++ = c < 10 ? '0'+c : 'A'+c-10; + } + *s++ = 0; + *nwritten = s - buffer; + } else { + *nwritten = 2*n + extra + negative + 1; + } + kfree(tmp); + return 0; + } else + return -EINVAL; +} +EXPORT_SYMBOL_GPL(mpi_print); diff --git a/lib/mpi/mpih-div.c b/lib/mpi/mpih-div.c index 913a519eb005..be70ee2e42d3 100644 --- a/lib/mpi/mpih-div.c +++ b/lib/mpi/mpih-div.c @@ -24,6 +24,150 @@ #define UDIV_TIME UMUL_TIME #endif + +mpi_limb_t +mpihelp_mod_1(mpi_ptr_t dividend_ptr, mpi_size_t dividend_size, + mpi_limb_t divisor_limb) +{ + mpi_size_t i; + mpi_limb_t n1, n0, r; + mpi_limb_t dummy __maybe_unused; + + /* Botch: Should this be handled at all? Rely on callers? */ + if (!dividend_size) + return 0; + + /* If multiplication is much faster than division, and the + * dividend is large, pre-invert the divisor, and use + * only multiplications in the inner loop. + * + * This test should be read: + * Does it ever help to use udiv_qrnnd_preinv? + * && Does what we save compensate for the inversion overhead? + */ + if (UDIV_TIME > (2 * UMUL_TIME + 6) + && (UDIV_TIME - (2 * UMUL_TIME + 6)) * dividend_size > UDIV_TIME) { + int normalization_steps; + + normalization_steps = count_leading_zeros(divisor_limb); + if (normalization_steps) { + mpi_limb_t divisor_limb_inverted; + + divisor_limb <<= normalization_steps; + + /* Compute (2**2N - 2**N * DIVISOR_LIMB) / DIVISOR_LIMB. The + * result is a (N+1)-bit approximation to 1/DIVISOR_LIMB, with the + * most significant bit (with weight 2**N) implicit. + * + * Special case for DIVISOR_LIMB == 100...000. + */ + if (!(divisor_limb << 1)) + divisor_limb_inverted = ~(mpi_limb_t)0; + else + udiv_qrnnd(divisor_limb_inverted, dummy, + -divisor_limb, 0, divisor_limb); + + n1 = dividend_ptr[dividend_size - 1]; + r = n1 >> (BITS_PER_MPI_LIMB - normalization_steps); + + /* Possible optimization: + * if (r == 0 + * && divisor_limb > ((n1 << normalization_steps) + * | (dividend_ptr[dividend_size - 2] >> ...))) + * ...one division less... + */ + for (i = dividend_size - 2; i >= 0; i--) { + n0 = dividend_ptr[i]; + UDIV_QRNND_PREINV(dummy, r, r, + ((n1 << normalization_steps) + | (n0 >> (BITS_PER_MPI_LIMB - normalization_steps))), + divisor_limb, divisor_limb_inverted); + n1 = n0; + } + UDIV_QRNND_PREINV(dummy, r, r, + n1 << normalization_steps, + divisor_limb, divisor_limb_inverted); + return r >> normalization_steps; + } else { + mpi_limb_t divisor_limb_inverted; + + /* Compute (2**2N - 2**N * DIVISOR_LIMB) / DIVISOR_LIMB. The + * result is a (N+1)-bit approximation to 1/DIVISOR_LIMB, with the + * most significant bit (with weight 2**N) implicit. + * + * Special case for DIVISOR_LIMB == 100...000. + */ + if (!(divisor_limb << 1)) + divisor_limb_inverted = ~(mpi_limb_t)0; + else + udiv_qrnnd(divisor_limb_inverted, dummy, + -divisor_limb, 0, divisor_limb); + + i = dividend_size - 1; + r = dividend_ptr[i]; + + if (r >= divisor_limb) + r = 0; + else + i--; + + for ( ; i >= 0; i--) { + n0 = dividend_ptr[i]; + UDIV_QRNND_PREINV(dummy, r, r, + n0, divisor_limb, divisor_limb_inverted); + } + return r; + } + } else { + if (UDIV_NEEDS_NORMALIZATION) { + int normalization_steps; + + normalization_steps = count_leading_zeros(divisor_limb); + if (normalization_steps) { + divisor_limb <<= normalization_steps; + + n1 = dividend_ptr[dividend_size - 1]; + r = n1 >> (BITS_PER_MPI_LIMB - normalization_steps); + + /* Possible optimization: + * if (r == 0 + * && divisor_limb > ((n1 << normalization_steps) + * | (dividend_ptr[dividend_size - 2] >> ...))) + * ...one division less... + */ + for (i = dividend_size - 2; i >= 0; i--) { + n0 = dividend_ptr[i]; + udiv_qrnnd(dummy, r, r, + ((n1 << normalization_steps) + | (n0 >> (BITS_PER_MPI_LIMB - normalization_steps))), + divisor_limb); + n1 = n0; + } + udiv_qrnnd(dummy, r, r, + n1 << normalization_steps, + divisor_limb); + return r >> normalization_steps; + } + } + /* No normalization needed, either because udiv_qrnnd doesn't require + * it, or because DIVISOR_LIMB is already normalized. + */ + i = dividend_size - 1; + r = dividend_ptr[i]; + + if (r >= divisor_limb) + r = 0; + else + i--; + + for (; i >= 0; i--) { + n0 = dividend_ptr[i]; + udiv_qrnnd(dummy, r, r, n0, divisor_limb); + } + return r; + } +} + /* Divide num (NP/NSIZE) by den (DP/DSIZE) and write * the NSIZE-DSIZE least significant quotient limbs at QP * and the DSIZE long remainder at NP. If QEXTRA_LIMBS is @@ -221,3 +365,153 @@ q_test: return most_significant_q_limb; } + +/**************** + * Divide (DIVIDEND_PTR,,DIVIDEND_SIZE) by DIVISOR_LIMB. + * Write DIVIDEND_SIZE limbs of quotient at QUOT_PTR. + * Return the single-limb remainder. + * There are no constraints on the value of the divisor. + * + * QUOT_PTR and DIVIDEND_PTR might point to the same limb. + */ + +mpi_limb_t +mpihelp_divmod_1(mpi_ptr_t quot_ptr, + mpi_ptr_t dividend_ptr, mpi_size_t dividend_size, + mpi_limb_t divisor_limb) +{ + mpi_size_t i; + mpi_limb_t n1, n0, r; + mpi_limb_t dummy __maybe_unused; + + if (!dividend_size) + return 0; + + /* If multiplication is much faster than division, and the + * dividend is large, pre-invert the divisor, and use + * only multiplications in the inner loop. + * + * This test should be read: + * Does it ever help to use udiv_qrnnd_preinv? + * && Does what we save compensate for the inversion overhead? + */ + if (UDIV_TIME > (2 * UMUL_TIME + 6) + && (UDIV_TIME - (2 * UMUL_TIME + 6)) * dividend_size > UDIV_TIME) { + int normalization_steps; + + normalization_steps = count_leading_zeros(divisor_limb); + if (normalization_steps) { + mpi_limb_t divisor_limb_inverted; + + divisor_limb <<= normalization_steps; + + /* Compute (2**2N - 2**N * DIVISOR_LIMB) / DIVISOR_LIMB. The + * result is a (N+1)-bit approximation to 1/DIVISOR_LIMB, with the + * most significant bit (with weight 2**N) implicit. + */ + /* Special case for DIVISOR_LIMB == 100...000. */ + if (!(divisor_limb << 1)) + divisor_limb_inverted = ~(mpi_limb_t)0; + else + udiv_qrnnd(divisor_limb_inverted, dummy, + -divisor_limb, 0, divisor_limb); + + n1 = dividend_ptr[dividend_size - 1]; + r = n1 >> (BITS_PER_MPI_LIMB - normalization_steps); + + /* Possible optimization: + * if (r == 0 + * && divisor_limb > ((n1 << normalization_steps) + * | (dividend_ptr[dividend_size - 2] >> ...))) + * ...one division less... + */ + for (i = dividend_size - 2; i >= 0; i--) { + n0 = dividend_ptr[i]; + UDIV_QRNND_PREINV(quot_ptr[i + 1], r, r, + ((n1 << normalization_steps) + | (n0 >> (BITS_PER_MPI_LIMB - normalization_steps))), + divisor_limb, divisor_limb_inverted); + n1 = n0; + } + UDIV_QRNND_PREINV(quot_ptr[0], r, r, + n1 << normalization_steps, + divisor_limb, divisor_limb_inverted); + return r >> normalization_steps; + } else { + mpi_limb_t divisor_limb_inverted; + + /* Compute (2**2N - 2**N * DIVISOR_LIMB) / DIVISOR_LIMB. The + * result is a (N+1)-bit approximation to 1/DIVISOR_LIMB, with the + * most significant bit (with weight 2**N) implicit. + */ + /* Special case for DIVISOR_LIMB == 100...000. */ + if (!(divisor_limb << 1)) + divisor_limb_inverted = ~(mpi_limb_t) 0; + else + udiv_qrnnd(divisor_limb_inverted, dummy, + -divisor_limb, 0, divisor_limb); + + i = dividend_size - 1; + r = dividend_ptr[i]; + + if (r >= divisor_limb) + r = 0; + else + quot_ptr[i--] = 0; + + for ( ; i >= 0; i--) { + n0 = dividend_ptr[i]; + UDIV_QRNND_PREINV(quot_ptr[i], r, r, + n0, divisor_limb, divisor_limb_inverted); + } + return r; + } + } else { + if (UDIV_NEEDS_NORMALIZATION) { + int normalization_steps; + + normalization_steps = count_leading_zeros(divisor_limb); + if (normalization_steps) { + divisor_limb <<= normalization_steps; + + n1 = dividend_ptr[dividend_size - 1]; + r = n1 >> (BITS_PER_MPI_LIMB - normalization_steps); + + /* Possible optimization: + * if (r == 0 + * && divisor_limb > ((n1 << normalization_steps) + * | (dividend_ptr[dividend_size - 2] >> ...))) + * ...one division less... + */ + for (i = dividend_size - 2; i >= 0; i--) { + n0 = dividend_ptr[i]; + udiv_qrnnd(quot_ptr[i + 1], r, r, + ((n1 << normalization_steps) + | (n0 >> (BITS_PER_MPI_LIMB - normalization_steps))), + divisor_limb); + n1 = n0; + } + udiv_qrnnd(quot_ptr[0], r, r, + n1 << normalization_steps, + divisor_limb); + return r >> normalization_steps; + } + } + /* No normalization needed, either because udiv_qrnnd doesn't require + * it, or because DIVISOR_LIMB is already normalized. + */ + i = dividend_size - 1; + r = dividend_ptr[i]; + + if (r >= divisor_limb) + r = 0; + else + quot_ptr[i--] = 0; + + for (; i >= 0; i--) { + n0 = dividend_ptr[i]; + udiv_qrnnd(quot_ptr[i], r, r, n0, divisor_limb); + } + return r; + } +} diff --git a/lib/mpi/mpih-mul.c b/lib/mpi/mpih-mul.c index a93647564054..e5f1c84e3c48 100644 --- a/lib/mpi/mpih-mul.c +++ b/lib/mpi/mpih-mul.c @@ -317,6 +317,31 @@ mpih_sqr_n(mpi_ptr_t prodp, mpi_ptr_t up, mpi_size_t size, mpi_ptr_t tspace) } } + +void mpihelp_mul_n(mpi_ptr_t prodp, + mpi_ptr_t up, mpi_ptr_t vp, mpi_size_t size) +{ + if (up == vp) { + if (size < KARATSUBA_THRESHOLD) + mpih_sqr_n_basecase(prodp, up, size); + else { + mpi_ptr_t tspace; + tspace = mpi_alloc_limb_space(2 * size); + mpih_sqr_n(prodp, up, size, tspace); + mpi_free_limb_space(tspace); + } + } else { + if (size < KARATSUBA_THRESHOLD) + mul_n_basecase(prodp, up, vp, size); + else { + mpi_ptr_t tspace; + tspace = mpi_alloc_limb_space(2 * size); + mul_n(prodp, up, vp, size, tspace); + mpi_free_limb_space(tspace); + } + } +} + int mpihelp_mul_karatsuba_case(mpi_ptr_t prodp, mpi_ptr_t up, mpi_size_t usize, diff --git a/lib/mpi/mpiutil.c b/lib/mpi/mpiutil.c index 4cd2b335cb7f..3c63710c20c6 100644 --- a/lib/mpi/mpiutil.c +++ b/lib/mpi/mpiutil.c @@ -20,6 +20,63 @@ #include "mpi-internal.h" +/* Constants allocated right away at startup. */ +static MPI constants[MPI_NUMBER_OF_CONSTANTS]; + +/* Initialize the MPI subsystem. This is called early and allows to + * do some initialization without taking care of threading issues. + */ +static int __init mpi_init(void) +{ + int idx; + unsigned long value; + + for (idx = 0; idx < MPI_NUMBER_OF_CONSTANTS; idx++) { + switch (idx) { + case MPI_C_ZERO: + value = 0; + break; + case MPI_C_ONE: + value = 1; + break; + case MPI_C_TWO: + value = 2; + break; + case MPI_C_THREE: + value = 3; + break; + case MPI_C_FOUR: + value = 4; + break; + case MPI_C_EIGHT: + value = 8; + break; + default: + pr_err("MPI: invalid mpi_const selector %d\n", idx); + return -EFAULT; + } + constants[idx] = mpi_alloc_set_ui(value); + constants[idx]->flags = (16|32); + } + + return 0; +} +postcore_initcall(mpi_init); + +/* Return a constant MPI descripbed by NO which is one of the + * MPI_C_xxx macros. There is no need to copy this returned value; it + * may be used directly. + */ +MPI mpi_const(enum gcry_mpi_constants no) +{ + if ((int)no < 0 || no > MPI_NUMBER_OF_CONSTANTS) + pr_err("MPI: invalid mpi_const selector %d\n", no); + if (!constants[no]) + pr_err("MPI: MPI subsystem not initialized\n"); + return constants[no]; +} +EXPORT_SYMBOL_GPL(mpi_const); + /**************** * Note: It was a bad idea to use the number of limbs to allocate * because on a alpha the limbs are large but we normally need @@ -106,6 +163,15 @@ int mpi_resize(MPI a, unsigned nlimbs) return 0; } +void mpi_clear(MPI a) +{ + if (!a) + return; + a->nlimbs = 0; + a->flags = 0; +} +EXPORT_SYMBOL_GPL(mpi_clear); + void mpi_free(MPI a) { if (!a) @@ -122,5 +188,143 @@ void mpi_free(MPI a) } EXPORT_SYMBOL_GPL(mpi_free); +/**************** + * Note: This copy function should not interpret the MPI + * but copy it transparently. + */ +MPI mpi_copy(MPI a) +{ + int i; + MPI b; + + if (a) { + b = mpi_alloc(a->nlimbs); + b->nlimbs = a->nlimbs; + b->sign = a->sign; + b->flags = a->flags; + b->flags &= ~(16|32); /* Reset the immutable and constant flags. */ + for (i = 0; i < b->nlimbs; i++) + b->d[i] = a->d[i]; + } else + b = NULL; + return b; +} + +/**************** + * This function allocates an MPI which is optimized to hold + * a value as large as the one given in the argument and allocates it + * with the same flags as A. + */ +MPI mpi_alloc_like(MPI a) +{ + MPI b; + + if (a) { + b = mpi_alloc(a->nlimbs); + b->nlimbs = 0; + b->sign = 0; + b->flags = a->flags; + } else + b = NULL; + + return b; +} + + +/* Set U into W and release U. If W is NULL only U will be released. */ +void mpi_snatch(MPI w, MPI u) +{ + if (w) { + mpi_assign_limb_space(w, u->d, u->alloced); + w->nlimbs = u->nlimbs; + w->sign = u->sign; + w->flags = u->flags; + u->alloced = 0; + u->nlimbs = 0; + u->d = NULL; + } + mpi_free(u); +} + + +MPI mpi_set(MPI w, MPI u) +{ + mpi_ptr_t wp, up; + mpi_size_t usize = u->nlimbs; + int usign = u->sign; + + if (!w) + w = mpi_alloc(mpi_get_nlimbs(u)); + RESIZE_IF_NEEDED(w, usize); + wp = w->d; + up = u->d; + MPN_COPY(wp, up, usize); + w->nlimbs = usize; + w->flags = u->flags; + w->flags &= ~(16|32); /* Reset the immutable and constant flags. */ + w->sign = usign; + return w; +} +EXPORT_SYMBOL_GPL(mpi_set); + +MPI mpi_set_ui(MPI w, unsigned long u) +{ + if (!w) + w = mpi_alloc(1); + /* FIXME: If U is 0 we have no need to resize and thus possible + * allocating the the limbs. + */ + RESIZE_IF_NEEDED(w, 1); + w->d[0] = u; + w->nlimbs = u ? 1 : 0; + w->sign = 0; + w->flags = 0; + return w; +} +EXPORT_SYMBOL_GPL(mpi_set_ui); + +MPI mpi_alloc_set_ui(unsigned long u) +{ + MPI w = mpi_alloc(1); + w->d[0] = u; + w->nlimbs = u ? 1 : 0; + w->sign = 0; + return w; +} + +/**************** + * Swap the value of A and B, when SWAP is 1. + * Leave the value when SWAP is 0. + * This implementation should be constant-time regardless of SWAP. + */ +void mpi_swap_cond(MPI a, MPI b, unsigned long swap) +{ + mpi_size_t i; + mpi_size_t nlimbs; + mpi_limb_t mask = ((mpi_limb_t)0) - swap; + mpi_limb_t x; + + if (a->alloced > b->alloced) + nlimbs = b->alloced; + else + nlimbs = a->alloced; + if (a->nlimbs > nlimbs || b->nlimbs > nlimbs) + return; + + for (i = 0; i < nlimbs; i++) { + x = mask & (a->d[i] ^ b->d[i]); + a->d[i] = a->d[i] ^ x; + b->d[i] = b->d[i] ^ x; + } + + x = mask & (a->nlimbs ^ b->nlimbs); + a->nlimbs = a->nlimbs ^ x; + b->nlimbs = b->nlimbs ^ x; + + x = mask & (a->sign ^ b->sign); + a->sign = a->sign ^ x; + b->sign = b->sign ^ x; +} + MODULE_DESCRIPTION("Multiprecision maths library"); MODULE_LICENSE("GPL"); diff --git a/net/sunrpc/Kconfig b/net/sunrpc/Kconfig index 3bcf985507be..bbbb5af0af13 100644 --- a/net/sunrpc/Kconfig +++ b/net/sunrpc/Kconfig @@ -21,7 +21,6 @@ config RPCSEC_GSS_KRB5 depends on SUNRPC && CRYPTO depends on CRYPTO_MD5 && CRYPTO_DES && CRYPTO_CBC && CRYPTO_CTS depends on CRYPTO_ECB && CRYPTO_HMAC && CRYPTO_SHA1 && CRYPTO_AES - depends on CRYPTO_ARC4 default y select SUNRPC_GSS help diff --git a/net/sunrpc/auth_gss/gss_krb5_crypto.c b/net/sunrpc/auth_gss/gss_krb5_crypto.c index 794fb3001880..634b6c6e0dcb 100644 --- a/net/sunrpc/auth_gss/gss_krb5_crypto.c +++ b/net/sunrpc/auth_gss/gss_krb5_crypto.c @@ -138,135 +138,6 @@ checksummer(struct scatterlist *sg, void *data) return crypto_ahash_update(req); } -static int -arcfour_hmac_md5_usage_to_salt(unsigned int usage, u8 salt[4]) -{ - unsigned int ms_usage; - - switch (usage) { - case KG_USAGE_SIGN: - ms_usage = 15; - break; - case KG_USAGE_SEAL: - ms_usage = 13; - break; - default: - return -EINVAL; - } - salt[0] = (ms_usage >> 0) & 0xff; - salt[1] = (ms_usage >> 8) & 0xff; - salt[2] = (ms_usage >> 16) & 0xff; - salt[3] = (ms_usage >> 24) & 0xff; - - return 0; -} - -static u32 -make_checksum_hmac_md5(struct krb5_ctx *kctx, char *header, int hdrlen, - struct xdr_buf *body, int body_offset, u8 *cksumkey, - unsigned int usage, struct xdr_netobj *cksumout) -{ - struct scatterlist sg[1]; - int err = -1; - u8 *checksumdata; - u8 *rc4salt; - struct crypto_ahash *md5; - struct crypto_ahash *hmac_md5; - struct ahash_request *req; - - if (cksumkey == NULL) - return GSS_S_FAILURE; - - if (cksumout->len < kctx->gk5e->cksumlength) { - dprintk("%s: checksum buffer length, %u, too small for %s\n", - __func__, cksumout->len, kctx->gk5e->name); - return GSS_S_FAILURE; - } - - rc4salt = kmalloc_array(4, sizeof(*rc4salt), GFP_NOFS); - if (!rc4salt) - return GSS_S_FAILURE; - - if (arcfour_hmac_md5_usage_to_salt(usage, rc4salt)) { - dprintk("%s: invalid usage value %u\n", __func__, usage); - goto out_free_rc4salt; - } - - checksumdata = kmalloc(GSS_KRB5_MAX_CKSUM_LEN, GFP_NOFS); - if (!checksumdata) - goto out_free_rc4salt; - - md5 = crypto_alloc_ahash("md5", 0, CRYPTO_ALG_ASYNC); - if (IS_ERR(md5)) - goto out_free_cksum; - - hmac_md5 = crypto_alloc_ahash(kctx->gk5e->cksum_name, 0, - CRYPTO_ALG_ASYNC); - if (IS_ERR(hmac_md5)) - goto out_free_md5; - - req = ahash_request_alloc(md5, GFP_NOFS); - if (!req) - goto out_free_hmac_md5; - - ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); - - err = crypto_ahash_init(req); - if (err) - goto out; - sg_init_one(sg, rc4salt, 4); - ahash_request_set_crypt(req, sg, NULL, 4); - err = crypto_ahash_update(req); - if (err) - goto out; - - sg_init_one(sg, header, hdrlen); - ahash_request_set_crypt(req, sg, NULL, hdrlen); - err = crypto_ahash_update(req); - if (err) - goto out; - err = xdr_process_buf(body, body_offset, body->len - body_offset, - checksummer, req); - if (err) - goto out; - ahash_request_set_crypt(req, NULL, checksumdata, 0); - err = crypto_ahash_final(req); - if (err) - goto out; - - ahash_request_free(req); - req = ahash_request_alloc(hmac_md5, GFP_NOFS); - if (!req) - goto out_free_hmac_md5; - - ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); - - err = crypto_ahash_setkey(hmac_md5, cksumkey, kctx->gk5e->keylength); - if (err) - goto out; - - sg_init_one(sg, checksumdata, crypto_ahash_digestsize(md5)); - ahash_request_set_crypt(req, sg, checksumdata, - crypto_ahash_digestsize(md5)); - err = crypto_ahash_digest(req); - if (err) - goto out; - - memcpy(cksumout->data, checksumdata, kctx->gk5e->cksumlength); - cksumout->len = kctx->gk5e->cksumlength; -out: - ahash_request_free(req); -out_free_hmac_md5: - crypto_free_ahash(hmac_md5); -out_free_md5: - crypto_free_ahash(md5); -out_free_cksum: - kfree(checksumdata); -out_free_rc4salt: - kfree(rc4salt); - return err ? GSS_S_FAILURE : 0; -} - /* * checksum the plaintext data and hdrlen bytes of the token header * The checksum is performed over the first 8 bytes of the @@ -284,11 +155,6 @@ make_checksum(struct krb5_ctx *kctx, char *header, int hdrlen, u8 *checksumdata; unsigned int checksumlen; - if (kctx->gk5e->ctype == CKSUMTYPE_HMAC_MD5_ARCFOUR) - return make_checksum_hmac_md5(kctx, header, hdrlen, - body, body_offset, - cksumkey, usage, cksumout); - if (cksumout->len < kctx->gk5e->cksumlength) { dprintk("%s: checksum buffer length, %u, too small for %s\n", __func__, cksumout->len, kctx->gk5e->name); @@ -942,145 +808,3 @@ out_err: ret = GSS_S_FAILURE; return ret; } - -/* - * Compute Kseq given the initial session key and the checksum. - * Set the key of the given cipher. - */ -int -krb5_rc4_setup_seq_key(struct krb5_ctx *kctx, - struct crypto_sync_skcipher *cipher, - unsigned char *cksum) -{ - struct crypto_shash *hmac; - struct shash_desc *desc; - u8 Kseq[GSS_KRB5_MAX_KEYLEN]; - u32 zeroconstant = 0; - int err; - - dprintk("%s: entered\n", __func__); - - hmac = crypto_alloc_shash(kctx->gk5e->cksum_name, 0, 0); - if (IS_ERR(hmac)) { - dprintk("%s: error %ld, allocating hash '%s'\n", - __func__, PTR_ERR(hmac), kctx->gk5e->cksum_name); - return PTR_ERR(hmac); - } - - desc = kmalloc(sizeof(*desc) + crypto_shash_descsize(hmac), - GFP_NOFS); - if (!desc) { - dprintk("%s: failed to allocate shash descriptor for '%s'\n", - __func__, kctx->gk5e->cksum_name); - crypto_free_shash(hmac); - return -ENOMEM; - } - - desc->tfm = hmac; - - /* Compute intermediate Kseq from session key */ - err = crypto_shash_setkey(hmac, kctx->Ksess, kctx->gk5e->keylength); - if (err) - goto out_err; - - err = crypto_shash_digest(desc, (u8 *)&zeroconstant, 4, Kseq); - if (err) - goto out_err; - - /* Compute final Kseq from the checksum and intermediate Kseq */ - err = crypto_shash_setkey(hmac, Kseq, kctx->gk5e->keylength); - if (err) - goto out_err; - - err = crypto_shash_digest(desc, cksum, 8, Kseq); - if (err) - goto out_err; - - err = crypto_sync_skcipher_setkey(cipher, Kseq, kctx->gk5e->keylength); - if (err) - goto out_err; - - err = 0; - -out_err: - kfree_sensitive(desc); - crypto_free_shash(hmac); - dprintk("%s: returning %d\n", __func__, err); - return err; -} - -/* - * Compute Kcrypt given the initial session key and the plaintext seqnum. - * Set the key of cipher kctx->enc. - */ -int -krb5_rc4_setup_enc_key(struct krb5_ctx *kctx, - struct crypto_sync_skcipher *cipher, - s32 seqnum) -{ - struct crypto_shash *hmac; - struct shash_desc *desc; - u8 Kcrypt[GSS_KRB5_MAX_KEYLEN]; - u8 zeroconstant[4] = {0}; - u8 seqnumarray[4]; - int err, i; - - dprintk("%s: entered, seqnum %u\n", __func__, seqnum); - - hmac = crypto_alloc_shash(kctx->gk5e->cksum_name, 0, 0); - if (IS_ERR(hmac)) { - dprintk("%s: error %ld, allocating hash '%s'\n", - __func__, PTR_ERR(hmac), kctx->gk5e->cksum_name); - return PTR_ERR(hmac); - } - - desc = kmalloc(sizeof(*desc) + crypto_shash_descsize(hmac), - GFP_NOFS); - if (!desc) { - dprintk("%s: failed to allocate shash descriptor for '%s'\n", - __func__, kctx->gk5e->cksum_name); - crypto_free_shash(hmac); - return -ENOMEM; - } - - desc->tfm = hmac; - - /* Compute intermediate Kcrypt from session key */ - for (i = 0; i < kctx->gk5e->keylength; i++) - Kcrypt[i] = kctx->Ksess[i] ^ 0xf0; - - err = crypto_shash_setkey(hmac, Kcrypt, kctx->gk5e->keylength); - if (err) - goto out_err; - - err = crypto_shash_digest(desc, zeroconstant, 4, Kcrypt); - if (err) - goto out_err; - - /* Compute final Kcrypt from the seqnum and intermediate Kcrypt */ - err = crypto_shash_setkey(hmac, Kcrypt, kctx->gk5e->keylength); - if (err) - goto out_err; - - seqnumarray[0] = (unsigned char) ((seqnum >> 24) & 0xff); - seqnumarray[1] = (unsigned char) ((seqnum >> 16) & 0xff); - seqnumarray[2] = (unsigned char) ((seqnum >> 8) & 0xff); - seqnumarray[3] = (unsigned char) ((seqnum >> 0) & 0xff); - - err = crypto_shash_digest(desc, seqnumarray, 4, Kcrypt); - if (err) - goto out_err; - - err = crypto_sync_skcipher_setkey(cipher, Kcrypt, - kctx->gk5e->keylength); - if (err) - goto out_err; - - err = 0; - -out_err: - kfree_sensitive(desc); - crypto_free_shash(hmac); - dprintk("%s: returning %d\n", __func__, err); - return err; -} diff --git a/net/sunrpc/auth_gss/gss_krb5_mech.c b/net/sunrpc/auth_gss/gss_krb5_mech.c index a84a5b289484..ae9acf3a7389 100644 --- a/net/sunrpc/auth_gss/gss_krb5_mech.c +++ b/net/sunrpc/auth_gss/gss_krb5_mech.c @@ -52,27 +52,6 @@ static const struct gss_krb5_enctype supported_gss_krb5_enctypes[] = { }, #endif /* CONFIG_SUNRPC_DISABLE_INSECURE_ENCTYPES */ /* - * RC4-HMAC - */ - { - .etype = ENCTYPE_ARCFOUR_HMAC, - .ctype = CKSUMTYPE_HMAC_MD5_ARCFOUR, - .name = "rc4-hmac", - .encrypt_name = "ecb(arc4)", - .cksum_name = "hmac(md5)", - .encrypt = krb5_encrypt, - .decrypt = krb5_decrypt, - .mk_key = NULL, - .signalg = SGN_ALG_HMAC_MD5, - .sealalg = SEAL_ALG_MICROSOFT_RC4, - .keybytes = 16, - .keylength = 16, - .blocksize = 1, - .conflen = 8, - .cksumlength = 8, - .keyed_cksum = 1, - }, - /* * 3DES */ { @@ -401,78 +380,6 @@ out_err: return -EINVAL; } -/* - * Note that RC4 depends on deriving keys using the sequence - * number or the checksum of a token. Therefore, the final keys - * cannot be calculated until the token is being constructed! - */ -static int -context_derive_keys_rc4(struct krb5_ctx *ctx) -{ - struct crypto_shash *hmac; - char sigkeyconstant[] = "signaturekey"; - int slen = strlen(sigkeyconstant) + 1; /* include null terminator */ - struct shash_desc *desc; - int err; - - dprintk("RPC: %s: entered\n", __func__); - /* - * derive cksum (aka Ksign) key - */ - hmac = crypto_alloc_shash(ctx->gk5e->cksum_name, 0, 0); - if (IS_ERR(hmac)) { - dprintk("%s: error %ld allocating hash '%s'\n", - __func__, PTR_ERR(hmac), ctx->gk5e->cksum_name); - err = PTR_ERR(hmac); - goto out_err; - } - - err = crypto_shash_setkey(hmac, ctx->Ksess, ctx->gk5e->keylength); - if (err) - goto out_err_free_hmac; - - - desc = kmalloc(sizeof(*desc) + crypto_shash_descsize(hmac), GFP_NOFS); - if (!desc) { - dprintk("%s: failed to allocate hash descriptor for '%s'\n", - __func__, ctx->gk5e->cksum_name); - err = -ENOMEM; - goto out_err_free_hmac; - } - - desc->tfm = hmac; - - err = crypto_shash_digest(desc, sigkeyconstant, slen, ctx->cksum); - kfree_sensitive(desc); - if (err) - goto out_err_free_hmac; - /* - * allocate hash, and skciphers for data and seqnum encryption - */ - ctx->enc = crypto_alloc_sync_skcipher(ctx->gk5e->encrypt_name, 0, 0); - if (IS_ERR(ctx->enc)) { - err = PTR_ERR(ctx->enc); - goto out_err_free_hmac; - } - - ctx->seq = crypto_alloc_sync_skcipher(ctx->gk5e->encrypt_name, 0, 0); - if (IS_ERR(ctx->seq)) { - crypto_free_sync_skcipher(ctx->enc); - err = PTR_ERR(ctx->seq); - goto out_err_free_hmac; - } - - dprintk("RPC: %s: returning success\n", __func__); - - err = 0; - -out_err_free_hmac: - crypto_free_shash(hmac); -out_err: - dprintk("RPC: %s: returning %d\n", __func__, err); - return err; -} - static int context_derive_keys_new(struct krb5_ctx *ctx, gfp_t gfp_mask) { @@ -649,8 +556,6 @@ gss_import_v2_context(const void *p, const void *end, struct krb5_ctx *ctx, switch (ctx->enctype) { case ENCTYPE_DES3_CBC_RAW: return context_derive_keys_des3(ctx, gfp_mask); - case ENCTYPE_ARCFOUR_HMAC: - return context_derive_keys_rc4(ctx); case ENCTYPE_AES128_CTS_HMAC_SHA1_96: case ENCTYPE_AES256_CTS_HMAC_SHA1_96: return context_derive_keys_new(ctx, gfp_mask); diff --git a/net/sunrpc/auth_gss/gss_krb5_seal.c b/net/sunrpc/auth_gss/gss_krb5_seal.c index f1d280accf43..33061417ec97 100644 --- a/net/sunrpc/auth_gss/gss_krb5_seal.c +++ b/net/sunrpc/auth_gss/gss_krb5_seal.c @@ -214,7 +214,6 @@ gss_get_mic_kerberos(struct gss_ctx *gss_ctx, struct xdr_buf *text, BUG(); case ENCTYPE_DES_CBC_RAW: case ENCTYPE_DES3_CBC_RAW: - case ENCTYPE_ARCFOUR_HMAC: return gss_get_mic_v1(ctx, text, token); case ENCTYPE_AES128_CTS_HMAC_SHA1_96: case ENCTYPE_AES256_CTS_HMAC_SHA1_96: diff --git a/net/sunrpc/auth_gss/gss_krb5_seqnum.c b/net/sunrpc/auth_gss/gss_krb5_seqnum.c index 507105127095..fb117817ff5d 100644 --- a/net/sunrpc/auth_gss/gss_krb5_seqnum.c +++ b/net/sunrpc/auth_gss/gss_krb5_seqnum.c @@ -39,42 +39,6 @@ # define RPCDBG_FACILITY RPCDBG_AUTH #endif -static s32 -krb5_make_rc4_seq_num(struct krb5_ctx *kctx, int direction, s32 seqnum, - unsigned char *cksum, unsigned char *buf) -{ - struct crypto_sync_skcipher *cipher; - unsigned char *plain; - s32 code; - - dprintk("RPC: %s:\n", __func__); - cipher = crypto_alloc_sync_skcipher(kctx->gk5e->encrypt_name, 0, 0); - if (IS_ERR(cipher)) - return PTR_ERR(cipher); - - plain = kmalloc(8, GFP_NOFS); - if (!plain) - return -ENOMEM; - - plain[0] = (unsigned char) ((seqnum >> 24) & 0xff); - plain[1] = (unsigned char) ((seqnum >> 16) & 0xff); - plain[2] = (unsigned char) ((seqnum >> 8) & 0xff); - plain[3] = (unsigned char) ((seqnum >> 0) & 0xff); - plain[4] = direction; - plain[5] = direction; - plain[6] = direction; - plain[7] = direction; - - code = krb5_rc4_setup_seq_key(kctx, cipher, cksum); - if (code) - goto out; - - code = krb5_encrypt(cipher, cksum, plain, buf, 8); -out: - kfree(plain); - crypto_free_sync_skcipher(cipher); - return code; -} s32 krb5_make_seq_num(struct krb5_ctx *kctx, struct crypto_sync_skcipher *key, @@ -85,10 +49,6 @@ krb5_make_seq_num(struct krb5_ctx *kctx, unsigned char *plain; s32 code; - if (kctx->enctype == ENCTYPE_ARCFOUR_HMAC) - return krb5_make_rc4_seq_num(kctx, direction, seqnum, - cksum, buf); - plain = kmalloc(8, GFP_NOFS); if (!plain) return -ENOMEM; @@ -108,50 +68,6 @@ krb5_make_seq_num(struct krb5_ctx *kctx, return code; } -static s32 -krb5_get_rc4_seq_num(struct krb5_ctx *kctx, unsigned char *cksum, - unsigned char *buf, int *direction, s32 *seqnum) -{ - struct crypto_sync_skcipher *cipher; - unsigned char *plain; - s32 code; - - dprintk("RPC: %s:\n", __func__); - cipher = crypto_alloc_sync_skcipher(kctx->gk5e->encrypt_name, 0, 0); - if (IS_ERR(cipher)) - return PTR_ERR(cipher); - - code = krb5_rc4_setup_seq_key(kctx, cipher, cksum); - if (code) - goto out; - - plain = kmalloc(8, GFP_NOFS); - if (!plain) { - code = -ENOMEM; - goto out; - } - - code = krb5_decrypt(cipher, cksum, buf, plain, 8); - if (code) - goto out_plain; - - if ((plain[4] != plain[5]) || (plain[4] != plain[6]) - || (plain[4] != plain[7])) { - code = (s32)KG_BAD_SEQ; - goto out_plain; - } - - *direction = plain[4]; - - *seqnum = ((plain[0] << 24) | (plain[1] << 16) | - (plain[2] << 8) | (plain[3])); -out_plain: - kfree(plain); -out: - crypto_free_sync_skcipher(cipher); - return code; -} - s32 krb5_get_seq_num(struct krb5_ctx *kctx, unsigned char *cksum, @@ -164,9 +80,6 @@ krb5_get_seq_num(struct krb5_ctx *kctx, dprintk("RPC: krb5_get_seq_num:\n"); - if (kctx->enctype == ENCTYPE_ARCFOUR_HMAC) - return krb5_get_rc4_seq_num(kctx, cksum, buf, - direction, seqnum); plain = kmalloc(8, GFP_NOFS); if (!plain) return -ENOMEM; diff --git a/net/sunrpc/auth_gss/gss_krb5_unseal.c b/net/sunrpc/auth_gss/gss_krb5_unseal.c index aaab91cf24c8..ba04e3ec970a 100644 --- a/net/sunrpc/auth_gss/gss_krb5_unseal.c +++ b/net/sunrpc/auth_gss/gss_krb5_unseal.c @@ -218,7 +218,6 @@ gss_verify_mic_kerberos(struct gss_ctx *gss_ctx, BUG(); case ENCTYPE_DES_CBC_RAW: case ENCTYPE_DES3_CBC_RAW: - case ENCTYPE_ARCFOUR_HMAC: return gss_verify_mic_v1(ctx, message_buffer, read_token); case ENCTYPE_AES128_CTS_HMAC_SHA1_96: case ENCTYPE_AES256_CTS_HMAC_SHA1_96: diff --git a/net/sunrpc/auth_gss/gss_krb5_wrap.c b/net/sunrpc/auth_gss/gss_krb5_wrap.c index 8b300b74a722..e95c009bb869 100644 --- a/net/sunrpc/auth_gss/gss_krb5_wrap.c +++ b/net/sunrpc/auth_gss/gss_krb5_wrap.c @@ -236,26 +236,9 @@ gss_wrap_kerberos_v1(struct krb5_ctx *kctx, int offset, seq_send, ptr + GSS_KRB5_TOK_HDR_LEN, ptr + 8))) return GSS_S_FAILURE; - if (kctx->enctype == ENCTYPE_ARCFOUR_HMAC) { - struct crypto_sync_skcipher *cipher; - int err; - cipher = crypto_alloc_sync_skcipher(kctx->gk5e->encrypt_name, - 0, 0); - if (IS_ERR(cipher)) - return GSS_S_FAILURE; - - krb5_rc4_setup_enc_key(kctx, cipher, seq_send); - - err = gss_encrypt_xdr_buf(cipher, buf, - offset + headlen - conflen, pages); - crypto_free_sync_skcipher(cipher); - if (err) - return GSS_S_FAILURE; - } else { - if (gss_encrypt_xdr_buf(kctx->enc, buf, - offset + headlen - conflen, pages)) - return GSS_S_FAILURE; - } + if (gss_encrypt_xdr_buf(kctx->enc, buf, + offset + headlen - conflen, pages)) + return GSS_S_FAILURE; return (kctx->endtime < now) ? GSS_S_CONTEXT_EXPIRED : GSS_S_COMPLETE; } @@ -316,37 +299,9 @@ gss_unwrap_kerberos_v1(struct krb5_ctx *kctx, int offset, int len, crypt_offset = ptr + (GSS_KRB5_TOK_HDR_LEN + kctx->gk5e->cksumlength) - (unsigned char *)buf->head[0].iov_base; - /* - * Need plaintext seqnum to derive encryption key for arcfour-hmac - */ - if (krb5_get_seq_num(kctx, ptr + GSS_KRB5_TOK_HDR_LEN, - ptr + 8, &direction, &seqnum)) - return GSS_S_BAD_SIG; - - if ((kctx->initiate && direction != 0xff) || - (!kctx->initiate && direction != 0)) - return GSS_S_BAD_SIG; - buf->len = len; - if (kctx->enctype == ENCTYPE_ARCFOUR_HMAC) { - struct crypto_sync_skcipher *cipher; - int err; - - cipher = crypto_alloc_sync_skcipher(kctx->gk5e->encrypt_name, - 0, 0); - if (IS_ERR(cipher)) - return GSS_S_FAILURE; - - krb5_rc4_setup_enc_key(kctx, cipher, seqnum); - - err = gss_decrypt_xdr_buf(cipher, buf, crypt_offset); - crypto_free_sync_skcipher(cipher); - if (err) - return GSS_S_DEFECTIVE_TOKEN; - } else { - if (gss_decrypt_xdr_buf(kctx->enc, buf, crypt_offset)) - return GSS_S_DEFECTIVE_TOKEN; - } + if (gss_decrypt_xdr_buf(kctx->enc, buf, crypt_offset)) + return GSS_S_DEFECTIVE_TOKEN; if (kctx->gk5e->keyed_cksum) cksumkey = kctx->cksum; @@ -370,6 +325,14 @@ gss_unwrap_kerberos_v1(struct krb5_ctx *kctx, int offset, int len, /* do sequencing checks */ + if (krb5_get_seq_num(kctx, ptr + GSS_KRB5_TOK_HDR_LEN, + ptr + 8, &direction, &seqnum)) + return GSS_S_BAD_SIG; + + if ((kctx->initiate && direction != 0xff) || + (!kctx->initiate && direction != 0)) + return GSS_S_BAD_SIG; + /* Copy the data back to the right position. XXX: Would probably be * better to copy and encrypt at the same time. */ @@ -605,7 +568,6 @@ gss_wrap_kerberos(struct gss_ctx *gctx, int offset, BUG(); case ENCTYPE_DES_CBC_RAW: case ENCTYPE_DES3_CBC_RAW: - case ENCTYPE_ARCFOUR_HMAC: return gss_wrap_kerberos_v1(kctx, offset, buf, pages); case ENCTYPE_AES128_CTS_HMAC_SHA1_96: case ENCTYPE_AES256_CTS_HMAC_SHA1_96: @@ -624,7 +586,6 @@ gss_unwrap_kerberos(struct gss_ctx *gctx, int offset, BUG(); case ENCTYPE_DES_CBC_RAW: case ENCTYPE_DES3_CBC_RAW: - case ENCTYPE_ARCFOUR_HMAC: return gss_unwrap_kerberos_v1(kctx, offset, len, buf, &gctx->slack, &gctx->align); case ENCTYPE_AES128_CTS_HMAC_SHA1_96: diff --git a/security/integrity/digsig_asymmetric.c b/security/integrity/digsig_asymmetric.c index cfa4127d0518..b86a4a8f61ab 100644 --- a/security/integrity/digsig_asymmetric.c +++ b/security/integrity/digsig_asymmetric.c @@ -99,14 +99,22 @@ int asymmetric_verify(struct key *keyring, const char *sig, memset(&pks, 0, sizeof(pks)); pks.hash_algo = hash_algo_name[hdr->hash_algo]; - if (hdr->hash_algo == HASH_ALGO_STREEBOG_256 || - hdr->hash_algo == HASH_ALGO_STREEBOG_512) { + switch (hdr->hash_algo) { + case HASH_ALGO_STREEBOG_256: + case HASH_ALGO_STREEBOG_512: /* EC-RDSA and Streebog should go together. */ pks.pkey_algo = "ecrdsa"; pks.encoding = "raw"; - } else { + break; + case HASH_ALGO_SM3_256: + /* SM2 and SM3 should go together. */ + pks.pkey_algo = "sm2"; + pks.encoding = "raw"; + break; + default: pks.pkey_algo = "rsa"; pks.encoding = "pkcs1"; + break; } pks.digest = (u8 *)data; pks.digest_size = datalen; |