diff options
Diffstat (limited to 'drivers/crypto/inside-secure/safexcel_cipher.c')
| -rw-r--r-- | drivers/crypto/inside-secure/safexcel_cipher.c | 1506 |
1 files changed, 1419 insertions, 87 deletions
diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c index ef51f8c2b473..c02995694b41 100644 --- a/drivers/crypto/inside-secure/safexcel_cipher.c +++ b/drivers/crypto/inside-secure/safexcel_cipher.c @@ -5,18 +5,22 @@ * Antoine Tenart <antoine.tenart@free-electrons.com> */ +#include <asm/unaligned.h> #include <linux/device.h> #include <linux/dma-mapping.h> #include <linux/dmapool.h> - #include <crypto/aead.h> #include <crypto/aes.h> #include <crypto/authenc.h> +#include <crypto/chacha.h> #include <crypto/ctr.h> #include <crypto/internal/des.h> #include <crypto/gcm.h> #include <crypto/ghash.h> +#include <crypto/poly1305.h> #include <crypto/sha.h> +#include <crypto/sm3.h> +#include <crypto/sm4.h> #include <crypto/xts.h> #include <crypto/skcipher.h> #include <crypto/internal/aead.h> @@ -33,6 +37,8 @@ enum safexcel_cipher_alg { SAFEXCEL_DES, SAFEXCEL_3DES, SAFEXCEL_AES, + SAFEXCEL_CHACHA20, + SAFEXCEL_SM4, }; struct safexcel_cipher_ctx { @@ -41,8 +47,8 @@ struct safexcel_cipher_ctx { u32 mode; enum safexcel_cipher_alg alg; - bool aead; - int xcm; /* 0=authenc, 1=GCM, 2 reserved for CCM */ + char aead; /* !=0=AEAD, 2=IPSec ESP AEAD, 3=IPsec ESP GMAC */ + char xcm; /* 0=authenc, 1=GCM, 2 reserved for CCM */ __le32 key[16]; u32 nonce; @@ -51,10 +57,11 @@ struct safexcel_cipher_ctx { /* All the below is AEAD specific */ u32 hash_alg; u32 state_sz; - u32 ipad[SHA512_DIGEST_SIZE / sizeof(u32)]; - u32 opad[SHA512_DIGEST_SIZE / sizeof(u32)]; + __be32 ipad[SHA512_DIGEST_SIZE / sizeof(u32)]; + __be32 opad[SHA512_DIGEST_SIZE / sizeof(u32)]; struct crypto_cipher *hkaes; + struct crypto_aead *fback; }; struct safexcel_cipher_req { @@ -70,24 +77,50 @@ static void safexcel_cipher_token(struct safexcel_cipher_ctx *ctx, u8 *iv, { u32 block_sz = 0; - if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD) { + if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD || + ctx->aead & EIP197_AEAD_TYPE_IPSEC_ESP) { /* _ESP and _ESP_GMAC */ cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD; /* 32 bit nonce */ cdesc->control_data.token[0] = ctx->nonce; /* 64 bit IV part */ memcpy(&cdesc->control_data.token[1], iv, 8); - /* 32 bit counter, start at 1 (big endian!) */ - cdesc->control_data.token[3] = cpu_to_be32(1); + + if (ctx->alg == SAFEXCEL_CHACHA20 || + ctx->xcm == EIP197_XCM_MODE_CCM) { + /* 32 bit counter, starting at 0 */ + cdesc->control_data.token[3] = 0; + } else { + /* 32 bit counter, start at 1 (big endian!) */ + cdesc->control_data.token[3] = + (__force u32)cpu_to_be32(1); + } return; - } else if (ctx->xcm == EIP197_XCM_MODE_GCM) { + } else if (ctx->xcm == EIP197_XCM_MODE_GCM || + (ctx->aead && ctx->alg == SAFEXCEL_CHACHA20)) { cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD; /* 96 bit IV part */ memcpy(&cdesc->control_data.token[0], iv, 12); - /* 32 bit counter, start at 1 (big endian!) */ - cdesc->control_data.token[3] = cpu_to_be32(1); + + if (ctx->alg == SAFEXCEL_CHACHA20) { + /* 32 bit counter, starting at 0 */ + cdesc->control_data.token[3] = 0; + } else { + /* 32 bit counter, start at 1 (big endian!) */ + *(__be32 *)&cdesc->control_data.token[3] = + cpu_to_be32(1); + } + + return; + } else if (ctx->alg == SAFEXCEL_CHACHA20) { + cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD; + + /* 96 bit nonce part */ + memcpy(&cdesc->control_data.token[0], &iv[4], 12); + /* 32 bit counter */ + cdesc->control_data.token[3] = *(u32 *)iv; return; } else if (ctx->xcm == EIP197_XCM_MODE_CCM) { @@ -112,10 +145,16 @@ static void safexcel_cipher_token(struct safexcel_cipher_ctx *ctx, u8 *iv, block_sz = DES3_EDE_BLOCK_SIZE; cdesc->control_data.options |= EIP197_OPTION_2_TOKEN_IV_CMD; break; + case SAFEXCEL_SM4: + block_sz = SM4_BLOCK_SIZE; + cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD; + break; case SAFEXCEL_AES: block_sz = AES_BLOCK_SIZE; cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD; break; + default: + break; } memcpy(cdesc->control_data.token, iv, block_sz); } @@ -153,72 +192,84 @@ static void safexcel_aead_token(struct safexcel_cipher_ctx *ctx, u8 *iv, if (direction == SAFEXCEL_ENCRYPT) { /* align end of instruction sequence to end of token */ token = (struct safexcel_token *)(cdesc->control_data.token + - EIP197_MAX_TOKENS - 13); + EIP197_MAX_TOKENS - 14); - token[12].opcode = EIP197_TOKEN_OPCODE_INSERT; - token[12].packet_length = digestsize; - token[12].stat = EIP197_TOKEN_STAT_LAST_HASH | + token[13].opcode = EIP197_TOKEN_OPCODE_INSERT; + token[13].packet_length = digestsize; + token[13].stat = EIP197_TOKEN_STAT_LAST_HASH | EIP197_TOKEN_STAT_LAST_PACKET; - token[12].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT | + token[13].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT | EIP197_TOKEN_INS_INSERT_HASH_DIGEST; } else { cryptlen -= digestsize; /* align end of instruction sequence to end of token */ token = (struct safexcel_token *)(cdesc->control_data.token + - EIP197_MAX_TOKENS - 14); + EIP197_MAX_TOKENS - 15); - token[12].opcode = EIP197_TOKEN_OPCODE_RETRIEVE; - token[12].packet_length = digestsize; - token[12].stat = EIP197_TOKEN_STAT_LAST_HASH | + token[13].opcode = EIP197_TOKEN_OPCODE_RETRIEVE; + token[13].packet_length = digestsize; + token[13].stat = EIP197_TOKEN_STAT_LAST_HASH | EIP197_TOKEN_STAT_LAST_PACKET; - token[12].instructions = EIP197_TOKEN_INS_INSERT_HASH_DIGEST; + token[13].instructions = EIP197_TOKEN_INS_INSERT_HASH_DIGEST; - token[13].opcode = EIP197_TOKEN_OPCODE_VERIFY; - token[13].packet_length = digestsize | + token[14].opcode = EIP197_TOKEN_OPCODE_VERIFY; + token[14].packet_length = digestsize | EIP197_TOKEN_HASH_RESULT_VERIFY; - token[13].stat = EIP197_TOKEN_STAT_LAST_HASH | + token[14].stat = EIP197_TOKEN_STAT_LAST_HASH | EIP197_TOKEN_STAT_LAST_PACKET; - token[13].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT; + token[14].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT; } - token[6].opcode = EIP197_TOKEN_OPCODE_DIRECTION; - token[6].packet_length = assoclen; + if (ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP) { + /* For ESP mode (and not GMAC), skip over the IV */ + token[8].opcode = EIP197_TOKEN_OPCODE_DIRECTION; + token[8].packet_length = EIP197_AEAD_IPSEC_IV_SIZE; - if (likely(cryptlen)) { - token[6].instructions = EIP197_TOKEN_INS_TYPE_HASH; + assoclen -= EIP197_AEAD_IPSEC_IV_SIZE; + } - token[10].opcode = EIP197_TOKEN_OPCODE_DIRECTION; - token[10].packet_length = cryptlen; - token[10].stat = EIP197_TOKEN_STAT_LAST_HASH; - token[10].instructions = EIP197_TOKEN_INS_LAST | - EIP197_TOKEN_INS_TYPE_CRYPTO | - EIP197_TOKEN_INS_TYPE_HASH | - EIP197_TOKEN_INS_TYPE_OUTPUT; + token[6].opcode = EIP197_TOKEN_OPCODE_DIRECTION; + token[6].packet_length = assoclen; + token[6].instructions = EIP197_TOKEN_INS_LAST | + EIP197_TOKEN_INS_TYPE_HASH; + + if (likely(cryptlen || ctx->alg == SAFEXCEL_CHACHA20)) { + token[11].opcode = EIP197_TOKEN_OPCODE_DIRECTION; + token[11].packet_length = cryptlen; + token[11].stat = EIP197_TOKEN_STAT_LAST_HASH; + if (unlikely(ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP_GMAC)) { + token[6].instructions = EIP197_TOKEN_INS_TYPE_HASH; + /* Do not send to crypt engine in case of GMAC */ + token[11].instructions = EIP197_TOKEN_INS_LAST | + EIP197_TOKEN_INS_TYPE_HASH | + EIP197_TOKEN_INS_TYPE_OUTPUT; + } else { + token[11].instructions = EIP197_TOKEN_INS_LAST | + EIP197_TOKEN_INS_TYPE_CRYPTO | + EIP197_TOKEN_INS_TYPE_HASH | + EIP197_TOKEN_INS_TYPE_OUTPUT; + } } else if (ctx->xcm != EIP197_XCM_MODE_CCM) { token[6].stat = EIP197_TOKEN_STAT_LAST_HASH; - token[6].instructions = EIP197_TOKEN_INS_LAST | - EIP197_TOKEN_INS_TYPE_HASH; } if (!ctx->xcm) return; - token[8].opcode = EIP197_TOKEN_OPCODE_INSERT_REMRES; - token[8].packet_length = 0; - token[8].instructions = AES_BLOCK_SIZE; + token[9].opcode = EIP197_TOKEN_OPCODE_INSERT_REMRES; + token[9].packet_length = 0; + token[9].instructions = AES_BLOCK_SIZE; - token[9].opcode = EIP197_TOKEN_OPCODE_INSERT; - token[9].packet_length = AES_BLOCK_SIZE; - token[9].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT | - EIP197_TOKEN_INS_TYPE_CRYPTO; + token[10].opcode = EIP197_TOKEN_OPCODE_INSERT; + token[10].packet_length = AES_BLOCK_SIZE; + token[10].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT | + EIP197_TOKEN_INS_TYPE_CRYPTO; - if (ctx->xcm == EIP197_XCM_MODE_GCM) { - token[6].instructions = EIP197_TOKEN_INS_LAST | - EIP197_TOKEN_INS_TYPE_HASH; - } else { + if (ctx->xcm != EIP197_XCM_MODE_GCM) { + u8 *final_iv = (u8 *)cdesc->control_data.token; u8 *cbcmaciv = (u8 *)&token[1]; - u32 *aadlen = (u32 *)&token[5]; + __le32 *aadlen = (__le32 *)&token[5]; /* Construct IV block B0 for the CBC-MAC */ token[0].opcode = EIP197_TOKEN_OPCODE_INSERT; @@ -227,17 +278,18 @@ static void safexcel_aead_token(struct safexcel_cipher_ctx *ctx, u8 *iv, token[0].instructions = EIP197_TOKEN_INS_ORIGIN_TOKEN | EIP197_TOKEN_INS_TYPE_HASH; /* Variable length IV part */ - memcpy(cbcmaciv, iv, 15 - iv[0]); + memcpy(cbcmaciv, final_iv, 15 - final_iv[0]); /* fixup flags byte */ cbcmaciv[0] |= ((assoclen > 0) << 6) | ((digestsize - 2) << 2); /* Clear upper bytes of variable message length to 0 */ - memset(cbcmaciv + 15 - iv[0], 0, iv[0] - 1); + memset(cbcmaciv + 15 - final_iv[0], 0, final_iv[0] - 1); /* insert lower 2 bytes of message length */ cbcmaciv[14] = cryptlen >> 8; cbcmaciv[15] = cryptlen & 255; if (assoclen) { - *aadlen = cpu_to_le32(cpu_to_be16(assoclen)); + *aadlen = cpu_to_le32((assoclen >> 8) | + ((assoclen & 0xff) << 8)); assoclen += 2; } @@ -252,13 +304,13 @@ static void safexcel_aead_token(struct safexcel_cipher_ctx *ctx, u8 *iv, token[7].instructions = EIP197_TOKEN_INS_TYPE_HASH; /* Align crypto data towards hash engine */ - token[10].stat = 0; + token[11].stat = 0; - token[11].opcode = EIP197_TOKEN_OPCODE_INSERT; + token[12].opcode = EIP197_TOKEN_OPCODE_INSERT; cryptlen &= 15; - token[11].packet_length = cryptlen ? 16 - cryptlen : 0; - token[11].stat = EIP197_TOKEN_STAT_LAST_HASH; - token[11].instructions = EIP197_TOKEN_INS_TYPE_HASH; + token[12].packet_length = cryptlen ? 16 - cryptlen : 0; + token[12].stat = EIP197_TOKEN_STAT_LAST_HASH; + token[12].instructions = EIP197_TOKEN_INS_TYPE_HASH; } else { token[7].stat = EIP197_TOKEN_STAT_LAST_HASH; token[7].instructions = EIP197_TOKEN_INS_LAST | @@ -284,7 +336,7 @@ static int safexcel_skcipher_aes_setkey(struct crypto_skcipher *ctfm, if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { for (i = 0; i < len / sizeof(u32); i++) { - if (ctx->key[i] != cpu_to_le32(aes.key_enc[i])) { + if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) { ctx->base.needs_inv = true; break; } @@ -309,25 +361,29 @@ static int safexcel_aead_setkey(struct crypto_aead *ctfm, const u8 *key, struct safexcel_crypto_priv *priv = ctx->priv; struct crypto_authenc_keys keys; struct crypto_aes_ctx aes; - int err = -EINVAL; + int err = -EINVAL, i; - if (crypto_authenc_extractkeys(&keys, key, len) != 0) + if (unlikely(crypto_authenc_extractkeys(&keys, key, len))) goto badkey; if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD) { - /* Minimum keysize is minimum AES key size + nonce size */ - if (keys.enckeylen < (AES_MIN_KEY_SIZE + - CTR_RFC3686_NONCE_SIZE)) + /* Must have at least space for the nonce here */ + if (unlikely(keys.enckeylen < CTR_RFC3686_NONCE_SIZE)) goto badkey; /* last 4 bytes of key are the nonce! */ ctx->nonce = *(u32 *)(keys.enckey + keys.enckeylen - CTR_RFC3686_NONCE_SIZE); /* exclude the nonce here */ - keys.enckeylen -= CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; + keys.enckeylen -= CTR_RFC3686_NONCE_SIZE; } /* Encryption key */ switch (ctx->alg) { + case SAFEXCEL_DES: + err = verify_aead_des_key(ctfm, keys.enckey, keys.enckeylen); + if (unlikely(err)) + goto badkey_expflags; + break; case SAFEXCEL_3DES: err = verify_aead_des3_key(ctfm, keys.enckey, keys.enckeylen); if (unlikely(err)) @@ -338,14 +394,24 @@ static int safexcel_aead_setkey(struct crypto_aead *ctfm, const u8 *key, if (unlikely(err)) goto badkey; break; + case SAFEXCEL_SM4: + if (unlikely(keys.enckeylen != SM4_KEY_SIZE)) + goto badkey; + break; default: dev_err(priv->dev, "aead: unsupported cipher algorithm\n"); goto badkey; } - if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma && - memcmp(ctx->key, keys.enckey, keys.enckeylen)) - ctx->base.needs_inv = true; + if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { + for (i = 0; i < keys.enckeylen / sizeof(u32); i++) { + if (le32_to_cpu(ctx->key[i]) != + ((u32 *)keys.enckey)[i]) { + ctx->base.needs_inv = true; + break; + } + } + } /* Auth key */ switch (ctx->hash_alg) { @@ -374,6 +440,11 @@ static int safexcel_aead_setkey(struct crypto_aead *ctfm, const u8 *key, keys.authkeylen, &istate, &ostate)) goto badkey; break; + case CONTEXT_CONTROL_CRYPTO_ALG_SM3: + if (safexcel_hmac_setkey("safexcel-sm3", keys.authkey, + keys.authkeylen, &istate, &ostate)) + goto badkey; + break; default: dev_err(priv->dev, "aead: unsupported hash algorithm\n"); goto badkey; @@ -388,7 +459,8 @@ static int safexcel_aead_setkey(struct crypto_aead *ctfm, const u8 *key, ctx->base.needs_inv = true; /* Now copy the keys into the context */ - memcpy(ctx->key, keys.enckey, keys.enckeylen); + for (i = 0; i < keys.enckeylen / sizeof(u32); i++) + ctx->key[i] = cpu_to_le32(((u32 *)keys.enckey)[i]); ctx->key_len = keys.enckeylen; memcpy(ctx->ipad, &istate.state, ctx->state_sz); @@ -423,6 +495,17 @@ static int safexcel_context_control(struct safexcel_cipher_ctx *ctx, CONTEXT_CONTROL_DIGEST_XCM | ctx->hash_alg | CONTEXT_CONTROL_SIZE(ctrl_size); + } else if (ctx->alg == SAFEXCEL_CHACHA20) { + /* Chacha20-Poly1305 */ + cdesc->control_data.control0 = + CONTEXT_CONTROL_KEY_EN | + CONTEXT_CONTROL_CRYPTO_ALG_CHACHA20 | + (sreq->direction == SAFEXCEL_ENCRYPT ? + CONTEXT_CONTROL_TYPE_ENCRYPT_HASH_OUT : + CONTEXT_CONTROL_TYPE_HASH_DECRYPT_IN) | + ctx->hash_alg | + CONTEXT_CONTROL_SIZE(ctrl_size); + return 0; } else { ctrl_size += ctx->state_sz / sizeof(u32) * 2; cdesc->control_data.control0 = @@ -431,17 +514,21 @@ static int safexcel_context_control(struct safexcel_cipher_ctx *ctx, ctx->hash_alg | CONTEXT_CONTROL_SIZE(ctrl_size); } - if (sreq->direction == SAFEXCEL_ENCRYPT) - cdesc->control_data.control0 |= - (ctx->xcm == EIP197_XCM_MODE_CCM) ? - CONTEXT_CONTROL_TYPE_HASH_ENCRYPT_OUT : - CONTEXT_CONTROL_TYPE_ENCRYPT_HASH_OUT; + if (sreq->direction == SAFEXCEL_ENCRYPT && + (ctx->xcm == EIP197_XCM_MODE_CCM || + ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP_GMAC)) + cdesc->control_data.control0 |= + CONTEXT_CONTROL_TYPE_HASH_ENCRYPT_OUT; + else if (sreq->direction == SAFEXCEL_ENCRYPT) + cdesc->control_data.control0 |= + CONTEXT_CONTROL_TYPE_ENCRYPT_HASH_OUT; + else if (ctx->xcm == EIP197_XCM_MODE_CCM) + cdesc->control_data.control0 |= + CONTEXT_CONTROL_TYPE_DECRYPT_HASH_IN; else cdesc->control_data.control0 |= - (ctx->xcm == EIP197_XCM_MODE_CCM) ? - CONTEXT_CONTROL_TYPE_DECRYPT_HASH_IN : - CONTEXT_CONTROL_TYPE_HASH_DECRYPT_IN; + CONTEXT_CONTROL_TYPE_HASH_DECRYPT_IN; } else { if (sreq->direction == SAFEXCEL_ENCRYPT) cdesc->control_data.control0 = @@ -480,6 +567,12 @@ static int safexcel_context_control(struct safexcel_cipher_ctx *ctx, ctx->key_len >> ctx->xts); return -EINVAL; } + } else if (ctx->alg == SAFEXCEL_CHACHA20) { + cdesc->control_data.control0 |= + CONTEXT_CONTROL_CRYPTO_ALG_CHACHA20; + } else if (ctx->alg == SAFEXCEL_SM4) { + cdesc->control_data.control0 |= + CONTEXT_CONTROL_CRYPTO_ALG_SM4; } return 0; @@ -1295,7 +1388,7 @@ static int safexcel_skcipher_aesctr_setkey(struct crypto_skcipher *ctfm, if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { for (i = 0; i < keylen / sizeof(u32); i++) { - if (ctx->key[i] != cpu_to_le32(aes.key_enc[i])) { + if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) { ctx->base.needs_inv = true; break; } @@ -1451,13 +1544,11 @@ static int safexcel_des3_ede_setkey(struct crypto_skcipher *ctfm, return err; /* if context exits and key changed, need to invalidate it */ - if (ctx->base.ctxr_dma) { + if (ctx->base.ctxr_dma) if (memcmp(ctx->key, key, len)) ctx->base.needs_inv = true; - } memcpy(ctx->key, key, len); - ctx->key_len = len; return 0; @@ -1777,6 +1868,312 @@ struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_des3_ede = { }, }; +static int safexcel_aead_sha256_des3_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_sha256_cra_init(tfm); + ctx->alg = SAFEXCEL_3DES; /* override default */ + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_des3_ede = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_256, + .alg.aead = { + .setkey = safexcel_aead_setkey, + .encrypt = safexcel_aead_encrypt, + .decrypt = safexcel_aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha256),cbc(des3_ede))", + .cra_driver_name = "safexcel-authenc-hmac-sha256-cbc-des3_ede", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_sha256_des3_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_sha224_des3_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_sha224_cra_init(tfm); + ctx->alg = SAFEXCEL_3DES; /* override default */ + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_des3_ede = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_256, + .alg.aead = { + .setkey = safexcel_aead_setkey, + .encrypt = safexcel_aead_encrypt, + .decrypt = safexcel_aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha224),cbc(des3_ede))", + .cra_driver_name = "safexcel-authenc-hmac-sha224-cbc-des3_ede", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_sha224_des3_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_sha512_des3_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_sha512_cra_init(tfm); + ctx->alg = SAFEXCEL_3DES; /* override default */ + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_des3_ede = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_512, + .alg.aead = { + .setkey = safexcel_aead_setkey, + .encrypt = safexcel_aead_encrypt, + .decrypt = safexcel_aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha512),cbc(des3_ede))", + .cra_driver_name = "safexcel-authenc-hmac-sha512-cbc-des3_ede", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_sha512_des3_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_sha384_des3_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_sha384_cra_init(tfm); + ctx->alg = SAFEXCEL_3DES; /* override default */ + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_des3_ede = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_512, + .alg.aead = { + .setkey = safexcel_aead_setkey, + .encrypt = safexcel_aead_encrypt, + .decrypt = safexcel_aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha384),cbc(des3_ede))", + .cra_driver_name = "safexcel-authenc-hmac-sha384-cbc-des3_ede", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_sha384_des3_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_sha1_des_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_sha1_cra_init(tfm); + ctx->alg = SAFEXCEL_DES; /* override default */ + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_des = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA1, + .alg.aead = { + .setkey = safexcel_aead_setkey, + .encrypt = safexcel_aead_encrypt, + .decrypt = safexcel_aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha1),cbc(des))", + .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-des", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_sha1_des_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_sha256_des_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_sha256_cra_init(tfm); + ctx->alg = SAFEXCEL_DES; /* override default */ + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_des = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_256, + .alg.aead = { + .setkey = safexcel_aead_setkey, + .encrypt = safexcel_aead_encrypt, + .decrypt = safexcel_aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha256),cbc(des))", + .cra_driver_name = "safexcel-authenc-hmac-sha256-cbc-des", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_sha256_des_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_sha224_des_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_sha224_cra_init(tfm); + ctx->alg = SAFEXCEL_DES; /* override default */ + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_des = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_256, + .alg.aead = { + .setkey = safexcel_aead_setkey, + .encrypt = safexcel_aead_encrypt, + .decrypt = safexcel_aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha224),cbc(des))", + .cra_driver_name = "safexcel-authenc-hmac-sha224-cbc-des", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_sha224_des_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_sha512_des_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_sha512_cra_init(tfm); + ctx->alg = SAFEXCEL_DES; /* override default */ + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_des = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_512, + .alg.aead = { + .setkey = safexcel_aead_setkey, + .encrypt = safexcel_aead_encrypt, + .decrypt = safexcel_aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha512),cbc(des))", + .cra_driver_name = "safexcel-authenc-hmac-sha512-cbc-des", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_sha512_des_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_sha384_des_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_sha384_cra_init(tfm); + ctx->alg = SAFEXCEL_DES; /* override default */ + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_des = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_512, + .alg.aead = { + .setkey = safexcel_aead_setkey, + .encrypt = safexcel_aead_encrypt, + .decrypt = safexcel_aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha384),cbc(des))", + .cra_driver_name = "safexcel-authenc-hmac-sha384-cbc-des", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_sha384_des_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + static int safexcel_aead_sha1_ctr_cra_init(struct crypto_tfm *tfm) { struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); @@ -1972,7 +2369,7 @@ static int safexcel_skcipher_aesxts_setkey(struct crypto_skcipher *ctfm, if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { for (i = 0; i < keylen / sizeof(u32); i++) { - if (ctx->key[i] != cpu_to_le32(aes.key_enc[i])) { + if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) { ctx->base.needs_inv = true; break; } @@ -1991,8 +2388,8 @@ static int safexcel_skcipher_aesxts_setkey(struct crypto_skcipher *ctfm, if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { for (i = 0; i < keylen / sizeof(u32); i++) { - if (ctx->key[i + keylen / sizeof(u32)] != - cpu_to_le32(aes.key_enc[i])) { + if (le32_to_cpu(ctx->key[i + keylen / sizeof(u32)]) != + aes.key_enc[i]) { ctx->base.needs_inv = true; break; } @@ -2082,7 +2479,7 @@ static int safexcel_aead_gcm_setkey(struct crypto_aead *ctfm, const u8 *key, if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { for (i = 0; i < len / sizeof(u32); i++) { - if (ctx->key[i] != cpu_to_le32(aes.key_enc[i])) { + if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) { ctx->base.needs_inv = true; break; } @@ -2109,7 +2506,7 @@ static int safexcel_aead_gcm_setkey(struct crypto_aead *ctfm, const u8 *key, if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++) { - if (ctx->ipad[i] != cpu_to_be32(hashkey[i])) { + if (be32_to_cpu(ctx->ipad[i]) != hashkey[i]) { ctx->base.needs_inv = true; break; } @@ -2199,7 +2596,7 @@ static int safexcel_aead_ccm_setkey(struct crypto_aead *ctfm, const u8 *key, if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { for (i = 0; i < len / sizeof(u32); i++) { - if (ctx->key[i] != cpu_to_le32(aes.key_enc[i])) { + if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) { ctx->base.needs_inv = true; break; } @@ -2303,3 +2700,938 @@ struct safexcel_alg_template safexcel_alg_ccm = { }, }, }; + +static void safexcel_chacha20_setkey(struct safexcel_cipher_ctx *ctx, + const u8 *key) +{ + struct safexcel_crypto_priv *priv = ctx->priv; + + if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) + if (memcmp(ctx->key, key, CHACHA_KEY_SIZE)) + ctx->base.needs_inv = true; + + memcpy(ctx->key, key, CHACHA_KEY_SIZE); + ctx->key_len = CHACHA_KEY_SIZE; +} + +static int safexcel_skcipher_chacha20_setkey(struct crypto_skcipher *ctfm, + const u8 *key, unsigned int len) +{ + struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(ctfm); + + if (len != CHACHA_KEY_SIZE) { + crypto_skcipher_set_flags(ctfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + safexcel_chacha20_setkey(ctx, key); + + return 0; +} + +static int safexcel_skcipher_chacha20_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_skcipher_cra_init(tfm); + ctx->alg = SAFEXCEL_CHACHA20; + ctx->mode = CONTEXT_CONTROL_CHACHA20_MODE_256_32; + return 0; +} + +struct safexcel_alg_template safexcel_alg_chacha20 = { + .type = SAFEXCEL_ALG_TYPE_SKCIPHER, + .algo_mask = SAFEXCEL_ALG_CHACHA20, + .alg.skcipher = { + .setkey = safexcel_skcipher_chacha20_setkey, + .encrypt = safexcel_encrypt, + .decrypt = safexcel_decrypt, + .min_keysize = CHACHA_KEY_SIZE, + .max_keysize = CHACHA_KEY_SIZE, + .ivsize = CHACHA_IV_SIZE, + .base = { + .cra_name = "chacha20", + .cra_driver_name = "safexcel-chacha20", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_skcipher_chacha20_cra_init, + .cra_exit = safexcel_skcipher_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_chachapoly_setkey(struct crypto_aead *ctfm, + const u8 *key, unsigned int len) +{ + struct safexcel_cipher_ctx *ctx = crypto_aead_ctx(ctfm); + + if (ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP && + len > EIP197_AEAD_IPSEC_NONCE_SIZE) { + /* ESP variant has nonce appended to key */ + len -= EIP197_AEAD_IPSEC_NONCE_SIZE; + ctx->nonce = *(u32 *)(key + len); + } + if (len != CHACHA_KEY_SIZE) { + crypto_aead_set_flags(ctfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + safexcel_chacha20_setkey(ctx, key); + + return 0; +} + +static int safexcel_aead_chachapoly_setauthsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + if (authsize != POLY1305_DIGEST_SIZE) + return -EINVAL; + return 0; +} + +static int safexcel_aead_chachapoly_crypt(struct aead_request *req, + enum safexcel_cipher_direction dir) +{ + struct safexcel_cipher_req *creq = aead_request_ctx(req); + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct crypto_tfm *tfm = crypto_aead_tfm(aead); + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + struct aead_request *subreq = aead_request_ctx(req); + u32 key[CHACHA_KEY_SIZE / sizeof(u32) + 1]; + int ret = 0; + + /* + * Instead of wasting time detecting umpteen silly corner cases, + * just dump all "small" requests to the fallback implementation. + * HW would not be faster on such small requests anyway. + */ + if (likely((ctx->aead != EIP197_AEAD_TYPE_IPSEC_ESP || + req->assoclen >= EIP197_AEAD_IPSEC_IV_SIZE) && + req->cryptlen > POLY1305_DIGEST_SIZE)) { + return safexcel_queue_req(&req->base, creq, dir); + } + + /* HW cannot do full (AAD+payload) zero length, use fallback */ + memcpy(key, ctx->key, CHACHA_KEY_SIZE); + if (ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP) { + /* ESP variant has nonce appended to the key */ + key[CHACHA_KEY_SIZE / sizeof(u32)] = ctx->nonce; + ret = crypto_aead_setkey(ctx->fback, (u8 *)key, + CHACHA_KEY_SIZE + + EIP197_AEAD_IPSEC_NONCE_SIZE); + } else { + ret = crypto_aead_setkey(ctx->fback, (u8 *)key, + CHACHA_KEY_SIZE); + } + if (ret) { + crypto_aead_clear_flags(aead, CRYPTO_TFM_REQ_MASK); + crypto_aead_set_flags(aead, crypto_aead_get_flags(ctx->fback) & + CRYPTO_TFM_REQ_MASK); + return ret; + } + + aead_request_set_tfm(subreq, ctx->fback); + aead_request_set_callback(subreq, req->base.flags, req->base.complete, + req->base.data); + aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, + req->iv); + aead_request_set_ad(subreq, req->assoclen); + + return (dir == SAFEXCEL_ENCRYPT) ? + crypto_aead_encrypt(subreq) : + crypto_aead_decrypt(subreq); +} + +static int safexcel_aead_chachapoly_encrypt(struct aead_request *req) +{ + return safexcel_aead_chachapoly_crypt(req, SAFEXCEL_ENCRYPT); +} + +static int safexcel_aead_chachapoly_decrypt(struct aead_request *req) +{ + return safexcel_aead_chachapoly_crypt(req, SAFEXCEL_DECRYPT); +} + +static int safexcel_aead_fallback_cra_init(struct crypto_tfm *tfm) +{ + struct crypto_aead *aead = __crypto_aead_cast(tfm); + struct aead_alg *alg = crypto_aead_alg(aead); + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_cra_init(tfm); + + /* Allocate fallback implementation */ + ctx->fback = crypto_alloc_aead(alg->base.cra_name, 0, + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->fback)) + return PTR_ERR(ctx->fback); + + crypto_aead_set_reqsize(aead, max(sizeof(struct safexcel_cipher_req), + sizeof(struct aead_request) + + crypto_aead_reqsize(ctx->fback))); + + return 0; +} + +static int safexcel_aead_chachapoly_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_fallback_cra_init(tfm); + ctx->alg = SAFEXCEL_CHACHA20; + ctx->mode = CONTEXT_CONTROL_CHACHA20_MODE_256_32 | + CONTEXT_CONTROL_CHACHA20_MODE_CALC_OTK; + ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_POLY1305; + ctx->state_sz = 0; /* Precomputed by HW */ + return 0; +} + +static void safexcel_aead_fallback_cra_exit(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + crypto_free_aead(ctx->fback); + safexcel_aead_cra_exit(tfm); +} + +struct safexcel_alg_template safexcel_alg_chachapoly = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_CHACHA20 | SAFEXCEL_ALG_POLY1305, + .alg.aead = { + .setkey = safexcel_aead_chachapoly_setkey, + .setauthsize = safexcel_aead_chachapoly_setauthsize, + .encrypt = safexcel_aead_chachapoly_encrypt, + .decrypt = safexcel_aead_chachapoly_decrypt, + .ivsize = CHACHAPOLY_IV_SIZE, + .maxauthsize = POLY1305_DIGEST_SIZE, + .base = { + .cra_name = "rfc7539(chacha20,poly1305)", + .cra_driver_name = "safexcel-chacha20-poly1305", + /* +1 to put it above HW chacha + SW poly */ + .cra_priority = SAFEXCEL_CRA_PRIORITY + 1, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_chachapoly_cra_init, + .cra_exit = safexcel_aead_fallback_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_chachapolyesp_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + int ret; + + ret = safexcel_aead_chachapoly_cra_init(tfm); + ctx->aead = EIP197_AEAD_TYPE_IPSEC_ESP; + return ret; +} + +struct safexcel_alg_template safexcel_alg_chachapoly_esp = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_CHACHA20 | SAFEXCEL_ALG_POLY1305, + .alg.aead = { + .setkey = safexcel_aead_chachapoly_setkey, + .setauthsize = safexcel_aead_chachapoly_setauthsize, + .encrypt = safexcel_aead_chachapoly_encrypt, + .decrypt = safexcel_aead_chachapoly_decrypt, + .ivsize = CHACHAPOLY_IV_SIZE - EIP197_AEAD_IPSEC_NONCE_SIZE, + .maxauthsize = POLY1305_DIGEST_SIZE, + .base = { + .cra_name = "rfc7539esp(chacha20,poly1305)", + .cra_driver_name = "safexcel-chacha20-poly1305-esp", + /* +1 to put it above HW chacha + SW poly */ + .cra_priority = SAFEXCEL_CRA_PRIORITY + 1, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_chachapolyesp_cra_init, + .cra_exit = safexcel_aead_fallback_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_skcipher_sm4_setkey(struct crypto_skcipher *ctfm, + const u8 *key, unsigned int len) +{ + struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + struct safexcel_crypto_priv *priv = ctx->priv; + + if (len != SM4_KEY_SIZE) { + crypto_skcipher_set_flags(ctfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + + if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) + if (memcmp(ctx->key, key, SM4_KEY_SIZE)) + ctx->base.needs_inv = true; + + memcpy(ctx->key, key, SM4_KEY_SIZE); + ctx->key_len = SM4_KEY_SIZE; + + return 0; +} + +static int safexcel_sm4_blk_encrypt(struct skcipher_request *req) +{ + /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ + if (req->cryptlen & (SM4_BLOCK_SIZE - 1)) + return -EINVAL; + else + return safexcel_queue_req(&req->base, skcipher_request_ctx(req), + SAFEXCEL_ENCRYPT); +} + +static int safexcel_sm4_blk_decrypt(struct skcipher_request *req) +{ + /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ + if (req->cryptlen & (SM4_BLOCK_SIZE - 1)) + return -EINVAL; + else + return safexcel_queue_req(&req->base, skcipher_request_ctx(req), + SAFEXCEL_DECRYPT); +} + +static int safexcel_skcipher_sm4_ecb_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_skcipher_cra_init(tfm); + ctx->alg = SAFEXCEL_SM4; + ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_ECB; + return 0; +} + +struct safexcel_alg_template safexcel_alg_ecb_sm4 = { + .type = SAFEXCEL_ALG_TYPE_SKCIPHER, + .algo_mask = SAFEXCEL_ALG_SM4, + .alg.skcipher = { + .setkey = safexcel_skcipher_sm4_setkey, + .encrypt = safexcel_sm4_blk_encrypt, + .decrypt = safexcel_sm4_blk_decrypt, + .min_keysize = SM4_KEY_SIZE, + .max_keysize = SM4_KEY_SIZE, + .base = { + .cra_name = "ecb(sm4)", + .cra_driver_name = "safexcel-ecb-sm4", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SM4_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_skcipher_sm4_ecb_cra_init, + .cra_exit = safexcel_skcipher_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_skcipher_sm4_cbc_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_skcipher_cra_init(tfm); + ctx->alg = SAFEXCEL_SM4; + ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC; + return 0; +} + +struct safexcel_alg_template safexcel_alg_cbc_sm4 = { + .type = SAFEXCEL_ALG_TYPE_SKCIPHER, + .algo_mask = SAFEXCEL_ALG_SM4, + .alg.skcipher = { + .setkey = safexcel_skcipher_sm4_setkey, + .encrypt = safexcel_sm4_blk_encrypt, + .decrypt = safexcel_sm4_blk_decrypt, + .min_keysize = SM4_KEY_SIZE, + .max_keysize = SM4_KEY_SIZE, + .ivsize = SM4_BLOCK_SIZE, + .base = { + .cra_name = "cbc(sm4)", + .cra_driver_name = "safexcel-cbc-sm4", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SM4_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_skcipher_sm4_cbc_cra_init, + .cra_exit = safexcel_skcipher_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_skcipher_sm4_ofb_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_skcipher_cra_init(tfm); + ctx->alg = SAFEXCEL_SM4; + ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_OFB; + return 0; +} + +struct safexcel_alg_template safexcel_alg_ofb_sm4 = { + .type = SAFEXCEL_ALG_TYPE_SKCIPHER, + .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_AES_XFB, + .alg.skcipher = { + .setkey = safexcel_skcipher_sm4_setkey, + .encrypt = safexcel_encrypt, + .decrypt = safexcel_decrypt, + .min_keysize = SM4_KEY_SIZE, + .max_keysize = SM4_KEY_SIZE, + .ivsize = SM4_BLOCK_SIZE, + .base = { + .cra_name = "ofb(sm4)", + .cra_driver_name = "safexcel-ofb-sm4", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_skcipher_sm4_ofb_cra_init, + .cra_exit = safexcel_skcipher_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_skcipher_sm4_cfb_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_skcipher_cra_init(tfm); + ctx->alg = SAFEXCEL_SM4; + ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CFB; + return 0; +} + +struct safexcel_alg_template safexcel_alg_cfb_sm4 = { + .type = SAFEXCEL_ALG_TYPE_SKCIPHER, + .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_AES_XFB, + .alg.skcipher = { + .setkey = safexcel_skcipher_sm4_setkey, + .encrypt = safexcel_encrypt, + .decrypt = safexcel_decrypt, + .min_keysize = SM4_KEY_SIZE, + .max_keysize = SM4_KEY_SIZE, + .ivsize = SM4_BLOCK_SIZE, + .base = { + .cra_name = "cfb(sm4)", + .cra_driver_name = "safexcel-cfb-sm4", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_skcipher_sm4_cfb_cra_init, + .cra_exit = safexcel_skcipher_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_skcipher_sm4ctr_setkey(struct crypto_skcipher *ctfm, + const u8 *key, unsigned int len) +{ + struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + /* last 4 bytes of key are the nonce! */ + ctx->nonce = *(u32 *)(key + len - CTR_RFC3686_NONCE_SIZE); + /* exclude the nonce here */ + len -= CTR_RFC3686_NONCE_SIZE; + + return safexcel_skcipher_sm4_setkey(ctfm, key, len); +} + +static int safexcel_skcipher_sm4_ctr_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_skcipher_cra_init(tfm); + ctx->alg = SAFEXCEL_SM4; + ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; + return 0; +} + +struct safexcel_alg_template safexcel_alg_ctr_sm4 = { + .type = SAFEXCEL_ALG_TYPE_SKCIPHER, + .algo_mask = SAFEXCEL_ALG_SM4, + .alg.skcipher = { + .setkey = safexcel_skcipher_sm4ctr_setkey, + .encrypt = safexcel_encrypt, + .decrypt = safexcel_decrypt, + /* Add nonce size */ + .min_keysize = SM4_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, + .max_keysize = SM4_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, + .ivsize = CTR_RFC3686_IV_SIZE, + .base = { + .cra_name = "rfc3686(ctr(sm4))", + .cra_driver_name = "safexcel-ctr-sm4", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_skcipher_sm4_ctr_cra_init, + .cra_exit = safexcel_skcipher_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_sm4_blk_encrypt(struct aead_request *req) +{ + /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ + if (req->cryptlen & (SM4_BLOCK_SIZE - 1)) + return -EINVAL; + + return safexcel_queue_req(&req->base, aead_request_ctx(req), + SAFEXCEL_ENCRYPT); +} + +static int safexcel_aead_sm4_blk_decrypt(struct aead_request *req) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + + /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ + if ((req->cryptlen - crypto_aead_authsize(tfm)) & (SM4_BLOCK_SIZE - 1)) + return -EINVAL; + + return safexcel_queue_req(&req->base, aead_request_ctx(req), + SAFEXCEL_DECRYPT); +} + +static int safexcel_aead_sm4cbc_sha1_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_cra_init(tfm); + ctx->alg = SAFEXCEL_SM4; + ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1; + ctx->state_sz = SHA1_DIGEST_SIZE; + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_sm4 = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_SHA1, + .alg.aead = { + .setkey = safexcel_aead_setkey, + .encrypt = safexcel_aead_sm4_blk_encrypt, + .decrypt = safexcel_aead_sm4_blk_decrypt, + .ivsize = SM4_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha1),cbc(sm4))", + .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-sm4", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SM4_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_sm4cbc_sha1_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_fallback_setkey(struct crypto_aead *ctfm, + const u8 *key, unsigned int len) +{ + struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + /* Keep fallback cipher synchronized */ + return crypto_aead_setkey(ctx->fback, (u8 *)key, len) ?: + safexcel_aead_setkey(ctfm, key, len); +} + +static int safexcel_aead_fallback_setauthsize(struct crypto_aead *ctfm, + unsigned int authsize) +{ + struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + /* Keep fallback cipher synchronized */ + return crypto_aead_setauthsize(ctx->fback, authsize); +} + +static int safexcel_aead_fallback_crypt(struct aead_request *req, + enum safexcel_cipher_direction dir) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct crypto_tfm *tfm = crypto_aead_tfm(aead); + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + struct aead_request *subreq = aead_request_ctx(req); + + aead_request_set_tfm(subreq, ctx->fback); + aead_request_set_callback(subreq, req->base.flags, req->base.complete, + req->base.data); + aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, + req->iv); + aead_request_set_ad(subreq, req->assoclen); + + return (dir == SAFEXCEL_ENCRYPT) ? + crypto_aead_encrypt(subreq) : + crypto_aead_decrypt(subreq); +} + +static int safexcel_aead_sm4cbc_sm3_encrypt(struct aead_request *req) +{ + struct safexcel_cipher_req *creq = aead_request_ctx(req); + + /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ + if (req->cryptlen & (SM4_BLOCK_SIZE - 1)) + return -EINVAL; + else if (req->cryptlen || req->assoclen) /* If input length > 0 only */ + return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT); + + /* HW cannot do full (AAD+payload) zero length, use fallback */ + return safexcel_aead_fallback_crypt(req, SAFEXCEL_ENCRYPT); +} + +static int safexcel_aead_sm4cbc_sm3_decrypt(struct aead_request *req) +{ + struct safexcel_cipher_req *creq = aead_request_ctx(req); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + + /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ + if ((req->cryptlen - crypto_aead_authsize(tfm)) & (SM4_BLOCK_SIZE - 1)) + return -EINVAL; + else if (req->cryptlen > crypto_aead_authsize(tfm) || req->assoclen) + /* If input length > 0 only */ + return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT); + + /* HW cannot do full (AAD+payload) zero length, use fallback */ + return safexcel_aead_fallback_crypt(req, SAFEXCEL_DECRYPT); +} + +static int safexcel_aead_sm4cbc_sm3_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_fallback_cra_init(tfm); + ctx->alg = SAFEXCEL_SM4; + ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SM3; + ctx->state_sz = SM3_DIGEST_SIZE; + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_sm3_cbc_sm4 = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_SM3, + .alg.aead = { + .setkey = safexcel_aead_fallback_setkey, + .setauthsize = safexcel_aead_fallback_setauthsize, + .encrypt = safexcel_aead_sm4cbc_sm3_encrypt, + .decrypt = safexcel_aead_sm4cbc_sm3_decrypt, + .ivsize = SM4_BLOCK_SIZE, + .maxauthsize = SM3_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sm3),cbc(sm4))", + .cra_driver_name = "safexcel-authenc-hmac-sm3-cbc-sm4", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SM4_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_sm4cbc_sm3_cra_init, + .cra_exit = safexcel_aead_fallback_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_sm4ctr_sha1_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_sm4cbc_sha1_cra_init(tfm); + ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_ctr_sm4 = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_SHA1, + .alg.aead = { + .setkey = safexcel_aead_setkey, + .encrypt = safexcel_aead_encrypt, + .decrypt = safexcel_aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha1),rfc3686(ctr(sm4)))", + .cra_driver_name = "safexcel-authenc-hmac-sha1-ctr-sm4", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_sm4ctr_sha1_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_sm4ctr_sm3_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_sm4cbc_sm3_cra_init(tfm); + ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_sm3_ctr_sm4 = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_SM3, + .alg.aead = { + .setkey = safexcel_aead_setkey, + .encrypt = safexcel_aead_encrypt, + .decrypt = safexcel_aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SM3_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sm3),rfc3686(ctr(sm4)))", + .cra_driver_name = "safexcel-authenc-hmac-sm3-ctr-sm4", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_sm4ctr_sm3_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_rfc4106_gcm_setkey(struct crypto_aead *ctfm, const u8 *key, + unsigned int len) +{ + struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + /* last 4 bytes of key are the nonce! */ + ctx->nonce = *(u32 *)(key + len - CTR_RFC3686_NONCE_SIZE); + + len -= CTR_RFC3686_NONCE_SIZE; + return safexcel_aead_gcm_setkey(ctfm, key, len); +} + +static int safexcel_rfc4106_gcm_setauthsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + return crypto_rfc4106_check_authsize(authsize); +} + +static int safexcel_rfc4106_encrypt(struct aead_request *req) +{ + return crypto_ipsec_check_assoclen(req->assoclen) ?: + safexcel_aead_encrypt(req); +} + +static int safexcel_rfc4106_decrypt(struct aead_request *req) +{ + return crypto_ipsec_check_assoclen(req->assoclen) ?: + safexcel_aead_decrypt(req); +} + +static int safexcel_rfc4106_gcm_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + int ret; + + ret = safexcel_aead_gcm_cra_init(tfm); + ctx->aead = EIP197_AEAD_TYPE_IPSEC_ESP; + return ret; +} + +struct safexcel_alg_template safexcel_alg_rfc4106_gcm = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_GHASH, + .alg.aead = { + .setkey = safexcel_rfc4106_gcm_setkey, + .setauthsize = safexcel_rfc4106_gcm_setauthsize, + .encrypt = safexcel_rfc4106_encrypt, + .decrypt = safexcel_rfc4106_decrypt, + .ivsize = GCM_RFC4106_IV_SIZE, + .maxauthsize = GHASH_DIGEST_SIZE, + .base = { + .cra_name = "rfc4106(gcm(aes))", + .cra_driver_name = "safexcel-rfc4106-gcm-aes", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_rfc4106_gcm_cra_init, + .cra_exit = safexcel_aead_gcm_cra_exit, + }, + }, +}; + +static int safexcel_rfc4543_gcm_setauthsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + if (authsize != GHASH_DIGEST_SIZE) + return -EINVAL; + + return 0; +} + +static int safexcel_rfc4543_gcm_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + int ret; + + ret = safexcel_aead_gcm_cra_init(tfm); + ctx->aead = EIP197_AEAD_TYPE_IPSEC_ESP_GMAC; + return ret; +} + +struct safexcel_alg_template safexcel_alg_rfc4543_gcm = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_GHASH, + .alg.aead = { + .setkey = safexcel_rfc4106_gcm_setkey, + .setauthsize = safexcel_rfc4543_gcm_setauthsize, + .encrypt = safexcel_rfc4106_encrypt, + .decrypt = safexcel_rfc4106_decrypt, + .ivsize = GCM_RFC4543_IV_SIZE, + .maxauthsize = GHASH_DIGEST_SIZE, + .base = { + .cra_name = "rfc4543(gcm(aes))", + .cra_driver_name = "safexcel-rfc4543-gcm-aes", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_rfc4543_gcm_cra_init, + .cra_exit = safexcel_aead_gcm_cra_exit, + }, + }, +}; + +static int safexcel_rfc4309_ccm_setkey(struct crypto_aead *ctfm, const u8 *key, + unsigned int len) +{ + struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + /* First byte of the nonce = L = always 3 for RFC4309 (4 byte ctr) */ + *(u8 *)&ctx->nonce = EIP197_AEAD_IPSEC_COUNTER_SIZE - 1; + /* last 3 bytes of key are the nonce! */ + memcpy((u8 *)&ctx->nonce + 1, key + len - + EIP197_AEAD_IPSEC_CCM_NONCE_SIZE, + EIP197_AEAD_IPSEC_CCM_NONCE_SIZE); + + len -= EIP197_AEAD_IPSEC_CCM_NONCE_SIZE; + return safexcel_aead_ccm_setkey(ctfm, key, len); +} + +static int safexcel_rfc4309_ccm_setauthsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + /* Borrowed from crypto/ccm.c */ + switch (authsize) { + case 8: + case 12: + case 16: + break; + default: + return -EINVAL; + } + + return 0; +} + +static int safexcel_rfc4309_ccm_encrypt(struct aead_request *req) +{ + struct safexcel_cipher_req *creq = aead_request_ctx(req); + + /* Borrowed from crypto/ccm.c */ + if (req->assoclen != 16 && req->assoclen != 20) + return -EINVAL; + + return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT); +} + +static int safexcel_rfc4309_ccm_decrypt(struct aead_request *req) +{ + struct safexcel_cipher_req *creq = aead_request_ctx(req); + + /* Borrowed from crypto/ccm.c */ + if (req->assoclen != 16 && req->assoclen != 20) + return -EINVAL; + + return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT); +} + +static int safexcel_rfc4309_ccm_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + int ret; + + ret = safexcel_aead_ccm_cra_init(tfm); + ctx->aead = EIP197_AEAD_TYPE_IPSEC_ESP; + return ret; +} + +struct safexcel_alg_template safexcel_alg_rfc4309_ccm = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_CBC_MAC_ALL, + .alg.aead = { + .setkey = safexcel_rfc4309_ccm_setkey, + .setauthsize = safexcel_rfc4309_ccm_setauthsize, + .encrypt = safexcel_rfc4309_ccm_encrypt, + .decrypt = safexcel_rfc4309_ccm_decrypt, + .ivsize = EIP197_AEAD_IPSEC_IV_SIZE, + .maxauthsize = AES_BLOCK_SIZE, + .base = { + .cra_name = "rfc4309(ccm(aes))", + .cra_driver_name = "safexcel-rfc4309-ccm-aes", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_rfc4309_ccm_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; |