diff options
Diffstat (limited to 'drivers/crypto')
40 files changed, 7725 insertions, 1630 deletions
diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index f4fd837bcb82..13857f5d28f7 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -289,16 +289,6 @@ config CRYPTO_DEV_SAHARA This option enables support for the SAHARA HW crypto accelerator found in some Freescale i.MX chips. -config CRYPTO_DEV_DCP - tristate "Support for the DCP engine" - depends on ARCH_MXS && OF - select CRYPTO_BLKCIPHER - select CRYPTO_AES - select CRYPTO_CBC - help - This options enables support for the hardware crypto-acceleration - capabilities of the DCP co-processor - config CRYPTO_DEV_S5P tristate "Support for Samsung S5PV210 crypto accelerator" depends on ARCH_S5PV210 @@ -399,4 +389,33 @@ config CRYPTO_DEV_ATMEL_SHA To compile this driver as a module, choose M here: the module will be called atmel-sha. +config CRYPTO_DEV_CCP + bool "Support for AMD Cryptographic Coprocessor" + depends on X86 && PCI + default n + help + The AMD Cryptographic Coprocessor provides hardware support + for encryption, hashing and related operations. + +if CRYPTO_DEV_CCP + source "drivers/crypto/ccp/Kconfig" +endif + +config CRYPTO_DEV_MXS_DCP + tristate "Support for Freescale MXS DCP" + depends on ARCH_MXS + select CRYPTO_SHA1 + select CRYPTO_SHA256 + select CRYPTO_CBC + select CRYPTO_ECB + select CRYPTO_AES + select CRYPTO_BLKCIPHER + select CRYPTO_ALGAPI + help + The Freescale i.MX23/i.MX28 has SHA1/SHA256 and AES128 CBC/ECB + co-processor on the die. + + To compile this driver as a module, choose M here: the module + will be called mxs-dcp. + endif # CRYPTO_HW diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile index b4946ddd2550..0bc6aa0a54d7 100644 --- a/drivers/crypto/Makefile +++ b/drivers/crypto/Makefile @@ -1,24 +1,25 @@ -obj-$(CONFIG_CRYPTO_DEV_PADLOCK_AES) += padlock-aes.o -obj-$(CONFIG_CRYPTO_DEV_PADLOCK_SHA) += padlock-sha.o +obj-$(CONFIG_CRYPTO_DEV_ATMEL_AES) += atmel-aes.o +obj-$(CONFIG_CRYPTO_DEV_ATMEL_SHA) += atmel-sha.o +obj-$(CONFIG_CRYPTO_DEV_ATMEL_TDES) += atmel-tdes.o +obj-$(CONFIG_CRYPTO_DEV_BFIN_CRC) += bfin_crc.o +obj-$(CONFIG_CRYPTO_DEV_CCP) += ccp/ +obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM) += caam/ obj-$(CONFIG_CRYPTO_DEV_GEODE) += geode-aes.o -obj-$(CONFIG_CRYPTO_DEV_NIAGARA2) += n2_crypto.o -n2_crypto-y := n2_core.o n2_asm.o obj-$(CONFIG_CRYPTO_DEV_HIFN_795X) += hifn_795x.o -obj-$(CONFIG_CRYPTO_DEV_MV_CESA) += mv_cesa.o -obj-$(CONFIG_CRYPTO_DEV_TALITOS) += talitos.o -obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM) += caam/ obj-$(CONFIG_CRYPTO_DEV_IXP4XX) += ixp4xx_crypto.o -obj-$(CONFIG_CRYPTO_DEV_PPC4XX) += amcc/ -obj-$(CONFIG_CRYPTO_DEV_OMAP_SHAM) += omap-sham.o +obj-$(CONFIG_CRYPTO_DEV_MV_CESA) += mv_cesa.o +obj-$(CONFIG_CRYPTO_DEV_MXS_DCP) += mxs-dcp.o +obj-$(CONFIG_CRYPTO_DEV_NIAGARA2) += n2_crypto.o +n2_crypto-y := n2_core.o n2_asm.o +obj-$(CONFIG_CRYPTO_DEV_NX) += nx/ obj-$(CONFIG_CRYPTO_DEV_OMAP_AES) += omap-aes.o +obj-$(CONFIG_CRYPTO_DEV_OMAP_SHAM) += omap-sham.o +obj-$(CONFIG_CRYPTO_DEV_PADLOCK_AES) += padlock-aes.o +obj-$(CONFIG_CRYPTO_DEV_PADLOCK_SHA) += padlock-sha.o obj-$(CONFIG_CRYPTO_DEV_PICOXCELL) += picoxcell_crypto.o -obj-$(CONFIG_CRYPTO_DEV_SAHARA) += sahara.o -obj-$(CONFIG_CRYPTO_DEV_DCP) += dcp.o +obj-$(CONFIG_CRYPTO_DEV_PPC4XX) += amcc/ obj-$(CONFIG_CRYPTO_DEV_S5P) += s5p-sss.o +obj-$(CONFIG_CRYPTO_DEV_SAHARA) += sahara.o +obj-$(CONFIG_CRYPTO_DEV_TALITOS) += talitos.o obj-$(CONFIG_CRYPTO_DEV_TEGRA_AES) += tegra-aes.o obj-$(CONFIG_CRYPTO_DEV_UX500) += ux500/ -obj-$(CONFIG_CRYPTO_DEV_BFIN_CRC) += bfin_crc.o -obj-$(CONFIG_CRYPTO_DEV_NX) += nx/ -obj-$(CONFIG_CRYPTO_DEV_ATMEL_AES) += atmel-aes.o -obj-$(CONFIG_CRYPTO_DEV_ATMEL_TDES) += atmel-tdes.o -obj-$(CONFIG_CRYPTO_DEV_ATMEL_SHA) += atmel-sha.o diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index efaf6302405f..37f9cc98ba17 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -724,7 +724,6 @@ static void crypto4xx_stop_all(struct crypto4xx_core_device *core_dev) crypto4xx_destroy_pdr(core_dev->dev); crypto4xx_destroy_gdr(core_dev->dev); crypto4xx_destroy_sdr(core_dev->dev); - dev_set_drvdata(core_dev->device, NULL); iounmap(core_dev->dev->ce_base); kfree(core_dev->dev); kfree(core_dev); diff --git a/drivers/crypto/atmel-aes.c b/drivers/crypto/atmel-aes.c index c1efd910d97b..d7c9e317423c 100644 --- a/drivers/crypto/atmel-aes.c +++ b/drivers/crypto/atmel-aes.c @@ -30,6 +30,7 @@ #include <linux/irq.h> #include <linux/scatterlist.h> #include <linux/dma-mapping.h> +#include <linux/of_device.h> #include <linux/delay.h> #include <linux/crypto.h> #include <linux/cryptohash.h> @@ -39,6 +40,7 @@ #include <crypto/hash.h> #include <crypto/internal/hash.h> #include <linux/platform_data/crypto-atmel.h> +#include <dt-bindings/dma/at91.h> #include "atmel-aes-regs.h" #define CFB8_BLOCK_SIZE 1 @@ -747,59 +749,50 @@ static int atmel_aes_dma_init(struct atmel_aes_dev *dd, struct crypto_platform_data *pdata) { int err = -ENOMEM; - dma_cap_mask_t mask_in, mask_out; + dma_cap_mask_t mask; + + dma_cap_zero(mask); + dma_cap_set(DMA_SLAVE, mask); + + /* Try to grab 2 DMA channels */ + dd->dma_lch_in.chan = dma_request_slave_channel_compat(mask, + atmel_aes_filter, &pdata->dma_slave->rxdata, dd->dev, "tx"); + if (!dd->dma_lch_in.chan) + goto err_dma_in; + + dd->dma_lch_in.dma_conf.direction = DMA_MEM_TO_DEV; + dd->dma_lch_in.dma_conf.dst_addr = dd->phys_base + + AES_IDATAR(0); + dd->dma_lch_in.dma_conf.src_maxburst = dd->caps.max_burst_size; + dd->dma_lch_in.dma_conf.src_addr_width = + DMA_SLAVE_BUSWIDTH_4_BYTES; + dd->dma_lch_in.dma_conf.dst_maxburst = dd->caps.max_burst_size; + dd->dma_lch_in.dma_conf.dst_addr_width = + DMA_SLAVE_BUSWIDTH_4_BYTES; + dd->dma_lch_in.dma_conf.device_fc = false; + + dd->dma_lch_out.chan = dma_request_slave_channel_compat(mask, + atmel_aes_filter, &pdata->dma_slave->txdata, dd->dev, "rx"); + if (!dd->dma_lch_out.chan) + goto err_dma_out; + + dd->dma_lch_out.dma_conf.direction = DMA_DEV_TO_MEM; + dd->dma_lch_out.dma_conf.src_addr = dd->phys_base + + AES_ODATAR(0); + dd->dma_lch_out.dma_conf.src_maxburst = dd->caps.max_burst_size; + dd->dma_lch_out.dma_conf.src_addr_width = + DMA_SLAVE_BUSWIDTH_4_BYTES; + dd->dma_lch_out.dma_conf.dst_maxburst = dd->caps.max_burst_size; + dd->dma_lch_out.dma_conf.dst_addr_width = + DMA_SLAVE_BUSWIDTH_4_BYTES; + dd->dma_lch_out.dma_conf.device_fc = false; - if (pdata && pdata->dma_slave->txdata.dma_dev && - pdata->dma_slave->rxdata.dma_dev) { - - /* Try to grab 2 DMA channels */ - dma_cap_zero(mask_in); - dma_cap_set(DMA_SLAVE, mask_in); - - dd->dma_lch_in.chan = dma_request_channel(mask_in, - atmel_aes_filter, &pdata->dma_slave->rxdata); - - if (!dd->dma_lch_in.chan) - goto err_dma_in; - - dd->dma_lch_in.dma_conf.direction = DMA_MEM_TO_DEV; - dd->dma_lch_in.dma_conf.dst_addr = dd->phys_base + - AES_IDATAR(0); - dd->dma_lch_in.dma_conf.src_maxburst = dd->caps.max_burst_size; - dd->dma_lch_in.dma_conf.src_addr_width = - DMA_SLAVE_BUSWIDTH_4_BYTES; - dd->dma_lch_in.dma_conf.dst_maxburst = dd->caps.max_burst_size; - dd->dma_lch_in.dma_conf.dst_addr_width = - DMA_SLAVE_BUSWIDTH_4_BYTES; - dd->dma_lch_in.dma_conf.device_fc = false; - - dma_cap_zero(mask_out); - dma_cap_set(DMA_SLAVE, mask_out); - dd->dma_lch_out.chan = dma_request_channel(mask_out, - atmel_aes_filter, &pdata->dma_slave->txdata); - - if (!dd->dma_lch_out.chan) - goto err_dma_out; - - dd->dma_lch_out.dma_conf.direction = DMA_DEV_TO_MEM; - dd->dma_lch_out.dma_conf.src_addr = dd->phys_base + - AES_ODATAR(0); - dd->dma_lch_out.dma_conf.src_maxburst = dd->caps.max_burst_size; - dd->dma_lch_out.dma_conf.src_addr_width = - DMA_SLAVE_BUSWIDTH_4_BYTES; - dd->dma_lch_out.dma_conf.dst_maxburst = dd->caps.max_burst_size; - dd->dma_lch_out.dma_conf.dst_addr_width = - DMA_SLAVE_BUSWIDTH_4_BYTES; - dd->dma_lch_out.dma_conf.device_fc = false; - - return 0; - } else { - return -ENODEV; - } + return 0; err_dma_out: dma_release_channel(dd->dma_lch_in.chan); err_dma_in: + dev_warn(dd->dev, "no DMA channel available\n"); return err; } @@ -1261,6 +1254,47 @@ static void atmel_aes_get_cap(struct atmel_aes_dev *dd) } } +#if defined(CONFIG_OF) +static const struct of_device_id atmel_aes_dt_ids[] = { + { .compatible = "atmel,at91sam9g46-aes" }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, atmel_aes_dt_ids); + +static struct crypto_platform_data *atmel_aes_of_init(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + struct crypto_platform_data *pdata; + + if (!np) { + dev_err(&pdev->dev, "device node not found\n"); + return ERR_PTR(-EINVAL); + } + + pdata = devm_kzalloc(&pdev->dev, sizeof(*pdata), GFP_KERNEL); + if (!pdata) { + dev_err(&pdev->dev, "could not allocate memory for pdata\n"); + return ERR_PTR(-ENOMEM); + } + + pdata->dma_slave = devm_kzalloc(&pdev->dev, + sizeof(*(pdata->dma_slave)), + GFP_KERNEL); + if (!pdata->dma_slave) { + dev_err(&pdev->dev, "could not allocate memory for dma_slave\n"); + devm_kfree(&pdev->dev, pdata); + return ERR_PTR(-ENOMEM); + } + + return pdata; +} +#else +static inline struct crypto_platform_data *atmel_aes_of_init(struct platform_device *pdev) +{ + return ERR_PTR(-EINVAL); +} +#endif + static int atmel_aes_probe(struct platform_device *pdev) { struct atmel_aes_dev *aes_dd; @@ -1272,6 +1306,14 @@ static int atmel_aes_probe(struct platform_device *pdev) pdata = pdev->dev.platform_data; if (!pdata) { + pdata = atmel_aes_of_init(pdev); + if (IS_ERR(pdata)) { + err = PTR_ERR(pdata); + goto aes_dd_err; + } + } + + if (!pdata->dma_slave) { err = -ENXIO; goto aes_dd_err; } @@ -1358,7 +1400,9 @@ static int atmel_aes_probe(struct platform_device *pdev) if (err) goto err_algs; - dev_info(dev, "Atmel AES\n"); + dev_info(dev, "Atmel AES - Using %s, %s for DMA transfers\n", + dma_chan_name(aes_dd->dma_lch_in.chan), + dma_chan_name(aes_dd->dma_lch_out.chan)); return 0; @@ -1424,6 +1468,7 @@ static struct platform_driver atmel_aes_driver = { .driver = { .name = "atmel_aes", .owner = THIS_MODULE, + .of_match_table = of_match_ptr(atmel_aes_dt_ids), }, }; diff --git a/drivers/crypto/atmel-sha.c b/drivers/crypto/atmel-sha.c index eaed8bf183bc..0618be06b9fb 100644 --- a/drivers/crypto/atmel-sha.c +++ b/drivers/crypto/atmel-sha.c @@ -30,6 +30,7 @@ #include <linux/irq.h> #include <linux/scatterlist.h> #include <linux/dma-mapping.h> +#include <linux/of_device.h> #include <linux/delay.h> #include <linux/crypto.h> #include <linux/cryptohash.h> @@ -1263,32 +1264,29 @@ static int atmel_sha_dma_init(struct atmel_sha_dev *dd, int err = -ENOMEM; dma_cap_mask_t mask_in; - if (pdata && pdata->dma_slave->rxdata.dma_dev) { - /* Try to grab DMA channel */ - dma_cap_zero(mask_in); - dma_cap_set(DMA_SLAVE, mask_in); + /* Try to grab DMA channel */ + dma_cap_zero(mask_in); + dma_cap_set(DMA_SLAVE, mask_in); - dd->dma_lch_in.chan = dma_request_channel(mask_in, - atmel_sha_filter, &pdata->dma_slave->rxdata); - - if (!dd->dma_lch_in.chan) - return err; - - dd->dma_lch_in.dma_conf.direction = DMA_MEM_TO_DEV; - dd->dma_lch_in.dma_conf.dst_addr = dd->phys_base + - SHA_REG_DIN(0); - dd->dma_lch_in.dma_conf.src_maxburst = 1; - dd->dma_lch_in.dma_conf.src_addr_width = - DMA_SLAVE_BUSWIDTH_4_BYTES; - dd->dma_lch_in.dma_conf.dst_maxburst = 1; - dd->dma_lch_in.dma_conf.dst_addr_width = - DMA_SLAVE_BUSWIDTH_4_BYTES; - dd->dma_lch_in.dma_conf.device_fc = false; - - return 0; + dd->dma_lch_in.chan = dma_request_slave_channel_compat(mask_in, + atmel_sha_filter, &pdata->dma_slave->rxdata, dd->dev, "tx"); + if (!dd->dma_lch_in.chan) { + dev_warn(dd->dev, "no DMA channel available\n"); + return err; } - return -ENODEV; + dd->dma_lch_in.dma_conf.direction = DMA_MEM_TO_DEV; + dd->dma_lch_in.dma_conf.dst_addr = dd->phys_base + + SHA_REG_DIN(0); + dd->dma_lch_in.dma_conf.src_maxburst = 1; + dd->dma_lch_in.dma_conf.src_addr_width = + DMA_SLAVE_BUSWIDTH_4_BYTES; + dd->dma_lch_in.dma_conf.dst_maxburst = 1; + dd->dma_lch_in.dma_conf.dst_addr_width = + DMA_SLAVE_BUSWIDTH_4_BYTES; + dd->dma_lch_in.dma_conf.device_fc = false; + + return 0; } static void atmel_sha_dma_cleanup(struct atmel_sha_dev *dd) @@ -1326,6 +1324,48 @@ static void atmel_sha_get_cap(struct atmel_sha_dev *dd) } } +#if defined(CONFIG_OF) +static const struct of_device_id atmel_sha_dt_ids[] = { + { .compatible = "atmel,at91sam9g46-sha" }, + { /* sentinel */ } +}; + +MODULE_DEVICE_TABLE(of, atmel_sha_dt_ids); + +static struct crypto_platform_data *atmel_sha_of_init(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + struct crypto_platform_data *pdata; + + if (!np) { + dev_err(&pdev->dev, "device node not found\n"); + return ERR_PTR(-EINVAL); + } + + pdata = devm_kzalloc(&pdev->dev, sizeof(*pdata), GFP_KERNEL); + if (!pdata) { + dev_err(&pdev->dev, "could not allocate memory for pdata\n"); + return ERR_PTR(-ENOMEM); + } + + pdata->dma_slave = devm_kzalloc(&pdev->dev, + sizeof(*(pdata->dma_slave)), + GFP_KERNEL); + if (!pdata->dma_slave) { + dev_err(&pdev->dev, "could not allocate memory for dma_slave\n"); + devm_kfree(&pdev->dev, pdata); + return ERR_PTR(-ENOMEM); + } + + return pdata; +} +#else /* CONFIG_OF */ +static inline struct crypto_platform_data *atmel_sha_of_init(struct platform_device *dev) +{ + return ERR_PTR(-EINVAL); +} +#endif + static int atmel_sha_probe(struct platform_device *pdev) { struct atmel_sha_dev *sha_dd; @@ -1402,13 +1442,23 @@ static int atmel_sha_probe(struct platform_device *pdev) if (sha_dd->caps.has_dma) { pdata = pdev->dev.platform_data; if (!pdata) { - dev_err(&pdev->dev, "platform data not available\n"); + pdata = atmel_sha_of_init(pdev); + if (IS_ERR(pdata)) { + dev_err(&pdev->dev, "platform data not available\n"); + err = PTR_ERR(pdata); + goto err_pdata; + } + } + if (!pdata->dma_slave) { err = -ENXIO; goto err_pdata; } err = atmel_sha_dma_init(sha_dd, pdata); if (err) goto err_sha_dma; + + dev_info(dev, "using %s for DMA transfers\n", + dma_chan_name(sha_dd->dma_lch_in.chan)); } spin_lock(&atmel_sha.lock); @@ -1419,7 +1469,9 @@ static int atmel_sha_probe(struct platform_device *pdev) if (err) goto err_algs; - dev_info(dev, "Atmel SHA1/SHA256\n"); + dev_info(dev, "Atmel SHA1/SHA256%s%s\n", + sha_dd->caps.has_sha224 ? "/SHA224" : "", + sha_dd->caps.has_sha_384_512 ? "/SHA384/SHA512" : ""); return 0; @@ -1483,6 +1535,7 @@ static struct platform_driver atmel_sha_driver = { .driver = { .name = "atmel_sha", .owner = THIS_MODULE, + .of_match_table = of_match_ptr(atmel_sha_dt_ids), }, }; diff --git a/drivers/crypto/atmel-tdes.c b/drivers/crypto/atmel-tdes.c index 4a99564a08e6..6cde5b530c69 100644 --- a/drivers/crypto/atmel-tdes.c +++ b/drivers/crypto/atmel-tdes.c @@ -30,6 +30,7 @@ #include <linux/irq.h> #include <linux/scatterlist.h> #include <linux/dma-mapping.h> +#include <linux/of_device.h> #include <linux/delay.h> #include <linux/crypto.h> #include <linux/cryptohash.h> @@ -716,59 +717,50 @@ static int atmel_tdes_dma_init(struct atmel_tdes_dev *dd, struct crypto_platform_data *pdata) { int err = -ENOMEM; - dma_cap_mask_t mask_in, mask_out; + dma_cap_mask_t mask; + + dma_cap_zero(mask); + dma_cap_set(DMA_SLAVE, mask); + + /* Try to grab 2 DMA channels */ + dd->dma_lch_in.chan = dma_request_slave_channel_compat(mask, + atmel_tdes_filter, &pdata->dma_slave->rxdata, dd->dev, "tx"); + if (!dd->dma_lch_in.chan) + goto err_dma_in; + + dd->dma_lch_in.dma_conf.direction = DMA_MEM_TO_DEV; + dd->dma_lch_in.dma_conf.dst_addr = dd->phys_base + + TDES_IDATA1R; + dd->dma_lch_in.dma_conf.src_maxburst = 1; + dd->dma_lch_in.dma_conf.src_addr_width = + DMA_SLAVE_BUSWIDTH_4_BYTES; + dd->dma_lch_in.dma_conf.dst_maxburst = 1; + dd->dma_lch_in.dma_conf.dst_addr_width = + DMA_SLAVE_BUSWIDTH_4_BYTES; + dd->dma_lch_in.dma_conf.device_fc = false; + + dd->dma_lch_out.chan = dma_request_slave_channel_compat(mask, + atmel_tdes_filter, &pdata->dma_slave->txdata, dd->dev, "rx"); + if (!dd->dma_lch_out.chan) + goto err_dma_out; + + dd->dma_lch_out.dma_conf.direction = DMA_DEV_TO_MEM; + dd->dma_lch_out.dma_conf.src_addr = dd->phys_base + + TDES_ODATA1R; + dd->dma_lch_out.dma_conf.src_maxburst = 1; + dd->dma_lch_out.dma_conf.src_addr_width = + DMA_SLAVE_BUSWIDTH_4_BYTES; + dd->dma_lch_out.dma_conf.dst_maxburst = 1; + dd->dma_lch_out.dma_conf.dst_addr_width = + DMA_SLAVE_BUSWIDTH_4_BYTES; + dd->dma_lch_out.dma_conf.device_fc = false; - if (pdata && pdata->dma_slave->txdata.dma_dev && - pdata->dma_slave->rxdata.dma_dev) { - - /* Try to grab 2 DMA channels */ - dma_cap_zero(mask_in); - dma_cap_set(DMA_SLAVE, mask_in); - - dd->dma_lch_in.chan = dma_request_channel(mask_in, - atmel_tdes_filter, &pdata->dma_slave->rxdata); - - if (!dd->dma_lch_in.chan) - goto err_dma_in; - - dd->dma_lch_in.dma_conf.direction = DMA_MEM_TO_DEV; - dd->dma_lch_in.dma_conf.dst_addr = dd->phys_base + - TDES_IDATA1R; - dd->dma_lch_in.dma_conf.src_maxburst = 1; - dd->dma_lch_in.dma_conf.src_addr_width = - DMA_SLAVE_BUSWIDTH_4_BYTES; - dd->dma_lch_in.dma_conf.dst_maxburst = 1; - dd->dma_lch_in.dma_conf.dst_addr_width = - DMA_SLAVE_BUSWIDTH_4_BYTES; - dd->dma_lch_in.dma_conf.device_fc = false; - - dma_cap_zero(mask_out); - dma_cap_set(DMA_SLAVE, mask_out); - dd->dma_lch_out.chan = dma_request_channel(mask_out, - atmel_tdes_filter, &pdata->dma_slave->txdata); - - if (!dd->dma_lch_out.chan) - goto err_dma_out; - - dd->dma_lch_out.dma_conf.direction = DMA_DEV_TO_MEM; - dd->dma_lch_out.dma_conf.src_addr = dd->phys_base + - TDES_ODATA1R; - dd->dma_lch_out.dma_conf.src_maxburst = 1; - dd->dma_lch_out.dma_conf.src_addr_width = - DMA_SLAVE_BUSWIDTH_4_BYTES; - dd->dma_lch_out.dma_conf.dst_maxburst = 1; - dd->dma_lch_out.dma_conf.dst_addr_width = - DMA_SLAVE_BUSWIDTH_4_BYTES; - dd->dma_lch_out.dma_conf.device_fc = false; - - return 0; - } else { - return -ENODEV; - } + return 0; err_dma_out: dma_release_channel(dd->dma_lch_in.chan); err_dma_in: + dev_warn(dd->dev, "no DMA channel available\n"); return err; } @@ -1317,6 +1309,47 @@ static void atmel_tdes_get_cap(struct atmel_tdes_dev *dd) } } +#if defined(CONFIG_OF) +static const struct of_device_id atmel_tdes_dt_ids[] = { + { .compatible = "atmel,at91sam9g46-tdes" }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, atmel_tdes_dt_ids); + +static struct crypto_platform_data *atmel_tdes_of_init(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + struct crypto_platform_data *pdata; + + if (!np) { + dev_err(&pdev->dev, "device node not found\n"); + return ERR_PTR(-EINVAL); + } + + pdata = devm_kzalloc(&pdev->dev, sizeof(*pdata), GFP_KERNEL); + if (!pdata) { + dev_err(&pdev->dev, "could not allocate memory for pdata\n"); + return ERR_PTR(-ENOMEM); + } + + pdata->dma_slave = devm_kzalloc(&pdev->dev, + sizeof(*(pdata->dma_slave)), + GFP_KERNEL); + if (!pdata->dma_slave) { + dev_err(&pdev->dev, "could not allocate memory for dma_slave\n"); + devm_kfree(&pdev->dev, pdata); + return ERR_PTR(-ENOMEM); + } + + return pdata; +} +#else /* CONFIG_OF */ +static inline struct crypto_platform_data *atmel_tdes_of_init(struct platform_device *pdev) +{ + return ERR_PTR(-EINVAL); +} +#endif + static int atmel_tdes_probe(struct platform_device *pdev) { struct atmel_tdes_dev *tdes_dd; @@ -1399,13 +1432,24 @@ static int atmel_tdes_probe(struct platform_device *pdev) if (tdes_dd->caps.has_dma) { pdata = pdev->dev.platform_data; if (!pdata) { - dev_err(&pdev->dev, "platform data not available\n"); + pdata = atmel_tdes_of_init(pdev); + if (IS_ERR(pdata)) { + dev_err(&pdev->dev, "platform data not available\n"); + err = PTR_ERR(pdata); + goto err_pdata; + } + } + if (!pdata->dma_slave) { err = -ENXIO; goto err_pdata; } err = atmel_tdes_dma_init(tdes_dd, pdata); if (err) goto err_tdes_dma; + + dev_info(dev, "using %s, %s for DMA transfers\n", + dma_chan_name(tdes_dd->dma_lch_in.chan), + dma_chan_name(tdes_dd->dma_lch_out.chan)); } spin_lock(&atmel_tdes.lock); @@ -1487,6 +1531,7 @@ static struct platform_driver atmel_tdes_driver = { .driver = { .name = "atmel_tdes", .owner = THIS_MODULE, + .of_match_table = of_match_ptr(atmel_tdes_dt_ids), }, }; diff --git a/drivers/crypto/caam/Kconfig b/drivers/crypto/caam/Kconfig index ca89f6b84b06..e7555ff4cafd 100644 --- a/drivers/crypto/caam/Kconfig +++ b/drivers/crypto/caam/Kconfig @@ -4,16 +4,29 @@ config CRYPTO_DEV_FSL_CAAM help Enables the driver module for Freescale's Cryptographic Accelerator and Assurance Module (CAAM), also known as the SEC version 4 (SEC4). - This module adds a job ring operation interface, and configures h/w + This module creates job ring devices, and configures h/w to operate as a DPAA component automatically, depending on h/w feature availability. To compile this driver as a module, choose M here: the module will be called caam. +config CRYPTO_DEV_FSL_CAAM_JR + tristate "Freescale CAAM Job Ring driver backend" + depends on CRYPTO_DEV_FSL_CAAM + default y + help + Enables the driver module for Job Rings which are part of + Freescale's Cryptographic Accelerator + and Assurance Module (CAAM). This module adds a job ring operation + interface. + + To compile this driver as a module, choose M here: the module + will be called caam_jr. + config CRYPTO_DEV_FSL_CAAM_RINGSIZE int "Job Ring size" - depends on CRYPTO_DEV_FSL_CAAM + depends on CRYPTO_DEV_FSL_CAAM_JR range 2 9 default "9" help @@ -31,7 +44,7 @@ config CRYPTO_DEV_FSL_CAAM_RINGSIZE config CRYPTO_DEV_FSL_CAAM_INTC bool "Job Ring interrupt coalescing" - depends on CRYPTO_DEV_FSL_CAAM + depends on CRYPTO_DEV_FSL_CAAM_JR default n help Enable the Job Ring's interrupt coalescing feature. @@ -62,7 +75,7 @@ config CRYPTO_DEV_FSL_CAAM_INTC_TIME_THLD config CRYPTO_DEV_FSL_CAAM_CRYPTO_API tristate "Register algorithm implementations with the Crypto API" - depends on CRYPTO_DEV_FSL_CAAM + depends on CRYPTO_DEV_FSL_CAAM && CRYPTO_DEV_FSL_CAAM_JR default y select CRYPTO_ALGAPI select CRYPTO_AUTHENC @@ -76,7 +89,7 @@ config CRYPTO_DEV_FSL_CAAM_CRYPTO_API config CRYPTO_DEV_FSL_CAAM_AHASH_API tristate "Register hash algorithm implementations with Crypto API" - depends on CRYPTO_DEV_FSL_CAAM + depends on CRYPTO_DEV_FSL_CAAM && CRYPTO_DEV_FSL_CAAM_JR default y select CRYPTO_HASH help @@ -88,7 +101,7 @@ config CRYPTO_DEV_FSL_CAAM_AHASH_API config CRYPTO_DEV_FSL_CAAM_RNG_API tristate "Register caam device for hwrng API" - depends on CRYPTO_DEV_FSL_CAAM + depends on CRYPTO_DEV_FSL_CAAM && CRYPTO_DEV_FSL_CAAM_JR default y select CRYPTO_RNG select HW_RANDOM diff --git a/drivers/crypto/caam/Makefile b/drivers/crypto/caam/Makefile index d56bd0ec65d8..550758a333e7 100644 --- a/drivers/crypto/caam/Makefile +++ b/drivers/crypto/caam/Makefile @@ -6,8 +6,10 @@ ifeq ($(CONFIG_CRYPTO_DEV_FSL_CAAM_DEBUG), y) endif obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM) += caam.o +obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_JR) += caam_jr.o obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API) += caamalg.o obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API) += caamhash.o obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_RNG_API) += caamrng.o -caam-objs := ctrl.o jr.o error.o key_gen.o +caam-objs := ctrl.o +caam_jr-objs := jr.o key_gen.o error.o diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index 7c63b72ecd75..b71f2fd749df 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -86,6 +86,7 @@ #else #define debug(format, arg...) #endif +static struct list_head alg_list; /* Set DK bit in class 1 operation if shared */ static inline void append_dec_op1(u32 *desc, u32 type) @@ -466,24 +467,10 @@ static int aead_setkey(struct crypto_aead *aead, static const u8 mdpadlen[] = { 16, 20, 32, 32, 64, 64 }; struct caam_ctx *ctx = crypto_aead_ctx(aead); struct device *jrdev = ctx->jrdev; - struct rtattr *rta = (void *)key; - struct crypto_authenc_key_param *param; - unsigned int authkeylen; - unsigned int enckeylen; + struct crypto_authenc_keys keys; int ret = 0; - param = RTA_DATA(rta); - enckeylen = be32_to_cpu(param->enckeylen); - - key += RTA_ALIGN(rta->rta_len); - keylen -= RTA_ALIGN(rta->rta_len); - - if (keylen < enckeylen) - goto badkey; - - authkeylen = keylen - enckeylen; - - if (keylen > CAAM_MAX_KEY_SIZE) + if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) goto badkey; /* Pick class 2 key length from algorithm submask */ @@ -491,25 +478,29 @@ static int aead_setkey(struct crypto_aead *aead, OP_ALG_ALGSEL_SHIFT] * 2; ctx->split_key_pad_len = ALIGN(ctx->split_key_len, 16); + if (ctx->split_key_pad_len + keys.enckeylen > CAAM_MAX_KEY_SIZE) + goto badkey; + #ifdef DEBUG printk(KERN_ERR "keylen %d enckeylen %d authkeylen %d\n", - keylen, enckeylen, authkeylen); + keys.authkeylen + keys.enckeylen, keys.enckeylen, + keys.authkeylen); printk(KERN_ERR "split_key_len %d split_key_pad_len %d\n", ctx->split_key_len, ctx->split_key_pad_len); print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); #endif - ret = gen_split_aead_key(ctx, key, authkeylen); + ret = gen_split_aead_key(ctx, keys.authkey, keys.authkeylen); if (ret) { goto badkey; } /* postpend encryption key to auth split key */ - memcpy(ctx->key + ctx->split_key_pad_len, key + authkeylen, enckeylen); + memcpy(ctx->key + ctx->split_key_pad_len, keys.enckey, keys.enckeylen); ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->split_key_pad_len + - enckeylen, DMA_TO_DEVICE); + keys.enckeylen, DMA_TO_DEVICE); if (dma_mapping_error(jrdev, ctx->key_dma)) { dev_err(jrdev, "unable to map key i/o memory\n"); return -ENOMEM; @@ -517,15 +508,15 @@ static int aead_setkey(struct crypto_aead *aead, #ifdef DEBUG print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, - ctx->split_key_pad_len + enckeylen, 1); + ctx->split_key_pad_len + keys.enckeylen, 1); #endif - ctx->enckeylen = enckeylen; + ctx->enckeylen = keys.enckeylen; ret = aead_set_sh_desc(aead); if (ret) { dma_unmap_single(jrdev, ctx->key_dma, ctx->split_key_pad_len + - enckeylen, DMA_TO_DEVICE); + keys.enckeylen, DMA_TO_DEVICE); } return ret; @@ -817,7 +808,7 @@ static void aead_decrypt_done(struct device *jrdev, u32 *desc, u32 err, ivsize, 1); print_hex_dump(KERN_ERR, "dst @"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->dst), - req->cryptlen, 1); + req->cryptlen - ctx->authsize, 1); #endif if (err) { @@ -971,12 +962,9 @@ static void init_aead_job(u32 *sh_desc, dma_addr_t ptr, (edesc->src_nents ? : 1); in_options = LDST_SGF; } - if (encrypt) - append_seq_in_ptr(desc, src_dma, req->assoclen + ivsize + - req->cryptlen - authsize, in_options); - else - append_seq_in_ptr(desc, src_dma, req->assoclen + ivsize + - req->cryptlen, in_options); + + append_seq_in_ptr(desc, src_dma, req->assoclen + ivsize + req->cryptlen, + in_options); if (likely(req->src == req->dst)) { if (all_contig) { @@ -997,7 +985,8 @@ static void init_aead_job(u32 *sh_desc, dma_addr_t ptr, } } if (encrypt) - append_seq_out_ptr(desc, dst_dma, req->cryptlen, out_options); + append_seq_out_ptr(desc, dst_dma, req->cryptlen + authsize, + out_options); else append_seq_out_ptr(desc, dst_dma, req->cryptlen - authsize, out_options); @@ -1047,8 +1036,8 @@ static void init_aead_giv_job(u32 *sh_desc, dma_addr_t ptr, sec4_sg_index += edesc->assoc_nents + 1 + edesc->src_nents; in_options = LDST_SGF; } - append_seq_in_ptr(desc, src_dma, req->assoclen + ivsize + - req->cryptlen - authsize, in_options); + append_seq_in_ptr(desc, src_dma, req->assoclen + ivsize + req->cryptlen, + in_options); if (contig & GIV_DST_CONTIG) { dst_dma = edesc->iv_dma; @@ -1065,7 +1054,8 @@ static void init_aead_giv_job(u32 *sh_desc, dma_addr_t ptr, } } - append_seq_out_ptr(desc, dst_dma, ivsize + req->cryptlen, out_options); + append_seq_out_ptr(desc, dst_dma, ivsize + req->cryptlen + authsize, + out_options); } /* @@ -1129,7 +1119,8 @@ static void init_ablkcipher_job(u32 *sh_desc, dma_addr_t ptr, * allocate and map the aead extended descriptor */ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, - int desc_bytes, bool *all_contig_ptr) + int desc_bytes, bool *all_contig_ptr, + bool encrypt) { struct crypto_aead *aead = crypto_aead_reqtfm(req); struct caam_ctx *ctx = crypto_aead_ctx(aead); @@ -1144,12 +1135,22 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, bool assoc_chained = false, src_chained = false, dst_chained = false; int ivsize = crypto_aead_ivsize(aead); int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes; + unsigned int authsize = ctx->authsize; assoc_nents = sg_count(req->assoc, req->assoclen, &assoc_chained); - src_nents = sg_count(req->src, req->cryptlen, &src_chained); - if (unlikely(req->dst != req->src)) - dst_nents = sg_count(req->dst, req->cryptlen, &dst_chained); + if (unlikely(req->dst != req->src)) { + src_nents = sg_count(req->src, req->cryptlen, &src_chained); + dst_nents = sg_count(req->dst, + req->cryptlen + + (encrypt ? authsize : (-authsize)), + &dst_chained); + } else { + src_nents = sg_count(req->src, + req->cryptlen + + (encrypt ? authsize : 0), + &src_chained); + } sgc = dma_map_sg_chained(jrdev, req->assoc, assoc_nents ? : 1, DMA_TO_DEVICE, assoc_chained); @@ -1233,11 +1234,9 @@ static int aead_encrypt(struct aead_request *req) u32 *desc; int ret = 0; - req->cryptlen += ctx->authsize; - /* allocate extended descriptor */ edesc = aead_edesc_alloc(req, DESC_JOB_IO_LEN * - CAAM_CMD_SZ, &all_contig); + CAAM_CMD_SZ, &all_contig, true); if (IS_ERR(edesc)) return PTR_ERR(edesc); @@ -1274,7 +1273,7 @@ static int aead_decrypt(struct aead_request *req) /* allocate extended descriptor */ edesc = aead_edesc_alloc(req, DESC_JOB_IO_LEN * - CAAM_CMD_SZ, &all_contig); + CAAM_CMD_SZ, &all_contig, false); if (IS_ERR(edesc)) return PTR_ERR(edesc); @@ -1331,7 +1330,8 @@ static struct aead_edesc *aead_giv_edesc_alloc(struct aead_givcrypt_request src_nents = sg_count(req->src, req->cryptlen, &src_chained); if (unlikely(req->dst != req->src)) - dst_nents = sg_count(req->dst, req->cryptlen, &dst_chained); + dst_nents = sg_count(req->dst, req->cryptlen + ctx->authsize, + &dst_chained); sgc = dma_map_sg_chained(jrdev, req->assoc, assoc_nents ? : 1, DMA_TO_DEVICE, assoc_chained); @@ -1425,8 +1425,6 @@ static int aead_givencrypt(struct aead_givcrypt_request *areq) u32 *desc; int ret = 0; - req->cryptlen += ctx->authsize; - /* allocate extended descriptor */ edesc = aead_giv_edesc_alloc(areq, DESC_JOB_IO_LEN * CAAM_CMD_SZ, &contig); @@ -2057,7 +2055,6 @@ static struct caam_alg_template driver_algs[] = { struct caam_crypto_alg { struct list_head entry; - struct device *ctrldev; int class1_alg_type; int class2_alg_type; int alg_op; @@ -2070,14 +2067,12 @@ static int caam_cra_init(struct crypto_tfm *tfm) struct caam_crypto_alg *caam_alg = container_of(alg, struct caam_crypto_alg, crypto_alg); struct caam_ctx *ctx = crypto_tfm_ctx(tfm); - struct caam_drv_private *priv = dev_get_drvdata(caam_alg->ctrldev); - int tgt_jr = atomic_inc_return(&priv->tfm_count); - /* - * distribute tfms across job rings to ensure in-order - * crypto request processing per tfm - */ - ctx->jrdev = priv->jrdev[(tgt_jr / 2) % priv->total_jobrs]; + ctx->jrdev = caam_jr_alloc(); + if (IS_ERR(ctx->jrdev)) { + pr_err("Job Ring Device allocation for transform failed\n"); + return PTR_ERR(ctx->jrdev); + } /* copy descriptor header template value */ ctx->class1_alg_type = OP_TYPE_CLASS1_ALG | caam_alg->class1_alg_type; @@ -2104,44 +2099,26 @@ static void caam_cra_exit(struct crypto_tfm *tfm) dma_unmap_single(ctx->jrdev, ctx->sh_desc_givenc_dma, desc_bytes(ctx->sh_desc_givenc), DMA_TO_DEVICE); + + caam_jr_free(ctx->jrdev); } static void __exit caam_algapi_exit(void) { - struct device_node *dev_node; - struct platform_device *pdev; - struct device *ctrldev; - struct caam_drv_private *priv; struct caam_crypto_alg *t_alg, *n; - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); - if (!dev_node) { - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0"); - if (!dev_node) - return; - } - - pdev = of_find_device_by_node(dev_node); - if (!pdev) + if (!alg_list.next) return; - ctrldev = &pdev->dev; - of_node_put(dev_node); - priv = dev_get_drvdata(ctrldev); - - if (!priv->alg_list.next) - return; - - list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) { + list_for_each_entry_safe(t_alg, n, &alg_list, entry) { crypto_unregister_alg(&t_alg->crypto_alg); list_del(&t_alg->entry); kfree(t_alg); } } -static struct caam_crypto_alg *caam_alg_alloc(struct device *ctrldev, - struct caam_alg_template +static struct caam_crypto_alg *caam_alg_alloc(struct caam_alg_template *template) { struct caam_crypto_alg *t_alg; @@ -2149,7 +2126,7 @@ static struct caam_crypto_alg *caam_alg_alloc(struct device *ctrldev, t_alg = kzalloc(sizeof(struct caam_crypto_alg), GFP_KERNEL); if (!t_alg) { - dev_err(ctrldev, "failed to allocate t_alg\n"); + pr_err("failed to allocate t_alg\n"); return ERR_PTR(-ENOMEM); } @@ -2181,62 +2158,39 @@ static struct caam_crypto_alg *caam_alg_alloc(struct device *ctrldev, t_alg->class1_alg_type = template->class1_alg_type; t_alg->class2_alg_type = template->class2_alg_type; t_alg->alg_op = template->alg_op; - t_alg->ctrldev = ctrldev; return t_alg; } static int __init caam_algapi_init(void) { - struct device_node *dev_node; - struct platform_device *pdev; - struct device *ctrldev; - struct caam_drv_private *priv; int i = 0, err = 0; - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); - if (!dev_node) { - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0"); - if (!dev_node) - return -ENODEV; - } - - pdev = of_find_device_by_node(dev_node); - if (!pdev) - return -ENODEV; - - ctrldev = &pdev->dev; - priv = dev_get_drvdata(ctrldev); - of_node_put(dev_node); - - INIT_LIST_HEAD(&priv->alg_list); - - atomic_set(&priv->tfm_count, -1); + INIT_LIST_HEAD(&alg_list); /* register crypto algorithms the device supports */ for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { /* TODO: check if h/w supports alg */ struct caam_crypto_alg *t_alg; - t_alg = caam_alg_alloc(ctrldev, &driver_algs[i]); + t_alg = caam_alg_alloc(&driver_algs[i]); if (IS_ERR(t_alg)) { err = PTR_ERR(t_alg); - dev_warn(ctrldev, "%s alg allocation failed\n", - driver_algs[i].driver_name); + pr_warn("%s alg allocation failed\n", + driver_algs[i].driver_name); continue; } err = crypto_register_alg(&t_alg->crypto_alg); if (err) { - dev_warn(ctrldev, "%s alg registration failed\n", + pr_warn("%s alg registration failed\n", t_alg->crypto_alg.cra_driver_name); kfree(t_alg); } else - list_add_tail(&t_alg->entry, &priv->alg_list); + list_add_tail(&t_alg->entry, &alg_list); } - if (!list_empty(&priv->alg_list)) - dev_info(ctrldev, "%s algorithms registered in /proc/crypto\n", - (char *)of_get_property(dev_node, "compatible", NULL)); + if (!list_empty(&alg_list)) + pr_info("caam algorithms registered in /proc/crypto\n"); return err; } diff --git a/drivers/crypto/caam/caamhash.c b/drivers/crypto/caam/caamhash.c index e732bd962e98..0378328f47a7 100644 --- a/drivers/crypto/caam/caamhash.c +++ b/drivers/crypto/caam/caamhash.c @@ -94,6 +94,9 @@ #define debug(format, arg...) #endif + +static struct list_head hash_list; + /* ahash per-session context */ struct caam_hash_ctx { struct device *jrdev; @@ -1653,7 +1656,6 @@ static struct caam_hash_template driver_hash[] = { struct caam_hash_alg { struct list_head entry; - struct device *ctrldev; int alg_type; int alg_op; struct ahash_alg ahash_alg; @@ -1670,7 +1672,6 @@ static int caam_hash_cra_init(struct crypto_tfm *tfm) struct caam_hash_alg *caam_hash = container_of(alg, struct caam_hash_alg, ahash_alg); struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm); - struct caam_drv_private *priv = dev_get_drvdata(caam_hash->ctrldev); /* Sizes for MDHA running digests: MD5, SHA1, 224, 256, 384, 512 */ static const u8 runninglen[] = { HASH_MSG_LEN + MD5_DIGEST_SIZE, HASH_MSG_LEN + SHA1_DIGEST_SIZE, @@ -1678,15 +1679,17 @@ static int caam_hash_cra_init(struct crypto_tfm *tfm) HASH_MSG_LEN + SHA256_DIGEST_SIZE, HASH_MSG_LEN + 64, HASH_MSG_LEN + SHA512_DIGEST_SIZE }; - int tgt_jr = atomic_inc_return(&priv->tfm_count); int ret = 0; /* - * distribute tfms across job rings to ensure in-order + * Get a Job ring from Job Ring driver to ensure in-order * crypto request processing per tfm */ - ctx->jrdev = priv->jrdev[tgt_jr % priv->total_jobrs]; - + ctx->jrdev = caam_jr_alloc(); + if (IS_ERR(ctx->jrdev)) { + pr_err("Job Ring Device allocation for transform failed\n"); + return PTR_ERR(ctx->jrdev); + } /* copy descriptor header template value */ ctx->alg_type = OP_TYPE_CLASS2_ALG | caam_hash->alg_type; ctx->alg_op = OP_TYPE_CLASS2_ALG | caam_hash->alg_op; @@ -1729,35 +1732,18 @@ static void caam_hash_cra_exit(struct crypto_tfm *tfm) !dma_mapping_error(ctx->jrdev, ctx->sh_desc_finup_dma)) dma_unmap_single(ctx->jrdev, ctx->sh_desc_finup_dma, desc_bytes(ctx->sh_desc_finup), DMA_TO_DEVICE); + + caam_jr_free(ctx->jrdev); } static void __exit caam_algapi_hash_exit(void) { - struct device_node *dev_node; - struct platform_device *pdev; - struct device *ctrldev; - struct caam_drv_private *priv; struct caam_hash_alg *t_alg, *n; - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); - if (!dev_node) { - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0"); - if (!dev_node) - return; - } - - pdev = of_find_device_by_node(dev_node); - if (!pdev) + if (!hash_list.next) return; - ctrldev = &pdev->dev; - of_node_put(dev_node); - priv = dev_get_drvdata(ctrldev); - - if (!priv->hash_list.next) - return; - - list_for_each_entry_safe(t_alg, n, &priv->hash_list, entry) { + list_for_each_entry_safe(t_alg, n, &hash_list, entry) { crypto_unregister_ahash(&t_alg->ahash_alg); list_del(&t_alg->entry); kfree(t_alg); @@ -1765,7 +1751,7 @@ static void __exit caam_algapi_hash_exit(void) } static struct caam_hash_alg * -caam_hash_alloc(struct device *ctrldev, struct caam_hash_template *template, +caam_hash_alloc(struct caam_hash_template *template, bool keyed) { struct caam_hash_alg *t_alg; @@ -1774,7 +1760,7 @@ caam_hash_alloc(struct device *ctrldev, struct caam_hash_template *template, t_alg = kzalloc(sizeof(struct caam_hash_alg), GFP_KERNEL); if (!t_alg) { - dev_err(ctrldev, "failed to allocate t_alg\n"); + pr_err("failed to allocate t_alg\n"); return ERR_PTR(-ENOMEM); } @@ -1805,37 +1791,15 @@ caam_hash_alloc(struct device *ctrldev, struct caam_hash_template *template, t_alg->alg_type = template->alg_type; t_alg->alg_op = template->alg_op; - t_alg->ctrldev = ctrldev; return t_alg; } static int __init caam_algapi_hash_init(void) { - struct device_node *dev_node; - struct platform_device *pdev; - struct device *ctrldev; - struct caam_drv_private *priv; int i = 0, err = 0; - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); - if (!dev_node) { - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0"); - if (!dev_node) - return -ENODEV; - } - - pdev = of_find_device_by_node(dev_node); - if (!pdev) - return -ENODEV; - - ctrldev = &pdev->dev; - priv = dev_get_drvdata(ctrldev); - of_node_put(dev_node); - - INIT_LIST_HEAD(&priv->hash_list); - - atomic_set(&priv->tfm_count, -1); + INIT_LIST_HEAD(&hash_list); /* register crypto algorithms the device supports */ for (i = 0; i < ARRAY_SIZE(driver_hash); i++) { @@ -1843,38 +1807,38 @@ static int __init caam_algapi_hash_init(void) struct caam_hash_alg *t_alg; /* register hmac version */ - t_alg = caam_hash_alloc(ctrldev, &driver_hash[i], true); + t_alg = caam_hash_alloc(&driver_hash[i], true); if (IS_ERR(t_alg)) { err = PTR_ERR(t_alg); - dev_warn(ctrldev, "%s alg allocation failed\n", - driver_hash[i].driver_name); + pr_warn("%s alg allocation failed\n", + driver_hash[i].driver_name); continue; } err = crypto_register_ahash(&t_alg->ahash_alg); if (err) { - dev_warn(ctrldev, "%s alg registration failed\n", + pr_warn("%s alg registration failed\n", t_alg->ahash_alg.halg.base.cra_driver_name); kfree(t_alg); } else - list_add_tail(&t_alg->entry, &priv->hash_list); + list_add_tail(&t_alg->entry, &hash_list); /* register unkeyed version */ - t_alg = caam_hash_alloc(ctrldev, &driver_hash[i], false); + t_alg = caam_hash_alloc(&driver_hash[i], false); if (IS_ERR(t_alg)) { err = PTR_ERR(t_alg); - dev_warn(ctrldev, "%s alg allocation failed\n", - driver_hash[i].driver_name); + pr_warn("%s alg allocation failed\n", + driver_hash[i].driver_name); continue; } err = crypto_register_ahash(&t_alg->ahash_alg); if (err) { - dev_warn(ctrldev, "%s alg registration failed\n", + pr_warn("%s alg registration failed\n", t_alg->ahash_alg.halg.base.cra_driver_name); kfree(t_alg); } else - list_add_tail(&t_alg->entry, &priv->hash_list); + list_add_tail(&t_alg->entry, &hash_list); } return err; diff --git a/drivers/crypto/caam/caamrng.c b/drivers/crypto/caam/caamrng.c index d1939a9539c0..28486b19fc36 100644 --- a/drivers/crypto/caam/caamrng.c +++ b/drivers/crypto/caam/caamrng.c @@ -273,34 +273,23 @@ static struct hwrng caam_rng = { static void __exit caam_rng_exit(void) { + caam_jr_free(rng_ctx.jrdev); hwrng_unregister(&caam_rng); } static int __init caam_rng_init(void) { - struct device_node *dev_node; - struct platform_device *pdev; - struct device *ctrldev; - struct caam_drv_private *priv; - - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); - if (!dev_node) { - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0"); - if (!dev_node) - return -ENODEV; - } - - pdev = of_find_device_by_node(dev_node); - if (!pdev) - return -ENODEV; + struct device *dev; - ctrldev = &pdev->dev; - priv = dev_get_drvdata(ctrldev); - of_node_put(dev_node); + dev = caam_jr_alloc(); + if (IS_ERR(dev)) { + pr_err("Job Ring Device allocation for transform failed\n"); + return PTR_ERR(dev); + } - caam_init_rng(&rng_ctx, priv->jrdev[0]); + caam_init_rng(&rng_ctx, dev); - dev_info(priv->jrdev[0], "registering rng-caam\n"); + dev_info(dev, "registering rng-caam\n"); return hwrng_register(&caam_rng); } diff --git a/drivers/crypto/caam/ctrl.c b/drivers/crypto/caam/ctrl.c index bc6d820812b6..63fb1af2c431 100644 --- a/drivers/crypto/caam/ctrl.c +++ b/drivers/crypto/caam/ctrl.c @@ -16,82 +16,75 @@ #include "error.h" #include "ctrl.h" -static int caam_remove(struct platform_device *pdev) -{ - struct device *ctrldev; - struct caam_drv_private *ctrlpriv; - struct caam_drv_private_jr *jrpriv; - struct caam_full __iomem *topregs; - int ring, ret = 0; - - ctrldev = &pdev->dev; - ctrlpriv = dev_get_drvdata(ctrldev); - topregs = (struct caam_full __iomem *)ctrlpriv->ctrl; - - /* shut down JobRs */ - for (ring = 0; ring < ctrlpriv->total_jobrs; ring++) { - ret |= caam_jr_shutdown(ctrlpriv->jrdev[ring]); - jrpriv = dev_get_drvdata(ctrlpriv->jrdev[ring]); - irq_dispose_mapping(jrpriv->irq); - } - - /* Shut down debug views */ -#ifdef CONFIG_DEBUG_FS - debugfs_remove_recursive(ctrlpriv->dfs_root); -#endif - - /* Unmap controller region */ - iounmap(&topregs->ctrl); - - kfree(ctrlpriv->jrdev); - kfree(ctrlpriv); - - return ret; -} - /* * Descriptor to instantiate RNG State Handle 0 in normal mode and * load the JDKEK, TDKEK and TDSK registers */ -static void build_instantiation_desc(u32 *desc) +static void build_instantiation_desc(u32 *desc, int handle, int do_sk) { - u32 *jump_cmd; + u32 *jump_cmd, op_flags; init_job_desc(desc, 0); + op_flags = OP_TYPE_CLASS1_ALG | OP_ALG_ALGSEL_RNG | + (handle << OP_ALG_AAI_SHIFT) | OP_ALG_AS_INIT; + /* INIT RNG in non-test mode */ - append_operation(desc, OP_TYPE_CLASS1_ALG | OP_ALG_ALGSEL_RNG | - OP_ALG_AS_INIT); + append_operation(desc, op_flags); + + if (!handle && do_sk) { + /* + * For SH0, Secure Keys must be generated as well + */ + + /* wait for done */ + jump_cmd = append_jump(desc, JUMP_CLASS_CLASS1); + set_jump_tgt_here(desc, jump_cmd); + + /* + * load 1 to clear written reg: + * resets the done interrrupt and returns the RNG to idle. + */ + append_load_imm_u32(desc, 1, LDST_SRCDST_WORD_CLRW); + + /* Initialize State Handle */ + append_operation(desc, OP_TYPE_CLASS1_ALG | OP_ALG_ALGSEL_RNG | + OP_ALG_AAI_RNG4_SK); + } - /* wait for done */ - jump_cmd = append_jump(desc, JUMP_CLASS_CLASS1); - set_jump_tgt_here(desc, jump_cmd); + append_jump(desc, JUMP_CLASS_CLASS1 | JUMP_TYPE_HALT); +} - /* - * load 1 to clear written reg: - * resets the done interrupt and returns the RNG to idle. - */ - append_load_imm_u32(desc, 1, LDST_SRCDST_WORD_CLRW); +/* Descriptor for deinstantiation of State Handle 0 of the RNG block. */ +static void build_deinstantiation_desc(u32 *desc, int handle) +{ + init_job_desc(desc, 0); - /* generate secure keys (non-test) */ + /* Uninstantiate State Handle 0 */ append_operation(desc, OP_TYPE_CLASS1_ALG | OP_ALG_ALGSEL_RNG | - OP_ALG_RNG4_SK); + (handle << OP_ALG_AAI_SHIFT) | OP_ALG_AS_INITFINAL); + + append_jump(desc, JUMP_CLASS_CLASS1 | JUMP_TYPE_HALT); } -static int instantiate_rng(struct device *ctrldev) +/* + * run_descriptor_deco0 - runs a descriptor on DECO0, under direct control of + * the software (no JR/QI used). + * @ctrldev - pointer to device + * @status - descriptor status, after being run + * + * Return: - 0 if no error occurred + * - -ENODEV if the DECO couldn't be acquired + * - -EAGAIN if an error occurred while executing the descriptor + */ +static inline int run_descriptor_deco0(struct device *ctrldev, u32 *desc, + u32 *status) { struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctrldev); struct caam_full __iomem *topregs; unsigned int timeout = 100000; - u32 *desc; - int i, ret = 0; - - desc = kmalloc(CAAM_CMD_SZ * 6, GFP_KERNEL | GFP_DMA); - if (!desc) { - dev_err(ctrldev, "can't allocate RNG init descriptor memory\n"); - return -ENOMEM; - } - build_instantiation_desc(desc); + u32 deco_dbg_reg, flags; + int i; /* Set the bit to request direct access to DECO0 */ topregs = (struct caam_full __iomem *)ctrlpriv->ctrl; @@ -103,36 +96,219 @@ static int instantiate_rng(struct device *ctrldev) if (!timeout) { dev_err(ctrldev, "failed to acquire DECO 0\n"); - ret = -EIO; - goto out; + clrbits32(&topregs->ctrl.deco_rq, DECORR_RQD0ENABLE); + return -ENODEV; } for (i = 0; i < desc_len(desc); i++) - topregs->deco.descbuf[i] = *(desc + i); + wr_reg32(&topregs->deco.descbuf[i], *(desc + i)); + + flags = DECO_JQCR_WHL; + /* + * If the descriptor length is longer than 4 words, then the + * FOUR bit in JRCTRL register must be set. + */ + if (desc_len(desc) >= 4) + flags |= DECO_JQCR_FOUR; - wr_reg32(&topregs->deco.jr_ctl_hi, DECO_JQCR_WHL | DECO_JQCR_FOUR); + /* Instruct the DECO to execute it */ + wr_reg32(&topregs->deco.jr_ctl_hi, flags); timeout = 10000000; - while ((rd_reg32(&topregs->deco.desc_dbg) & DECO_DBG_VALID) && - --timeout) + do { + deco_dbg_reg = rd_reg32(&topregs->deco.desc_dbg); + /* + * If an error occured in the descriptor, then + * the DECO status field will be set to 0x0D + */ + if ((deco_dbg_reg & DESC_DBG_DECO_STAT_MASK) == + DESC_DBG_DECO_STAT_HOST_ERR) + break; cpu_relax(); + } while ((deco_dbg_reg & DESC_DBG_DECO_STAT_VALID) && --timeout); - if (!timeout) { - dev_err(ctrldev, "failed to instantiate RNG\n"); - ret = -EIO; - } + *status = rd_reg32(&topregs->deco.op_status_hi) & + DECO_OP_STATUS_HI_ERR_MASK; + /* Mark the DECO as free */ clrbits32(&topregs->ctrl.deco_rq, DECORR_RQD0ENABLE); -out: + + if (!timeout) + return -EAGAIN; + + return 0; +} + +/* + * instantiate_rng - builds and executes a descriptor on DECO0, + * which initializes the RNG block. + * @ctrldev - pointer to device + * @state_handle_mask - bitmask containing the instantiation status + * for the RNG4 state handles which exist in + * the RNG4 block: 1 if it's been instantiated + * by an external entry, 0 otherwise. + * @gen_sk - generate data to be loaded into the JDKEK, TDKEK and TDSK; + * Caution: this can be done only once; if the keys need to be + * regenerated, a POR is required + * + * Return: - 0 if no error occurred + * - -ENOMEM if there isn't enough memory to allocate the descriptor + * - -ENODEV if DECO0 couldn't be acquired + * - -EAGAIN if an error occurred when executing the descriptor + * f.i. there was a RNG hardware error due to not "good enough" + * entropy being aquired. + */ +static int instantiate_rng(struct device *ctrldev, int state_handle_mask, + int gen_sk) +{ + struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctrldev); + struct caam_full __iomem *topregs; + struct rng4tst __iomem *r4tst; + u32 *desc, status, rdsta_val; + int ret = 0, sh_idx; + + topregs = (struct caam_full __iomem *)ctrlpriv->ctrl; + r4tst = &topregs->ctrl.r4tst[0]; + + desc = kmalloc(CAAM_CMD_SZ * 7, GFP_KERNEL); + if (!desc) + return -ENOMEM; + + for (sh_idx = 0; sh_idx < RNG4_MAX_HANDLES; sh_idx++) { + /* + * If the corresponding bit is set, this state handle + * was initialized by somebody else, so it's left alone. + */ + if ((1 << sh_idx) & state_handle_mask) + continue; + + /* Create the descriptor for instantiating RNG State Handle */ + build_instantiation_desc(desc, sh_idx, gen_sk); + + /* Try to run it through DECO0 */ + ret = run_descriptor_deco0(ctrldev, desc, &status); + + /* + * If ret is not 0, or descriptor status is not 0, then + * something went wrong. No need to try the next state + * handle (if available), bail out here. + * Also, if for some reason, the State Handle didn't get + * instantiated although the descriptor has finished + * without any error (HW optimizations for later + * CAAM eras), then try again. + */ + rdsta_val = + rd_reg32(&topregs->ctrl.r4tst[0].rdsta) & RDSTA_IFMASK; + if (status || !(rdsta_val & (1 << sh_idx))) + ret = -EAGAIN; + if (ret) + break; + + dev_info(ctrldev, "Instantiated RNG4 SH%d\n", sh_idx); + /* Clear the contents before recreating the descriptor */ + memset(desc, 0x00, CAAM_CMD_SZ * 7); + } + kfree(desc); + return ret; } /* - * By default, the TRNG runs for 200 clocks per sample; - * 1600 clocks per sample generates better entropy. + * deinstantiate_rng - builds and executes a descriptor on DECO0, + * which deinitializes the RNG block. + * @ctrldev - pointer to device + * @state_handle_mask - bitmask containing the instantiation status + * for the RNG4 state handles which exist in + * the RNG4 block: 1 if it's been instantiated + * + * Return: - 0 if no error occurred + * - -ENOMEM if there isn't enough memory to allocate the descriptor + * - -ENODEV if DECO0 couldn't be acquired + * - -EAGAIN if an error occurred when executing the descriptor */ -static void kick_trng(struct platform_device *pdev) +static int deinstantiate_rng(struct device *ctrldev, int state_handle_mask) +{ + u32 *desc, status; + int sh_idx, ret = 0; + + desc = kmalloc(CAAM_CMD_SZ * 3, GFP_KERNEL); + if (!desc) + return -ENOMEM; + + for (sh_idx = 0; sh_idx < RNG4_MAX_HANDLES; sh_idx++) { + /* + * If the corresponding bit is set, then it means the state + * handle was initialized by us, and thus it needs to be + * deintialized as well + */ + if ((1 << sh_idx) & state_handle_mask) { + /* + * Create the descriptor for deinstantating this state + * handle + */ + build_deinstantiation_desc(desc, sh_idx); + + /* Try to run it through DECO0 */ + ret = run_descriptor_deco0(ctrldev, desc, &status); + + if (ret || status) { + dev_err(ctrldev, + "Failed to deinstantiate RNG4 SH%d\n", + sh_idx); + break; + } + dev_info(ctrldev, "Deinstantiated RNG4 SH%d\n", sh_idx); + } + } + + kfree(desc); + + return ret; +} + +static int caam_remove(struct platform_device *pdev) +{ + struct device *ctrldev; + struct caam_drv_private *ctrlpriv; + struct caam_full __iomem *topregs; + int ring, ret = 0; + + ctrldev = &pdev->dev; + ctrlpriv = dev_get_drvdata(ctrldev); + topregs = (struct caam_full __iomem *)ctrlpriv->ctrl; + + /* Remove platform devices for JobRs */ + for (ring = 0; ring < ctrlpriv->total_jobrs; ring++) { + if (ctrlpriv->jrpdev[ring]) + of_device_unregister(ctrlpriv->jrpdev[ring]); + } + + /* De-initialize RNG state handles initialized by this driver. */ + if (ctrlpriv->rng4_sh_init) + deinstantiate_rng(ctrldev, ctrlpriv->rng4_sh_init); + + /* Shut down debug views */ +#ifdef CONFIG_DEBUG_FS + debugfs_remove_recursive(ctrlpriv->dfs_root); +#endif + + /* Unmap controller region */ + iounmap(&topregs->ctrl); + + kfree(ctrlpriv->jrpdev); + kfree(ctrlpriv); + + return ret; +} + +/* + * kick_trng - sets the various parameters for enabling the initialization + * of the RNG4 block in CAAM + * @pdev - pointer to the platform device + * @ent_delay - Defines the length (in system clocks) of each entropy sample. + */ +static void kick_trng(struct platform_device *pdev, int ent_delay) { struct device *ctrldev = &pdev->dev; struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctrldev); @@ -145,14 +321,31 @@ static void kick_trng(struct platform_device *pdev) /* put RNG4 into program mode */ setbits32(&r4tst->rtmctl, RTMCTL_PRGM); - /* 1600 clocks per sample */ + + /* + * Performance-wise, it does not make sense to + * set the delay to a value that is lower + * than the last one that worked (i.e. the state handles + * were instantiated properly. Thus, instead of wasting + * time trying to set the values controlling the sample + * frequency, the function simply returns. + */ + val = (rd_reg32(&r4tst->rtsdctl) & RTSDCTL_ENT_DLY_MASK) + >> RTSDCTL_ENT_DLY_SHIFT; + if (ent_delay <= val) { + /* put RNG4 into run mode */ + clrbits32(&r4tst->rtmctl, RTMCTL_PRGM); + return; + } + val = rd_reg32(&r4tst->rtsdctl); - val = (val & ~RTSDCTL_ENT_DLY_MASK) | (1600 << RTSDCTL_ENT_DLY_SHIFT); + val = (val & ~RTSDCTL_ENT_DLY_MASK) | + (ent_delay << RTSDCTL_ENT_DLY_SHIFT); wr_reg32(&r4tst->rtsdctl, val); - /* min. freq. count */ - wr_reg32(&r4tst->rtfrqmin, 400); - /* max. freq. count */ - wr_reg32(&r4tst->rtfrqmax, 6400); + /* min. freq. count, equal to 1/4 of the entropy sample length */ + wr_reg32(&r4tst->rtfrqmin, ent_delay >> 2); + /* max. freq. count, equal to 8 times the entropy sample length */ + wr_reg32(&r4tst->rtfrqmax, ent_delay << 3); /* put RNG4 into run mode */ clrbits32(&r4tst->rtmctl, RTMCTL_PRGM); } @@ -193,7 +386,7 @@ EXPORT_SYMBOL(caam_get_era); /* Probe routine for CAAM top (controller) level */ static int caam_probe(struct platform_device *pdev) { - int ret, ring, rspec; + int ret, ring, rspec, gen_sk, ent_delay = RTSDCTL_ENT_DLY_MIN; u64 caam_id; struct device *dev; struct device_node *nprop, *np; @@ -258,8 +451,9 @@ static int caam_probe(struct platform_device *pdev) rspec++; } - ctrlpriv->jrdev = kzalloc(sizeof(struct device *) * rspec, GFP_KERNEL); - if (ctrlpriv->jrdev == NULL) { + ctrlpriv->jrpdev = kzalloc(sizeof(struct platform_device *) * rspec, + GFP_KERNEL); + if (ctrlpriv->jrpdev == NULL) { iounmap(&topregs->ctrl); return -ENOMEM; } @@ -267,13 +461,24 @@ static int caam_probe(struct platform_device *pdev) ring = 0; ctrlpriv->total_jobrs = 0; for_each_compatible_node(np, NULL, "fsl,sec-v4.0-job-ring") { - caam_jr_probe(pdev, np, ring); + ctrlpriv->jrpdev[ring] = + of_platform_device_create(np, NULL, dev); + if (!ctrlpriv->jrpdev[ring]) { + pr_warn("JR%d Platform device creation error\n", ring); + continue; + } ctrlpriv->total_jobrs++; ring++; } if (!ring) { for_each_compatible_node(np, NULL, "fsl,sec4.0-job-ring") { - caam_jr_probe(pdev, np, ring); + ctrlpriv->jrpdev[ring] = + of_platform_device_create(np, NULL, dev); + if (!ctrlpriv->jrpdev[ring]) { + pr_warn("JR%d Platform device creation error\n", + ring); + continue; + } ctrlpriv->total_jobrs++; ring++; } @@ -299,16 +504,55 @@ static int caam_probe(struct platform_device *pdev) /* * If SEC has RNG version >= 4 and RNG state handle has not been - * already instantiated ,do RNG instantiation + * already instantiated, do RNG instantiation */ - if ((cha_vid & CHA_ID_RNG_MASK) >> CHA_ID_RNG_SHIFT >= 4 && - !(rd_reg32(&topregs->ctrl.r4tst[0].rdsta) & RDSTA_IF0)) { - kick_trng(pdev); - ret = instantiate_rng(dev); + if ((cha_vid & CHA_ID_RNG_MASK) >> CHA_ID_RNG_SHIFT >= 4) { + ctrlpriv->rng4_sh_init = + rd_reg32(&topregs->ctrl.r4tst[0].rdsta); + /* + * If the secure keys (TDKEK, JDKEK, TDSK), were already + * generated, signal this to the function that is instantiating + * the state handles. An error would occur if RNG4 attempts + * to regenerate these keys before the next POR. + */ + gen_sk = ctrlpriv->rng4_sh_init & RDSTA_SKVN ? 0 : 1; + ctrlpriv->rng4_sh_init &= RDSTA_IFMASK; + do { + int inst_handles = + rd_reg32(&topregs->ctrl.r4tst[0].rdsta) & + RDSTA_IFMASK; + /* + * If either SH were instantiated by somebody else + * (e.g. u-boot) then it is assumed that the entropy + * parameters are properly set and thus the function + * setting these (kick_trng(...)) is skipped. + * Also, if a handle was instantiated, do not change + * the TRNG parameters. + */ + if (!(ctrlpriv->rng4_sh_init || inst_handles)) { + kick_trng(pdev, ent_delay); + ent_delay += 400; + } + /* + * if instantiate_rng(...) fails, the loop will rerun + * and the kick_trng(...) function will modfiy the + * upper and lower limits of the entropy sampling + * interval, leading to a sucessful initialization of + * the RNG. + */ + ret = instantiate_rng(dev, inst_handles, + gen_sk); + } while ((ret == -EAGAIN) && (ent_delay < RTSDCTL_ENT_DLY_MAX)); if (ret) { + dev_err(dev, "failed to instantiate RNG"); caam_remove(pdev); return ret; } + /* + * Set handles init'ed by this module as the complement of the + * already initialized ones + */ + ctrlpriv->rng4_sh_init = ~ctrlpriv->rng4_sh_init & RDSTA_IFMASK; /* Enable RDB bit so that RNG works faster */ setbits32(&topregs->ctrl.scfgr, SCFGR_RDBENABLE); diff --git a/drivers/crypto/caam/desc.h b/drivers/crypto/caam/desc.h index 53b296f78b0d..7e4500f18df6 100644 --- a/drivers/crypto/caam/desc.h +++ b/drivers/crypto/caam/desc.h @@ -1155,8 +1155,15 @@ struct sec4_sg_entry { /* randomizer AAI set */ #define OP_ALG_AAI_RNG (0x00 << OP_ALG_AAI_SHIFT) -#define OP_ALG_AAI_RNG_NOZERO (0x10 << OP_ALG_AAI_SHIFT) -#define OP_ALG_AAI_RNG_ODD (0x20 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_RNG_NZB (0x10 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_RNG_OBP (0x20 << OP_ALG_AAI_SHIFT) + +/* RNG4 AAI set */ +#define OP_ALG_AAI_RNG4_SH_0 (0x00 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_RNG4_SH_1 (0x01 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_RNG4_PS (0x40 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_RNG4_AI (0x80 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_RNG4_SK (0x100 << OP_ALG_AAI_SHIFT) /* hmac/smac AAI set */ #define OP_ALG_AAI_HASH (0x00 << OP_ALG_AAI_SHIFT) @@ -1178,12 +1185,6 @@ struct sec4_sg_entry { #define OP_ALG_AAI_GSM (0x10 << OP_ALG_AAI_SHIFT) #define OP_ALG_AAI_EDGE (0x20 << OP_ALG_AAI_SHIFT) -/* RNG4 set */ -#define OP_ALG_RNG4_SHIFT 4 -#define OP_ALG_RNG4_MASK (0x1f3 << OP_ALG_RNG4_SHIFT) - -#define OP_ALG_RNG4_SK (0x100 << OP_ALG_RNG4_SHIFT) - #define OP_ALG_AS_SHIFT 2 #define OP_ALG_AS_MASK (0x3 << OP_ALG_AS_SHIFT) #define OP_ALG_AS_UPDATE (0 << OP_ALG_AS_SHIFT) diff --git a/drivers/crypto/caam/intern.h b/drivers/crypto/caam/intern.h index 34c4b9f7fbfa..6d85fcc5bd0a 100644 --- a/drivers/crypto/caam/intern.h +++ b/drivers/crypto/caam/intern.h @@ -37,13 +37,16 @@ struct caam_jrentry_info { /* Private sub-storage for a single JobR */ struct caam_drv_private_jr { - struct device *parentdev; /* points back to controller dev */ - struct platform_device *jr_pdev;/* points to platform device for JR */ + struct list_head list_node; /* Job Ring device list */ + struct device *dev; int ridx; struct caam_job_ring __iomem *rregs; /* JobR's register space */ struct tasklet_struct irqtask; int irq; /* One per queue */ + /* Number of scatterlist crypt transforms active on the JobR */ + atomic_t tfm_count ____cacheline_aligned; + /* Job ring info */ int ringsize; /* Size of rings (assume input = output) */ struct caam_jrentry_info *entinfo; /* Alloc'ed 1 per ring entry */ @@ -63,7 +66,7 @@ struct caam_drv_private_jr { struct caam_drv_private { struct device *dev; - struct device **jrdev; /* Alloc'ed array per sub-device */ + struct platform_device **jrpdev; /* Alloc'ed array per sub-device */ struct platform_device *pdev; /* Physical-presence section */ @@ -80,12 +83,11 @@ struct caam_drv_private { u8 qi_present; /* Nonzero if QI present in device */ int secvio_irq; /* Security violation interrupt number */ - /* which jr allocated to scatterlist crypto */ - atomic_t tfm_count ____cacheline_aligned; - /* list of registered crypto algorithms (mk generic context handle?) */ - struct list_head alg_list; - /* list of registered hash algorithms (mk generic context handle?) */ - struct list_head hash_list; +#define RNG4_MAX_HANDLES 2 + /* RNG4 block */ + u32 rng4_sh_init; /* This bitmap shows which of the State + Handles of the RNG4 block are initialized + by this driver */ /* * debugfs entries for developer view into driver/device diff --git a/drivers/crypto/caam/jr.c b/drivers/crypto/caam/jr.c index bdb786d5a5e5..1d80bd3636c5 100644 --- a/drivers/crypto/caam/jr.c +++ b/drivers/crypto/caam/jr.c @@ -6,6 +6,7 @@ */ #include <linux/of_irq.h> +#include <linux/of_address.h> #include "compat.h" #include "regs.h" @@ -13,6 +14,113 @@ #include "desc.h" #include "intern.h" +struct jr_driver_data { + /* List of Physical JobR's with the Driver */ + struct list_head jr_list; + spinlock_t jr_alloc_lock; /* jr_list lock */ +} ____cacheline_aligned; + +static struct jr_driver_data driver_data; + +static int caam_reset_hw_jr(struct device *dev) +{ + struct caam_drv_private_jr *jrp = dev_get_drvdata(dev); + unsigned int timeout = 100000; + + /* + * mask interrupts since we are going to poll + * for reset completion status + */ + setbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK); + + /* initiate flush (required prior to reset) */ + wr_reg32(&jrp->rregs->jrcommand, JRCR_RESET); + while (((rd_reg32(&jrp->rregs->jrintstatus) & JRINT_ERR_HALT_MASK) == + JRINT_ERR_HALT_INPROGRESS) && --timeout) + cpu_relax(); + + if ((rd_reg32(&jrp->rregs->jrintstatus) & JRINT_ERR_HALT_MASK) != + JRINT_ERR_HALT_COMPLETE || timeout == 0) { + dev_err(dev, "failed to flush job ring %d\n", jrp->ridx); + return -EIO; + } + + /* initiate reset */ + timeout = 100000; + wr_reg32(&jrp->rregs->jrcommand, JRCR_RESET); + while ((rd_reg32(&jrp->rregs->jrcommand) & JRCR_RESET) && --timeout) + cpu_relax(); + + if (timeout == 0) { + dev_err(dev, "failed to reset job ring %d\n", jrp->ridx); + return -EIO; + } + + /* unmask interrupts */ + clrbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK); + + return 0; +} + +/* + * Shutdown JobR independent of platform property code + */ +int caam_jr_shutdown(struct device *dev) +{ + struct caam_drv_private_jr *jrp = dev_get_drvdata(dev); + dma_addr_t inpbusaddr, outbusaddr; + int ret; + + ret = caam_reset_hw_jr(dev); + + tasklet_kill(&jrp->irqtask); + + /* Release interrupt */ + free_irq(jrp->irq, dev); + + /* Free rings */ + inpbusaddr = rd_reg64(&jrp->rregs->inpring_base); + outbusaddr = rd_reg64(&jrp->rregs->outring_base); + dma_free_coherent(dev, sizeof(dma_addr_t) * JOBR_DEPTH, + jrp->inpring, inpbusaddr); + dma_free_coherent(dev, sizeof(struct jr_outentry) * JOBR_DEPTH, + jrp->outring, outbusaddr); + kfree(jrp->entinfo); + + return ret; +} + +static int caam_jr_remove(struct platform_device *pdev) +{ + int ret; + struct device *jrdev; + struct caam_drv_private_jr *jrpriv; + + jrdev = &pdev->dev; + jrpriv = dev_get_drvdata(jrdev); + + /* + * Return EBUSY if job ring already allocated. + */ + if (atomic_read(&jrpriv->tfm_count)) { + dev_err(jrdev, "Device is busy\n"); + return -EBUSY; + } + + /* Remove the node from Physical JobR list maintained by driver */ + spin_lock(&driver_data.jr_alloc_lock); + list_del(&jrpriv->list_node); + spin_unlock(&driver_data.jr_alloc_lock); + + /* Release ring */ + ret = caam_jr_shutdown(jrdev); + if (ret) + dev_err(jrdev, "Failed to shut down job ring\n"); + irq_dispose_mapping(jrpriv->irq); + + return ret; +} + /* Main per-ring interrupt handler */ static irqreturn_t caam_jr_interrupt(int irq, void *st_dev) { @@ -128,6 +236,59 @@ static void caam_jr_dequeue(unsigned long devarg) } /** + * caam_jr_alloc() - Alloc a job ring for someone to use as needed. + * + * returns : pointer to the newly allocated physical + * JobR dev can be written to if successful. + **/ +struct device *caam_jr_alloc(void) +{ + struct caam_drv_private_jr *jrpriv, *min_jrpriv = NULL; + struct device *dev = NULL; + int min_tfm_cnt = INT_MAX; + int tfm_cnt; + + spin_lock(&driver_data.jr_alloc_lock); + + if (list_empty(&driver_data.jr_list)) { + spin_unlock(&driver_data.jr_alloc_lock); + return ERR_PTR(-ENODEV); + } + + list_for_each_entry(jrpriv, &driver_data.jr_list, list_node) { + tfm_cnt = atomic_read(&jrpriv->tfm_count); + if (tfm_cnt < min_tfm_cnt) { + min_tfm_cnt = tfm_cnt; + min_jrpriv = jrpriv; + } + if (!min_tfm_cnt) + break; + } + + if (min_jrpriv) { + atomic_inc(&min_jrpriv->tfm_count); + dev = min_jrpriv->dev; + } + spin_unlock(&driver_data.jr_alloc_lock); + + return dev; +} +EXPORT_SYMBOL(caam_jr_alloc); + +/** + * caam_jr_free() - Free the Job Ring + * @rdev - points to the dev that identifies the Job ring to + * be released. + **/ +void caam_jr_free(struct device *rdev) +{ + struct caam_drv_private_jr *jrpriv = dev_get_drvdata(rdev); + + atomic_dec(&jrpriv->tfm_count); +} +EXPORT_SYMBOL(caam_jr_free); + +/** * caam_jr_enqueue() - Enqueue a job descriptor head. Returns 0 if OK, * -EBUSY if the queue is full, -EIO if it cannot map the caller's * descriptor. @@ -207,46 +368,6 @@ int caam_jr_enqueue(struct device *dev, u32 *desc, } EXPORT_SYMBOL(caam_jr_enqueue); -static int caam_reset_hw_jr(struct device *dev) -{ - struct caam_drv_private_jr *jrp = dev_get_drvdata(dev); - unsigned int timeout = 100000; - - /* - * mask interrupts since we are going to poll - * for reset completion status - */ - setbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK); - - /* initiate flush (required prior to reset) */ - wr_reg32(&jrp->rregs->jrcommand, JRCR_RESET); - while (((rd_reg32(&jrp->rregs->jrintstatus) & JRINT_ERR_HALT_MASK) == - JRINT_ERR_HALT_INPROGRESS) && --timeout) - cpu_relax(); - - if ((rd_reg32(&jrp->rregs->jrintstatus) & JRINT_ERR_HALT_MASK) != - JRINT_ERR_HALT_COMPLETE || timeout == 0) { - dev_err(dev, "failed to flush job ring %d\n", jrp->ridx); - return -EIO; - } - - /* initiate reset */ - timeout = 100000; - wr_reg32(&jrp->rregs->jrcommand, JRCR_RESET); - while ((rd_reg32(&jrp->rregs->jrcommand) & JRCR_RESET) && --timeout) - cpu_relax(); - - if (timeout == 0) { - dev_err(dev, "failed to reset job ring %d\n", jrp->ridx); - return -EIO; - } - - /* unmask interrupts */ - clrbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK); - - return 0; -} - /* * Init JobR independent of platform property detection */ @@ -262,7 +383,7 @@ static int caam_jr_init(struct device *dev) /* Connect job ring interrupt handler. */ error = request_irq(jrp->irq, caam_jr_interrupt, IRQF_SHARED, - "caam-jobr", dev); + dev_name(dev), dev); if (error) { dev_err(dev, "can't connect JobR %d interrupt (%d)\n", jrp->ridx, jrp->irq); @@ -318,86 +439,43 @@ static int caam_jr_init(struct device *dev) return 0; } -/* - * Shutdown JobR independent of platform property code - */ -int caam_jr_shutdown(struct device *dev) -{ - struct caam_drv_private_jr *jrp = dev_get_drvdata(dev); - dma_addr_t inpbusaddr, outbusaddr; - int ret; - - ret = caam_reset_hw_jr(dev); - - tasklet_kill(&jrp->irqtask); - - /* Release interrupt */ - free_irq(jrp->irq, dev); - - /* Free rings */ - inpbusaddr = rd_reg64(&jrp->rregs->inpring_base); - outbusaddr = rd_reg64(&jrp->rregs->outring_base); - dma_free_coherent(dev, sizeof(dma_addr_t) * JOBR_DEPTH, - jrp->inpring, inpbusaddr); - dma_free_coherent(dev, sizeof(struct jr_outentry) * JOBR_DEPTH, - jrp->outring, outbusaddr); - kfree(jrp->entinfo); - of_device_unregister(jrp->jr_pdev); - - return ret; -} /* - * Probe routine for each detected JobR subsystem. It assumes that - * property detection was picked up externally. + * Probe routine for each detected JobR subsystem. */ -int caam_jr_probe(struct platform_device *pdev, struct device_node *np, - int ring) +static int caam_jr_probe(struct platform_device *pdev) { - struct device *ctrldev, *jrdev; - struct platform_device *jr_pdev; - struct caam_drv_private *ctrlpriv; + struct device *jrdev; + struct device_node *nprop; + struct caam_job_ring __iomem *ctrl; struct caam_drv_private_jr *jrpriv; - u32 *jroffset; + static int total_jobrs; int error; - ctrldev = &pdev->dev; - ctrlpriv = dev_get_drvdata(ctrldev); - + jrdev = &pdev->dev; jrpriv = kmalloc(sizeof(struct caam_drv_private_jr), GFP_KERNEL); - if (jrpriv == NULL) { - dev_err(ctrldev, "can't alloc private mem for job ring %d\n", - ring); + if (!jrpriv) return -ENOMEM; - } - jrpriv->parentdev = ctrldev; /* point back to parent */ - jrpriv->ridx = ring; /* save ring identity relative to detection */ - /* - * Derive a pointer to the detected JobRs regs - * Driver has already iomapped the entire space, we just - * need to add in the offset to this JobR. Don't know if I - * like this long-term, but it'll run - */ - jroffset = (u32 *)of_get_property(np, "reg", NULL); - jrpriv->rregs = (struct caam_job_ring __iomem *)((void *)ctrlpriv->ctrl - + *jroffset); + dev_set_drvdata(jrdev, jrpriv); - /* Build a local dev for each detected queue */ - jr_pdev = of_platform_device_create(np, NULL, ctrldev); - if (jr_pdev == NULL) { - kfree(jrpriv); - return -EINVAL; + /* save ring identity relative to detection */ + jrpriv->ridx = total_jobrs++; + + nprop = pdev->dev.of_node; + /* Get configuration properties from device tree */ + /* First, get register page */ + ctrl = of_iomap(nprop, 0); + if (!ctrl) { + dev_err(jrdev, "of_iomap() failed\n"); + return -ENOMEM; } - jrpriv->jr_pdev = jr_pdev; - jrdev = &jr_pdev->dev; - dev_set_drvdata(jrdev, jrpriv); - ctrlpriv->jrdev[ring] = jrdev; + jrpriv->rregs = (struct caam_job_ring __force *)ctrl; if (sizeof(dma_addr_t) == sizeof(u64)) - if (of_device_is_compatible(np, "fsl,sec-v5.0-job-ring")) + if (of_device_is_compatible(nprop, "fsl,sec-v5.0-job-ring")) dma_set_mask(jrdev, DMA_BIT_MASK(40)); else dma_set_mask(jrdev, DMA_BIT_MASK(36)); @@ -405,15 +483,61 @@ int caam_jr_probe(struct platform_device *pdev, struct device_node *np, dma_set_mask(jrdev, DMA_BIT_MASK(32)); /* Identify the interrupt */ - jrpriv->irq = irq_of_parse_and_map(np, 0); + jrpriv->irq = irq_of_parse_and_map(nprop, 0); /* Now do the platform independent part */ error = caam_jr_init(jrdev); /* now turn on hardware */ if (error) { - of_device_unregister(jr_pdev); kfree(jrpriv); return error; } - return error; + jrpriv->dev = jrdev; + spin_lock(&driver_data.jr_alloc_lock); + list_add_tail(&jrpriv->list_node, &driver_data.jr_list); + spin_unlock(&driver_data.jr_alloc_lock); + + atomic_set(&jrpriv->tfm_count, 0); + + return 0; +} + +static struct of_device_id caam_jr_match[] = { + { + .compatible = "fsl,sec-v4.0-job-ring", + }, + { + .compatible = "fsl,sec4.0-job-ring", + }, + {}, +}; +MODULE_DEVICE_TABLE(of, caam_jr_match); + +static struct platform_driver caam_jr_driver = { + .driver = { + .name = "caam_jr", + .owner = THIS_MODULE, + .of_match_table = caam_jr_match, + }, + .probe = caam_jr_probe, + .remove = caam_jr_remove, +}; + +static int __init jr_driver_init(void) +{ + spin_lock_init(&driver_data.jr_alloc_lock); + INIT_LIST_HEAD(&driver_data.jr_list); + return platform_driver_register(&caam_jr_driver); +} + +static void __exit jr_driver_exit(void) +{ + platform_driver_unregister(&caam_jr_driver); } + +module_init(jr_driver_init); +module_exit(jr_driver_exit); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("FSL CAAM JR request backend"); +MODULE_AUTHOR("Freescale Semiconductor - NMG/STC"); diff --git a/drivers/crypto/caam/jr.h b/drivers/crypto/caam/jr.h index 9d8741a59037..97113a6d6c58 100644 --- a/drivers/crypto/caam/jr.h +++ b/drivers/crypto/caam/jr.h @@ -8,12 +8,11 @@ #define JR_H /* Prototypes for backend-level services exposed to APIs */ +struct device *caam_jr_alloc(void); +void caam_jr_free(struct device *rdev); int caam_jr_enqueue(struct device *dev, u32 *desc, void (*cbk)(struct device *dev, u32 *desc, u32 status, void *areq), void *areq); -extern int caam_jr_probe(struct platform_device *pdev, struct device_node *np, - int ring); -extern int caam_jr_shutdown(struct device *dev); #endif /* JR_H */ diff --git a/drivers/crypto/caam/regs.h b/drivers/crypto/caam/regs.h index 4455396918de..d50174f45b21 100644 --- a/drivers/crypto/caam/regs.h +++ b/drivers/crypto/caam/regs.h @@ -245,7 +245,7 @@ struct rngtst { /* RNG4 TRNG test registers */ struct rng4tst { -#define RTMCTL_PRGM 0x00010000 /* 1 -> program mode, 0 -> run mode */ +#define RTMCTL_PRGM 0x00010000 /* 1 -> program mode, 0 -> run mode */ u32 rtmctl; /* misc. control register */ u32 rtscmisc; /* statistical check misc. register */ u32 rtpkrrng; /* poker range register */ @@ -255,6 +255,8 @@ struct rng4tst { }; #define RTSDCTL_ENT_DLY_SHIFT 16 #define RTSDCTL_ENT_DLY_MASK (0xffff << RTSDCTL_ENT_DLY_SHIFT) +#define RTSDCTL_ENT_DLY_MIN 1200 +#define RTSDCTL_ENT_DLY_MAX 12800 u32 rtsdctl; /* seed control register */ union { u32 rtsblim; /* PRGM=1: sparse bit limit register */ @@ -266,7 +268,11 @@ struct rng4tst { u32 rtfrqcnt; /* PRGM=0: freq. count register */ }; u32 rsvd1[40]; +#define RDSTA_SKVT 0x80000000 +#define RDSTA_SKVN 0x40000000 #define RDSTA_IF0 0x00000001 +#define RDSTA_IF1 0x00000002 +#define RDSTA_IFMASK (RDSTA_IF1 | RDSTA_IF0) u32 rdsta; u32 rsvd2[15]; }; @@ -692,6 +698,7 @@ struct caam_deco { u32 jr_ctl_hi; /* CxJRR - JobR Control Register @800 */ u32 jr_ctl_lo; u64 jr_descaddr; /* CxDADR - JobR Descriptor Address */ +#define DECO_OP_STATUS_HI_ERR_MASK 0xF00000FF u32 op_status_hi; /* DxOPSTA - DECO Operation Status */ u32 op_status_lo; u32 rsvd24[2]; @@ -706,12 +713,13 @@ struct caam_deco { u32 rsvd29[48]; u32 descbuf[64]; /* DxDESB - Descriptor buffer */ u32 rscvd30[193]; +#define DESC_DBG_DECO_STAT_HOST_ERR 0x00D00000 +#define DESC_DBG_DECO_STAT_VALID 0x80000000 +#define DESC_DBG_DECO_STAT_MASK 0x00F00000 u32 desc_dbg; /* DxDDR - DECO Debug Register */ u32 rsvd31[126]; }; -/* DECO DBG Register Valid Bit*/ -#define DECO_DBG_VALID 0x80000000 #define DECO_JQCR_WHL 0x20000000 #define DECO_JQCR_FOUR 0x10000000 diff --git a/drivers/crypto/caam/sg_sw_sec4.h b/drivers/crypto/caam/sg_sw_sec4.h index e0037c8ee243..b12ff85f4241 100644 --- a/drivers/crypto/caam/sg_sw_sec4.h +++ b/drivers/crypto/caam/sg_sw_sec4.h @@ -117,6 +117,21 @@ static int dma_unmap_sg_chained(struct device *dev, struct scatterlist *sg, return nents; } +/* Map SG page in kernel virtual address space and copy */ +static inline void sg_map_copy(u8 *dest, struct scatterlist *sg, + int len, int offset) +{ + u8 *mapped_addr; + + /* + * Page here can be user-space pinned using get_user_pages + * Same must be kmapped before use and kunmapped subsequently + */ + mapped_addr = kmap_atomic(sg_page(sg)); + memcpy(dest, mapped_addr + offset, len); + kunmap_atomic(mapped_addr); +} + /* Copy from len bytes of sg to dest, starting from beginning */ static inline void sg_copy(u8 *dest, struct scatterlist *sg, unsigned int len) { @@ -124,15 +139,15 @@ static inline void sg_copy(u8 *dest, struct scatterlist *sg, unsigned int len) int cpy_index = 0, next_cpy_index = current_sg->length; while (next_cpy_index < len) { - memcpy(dest + cpy_index, (u8 *) sg_virt(current_sg), - current_sg->length); + sg_map_copy(dest + cpy_index, current_sg, current_sg->length, + current_sg->offset); current_sg = scatterwalk_sg_next(current_sg); cpy_index = next_cpy_index; next_cpy_index += current_sg->length; } if (cpy_index < len) - memcpy(dest + cpy_index, (u8 *) sg_virt(current_sg), - len - cpy_index); + sg_map_copy(dest + cpy_index, current_sg, len-cpy_index, + current_sg->offset); } /* Copy sg data, from to_skip to end, to dest */ @@ -140,7 +155,7 @@ static inline void sg_copy_part(u8 *dest, struct scatterlist *sg, int to_skip, unsigned int end) { struct scatterlist *current_sg = sg; - int sg_index, cpy_index; + int sg_index, cpy_index, offset; sg_index = current_sg->length; while (sg_index <= to_skip) { @@ -148,9 +163,10 @@ static inline void sg_copy_part(u8 *dest, struct scatterlist *sg, sg_index += current_sg->length; } cpy_index = sg_index - to_skip; - memcpy(dest, (u8 *) sg_virt(current_sg) + - current_sg->length - cpy_index, cpy_index); - current_sg = scatterwalk_sg_next(current_sg); - if (end - sg_index) + offset = current_sg->offset + current_sg->length - cpy_index; + sg_map_copy(dest, current_sg, cpy_index, offset); + if (end - sg_index) { + current_sg = scatterwalk_sg_next(current_sg); sg_copy(dest + cpy_index, current_sg, end - sg_index); + } } diff --git a/drivers/crypto/ccp/Kconfig b/drivers/crypto/ccp/Kconfig new file mode 100644 index 000000000000..7639ffc36c68 --- /dev/null +++ b/drivers/crypto/ccp/Kconfig @@ -0,0 +1,24 @@ +config CRYPTO_DEV_CCP_DD + tristate "Cryptographic Coprocessor device driver" + depends on CRYPTO_DEV_CCP + default m + select HW_RANDOM + help + Provides the interface to use the AMD Cryptographic Coprocessor + which can be used to accelerate or offload encryption operations + such as SHA, AES and more. If you choose 'M' here, this module + will be called ccp. + +config CRYPTO_DEV_CCP_CRYPTO + tristate "Encryption and hashing acceleration support" + depends on CRYPTO_DEV_CCP_DD + default m + select CRYPTO_ALGAPI + select CRYPTO_HASH + select CRYPTO_BLKCIPHER + select CRYPTO_AUTHENC + help + Support for using the cryptographic API with the AMD Cryptographic + Coprocessor. This module supports acceleration and offload of SHA + and AES algorithms. If you choose 'M' here, this module will be + called ccp_crypto. diff --git a/drivers/crypto/ccp/Makefile b/drivers/crypto/ccp/Makefile new file mode 100644 index 000000000000..d3505a018720 --- /dev/null +++ b/drivers/crypto/ccp/Makefile @@ -0,0 +1,10 @@ +obj-$(CONFIG_CRYPTO_DEV_CCP_DD) += ccp.o +ccp-objs := ccp-dev.o ccp-ops.o +ccp-objs += ccp-pci.o + +obj-$(CONFIG_CRYPTO_DEV_CCP_CRYPTO) += ccp-crypto.o +ccp-crypto-objs := ccp-crypto-main.o \ + ccp-crypto-aes.o \ + ccp-crypto-aes-cmac.o \ + ccp-crypto-aes-xts.o \ + ccp-crypto-sha.o diff --git a/drivers/crypto/ccp/ccp-crypto-aes-cmac.c b/drivers/crypto/ccp/ccp-crypto-aes-cmac.c new file mode 100644 index 000000000000..8e162ad82085 --- /dev/null +++ b/drivers/crypto/ccp/ccp-crypto-aes-cmac.c @@ -0,0 +1,365 @@ +/* + * AMD Cryptographic Coprocessor (CCP) AES CMAC crypto API support + * + * Copyright (C) 2013 Advanced Micro Devices, Inc. + * + * Author: Tom Lendacky <thomas.lendacky@amd.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/module.h> +#include <linux/sched.h> +#include <linux/delay.h> +#include <linux/scatterlist.h> +#include <linux/crypto.h> +#include <crypto/algapi.h> +#include <crypto/aes.h> +#include <crypto/hash.h> +#include <crypto/internal/hash.h> +#include <crypto/scatterwalk.h> + +#include "ccp-crypto.h" + + +static int ccp_aes_cmac_complete(struct crypto_async_request *async_req, + int ret) +{ + struct ahash_request *req = ahash_request_cast(async_req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct ccp_aes_cmac_req_ctx *rctx = ahash_request_ctx(req); + unsigned int digest_size = crypto_ahash_digestsize(tfm); + + if (ret) + goto e_free; + + if (rctx->hash_rem) { + /* Save remaining data to buffer */ + unsigned int offset = rctx->nbytes - rctx->hash_rem; + scatterwalk_map_and_copy(rctx->buf, rctx->src, + offset, rctx->hash_rem, 0); + rctx->buf_count = rctx->hash_rem; + } else + rctx->buf_count = 0; + + /* Update result area if supplied */ + if (req->result) + memcpy(req->result, rctx->iv, digest_size); + +e_free: + sg_free_table(&rctx->data_sg); + + return ret; +} + +static int ccp_do_cmac_update(struct ahash_request *req, unsigned int nbytes, + unsigned int final) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct ccp_ctx *ctx = crypto_ahash_ctx(tfm); + struct ccp_aes_cmac_req_ctx *rctx = ahash_request_ctx(req); + struct scatterlist *sg, *cmac_key_sg = NULL; + unsigned int block_size = + crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); + unsigned int need_pad, sg_count; + gfp_t gfp; + u64 len; + int ret; + + if (!ctx->u.aes.key_len) + return -EINVAL; + + if (nbytes) + rctx->null_msg = 0; + + len = (u64)rctx->buf_count + (u64)nbytes; + + if (!final && (len <= block_size)) { + scatterwalk_map_and_copy(rctx->buf + rctx->buf_count, req->src, + 0, nbytes, 0); + rctx->buf_count += nbytes; + + return 0; + } + + rctx->src = req->src; + rctx->nbytes = nbytes; + + rctx->final = final; + rctx->hash_rem = final ? 0 : len & (block_size - 1); + rctx->hash_cnt = len - rctx->hash_rem; + if (!final && !rctx->hash_rem) { + /* CCP can't do zero length final, so keep some data around */ + rctx->hash_cnt -= block_size; + rctx->hash_rem = block_size; + } + + if (final && (rctx->null_msg || (len & (block_size - 1)))) + need_pad = 1; + else + need_pad = 0; + + sg_init_one(&rctx->iv_sg, rctx->iv, sizeof(rctx->iv)); + + /* Build the data scatterlist table - allocate enough entries for all + * possible data pieces (buffer, input data, padding) + */ + sg_count = (nbytes) ? sg_nents(req->src) + 2 : 2; + gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + ret = sg_alloc_table(&rctx->data_sg, sg_count, gfp); + if (ret) + return ret; + + sg = NULL; + if (rctx->buf_count) { + sg_init_one(&rctx->buf_sg, rctx->buf, rctx->buf_count); + sg = ccp_crypto_sg_table_add(&rctx->data_sg, &rctx->buf_sg); + } + + if (nbytes) + sg = ccp_crypto_sg_table_add(&rctx->data_sg, req->src); + + if (need_pad) { + int pad_length = block_size - (len & (block_size - 1)); + + rctx->hash_cnt += pad_length; + + memset(rctx->pad, 0, sizeof(rctx->pad)); + rctx->pad[0] = 0x80; + sg_init_one(&rctx->pad_sg, rctx->pad, pad_length); + sg = ccp_crypto_sg_table_add(&rctx->data_sg, &rctx->pad_sg); + } + if (sg) { + sg_mark_end(sg); + sg = rctx->data_sg.sgl; + } + + /* Initialize the K1/K2 scatterlist */ + if (final) + cmac_key_sg = (need_pad) ? &ctx->u.aes.k2_sg + : &ctx->u.aes.k1_sg; + + memset(&rctx->cmd, 0, sizeof(rctx->cmd)); + INIT_LIST_HEAD(&rctx->cmd.entry); + rctx->cmd.engine = CCP_ENGINE_AES; + rctx->cmd.u.aes.type = ctx->u.aes.type; + rctx->cmd.u.aes.mode = ctx->u.aes.mode; + rctx->cmd.u.aes.action = CCP_AES_ACTION_ENCRYPT; + rctx->cmd.u.aes.key = &ctx->u.aes.key_sg; + rctx->cmd.u.aes.key_len = ctx->u.aes.key_len; + rctx->cmd.u.aes.iv = &rctx->iv_sg; + rctx->cmd.u.aes.iv_len = AES_BLOCK_SIZE; + rctx->cmd.u.aes.src = sg; + rctx->cmd.u.aes.src_len = rctx->hash_cnt; + rctx->cmd.u.aes.dst = NULL; + rctx->cmd.u.aes.cmac_key = cmac_key_sg; + rctx->cmd.u.aes.cmac_key_len = ctx->u.aes.kn_len; + rctx->cmd.u.aes.cmac_final = final; + + ret = ccp_crypto_enqueue_request(&req->base, &rctx->cmd); + + return ret; +} + +static int ccp_aes_cmac_init(struct ahash_request *req) +{ + struct ccp_aes_cmac_req_ctx *rctx = ahash_request_ctx(req); + + memset(rctx, 0, sizeof(*rctx)); + + rctx->null_msg = 1; + + return 0; +} + +static int ccp_aes_cmac_update(struct ahash_request *req) +{ + return ccp_do_cmac_update(req, req->nbytes, 0); +} + +static int ccp_aes_cmac_final(struct ahash_request *req) +{ + return ccp_do_cmac_update(req, 0, 1); +} + +static int ccp_aes_cmac_finup(struct ahash_request *req) +{ + return ccp_do_cmac_update(req, req->nbytes, 1); +} + +static int ccp_aes_cmac_digest(struct ahash_request *req) +{ + int ret; + + ret = ccp_aes_cmac_init(req); + if (ret) + return ret; + + return ccp_aes_cmac_finup(req); +} + +static int ccp_aes_cmac_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int key_len) +{ + struct ccp_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); + struct ccp_crypto_ahash_alg *alg = + ccp_crypto_ahash_alg(crypto_ahash_tfm(tfm)); + u64 k0_hi, k0_lo, k1_hi, k1_lo, k2_hi, k2_lo; + u64 rb_hi = 0x00, rb_lo = 0x87; + __be64 *gk; + int ret; + + switch (key_len) { + case AES_KEYSIZE_128: + ctx->u.aes.type = CCP_AES_TYPE_128; + break; + case AES_KEYSIZE_192: + ctx->u.aes.type = CCP_AES_TYPE_192; + break; + case AES_KEYSIZE_256: + ctx->u.aes.type = CCP_AES_TYPE_256; + break; + default: + crypto_ahash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + ctx->u.aes.mode = alg->mode; + + /* Set to zero until complete */ + ctx->u.aes.key_len = 0; + + /* Set the key for the AES cipher used to generate the keys */ + ret = crypto_cipher_setkey(ctx->u.aes.tfm_cipher, key, key_len); + if (ret) + return ret; + + /* Encrypt a block of zeroes - use key area in context */ + memset(ctx->u.aes.key, 0, sizeof(ctx->u.aes.key)); + crypto_cipher_encrypt_one(ctx->u.aes.tfm_cipher, ctx->u.aes.key, + ctx->u.aes.key); + + /* Generate K1 and K2 */ + k0_hi = be64_to_cpu(*((__be64 *)ctx->u.aes.key)); + k0_lo = be64_to_cpu(*((__be64 *)ctx->u.aes.key + 1)); + + k1_hi = (k0_hi << 1) | (k0_lo >> 63); + k1_lo = k0_lo << 1; + if (ctx->u.aes.key[0] & 0x80) { + k1_hi ^= rb_hi; + k1_lo ^= rb_lo; + } + gk = (__be64 *)ctx->u.aes.k1; + *gk = cpu_to_be64(k1_hi); + gk++; + *gk = cpu_to_be64(k1_lo); + + k2_hi = (k1_hi << 1) | (k1_lo >> 63); + k2_lo = k1_lo << 1; + if (ctx->u.aes.k1[0] & 0x80) { + k2_hi ^= rb_hi; + k2_lo ^= rb_lo; + } + gk = (__be64 *)ctx->u.aes.k2; + *gk = cpu_to_be64(k2_hi); + gk++; + *gk = cpu_to_be64(k2_lo); + + ctx->u.aes.kn_len = sizeof(ctx->u.aes.k1); + sg_init_one(&ctx->u.aes.k1_sg, ctx->u.aes.k1, sizeof(ctx->u.aes.k1)); + sg_init_one(&ctx->u.aes.k2_sg, ctx->u.aes.k2, sizeof(ctx->u.aes.k2)); + + /* Save the supplied key */ + memset(ctx->u.aes.key, 0, sizeof(ctx->u.aes.key)); + memcpy(ctx->u.aes.key, key, key_len); + ctx->u.aes.key_len = key_len; + sg_init_one(&ctx->u.aes.key_sg, ctx->u.aes.key, key_len); + + return ret; +} + +static int ccp_aes_cmac_cra_init(struct crypto_tfm *tfm) +{ + struct ccp_ctx *ctx = crypto_tfm_ctx(tfm); + struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); + struct crypto_cipher *cipher_tfm; + + ctx->complete = ccp_aes_cmac_complete; + ctx->u.aes.key_len = 0; + + crypto_ahash_set_reqsize(ahash, sizeof(struct ccp_aes_cmac_req_ctx)); + + cipher_tfm = crypto_alloc_cipher("aes", 0, + CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(cipher_tfm)) { + pr_warn("could not load aes cipher driver\n"); + return PTR_ERR(cipher_tfm); + } + ctx->u.aes.tfm_cipher = cipher_tfm; + + return 0; +} + +static void ccp_aes_cmac_cra_exit(struct crypto_tfm *tfm) +{ + struct ccp_ctx *ctx = crypto_tfm_ctx(tfm); + + if (ctx->u.aes.tfm_cipher) + crypto_free_cipher(ctx->u.aes.tfm_cipher); + ctx->u.aes.tfm_cipher = NULL; +} + +int ccp_register_aes_cmac_algs(struct list_head *head) +{ + struct ccp_crypto_ahash_alg *ccp_alg; + struct ahash_alg *alg; + struct hash_alg_common *halg; + struct crypto_alg *base; + int ret; + + ccp_alg = kzalloc(sizeof(*ccp_alg), GFP_KERNEL); + if (!ccp_alg) + return -ENOMEM; + + INIT_LIST_HEAD(&ccp_alg->entry); + ccp_alg->mode = CCP_AES_MODE_CMAC; + + alg = &ccp_alg->alg; + alg->init = ccp_aes_cmac_init; + alg->update = ccp_aes_cmac_update; + alg->final = ccp_aes_cmac_final; + alg->finup = ccp_aes_cmac_finup; + alg->digest = ccp_aes_cmac_digest; + alg->setkey = ccp_aes_cmac_setkey; + + halg = &alg->halg; + halg->digestsize = AES_BLOCK_SIZE; + + base = &halg->base; + snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "cmac(aes)"); + snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "cmac-aes-ccp"); + base->cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK; + base->cra_blocksize = AES_BLOCK_SIZE; + base->cra_ctxsize = sizeof(struct ccp_ctx); + base->cra_priority = CCP_CRA_PRIORITY; + base->cra_type = &crypto_ahash_type; + base->cra_init = ccp_aes_cmac_cra_init; + base->cra_exit = ccp_aes_cmac_cra_exit; + base->cra_module = THIS_MODULE; + + ret = crypto_register_ahash(alg); + if (ret) { + pr_err("%s ahash algorithm registration error (%d)\n", + base->cra_name, ret); + kfree(ccp_alg); + return ret; + } + + list_add(&ccp_alg->entry, head); + + return 0; +} diff --git a/drivers/crypto/ccp/ccp-crypto-aes-xts.c b/drivers/crypto/ccp/ccp-crypto-aes-xts.c new file mode 100644 index 000000000000..0237ab58f242 --- /dev/null +++ b/drivers/crypto/ccp/ccp-crypto-aes-xts.c @@ -0,0 +1,279 @@ +/* + * AMD Cryptographic Coprocessor (CCP) AES XTS crypto API support + * + * Copyright (C) 2013 Advanced Micro Devices, Inc. + * + * Author: Tom Lendacky <thomas.lendacky@amd.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/module.h> +#include <linux/sched.h> +#include <linux/delay.h> +#include <linux/scatterlist.h> +#include <linux/crypto.h> +#include <crypto/algapi.h> +#include <crypto/aes.h> +#include <crypto/scatterwalk.h> + +#include "ccp-crypto.h" + + +struct ccp_aes_xts_def { + const char *name; + const char *drv_name; +}; + +static struct ccp_aes_xts_def aes_xts_algs[] = { + { + .name = "xts(aes)", + .drv_name = "xts-aes-ccp", + }, +}; + +struct ccp_unit_size_map { + unsigned int size; + u32 value; +}; + +static struct ccp_unit_size_map unit_size_map[] = { + { + .size = 4096, + .value = CCP_XTS_AES_UNIT_SIZE_4096, + }, + { + .size = 2048, + .value = CCP_XTS_AES_UNIT_SIZE_2048, + }, + { + .size = 1024, + .value = CCP_XTS_AES_UNIT_SIZE_1024, + }, + { + .size = 512, + .value = CCP_XTS_AES_UNIT_SIZE_512, + }, + { + .size = 256, + .value = CCP_XTS_AES_UNIT_SIZE__LAST, + }, + { + .size = 128, + .value = CCP_XTS_AES_UNIT_SIZE__LAST, + }, + { + .size = 64, + .value = CCP_XTS_AES_UNIT_SIZE__LAST, + }, + { + .size = 32, + .value = CCP_XTS_AES_UNIT_SIZE__LAST, + }, + { + .size = 16, + .value = CCP_XTS_AES_UNIT_SIZE_16, + }, + { + .size = 1, + .value = CCP_XTS_AES_UNIT_SIZE__LAST, + }, +}; + +static int ccp_aes_xts_complete(struct crypto_async_request *async_req, int ret) +{ + struct ablkcipher_request *req = ablkcipher_request_cast(async_req); + struct ccp_aes_req_ctx *rctx = ablkcipher_request_ctx(req); + + if (ret) + return ret; + + memcpy(req->info, rctx->iv, AES_BLOCK_SIZE); + + return 0; +} + +static int ccp_aes_xts_setkey(struct crypto_ablkcipher *tfm, const u8 *key, + unsigned int key_len) +{ + struct ccp_ctx *ctx = crypto_tfm_ctx(crypto_ablkcipher_tfm(tfm)); + + /* Only support 128-bit AES key with a 128-bit Tweak key, + * otherwise use the fallback + */ + switch (key_len) { + case AES_KEYSIZE_128 * 2: + memcpy(ctx->u.aes.key, key, key_len); + break; + } + ctx->u.aes.key_len = key_len / 2; + sg_init_one(&ctx->u.aes.key_sg, ctx->u.aes.key, key_len); + + return crypto_ablkcipher_setkey(ctx->u.aes.tfm_ablkcipher, key, + key_len); +} + +static int ccp_aes_xts_crypt(struct ablkcipher_request *req, + unsigned int encrypt) +{ + struct crypto_tfm *tfm = + crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req)); + struct ccp_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct ccp_aes_req_ctx *rctx = ablkcipher_request_ctx(req); + unsigned int unit; + int ret; + + if (!ctx->u.aes.key_len) + return -EINVAL; + + if (req->nbytes & (AES_BLOCK_SIZE - 1)) + return -EINVAL; + + if (!req->info) + return -EINVAL; + + for (unit = 0; unit < ARRAY_SIZE(unit_size_map); unit++) + if (!(req->nbytes & (unit_size_map[unit].size - 1))) + break; + + if ((unit_size_map[unit].value == CCP_XTS_AES_UNIT_SIZE__LAST) || + (ctx->u.aes.key_len != AES_KEYSIZE_128)) { + /* Use the fallback to process the request for any + * unsupported unit sizes or key sizes + */ + ablkcipher_request_set_tfm(req, ctx->u.aes.tfm_ablkcipher); + ret = (encrypt) ? crypto_ablkcipher_encrypt(req) : + crypto_ablkcipher_decrypt(req); + ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm)); + + return ret; + } + + memcpy(rctx->iv, req->info, AES_BLOCK_SIZE); + sg_init_one(&rctx->iv_sg, rctx->iv, AES_BLOCK_SIZE); + + memset(&rctx->cmd, 0, sizeof(rctx->cmd)); + INIT_LIST_HEAD(&rctx->cmd.entry); + rctx->cmd.engine = CCP_ENGINE_XTS_AES_128; + rctx->cmd.u.xts.action = (encrypt) ? CCP_AES_ACTION_ENCRYPT + : CCP_AES_ACTION_DECRYPT; + rctx->cmd.u.xts.unit_size = unit_size_map[unit].value; + rctx->cmd.u.xts.key = &ctx->u.aes.key_sg; + rctx->cmd.u.xts.key_len = ctx->u.aes.key_len; + rctx->cmd.u.xts.iv = &rctx->iv_sg; + rctx->cmd.u.xts.iv_len = AES_BLOCK_SIZE; + rctx->cmd.u.xts.src = req->src; + rctx->cmd.u.xts.src_len = req->nbytes; + rctx->cmd.u.xts.dst = req->dst; + + ret = ccp_crypto_enqueue_request(&req->base, &rctx->cmd); + + return ret; +} + +static int ccp_aes_xts_encrypt(struct ablkcipher_request *req) +{ + return ccp_aes_xts_crypt(req, 1); +} + +static int ccp_aes_xts_decrypt(struct ablkcipher_request *req) +{ + return ccp_aes_xts_crypt(req, 0); +} + +static int ccp_aes_xts_cra_init(struct crypto_tfm *tfm) +{ + struct ccp_ctx *ctx = crypto_tfm_ctx(tfm); + struct crypto_ablkcipher *fallback_tfm; + + ctx->complete = ccp_aes_xts_complete; + ctx->u.aes.key_len = 0; + + fallback_tfm = crypto_alloc_ablkcipher(tfm->__crt_alg->cra_name, 0, + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(fallback_tfm)) { + pr_warn("could not load fallback driver %s\n", + tfm->__crt_alg->cra_name); + return PTR_ERR(fallback_tfm); + } + ctx->u.aes.tfm_ablkcipher = fallback_tfm; + + tfm->crt_ablkcipher.reqsize = sizeof(struct ccp_aes_req_ctx) + + fallback_tfm->base.crt_ablkcipher.reqsize; + + return 0; +} + +static void ccp_aes_xts_cra_exit(struct crypto_tfm *tfm) +{ + struct ccp_ctx *ctx = crypto_tfm_ctx(tfm); + + if (ctx->u.aes.tfm_ablkcipher) + crypto_free_ablkcipher(ctx->u.aes.tfm_ablkcipher); + ctx->u.aes.tfm_ablkcipher = NULL; +} + + +static int ccp_register_aes_xts_alg(struct list_head *head, + const struct ccp_aes_xts_def *def) +{ + struct ccp_crypto_ablkcipher_alg *ccp_alg; + struct crypto_alg *alg; + int ret; + + ccp_alg = kzalloc(sizeof(*ccp_alg), GFP_KERNEL); + if (!ccp_alg) + return -ENOMEM; + + INIT_LIST_HEAD(&ccp_alg->entry); + + alg = &ccp_alg->alg; + + snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", def->name); + snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", + def->drv_name); + alg->cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK; + alg->cra_blocksize = AES_BLOCK_SIZE; + alg->cra_ctxsize = sizeof(struct ccp_ctx); + alg->cra_priority = CCP_CRA_PRIORITY; + alg->cra_type = &crypto_ablkcipher_type; + alg->cra_ablkcipher.setkey = ccp_aes_xts_setkey; + alg->cra_ablkcipher.encrypt = ccp_aes_xts_encrypt; + alg->cra_ablkcipher.decrypt = ccp_aes_xts_decrypt; + alg->cra_ablkcipher.min_keysize = AES_MIN_KEY_SIZE * 2; + alg->cra_ablkcipher.max_keysize = AES_MAX_KEY_SIZE * 2; + alg->cra_ablkcipher.ivsize = AES_BLOCK_SIZE; + alg->cra_init = ccp_aes_xts_cra_init; + alg->cra_exit = ccp_aes_xts_cra_exit; + alg->cra_module = THIS_MODULE; + + ret = crypto_register_alg(alg); + if (ret) { + pr_err("%s ablkcipher algorithm registration error (%d)\n", + alg->cra_name, ret); + kfree(ccp_alg); + return ret; + } + + list_add(&ccp_alg->entry, head); + + return 0; +} + +int ccp_register_aes_xts_algs(struct list_head *head) +{ + int i, ret; + + for (i = 0; i < ARRAY_SIZE(aes_xts_algs); i++) { + ret = ccp_register_aes_xts_alg(head, &aes_xts_algs[i]); + if (ret) + return ret; + } + + return 0; +} diff --git a/drivers/crypto/ccp/ccp-crypto-aes.c b/drivers/crypto/ccp/ccp-crypto-aes.c new file mode 100644 index 000000000000..e46490db0f63 --- /dev/null +++ b/drivers/crypto/ccp/ccp-crypto-aes.c @@ -0,0 +1,369 @@ +/* + * AMD Cryptographic Coprocessor (CCP) AES crypto API support + * + * Copyright (C) 2013 Advanced Micro Devices, Inc. + * + * Author: Tom Lendacky <thomas.lendacky@amd.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/module.h> +#include <linux/sched.h> +#include <linux/delay.h> +#include <linux/scatterlist.h> +#include <linux/crypto.h> +#include <crypto/algapi.h> +#include <crypto/aes.h> +#include <crypto/ctr.h> +#include <crypto/scatterwalk.h> + +#include "ccp-crypto.h" + + +static int ccp_aes_complete(struct crypto_async_request *async_req, int ret) +{ + struct ablkcipher_request *req = ablkcipher_request_cast(async_req); + struct ccp_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct ccp_aes_req_ctx *rctx = ablkcipher_request_ctx(req); + + if (ret) + return ret; + + if (ctx->u.aes.mode != CCP_AES_MODE_ECB) + memcpy(req->info, rctx->iv, AES_BLOCK_SIZE); + + return 0; +} + +static int ccp_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key, + unsigned int key_len) +{ + struct ccp_ctx *ctx = crypto_tfm_ctx(crypto_ablkcipher_tfm(tfm)); + struct ccp_crypto_ablkcipher_alg *alg = + ccp_crypto_ablkcipher_alg(crypto_ablkcipher_tfm(tfm)); + + switch (key_len) { + case AES_KEYSIZE_128: + ctx->u.aes.type = CCP_AES_TYPE_128; + break; + case AES_KEYSIZE_192: + ctx->u.aes.type = CCP_AES_TYPE_192; + break; + case AES_KEYSIZE_256: + ctx->u.aes.type = CCP_AES_TYPE_256; + break; + default: + crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + ctx->u.aes.mode = alg->mode; + ctx->u.aes.key_len = key_len; + + memcpy(ctx->u.aes.key, key, key_len); + sg_init_one(&ctx->u.aes.key_sg, ctx->u.aes.key, key_len); + + return 0; +} + +static int ccp_aes_crypt(struct ablkcipher_request *req, bool encrypt) +{ + struct ccp_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct ccp_aes_req_ctx *rctx = ablkcipher_request_ctx(req); + struct scatterlist *iv_sg = NULL; + unsigned int iv_len = 0; + int ret; + + if (!ctx->u.aes.key_len) + return -EINVAL; + + if (((ctx->u.aes.mode == CCP_AES_MODE_ECB) || + (ctx->u.aes.mode == CCP_AES_MODE_CBC) || + (ctx->u.aes.mode == CCP_AES_MODE_CFB)) && + (req->nbytes & (AES_BLOCK_SIZE - 1))) + return -EINVAL; + + if (ctx->u.aes.mode != CCP_AES_MODE_ECB) { + if (!req->info) + return -EINVAL; + + memcpy(rctx->iv, req->info, AES_BLOCK_SIZE); + iv_sg = &rctx->iv_sg; + iv_len = AES_BLOCK_SIZE; + sg_init_one(iv_sg, rctx->iv, iv_len); + } + + memset(&rctx->cmd, 0, sizeof(rctx->cmd)); + INIT_LIST_HEAD(&rctx->cmd.entry); + rctx->cmd.engine = CCP_ENGINE_AES; + rctx->cmd.u.aes.type = ctx->u.aes.type; + rctx->cmd.u.aes.mode = ctx->u.aes.mode; + rctx->cmd.u.aes.action = + (encrypt) ? CCP_AES_ACTION_ENCRYPT : CCP_AES_ACTION_DECRYPT; + rctx->cmd.u.aes.key = &ctx->u.aes.key_sg; + rctx->cmd.u.aes.key_len = ctx->u.aes.key_len; + rctx->cmd.u.aes.iv = iv_sg; + rctx->cmd.u.aes.iv_len = iv_len; + rctx->cmd.u.aes.src = req->src; + rctx->cmd.u.aes.src_len = req->nbytes; + rctx->cmd.u.aes.dst = req->dst; + + ret = ccp_crypto_enqueue_request(&req->base, &rctx->cmd); + + return ret; +} + +static int ccp_aes_encrypt(struct ablkcipher_request *req) +{ + return ccp_aes_crypt(req, true); +} + +static int ccp_aes_decrypt(struct ablkcipher_request *req) +{ + return ccp_aes_crypt(req, false); +} + +static int ccp_aes_cra_init(struct crypto_tfm *tfm) +{ + struct ccp_ctx *ctx = crypto_tfm_ctx(tfm); + + ctx->complete = ccp_aes_complete; + ctx->u.aes.key_len = 0; + + tfm->crt_ablkcipher.reqsize = sizeof(struct ccp_aes_req_ctx); + + return 0; +} + +static void ccp_aes_cra_exit(struct crypto_tfm *tfm) +{ +} + +static int ccp_aes_rfc3686_complete(struct crypto_async_request *async_req, + int ret) +{ + struct ablkcipher_request *req = ablkcipher_request_cast(async_req); + struct ccp_aes_req_ctx *rctx = ablkcipher_request_ctx(req); + + /* Restore the original pointer */ + req->info = rctx->rfc3686_info; + + return ccp_aes_complete(async_req, ret); +} + +static int ccp_aes_rfc3686_setkey(struct crypto_ablkcipher *tfm, const u8 *key, + unsigned int key_len) +{ + struct ccp_ctx *ctx = crypto_tfm_ctx(crypto_ablkcipher_tfm(tfm)); + + if (key_len < CTR_RFC3686_NONCE_SIZE) + return -EINVAL; + + key_len -= CTR_RFC3686_NONCE_SIZE; + memcpy(ctx->u.aes.nonce, key + key_len, CTR_RFC3686_NONCE_SIZE); + + return ccp_aes_setkey(tfm, key, key_len); +} + +static int ccp_aes_rfc3686_crypt(struct ablkcipher_request *req, bool encrypt) +{ + struct ccp_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct ccp_aes_req_ctx *rctx = ablkcipher_request_ctx(req); + u8 *iv; + + /* Initialize the CTR block */ + iv = rctx->rfc3686_iv; + memcpy(iv, ctx->u.aes.nonce, CTR_RFC3686_NONCE_SIZE); + + iv += CTR_RFC3686_NONCE_SIZE; + memcpy(iv, req->info, CTR_RFC3686_IV_SIZE); + + iv += CTR_RFC3686_IV_SIZE; + *(__be32 *)iv = cpu_to_be32(1); + + /* Point to the new IV */ + rctx->rfc3686_info = req->info; + req->info = rctx->rfc3686_iv; + + return ccp_aes_crypt(req, encrypt); +} + +static int ccp_aes_rfc3686_encrypt(struct ablkcipher_request *req) +{ + return ccp_aes_rfc3686_crypt(req, true); +} + +static int ccp_aes_rfc3686_decrypt(struct ablkcipher_request *req) +{ + return ccp_aes_rfc3686_crypt(req, false); +} + +static int ccp_aes_rfc3686_cra_init(struct crypto_tfm *tfm) +{ + struct ccp_ctx *ctx = crypto_tfm_ctx(tfm); + + ctx->complete = ccp_aes_rfc3686_complete; + ctx->u.aes.key_len = 0; + + tfm->crt_ablkcipher.reqsize = sizeof(struct ccp_aes_req_ctx); + + return 0; +} + +static void ccp_aes_rfc3686_cra_exit(struct crypto_tfm *tfm) +{ +} + +static struct crypto_alg ccp_aes_defaults = { + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ccp_ctx), + .cra_priority = CCP_CRA_PRIORITY, + .cra_type = &crypto_ablkcipher_type, + .cra_init = ccp_aes_cra_init, + .cra_exit = ccp_aes_cra_exit, + .cra_module = THIS_MODULE, + .cra_ablkcipher = { + .setkey = ccp_aes_setkey, + .encrypt = ccp_aes_encrypt, + .decrypt = ccp_aes_decrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + }, +}; + +static struct crypto_alg ccp_aes_rfc3686_defaults = { + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = CTR_RFC3686_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ccp_ctx), + .cra_priority = CCP_CRA_PRIORITY, + .cra_type = &crypto_ablkcipher_type, + .cra_init = ccp_aes_rfc3686_cra_init, + .cra_exit = ccp_aes_rfc3686_cra_exit, + .cra_module = THIS_MODULE, + .cra_ablkcipher = { + .setkey = ccp_aes_rfc3686_setkey, + .encrypt = ccp_aes_rfc3686_encrypt, + .decrypt = ccp_aes_rfc3686_decrypt, + .min_keysize = AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, + .max_keysize = AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, + }, +}; + +struct ccp_aes_def { + enum ccp_aes_mode mode; + const char *name; + const char *driver_name; + unsigned int blocksize; + unsigned int ivsize; + struct crypto_alg *alg_defaults; +}; + +static struct ccp_aes_def aes_algs[] = { + { + .mode = CCP_AES_MODE_ECB, + .name = "ecb(aes)", + .driver_name = "ecb-aes-ccp", + .blocksize = AES_BLOCK_SIZE, + .ivsize = 0, + .alg_defaults = &ccp_aes_defaults, + }, + { + .mode = CCP_AES_MODE_CBC, + .name = "cbc(aes)", + .driver_name = "cbc-aes-ccp", + .blocksize = AES_BLOCK_SIZE, + .ivsize = AES_BLOCK_SIZE, + .alg_defaults = &ccp_aes_defaults, + }, + { + .mode = CCP_AES_MODE_CFB, + .name = "cfb(aes)", + .driver_name = "cfb-aes-ccp", + .blocksize = AES_BLOCK_SIZE, + .ivsize = AES_BLOCK_SIZE, + .alg_defaults = &ccp_aes_defaults, + }, + { + .mode = CCP_AES_MODE_OFB, + .name = "ofb(aes)", + .driver_name = "ofb-aes-ccp", + .blocksize = 1, + .ivsize = AES_BLOCK_SIZE, + .alg_defaults = &ccp_aes_defaults, + }, + { + .mode = CCP_AES_MODE_CTR, + .name = "ctr(aes)", + .driver_name = "ctr-aes-ccp", + .blocksize = 1, + .ivsize = AES_BLOCK_SIZE, + .alg_defaults = &ccp_aes_defaults, + }, + { + .mode = CCP_AES_MODE_CTR, + .name = "rfc3686(ctr(aes))", + .driver_name = "rfc3686-ctr-aes-ccp", + .blocksize = 1, + .ivsize = CTR_RFC3686_IV_SIZE, + .alg_defaults = &ccp_aes_rfc3686_defaults, + }, +}; + +static int ccp_register_aes_alg(struct list_head *head, + const struct ccp_aes_def *def) +{ + struct ccp_crypto_ablkcipher_alg *ccp_alg; + struct crypto_alg *alg; + int ret; + + ccp_alg = kzalloc(sizeof(*ccp_alg), GFP_KERNEL); + if (!ccp_alg) + return -ENOMEM; + + INIT_LIST_HEAD(&ccp_alg->entry); + + ccp_alg->mode = def->mode; + + /* Copy the defaults and override as necessary */ + alg = &ccp_alg->alg; + *alg = *def->alg_defaults; + snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", def->name); + snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", + def->driver_name); + alg->cra_blocksize = def->blocksize; + alg->cra_ablkcipher.ivsize = def->ivsize; + + ret = crypto_register_alg(alg); + if (ret) { + pr_err("%s ablkcipher algorithm registration error (%d)\n", + alg->cra_name, ret); + kfree(ccp_alg); + return ret; + } + + list_add(&ccp_alg->entry, head); + + return 0; +} + +int ccp_register_aes_algs(struct list_head *head) +{ + int i, ret; + + for (i = 0; i < ARRAY_SIZE(aes_algs); i++) { + ret = ccp_register_aes_alg(head, &aes_algs[i]); + if (ret) + return ret; + } + + return 0; +} diff --git a/drivers/crypto/ccp/ccp-crypto-main.c b/drivers/crypto/ccp/ccp-crypto-main.c new file mode 100644 index 000000000000..2636f044789d --- /dev/null +++ b/drivers/crypto/ccp/ccp-crypto-main.c @@ -0,0 +1,432 @@ +/* + * AMD Cryptographic Coprocessor (CCP) crypto API support + * + * Copyright (C) 2013 Advanced Micro Devices, Inc. + * + * Author: Tom Lendacky <thomas.lendacky@amd.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/list.h> +#include <linux/ccp.h> +#include <linux/scatterlist.h> +#include <crypto/internal/hash.h> + +#include "ccp-crypto.h" + +MODULE_AUTHOR("Tom Lendacky <thomas.lendacky@amd.com>"); +MODULE_LICENSE("GPL"); +MODULE_VERSION("1.0.0"); +MODULE_DESCRIPTION("AMD Cryptographic Coprocessor crypto API support"); + + +/* List heads for the supported algorithms */ +static LIST_HEAD(hash_algs); +static LIST_HEAD(cipher_algs); + +/* For any tfm, requests for that tfm on the same CPU must be returned + * in the order received. With multiple queues available, the CCP can + * process more than one cmd at a time. Therefore we must maintain + * a cmd list to insure the proper ordering of requests on a given tfm/cpu + * combination. + */ +struct ccp_crypto_cpu_queue { + struct list_head cmds; + struct list_head *backlog; + unsigned int cmd_count; +}; +#define CCP_CRYPTO_MAX_QLEN 50 + +struct ccp_crypto_percpu_queue { + struct ccp_crypto_cpu_queue __percpu *cpu_queue; +}; +static struct ccp_crypto_percpu_queue req_queue; + +struct ccp_crypto_cmd { + struct list_head entry; + + struct ccp_cmd *cmd; + + /* Save the crypto_tfm and crypto_async_request addresses + * separately to avoid any reference to a possibly invalid + * crypto_async_request structure after invoking the request + * callback + */ + struct crypto_async_request *req; + struct crypto_tfm *tfm; + + /* Used for held command processing to determine state */ + int ret; + + int cpu; +}; + +struct ccp_crypto_cpu { + struct work_struct work; + struct completion completion; + struct ccp_crypto_cmd *crypto_cmd; + int err; +}; + + +static inline bool ccp_crypto_success(int err) +{ + if (err && (err != -EINPROGRESS) && (err != -EBUSY)) + return false; + + return true; +} + +/* + * ccp_crypto_cmd_complete must be called while running on the appropriate + * cpu and the caller must have done a get_cpu to disable preemption + */ +static struct ccp_crypto_cmd *ccp_crypto_cmd_complete( + struct ccp_crypto_cmd *crypto_cmd, struct ccp_crypto_cmd **backlog) +{ + struct ccp_crypto_cpu_queue *cpu_queue; + struct ccp_crypto_cmd *held = NULL, *tmp; + + *backlog = NULL; + + cpu_queue = this_cpu_ptr(req_queue.cpu_queue); + + /* Held cmds will be after the current cmd in the queue so start + * searching for a cmd with a matching tfm for submission. + */ + tmp = crypto_cmd; + list_for_each_entry_continue(tmp, &cpu_queue->cmds, entry) { + if (crypto_cmd->tfm != tmp->tfm) + continue; + held = tmp; + break; + } + + /* Process the backlog: + * Because cmds can be executed from any point in the cmd list + * special precautions have to be taken when handling the backlog. + */ + if (cpu_queue->backlog != &cpu_queue->cmds) { + /* Skip over this cmd if it is the next backlog cmd */ + if (cpu_queue->backlog == &crypto_cmd->entry) + cpu_queue->backlog = crypto_cmd->entry.next; + + *backlog = container_of(cpu_queue->backlog, + struct ccp_crypto_cmd, entry); + cpu_queue->backlog = cpu_queue->backlog->next; + + /* Skip over this cmd if it is now the next backlog cmd */ + if (cpu_queue->backlog == &crypto_cmd->entry) + cpu_queue->backlog = crypto_cmd->entry.next; + } + + /* Remove the cmd entry from the list of cmds */ + cpu_queue->cmd_count--; + list_del(&crypto_cmd->entry); + + return held; +} + +static void ccp_crypto_complete_on_cpu(struct work_struct *work) +{ + struct ccp_crypto_cpu *cpu_work = + container_of(work, struct ccp_crypto_cpu, work); + struct ccp_crypto_cmd *crypto_cmd = cpu_work->crypto_cmd; + struct ccp_crypto_cmd *held, *next, *backlog; + struct crypto_async_request *req = crypto_cmd->req; + struct ccp_ctx *ctx = crypto_tfm_ctx(req->tfm); + int cpu, ret; + + cpu = get_cpu(); + + if (cpu_work->err == -EINPROGRESS) { + /* Only propogate the -EINPROGRESS if necessary */ + if (crypto_cmd->ret == -EBUSY) { + crypto_cmd->ret = -EINPROGRESS; + req->complete(req, -EINPROGRESS); + } + + goto e_cpu; + } + + /* Operation has completed - update the queue before invoking + * the completion callbacks and retrieve the next cmd (cmd with + * a matching tfm) that can be submitted to the CCP. + */ + held = ccp_crypto_cmd_complete(crypto_cmd, &backlog); + if (backlog) { + backlog->ret = -EINPROGRESS; + backlog->req->complete(backlog->req, -EINPROGRESS); + } + + /* Transition the state from -EBUSY to -EINPROGRESS first */ + if (crypto_cmd->ret == -EBUSY) + req->complete(req, -EINPROGRESS); + + /* Completion callbacks */ + ret = cpu_work->err; + if (ctx->complete) + ret = ctx->complete(req, ret); + req->complete(req, ret); + + /* Submit the next cmd */ + while (held) { + ret = ccp_enqueue_cmd(held->cmd); + if (ccp_crypto_success(ret)) + break; + + /* Error occurred, report it and get the next entry */ + held->req->complete(held->req, ret); + + next = ccp_crypto_cmd_complete(held, &backlog); + if (backlog) { + backlog->ret = -EINPROGRESS; + backlog->req->complete(backlog->req, -EINPROGRESS); + } + + kfree(held); + held = next; + } + + kfree(crypto_cmd); + +e_cpu: + put_cpu(); + + complete(&cpu_work->completion); +} + +static void ccp_crypto_complete(void *data, int err) +{ + struct ccp_crypto_cmd *crypto_cmd = data; + struct ccp_crypto_cpu cpu_work; + + INIT_WORK(&cpu_work.work, ccp_crypto_complete_on_cpu); + init_completion(&cpu_work.completion); + cpu_work.crypto_cmd = crypto_cmd; + cpu_work.err = err; + + schedule_work_on(crypto_cmd->cpu, &cpu_work.work); + + /* Keep the completion call synchronous */ + wait_for_completion(&cpu_work.completion); +} + +static int ccp_crypto_enqueue_cmd(struct ccp_crypto_cmd *crypto_cmd) +{ + struct ccp_crypto_cpu_queue *cpu_queue; + struct ccp_crypto_cmd *active = NULL, *tmp; + int cpu, ret; + + cpu = get_cpu(); + crypto_cmd->cpu = cpu; + + cpu_queue = this_cpu_ptr(req_queue.cpu_queue); + + /* Check if the cmd can/should be queued */ + if (cpu_queue->cmd_count >= CCP_CRYPTO_MAX_QLEN) { + ret = -EBUSY; + if (!(crypto_cmd->cmd->flags & CCP_CMD_MAY_BACKLOG)) + goto e_cpu; + } + + /* Look for an entry with the same tfm. If there is a cmd + * with the same tfm in the list for this cpu then the current + * cmd cannot be submitted to the CCP yet. + */ + list_for_each_entry(tmp, &cpu_queue->cmds, entry) { + if (crypto_cmd->tfm != tmp->tfm) + continue; + active = tmp; + break; + } + + ret = -EINPROGRESS; + if (!active) { + ret = ccp_enqueue_cmd(crypto_cmd->cmd); + if (!ccp_crypto_success(ret)) + goto e_cpu; + } + + if (cpu_queue->cmd_count >= CCP_CRYPTO_MAX_QLEN) { + ret = -EBUSY; + if (cpu_queue->backlog == &cpu_queue->cmds) + cpu_queue->backlog = &crypto_cmd->entry; + } + crypto_cmd->ret = ret; + + cpu_queue->cmd_count++; + list_add_tail(&crypto_cmd->entry, &cpu_queue->cmds); + +e_cpu: + put_cpu(); + + return ret; +} + +/** + * ccp_crypto_enqueue_request - queue an crypto async request for processing + * by the CCP + * + * @req: crypto_async_request struct to be processed + * @cmd: ccp_cmd struct to be sent to the CCP + */ +int ccp_crypto_enqueue_request(struct crypto_async_request *req, + struct ccp_cmd *cmd) +{ + struct ccp_crypto_cmd *crypto_cmd; + gfp_t gfp; + int ret; + + gfp = req->flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; + + crypto_cmd = kzalloc(sizeof(*crypto_cmd), gfp); + if (!crypto_cmd) + return -ENOMEM; + + /* The tfm pointer must be saved and not referenced from the + * crypto_async_request (req) pointer because it is used after + * completion callback for the request and the req pointer + * might not be valid anymore. + */ + crypto_cmd->cmd = cmd; + crypto_cmd->req = req; + crypto_cmd->tfm = req->tfm; + + cmd->callback = ccp_crypto_complete; + cmd->data = crypto_cmd; + + if (req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG) + cmd->flags |= CCP_CMD_MAY_BACKLOG; + else + cmd->flags &= ~CCP_CMD_MAY_BACKLOG; + + ret = ccp_crypto_enqueue_cmd(crypto_cmd); + if (!ccp_crypto_success(ret)) + kfree(crypto_cmd); + + return ret; +} + +struct scatterlist *ccp_crypto_sg_table_add(struct sg_table *table, + struct scatterlist *sg_add) +{ + struct scatterlist *sg, *sg_last = NULL; + + for (sg = table->sgl; sg; sg = sg_next(sg)) + if (!sg_page(sg)) + break; + BUG_ON(!sg); + + for (; sg && sg_add; sg = sg_next(sg), sg_add = sg_next(sg_add)) { + sg_set_page(sg, sg_page(sg_add), sg_add->length, + sg_add->offset); + sg_last = sg; + } + BUG_ON(sg_add); + + return sg_last; +} + +static int ccp_register_algs(void) +{ + int ret; + + ret = ccp_register_aes_algs(&cipher_algs); + if (ret) + return ret; + + ret = ccp_register_aes_cmac_algs(&hash_algs); + if (ret) + return ret; + + ret = ccp_register_aes_xts_algs(&cipher_algs); + if (ret) + return ret; + + ret = ccp_register_sha_algs(&hash_algs); + if (ret) + return ret; + + return 0; +} + +static void ccp_unregister_algs(void) +{ + struct ccp_crypto_ahash_alg *ahash_alg, *ahash_tmp; + struct ccp_crypto_ablkcipher_alg *ablk_alg, *ablk_tmp; + + list_for_each_entry_safe(ahash_alg, ahash_tmp, &hash_algs, entry) { + crypto_unregister_ahash(&ahash_alg->alg); + list_del(&ahash_alg->entry); + kfree(ahash_alg); + } + + list_for_each_entry_safe(ablk_alg, ablk_tmp, &cipher_algs, entry) { + crypto_unregister_alg(&ablk_alg->alg); + list_del(&ablk_alg->entry); + kfree(ablk_alg); + } +} + +static int ccp_init_queues(void) +{ + struct ccp_crypto_cpu_queue *cpu_queue; + int cpu; + + req_queue.cpu_queue = alloc_percpu(struct ccp_crypto_cpu_queue); + if (!req_queue.cpu_queue) + return -ENOMEM; + + for_each_possible_cpu(cpu) { + cpu_queue = per_cpu_ptr(req_queue.cpu_queue, cpu); + INIT_LIST_HEAD(&cpu_queue->cmds); + cpu_queue->backlog = &cpu_queue->cmds; + cpu_queue->cmd_count = 0; + } + + return 0; +} + +static void ccp_fini_queue(void) +{ + struct ccp_crypto_cpu_queue *cpu_queue; + int cpu; + + for_each_possible_cpu(cpu) { + cpu_queue = per_cpu_ptr(req_queue.cpu_queue, cpu); + BUG_ON(!list_empty(&cpu_queue->cmds)); + } + free_percpu(req_queue.cpu_queue); +} + +static int ccp_crypto_init(void) +{ + int ret; + + ret = ccp_init_queues(); + if (ret) + return ret; + + ret = ccp_register_algs(); + if (ret) { + ccp_unregister_algs(); + ccp_fini_queue(); + } + + return ret; +} + +static void ccp_crypto_exit(void) +{ + ccp_unregister_algs(); + ccp_fini_queue(); +} + +module_init(ccp_crypto_init); +module_exit(ccp_crypto_exit); diff --git a/drivers/crypto/ccp/ccp-crypto-sha.c b/drivers/crypto/ccp/ccp-crypto-sha.c new file mode 100644 index 000000000000..3867290b3531 --- /dev/null +++ b/drivers/crypto/ccp/ccp-crypto-sha.c @@ -0,0 +1,517 @@ +/* + * AMD Cryptographic Coprocessor (CCP) SHA crypto API support + * + * Copyright (C) 2013 Advanced Micro Devices, Inc. + * + * Author: Tom Lendacky <thomas.lendacky@amd.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/module.h> +#include <linux/sched.h> +#include <linux/delay.h> +#include <linux/scatterlist.h> +#include <linux/crypto.h> +#include <crypto/algapi.h> +#include <crypto/hash.h> +#include <crypto/internal/hash.h> +#include <crypto/sha.h> +#include <crypto/scatterwalk.h> + +#include "ccp-crypto.h" + + +struct ccp_sha_result { + struct completion completion; + int err; +}; + +static void ccp_sync_hash_complete(struct crypto_async_request *req, int err) +{ + struct ccp_sha_result *result = req->data; + + if (err == -EINPROGRESS) + return; + + result->err = err; + complete(&result->completion); +} + +static int ccp_sync_hash(struct crypto_ahash *tfm, u8 *buf, + struct scatterlist *sg, unsigned int len) +{ + struct ccp_sha_result result; + struct ahash_request *req; + int ret; + + init_completion(&result.completion); + + req = ahash_request_alloc(tfm, GFP_KERNEL); + if (!req) + return -ENOMEM; + + ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + ccp_sync_hash_complete, &result); + ahash_request_set_crypt(req, sg, buf, len); + + ret = crypto_ahash_digest(req); + if ((ret == -EINPROGRESS) || (ret == -EBUSY)) { + ret = wait_for_completion_interruptible(&result.completion); + if (!ret) + ret = result.err; + } + + ahash_request_free(req); + + return ret; +} + +static int ccp_sha_finish_hmac(struct crypto_async_request *async_req) +{ + struct ahash_request *req = ahash_request_cast(async_req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct ccp_ctx *ctx = crypto_ahash_ctx(tfm); + struct ccp_sha_req_ctx *rctx = ahash_request_ctx(req); + struct scatterlist sg[2]; + unsigned int block_size = + crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); + unsigned int digest_size = crypto_ahash_digestsize(tfm); + + sg_init_table(sg, ARRAY_SIZE(sg)); + sg_set_buf(&sg[0], ctx->u.sha.opad, block_size); + sg_set_buf(&sg[1], rctx->ctx, digest_size); + + return ccp_sync_hash(ctx->u.sha.hmac_tfm, req->result, sg, + block_size + digest_size); +} + +static int ccp_sha_complete(struct crypto_async_request *async_req, int ret) +{ + struct ahash_request *req = ahash_request_cast(async_req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct ccp_ctx *ctx = crypto_ahash_ctx(tfm); + struct ccp_sha_req_ctx *rctx = ahash_request_ctx(req); + unsigned int digest_size = crypto_ahash_digestsize(tfm); + + if (ret) + goto e_free; + + if (rctx->hash_rem) { + /* Save remaining data to buffer */ + unsigned int offset = rctx->nbytes - rctx->hash_rem; + scatterwalk_map_and_copy(rctx->buf, rctx->src, + offset, rctx->hash_rem, 0); + rctx->buf_count = rctx->hash_rem; + } else + rctx->buf_count = 0; + + /* Update result area if supplied */ + if (req->result) + memcpy(req->result, rctx->ctx, digest_size); + + /* If we're doing an HMAC, we need to perform that on the final op */ + if (rctx->final && ctx->u.sha.key_len) + ret = ccp_sha_finish_hmac(async_req); + +e_free: + sg_free_table(&rctx->data_sg); + + return ret; +} + +static int ccp_do_sha_update(struct ahash_request *req, unsigned int nbytes, + unsigned int final) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct ccp_sha_req_ctx *rctx = ahash_request_ctx(req); + struct scatterlist *sg; + unsigned int block_size = + crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); + unsigned int sg_count; + gfp_t gfp; + u64 len; + int ret; + + len = (u64)rctx->buf_count + (u64)nbytes; + + if (!final && (len <= block_size)) { + scatterwalk_map_and_copy(rctx->buf + rctx->buf_count, req->src, + 0, nbytes, 0); + rctx->buf_count += nbytes; + + return 0; + } + + rctx->src = req->src; + rctx->nbytes = nbytes; + + rctx->final = final; + rctx->hash_rem = final ? 0 : len & (block_size - 1); + rctx->hash_cnt = len - rctx->hash_rem; + if (!final && !rctx->hash_rem) { + /* CCP can't do zero length final, so keep some data around */ + rctx->hash_cnt -= block_size; + rctx->hash_rem = block_size; + } + + /* Initialize the context scatterlist */ + sg_init_one(&rctx->ctx_sg, rctx->ctx, sizeof(rctx->ctx)); + + sg = NULL; + if (rctx->buf_count && nbytes) { + /* Build the data scatterlist table - allocate enough entries + * for both data pieces (buffer and input data) + */ + gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + sg_count = sg_nents(req->src) + 1; + ret = sg_alloc_table(&rctx->data_sg, sg_count, gfp); + if (ret) + return ret; + + sg_init_one(&rctx->buf_sg, rctx->buf, rctx->buf_count); + sg = ccp_crypto_sg_table_add(&rctx->data_sg, &rctx->buf_sg); + sg = ccp_crypto_sg_table_add(&rctx->data_sg, req->src); + sg_mark_end(sg); + + sg = rctx->data_sg.sgl; + } else if (rctx->buf_count) { + sg_init_one(&rctx->buf_sg, rctx->buf, rctx->buf_count); + + sg = &rctx->buf_sg; + } else if (nbytes) { + sg = req->src; + } + + rctx->msg_bits += (rctx->hash_cnt << 3); /* Total in bits */ + + memset(&rctx->cmd, 0, sizeof(rctx->cmd)); + INIT_LIST_HEAD(&rctx->cmd.entry); + rctx->cmd.engine = CCP_ENGINE_SHA; + rctx->cmd.u.sha.type = rctx->type; + rctx->cmd.u.sha.ctx = &rctx->ctx_sg; + rctx->cmd.u.sha.ctx_len = sizeof(rctx->ctx); + rctx->cmd.u.sha.src = sg; + rctx->cmd.u.sha.src_len = rctx->hash_cnt; + rctx->cmd.u.sha.final = rctx->final; + rctx->cmd.u.sha.msg_bits = rctx->msg_bits; + + rctx->first = 0; + + ret = ccp_crypto_enqueue_request(&req->base, &rctx->cmd); + + return ret; +} + +static int ccp_sha_init(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct ccp_ctx *ctx = crypto_ahash_ctx(tfm); + struct ccp_sha_req_ctx *rctx = ahash_request_ctx(req); + struct ccp_crypto_ahash_alg *alg = + ccp_crypto_ahash_alg(crypto_ahash_tfm(tfm)); + unsigned int block_size = + crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); + + memset(rctx, 0, sizeof(*rctx)); + + memcpy(rctx->ctx, alg->init, sizeof(rctx->ctx)); + rctx->type = alg->type; + rctx->first = 1; + + if (ctx->u.sha.key_len) { + /* Buffer the HMAC key for first update */ + memcpy(rctx->buf, ctx->u.sha.ipad, block_size); + rctx->buf_count = block_size; + } + + return 0; +} + +static int ccp_sha_update(struct ahash_request *req) +{ + return ccp_do_sha_update(req, req->nbytes, 0); +} + +static int ccp_sha_final(struct ahash_request *req) +{ + return ccp_do_sha_update(req, 0, 1); +} + +static int ccp_sha_finup(struct ahash_request *req) +{ + return ccp_do_sha_update(req, req->nbytes, 1); +} + +static int ccp_sha_digest(struct ahash_request *req) +{ + int ret; + + ret = ccp_sha_init(req); + if (ret) + return ret; + + return ccp_sha_finup(req); +} + +static int ccp_sha_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int key_len) +{ + struct ccp_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); + struct scatterlist sg; + unsigned int block_size = + crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); + unsigned int digest_size = crypto_ahash_digestsize(tfm); + int i, ret; + + /* Set to zero until complete */ + ctx->u.sha.key_len = 0; + + /* Clear key area to provide zero padding for keys smaller + * than the block size + */ + memset(ctx->u.sha.key, 0, sizeof(ctx->u.sha.key)); + + if (key_len > block_size) { + /* Must hash the input key */ + sg_init_one(&sg, key, key_len); + ret = ccp_sync_hash(tfm, ctx->u.sha.key, &sg, key_len); + if (ret) { + crypto_ahash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + + key_len = digest_size; + } else + memcpy(ctx->u.sha.key, key, key_len); + + for (i = 0; i < block_size; i++) { + ctx->u.sha.ipad[i] = ctx->u.sha.key[i] ^ 0x36; + ctx->u.sha.opad[i] = ctx->u.sha.key[i] ^ 0x5c; + } + + ctx->u.sha.key_len = key_len; + + return 0; +} + +static int ccp_sha_cra_init(struct crypto_tfm *tfm) +{ + struct ccp_ctx *ctx = crypto_tfm_ctx(tfm); + struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); + + ctx->complete = ccp_sha_complete; + ctx->u.sha.key_len = 0; + + crypto_ahash_set_reqsize(ahash, sizeof(struct ccp_sha_req_ctx)); + + return 0; +} + +static void ccp_sha_cra_exit(struct crypto_tfm *tfm) +{ +} + +static int ccp_hmac_sha_cra_init(struct crypto_tfm *tfm) +{ + struct ccp_ctx *ctx = crypto_tfm_ctx(tfm); + struct ccp_crypto_ahash_alg *alg = ccp_crypto_ahash_alg(tfm); + struct crypto_ahash *hmac_tfm; + + hmac_tfm = crypto_alloc_ahash(alg->child_alg, + CRYPTO_ALG_TYPE_AHASH, 0); + if (IS_ERR(hmac_tfm)) { + pr_warn("could not load driver %s need for HMAC support\n", + alg->child_alg); + return PTR_ERR(hmac_tfm); + } + + ctx->u.sha.hmac_tfm = hmac_tfm; + + return ccp_sha_cra_init(tfm); +} + +static void ccp_hmac_sha_cra_exit(struct crypto_tfm *tfm) +{ + struct ccp_ctx *ctx = crypto_tfm_ctx(tfm); + + if (ctx->u.sha.hmac_tfm) + crypto_free_ahash(ctx->u.sha.hmac_tfm); + + ccp_sha_cra_exit(tfm); +} + +static const __be32 sha1_init[CCP_SHA_CTXSIZE / sizeof(__be32)] = { + cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1), + cpu_to_be32(SHA1_H2), cpu_to_be32(SHA1_H3), + cpu_to_be32(SHA1_H4), 0, 0, 0, +}; + +static const __be32 sha224_init[CCP_SHA_CTXSIZE / sizeof(__be32)] = { + cpu_to_be32(SHA224_H0), cpu_to_be32(SHA224_H1), + cpu_to_be32(SHA224_H2), cpu_to_be32(SHA224_H3), + cpu_to_be32(SHA224_H4), cpu_to_be32(SHA224_H5), + cpu_to_be32(SHA224_H6), cpu_to_be32(SHA224_H7), +}; + +static const __be32 sha256_init[CCP_SHA_CTXSIZE / sizeof(__be32)] = { + cpu_to_be32(SHA256_H0), cpu_to_be32(SHA256_H1), + cpu_to_be32(SHA256_H2), cpu_to_be32(SHA256_H3), + cpu_to_be32(SHA256_H4), cpu_to_be32(SHA256_H5), + cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7), +}; + +struct ccp_sha_def { + const char *name; + const char *drv_name; + const __be32 *init; + enum ccp_sha_type type; + u32 digest_size; + u32 block_size; +}; + +static struct ccp_sha_def sha_algs[] = { + { + .name = "sha1", + .drv_name = "sha1-ccp", + .init = sha1_init, + .type = CCP_SHA_TYPE_1, + .digest_size = SHA1_DIGEST_SIZE, + .block_size = SHA1_BLOCK_SIZE, + }, + { + .name = "sha224", + .drv_name = "sha224-ccp", + .init = sha224_init, + .type = CCP_SHA_TYPE_224, + .digest_size = SHA224_DIGEST_SIZE, + .block_size = SHA224_BLOCK_SIZE, + }, + { + .name = "sha256", + .drv_name = "sha256-ccp", + .init = sha256_init, + .type = CCP_SHA_TYPE_256, + .digest_size = SHA256_DIGEST_SIZE, + .block_size = SHA256_BLOCK_SIZE, + }, +}; + +static int ccp_register_hmac_alg(struct list_head *head, + const struct ccp_sha_def *def, + const struct ccp_crypto_ahash_alg *base_alg) +{ + struct ccp_crypto_ahash_alg *ccp_alg; + struct ahash_alg *alg; + struct hash_alg_common *halg; + struct crypto_alg *base; + int ret; + + ccp_alg = kzalloc(sizeof(*ccp_alg), GFP_KERNEL); + if (!ccp_alg) + return -ENOMEM; + + /* Copy the base algorithm and only change what's necessary */ + *ccp_alg = *base_alg; + INIT_LIST_HEAD(&ccp_alg->entry); + + strncpy(ccp_alg->child_alg, def->name, CRYPTO_MAX_ALG_NAME); + + alg = &ccp_alg->alg; + alg->setkey = ccp_sha_setkey; + + halg = &alg->halg; + + base = &halg->base; + snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "hmac(%s)", def->name); + snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "hmac-%s", + def->drv_name); + base->cra_init = ccp_hmac_sha_cra_init; + base->cra_exit = ccp_hmac_sha_cra_exit; + + ret = crypto_register_ahash(alg); + if (ret) { + pr_err("%s ahash algorithm registration error (%d)\n", + base->cra_name, ret); + kfree(ccp_alg); + return ret; + } + + list_add(&ccp_alg->entry, head); + + return ret; +} + +static int ccp_register_sha_alg(struct list_head *head, + const struct ccp_sha_def *def) +{ + struct ccp_crypto_ahash_alg *ccp_alg; + struct ahash_alg *alg; + struct hash_alg_common *halg; + struct crypto_alg *base; + int ret; + + ccp_alg = kzalloc(sizeof(*ccp_alg), GFP_KERNEL); + if (!ccp_alg) + return -ENOMEM; + + INIT_LIST_HEAD(&ccp_alg->entry); + + ccp_alg->init = def->init; + ccp_alg->type = def->type; + + alg = &ccp_alg->alg; + alg->init = ccp_sha_init; + alg->update = ccp_sha_update; + alg->final = ccp_sha_final; + alg->finup = ccp_sha_finup; + alg->digest = ccp_sha_digest; + + halg = &alg->halg; + halg->digestsize = def->digest_size; + + base = &halg->base; + snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "%s", def->name); + snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", + def->drv_name); + base->cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK; + base->cra_blocksize = def->block_size; + base->cra_ctxsize = sizeof(struct ccp_ctx); + base->cra_priority = CCP_CRA_PRIORITY; + base->cra_type = &crypto_ahash_type; + base->cra_init = ccp_sha_cra_init; + base->cra_exit = ccp_sha_cra_exit; + base->cra_module = THIS_MODULE; + + ret = crypto_register_ahash(alg); + if (ret) { + pr_err("%s ahash algorithm registration error (%d)\n", + base->cra_name, ret); + kfree(ccp_alg); + return ret; + } + + list_add(&ccp_alg->entry, head); + + ret = ccp_register_hmac_alg(head, def, ccp_alg); + + return ret; +} + +int ccp_register_sha_algs(struct list_head *head) +{ + int i, ret; + + for (i = 0; i < ARRAY_SIZE(sha_algs); i++) { + ret = ccp_register_sha_alg(head, &sha_algs[i]); + if (ret) + return ret; + } + + return 0; +} diff --git a/drivers/crypto/ccp/ccp-crypto.h b/drivers/crypto/ccp/ccp-crypto.h new file mode 100644 index 000000000000..b222231b6169 --- /dev/null +++ b/drivers/crypto/ccp/ccp-crypto.h @@ -0,0 +1,197 @@ +/* + * AMD Cryptographic Coprocessor (CCP) crypto API support + * + * Copyright (C) 2013 Advanced Micro Devices, Inc. + * + * Author: Tom Lendacky <thomas.lendacky@amd.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#ifndef __CCP_CRYPTO_H__ +#define __CCP_CRYPTO_H__ + + +#include <linux/list.h> +#include <linux/wait.h> +#include <linux/pci.h> +#include <linux/ccp.h> +#include <linux/crypto.h> +#include <crypto/algapi.h> +#include <crypto/aes.h> +#include <crypto/ctr.h> +#include <crypto/hash.h> +#include <crypto/sha.h> + + +#define CCP_CRA_PRIORITY 300 + +struct ccp_crypto_ablkcipher_alg { + struct list_head entry; + + u32 mode; + + struct crypto_alg alg; +}; + +struct ccp_crypto_ahash_alg { + struct list_head entry; + + const __be32 *init; + u32 type; + u32 mode; + + /* Child algorithm used for HMAC, CMAC, etc */ + char child_alg[CRYPTO_MAX_ALG_NAME]; + + struct ahash_alg alg; +}; + +static inline struct ccp_crypto_ablkcipher_alg * + ccp_crypto_ablkcipher_alg(struct crypto_tfm *tfm) +{ + struct crypto_alg *alg = tfm->__crt_alg; + + return container_of(alg, struct ccp_crypto_ablkcipher_alg, alg); +} + +static inline struct ccp_crypto_ahash_alg * + ccp_crypto_ahash_alg(struct crypto_tfm *tfm) +{ + struct crypto_alg *alg = tfm->__crt_alg; + struct ahash_alg *ahash_alg; + + ahash_alg = container_of(alg, struct ahash_alg, halg.base); + + return container_of(ahash_alg, struct ccp_crypto_ahash_alg, alg); +} + + +/***** AES related defines *****/ +struct ccp_aes_ctx { + /* Fallback cipher for XTS with unsupported unit sizes */ + struct crypto_ablkcipher *tfm_ablkcipher; + + /* Cipher used to generate CMAC K1/K2 keys */ + struct crypto_cipher *tfm_cipher; + + enum ccp_engine engine; + enum ccp_aes_type type; + enum ccp_aes_mode mode; + + struct scatterlist key_sg; + unsigned int key_len; + u8 key[AES_MAX_KEY_SIZE]; + + u8 nonce[CTR_RFC3686_NONCE_SIZE]; + + /* CMAC key structures */ + struct scatterlist k1_sg; + struct scatterlist k2_sg; + unsigned int kn_len; + u8 k1[AES_BLOCK_SIZE]; + u8 k2[AES_BLOCK_SIZE]; +}; + +struct ccp_aes_req_ctx { + struct scatterlist iv_sg; + u8 iv[AES_BLOCK_SIZE]; + + /* Fields used for RFC3686 requests */ + u8 *rfc3686_info; + u8 rfc3686_iv[AES_BLOCK_SIZE]; + + struct ccp_cmd cmd; +}; + +struct ccp_aes_cmac_req_ctx { + unsigned int null_msg; + unsigned int final; + + struct scatterlist *src; + unsigned int nbytes; + + u64 hash_cnt; + unsigned int hash_rem; + + struct sg_table data_sg; + + struct scatterlist iv_sg; + u8 iv[AES_BLOCK_SIZE]; + + struct scatterlist buf_sg; + unsigned int buf_count; + u8 buf[AES_BLOCK_SIZE]; + + struct scatterlist pad_sg; + unsigned int pad_count; + u8 pad[AES_BLOCK_SIZE]; + + struct ccp_cmd cmd; +}; + +/***** SHA related defines *****/ +#define MAX_SHA_CONTEXT_SIZE SHA256_DIGEST_SIZE +#define MAX_SHA_BLOCK_SIZE SHA256_BLOCK_SIZE + +struct ccp_sha_ctx { + unsigned int key_len; + u8 key[MAX_SHA_BLOCK_SIZE]; + u8 ipad[MAX_SHA_BLOCK_SIZE]; + u8 opad[MAX_SHA_BLOCK_SIZE]; + struct crypto_ahash *hmac_tfm; +}; + +struct ccp_sha_req_ctx { + enum ccp_sha_type type; + + u64 msg_bits; + + unsigned int first; + unsigned int final; + + struct scatterlist *src; + unsigned int nbytes; + + u64 hash_cnt; + unsigned int hash_rem; + + struct sg_table data_sg; + + struct scatterlist ctx_sg; + u8 ctx[MAX_SHA_CONTEXT_SIZE]; + + struct scatterlist buf_sg; + unsigned int buf_count; + u8 buf[MAX_SHA_BLOCK_SIZE]; + + /* HMAC support field */ + struct scatterlist pad_sg; + + /* CCP driver command */ + struct ccp_cmd cmd; +}; + +/***** Common Context Structure *****/ +struct ccp_ctx { + int (*complete)(struct crypto_async_request *req, int ret); + + union { + struct ccp_aes_ctx aes; + struct ccp_sha_ctx sha; + } u; +}; + +int ccp_crypto_enqueue_request(struct crypto_async_request *req, + struct ccp_cmd *cmd); +struct scatterlist *ccp_crypto_sg_table_add(struct sg_table *table, + struct scatterlist *sg_add); + +int ccp_register_aes_algs(struct list_head *head); +int ccp_register_aes_cmac_algs(struct list_head *head); +int ccp_register_aes_xts_algs(struct list_head *head); +int ccp_register_sha_algs(struct list_head *head); + +#endif diff --git a/drivers/crypto/ccp/ccp-dev.c b/drivers/crypto/ccp/ccp-dev.c new file mode 100644 index 000000000000..c3bc21264600 --- /dev/null +++ b/drivers/crypto/ccp/ccp-dev.c @@ -0,0 +1,595 @@ +/* + * AMD Cryptographic Coprocessor (CCP) driver + * + * Copyright (C) 2013 Advanced Micro Devices, Inc. + * + * Author: Tom Lendacky <thomas.lendacky@amd.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/kthread.h> +#include <linux/sched.h> +#include <linux/interrupt.h> +#include <linux/spinlock.h> +#include <linux/mutex.h> +#include <linux/delay.h> +#include <linux/hw_random.h> +#include <linux/cpu.h> +#include <asm/cpu_device_id.h> +#include <linux/ccp.h> + +#include "ccp-dev.h" + +MODULE_AUTHOR("Tom Lendacky <thomas.lendacky@amd.com>"); +MODULE_LICENSE("GPL"); +MODULE_VERSION("1.0.0"); +MODULE_DESCRIPTION("AMD Cryptographic Coprocessor driver"); + + +static struct ccp_device *ccp_dev; +static inline struct ccp_device *ccp_get_device(void) +{ + return ccp_dev; +} + +static inline void ccp_add_device(struct ccp_device *ccp) +{ + ccp_dev = ccp; +} + +static inline void ccp_del_device(struct ccp_device *ccp) +{ + ccp_dev = NULL; +} + +/** + * ccp_enqueue_cmd - queue an operation for processing by the CCP + * + * @cmd: ccp_cmd struct to be processed + * + * Queue a cmd to be processed by the CCP. If queueing the cmd + * would exceed the defined length of the cmd queue the cmd will + * only be queued if the CCP_CMD_MAY_BACKLOG flag is set and will + * result in a return code of -EBUSY. + * + * The callback routine specified in the ccp_cmd struct will be + * called to notify the caller of completion (if the cmd was not + * backlogged) or advancement out of the backlog. If the cmd has + * advanced out of the backlog the "err" value of the callback + * will be -EINPROGRESS. Any other "err" value during callback is + * the result of the operation. + * + * The cmd has been successfully queued if: + * the return code is -EINPROGRESS or + * the return code is -EBUSY and CCP_CMD_MAY_BACKLOG flag is set + */ +int ccp_enqueue_cmd(struct ccp_cmd *cmd) +{ + struct ccp_device *ccp = ccp_get_device(); + unsigned long flags; + unsigned int i; + int ret; + + if (!ccp) + return -ENODEV; + + /* Caller must supply a callback routine */ + if (!cmd->callback) + return -EINVAL; + + cmd->ccp = ccp; + + spin_lock_irqsave(&ccp->cmd_lock, flags); + + i = ccp->cmd_q_count; + + if (ccp->cmd_count >= MAX_CMD_QLEN) { + ret = -EBUSY; + if (cmd->flags & CCP_CMD_MAY_BACKLOG) + list_add_tail(&cmd->entry, &ccp->backlog); + } else { + ret = -EINPROGRESS; + ccp->cmd_count++; + list_add_tail(&cmd->entry, &ccp->cmd); + + /* Find an idle queue */ + if (!ccp->suspending) { + for (i = 0; i < ccp->cmd_q_count; i++) { + if (ccp->cmd_q[i].active) + continue; + + break; + } + } + } + + spin_unlock_irqrestore(&ccp->cmd_lock, flags); + + /* If we found an idle queue, wake it up */ + if (i < ccp->cmd_q_count) + wake_up_process(ccp->cmd_q[i].kthread); + + return ret; +} +EXPORT_SYMBOL_GPL(ccp_enqueue_cmd); + +static void ccp_do_cmd_backlog(struct work_struct *work) +{ + struct ccp_cmd *cmd = container_of(work, struct ccp_cmd, work); + struct ccp_device *ccp = cmd->ccp; + unsigned long flags; + unsigned int i; + + cmd->callback(cmd->data, -EINPROGRESS); + + spin_lock_irqsave(&ccp->cmd_lock, flags); + + ccp->cmd_count++; + list_add_tail(&cmd->entry, &ccp->cmd); + + /* Find an idle queue */ + for (i = 0; i < ccp->cmd_q_count; i++) { + if (ccp->cmd_q[i].active) + continue; + + break; + } + + spin_unlock_irqrestore(&ccp->cmd_lock, flags); + + /* If we found an idle queue, wake it up */ + if (i < ccp->cmd_q_count) + wake_up_process(ccp->cmd_q[i].kthread); +} + +static struct ccp_cmd *ccp_dequeue_cmd(struct ccp_cmd_queue *cmd_q) +{ + struct ccp_device *ccp = cmd_q->ccp; + struct ccp_cmd *cmd = NULL; + struct ccp_cmd *backlog = NULL; + unsigned long flags; + + spin_lock_irqsave(&ccp->cmd_lock, flags); + + cmd_q->active = 0; + + if (ccp->suspending) { + cmd_q->suspended = 1; + + spin_unlock_irqrestore(&ccp->cmd_lock, flags); + wake_up_interruptible(&ccp->suspend_queue); + + return NULL; + } + + if (ccp->cmd_count) { + cmd_q->active = 1; + + cmd = list_first_entry(&ccp->cmd, struct ccp_cmd, entry); + list_del(&cmd->entry); + + ccp->cmd_count--; + } + + if (!list_empty(&ccp->backlog)) { + backlog = list_first_entry(&ccp->backlog, struct ccp_cmd, + entry); + list_del(&backlog->entry); + } + + spin_unlock_irqrestore(&ccp->cmd_lock, flags); + + if (backlog) { + INIT_WORK(&backlog->work, ccp_do_cmd_backlog); + schedule_work(&backlog->work); + } + + return cmd; +} + +static void ccp_do_cmd_complete(struct work_struct *work) +{ + struct ccp_cmd *cmd = container_of(work, struct ccp_cmd, work); + + cmd->callback(cmd->data, cmd->ret); +} + +static int ccp_cmd_queue_thread(void *data) +{ + struct ccp_cmd_queue *cmd_q = (struct ccp_cmd_queue *)data; + struct ccp_cmd *cmd; + + set_current_state(TASK_INTERRUPTIBLE); + while (!kthread_should_stop()) { + schedule(); + + set_current_state(TASK_INTERRUPTIBLE); + + cmd = ccp_dequeue_cmd(cmd_q); + if (!cmd) + continue; + + __set_current_state(TASK_RUNNING); + + /* Execute the command */ + cmd->ret = ccp_run_cmd(cmd_q, cmd); + + /* Schedule the completion callback */ + INIT_WORK(&cmd->work, ccp_do_cmd_complete); + schedule_work(&cmd->work); + } + + __set_current_state(TASK_RUNNING); + + return 0; +} + +static int ccp_trng_read(struct hwrng *rng, void *data, size_t max, bool wait) +{ + struct ccp_device *ccp = container_of(rng, struct ccp_device, hwrng); + u32 trng_value; + int len = min_t(int, sizeof(trng_value), max); + + /* + * Locking is provided by the caller so we can update device + * hwrng-related fields safely + */ + trng_value = ioread32(ccp->io_regs + TRNG_OUT_REG); + if (!trng_value) { + /* Zero is returned if not data is available or if a + * bad-entropy error is present. Assume an error if + * we exceed TRNG_RETRIES reads of zero. + */ + if (ccp->hwrng_retries++ > TRNG_RETRIES) + return -EIO; + + return 0; + } + + /* Reset the counter and save the rng value */ + ccp->hwrng_retries = 0; + memcpy(data, &trng_value, len); + + return len; +} + +/** + * ccp_alloc_struct - allocate and initialize the ccp_device struct + * + * @dev: device struct of the CCP + */ +struct ccp_device *ccp_alloc_struct(struct device *dev) +{ + struct ccp_device *ccp; + + ccp = kzalloc(sizeof(*ccp), GFP_KERNEL); + if (ccp == NULL) { + dev_err(dev, "unable to allocate device struct\n"); + return NULL; + } + ccp->dev = dev; + + INIT_LIST_HEAD(&ccp->cmd); + INIT_LIST_HEAD(&ccp->backlog); + + spin_lock_init(&ccp->cmd_lock); + mutex_init(&ccp->req_mutex); + mutex_init(&ccp->ksb_mutex); + ccp->ksb_count = KSB_COUNT; + ccp->ksb_start = 0; + + return ccp; +} + +/** + * ccp_init - initialize the CCP device + * + * @ccp: ccp_device struct + */ +int ccp_init(struct ccp_device *ccp) +{ + struct device *dev = ccp->dev; + struct ccp_cmd_queue *cmd_q; + struct dma_pool *dma_pool; + char dma_pool_name[MAX_DMAPOOL_NAME_LEN]; + unsigned int qmr, qim, i; + int ret; + + /* Find available queues */ + qim = 0; + qmr = ioread32(ccp->io_regs + Q_MASK_REG); + for (i = 0; i < MAX_HW_QUEUES; i++) { + if (!(qmr & (1 << i))) + continue; + + /* Allocate a dma pool for this queue */ + snprintf(dma_pool_name, sizeof(dma_pool_name), "ccp_q%d", i); + dma_pool = dma_pool_create(dma_pool_name, dev, + CCP_DMAPOOL_MAX_SIZE, + CCP_DMAPOOL_ALIGN, 0); + if (!dma_pool) { + dev_err(dev, "unable to allocate dma pool\n"); + ret = -ENOMEM; + goto e_pool; + } + + cmd_q = &ccp->cmd_q[ccp->cmd_q_count]; + ccp->cmd_q_count++; + + cmd_q->ccp = ccp; + cmd_q->id = i; + cmd_q->dma_pool = dma_pool; + + /* Reserve 2 KSB regions for the queue */ + cmd_q->ksb_key = KSB_START + ccp->ksb_start++; + cmd_q->ksb_ctx = KSB_START + ccp->ksb_start++; + ccp->ksb_count -= 2; + + /* Preset some register values and masks that are queue + * number dependent + */ + cmd_q->reg_status = ccp->io_regs + CMD_Q_STATUS_BASE + + (CMD_Q_STATUS_INCR * i); + cmd_q->reg_int_status = ccp->io_regs + CMD_Q_INT_STATUS_BASE + + (CMD_Q_STATUS_INCR * i); + cmd_q->int_ok = 1 << (i * 2); + cmd_q->int_err = 1 << ((i * 2) + 1); + + cmd_q->free_slots = CMD_Q_DEPTH(ioread32(cmd_q->reg_status)); + + init_waitqueue_head(&cmd_q->int_queue); + + /* Build queue interrupt mask (two interrupts per queue) */ + qim |= cmd_q->int_ok | cmd_q->int_err; + + dev_dbg(dev, "queue #%u available\n", i); + } + if (ccp->cmd_q_count == 0) { + dev_notice(dev, "no command queues available\n"); + ret = -EIO; + goto e_pool; + } + dev_notice(dev, "%u command queues available\n", ccp->cmd_q_count); + + /* Disable and clear interrupts until ready */ + iowrite32(0x00, ccp->io_regs + IRQ_MASK_REG); + for (i = 0; i < ccp->cmd_q_count; i++) { + cmd_q = &ccp->cmd_q[i]; + + ioread32(cmd_q->reg_int_status); + ioread32(cmd_q->reg_status); + } + iowrite32(qim, ccp->io_regs + IRQ_STATUS_REG); + + /* Request an irq */ + ret = ccp->get_irq(ccp); + if (ret) { + dev_err(dev, "unable to allocate an IRQ\n"); + goto e_pool; + } + + /* Initialize the queues used to wait for KSB space and suspend */ + init_waitqueue_head(&ccp->ksb_queue); + init_waitqueue_head(&ccp->suspend_queue); + + /* Create a kthread for each queue */ + for (i = 0; i < ccp->cmd_q_count; i++) { + struct task_struct *kthread; + + cmd_q = &ccp->cmd_q[i]; + + kthread = kthread_create(ccp_cmd_queue_thread, cmd_q, + "ccp-q%u", cmd_q->id); + if (IS_ERR(kthread)) { + dev_err(dev, "error creating queue thread (%ld)\n", + PTR_ERR(kthread)); + ret = PTR_ERR(kthread); + goto e_kthread; + } + + cmd_q->kthread = kthread; + wake_up_process(kthread); + } + + /* Register the RNG */ + ccp->hwrng.name = "ccp-rng"; + ccp->hwrng.read = ccp_trng_read; + ret = hwrng_register(&ccp->hwrng); + if (ret) { + dev_err(dev, "error registering hwrng (%d)\n", ret); + goto e_kthread; + } + + /* Make the device struct available before enabling interrupts */ + ccp_add_device(ccp); + + /* Enable interrupts */ + iowrite32(qim, ccp->io_regs + IRQ_MASK_REG); + + return 0; + +e_kthread: + for (i = 0; i < ccp->cmd_q_count; i++) + if (ccp->cmd_q[i].kthread) + kthread_stop(ccp->cmd_q[i].kthread); + + ccp->free_irq(ccp); + +e_pool: + for (i = 0; i < ccp->cmd_q_count; i++) + dma_pool_destroy(ccp->cmd_q[i].dma_pool); + + return ret; +} + +/** + * ccp_destroy - tear down the CCP device + * + * @ccp: ccp_device struct + */ +void ccp_destroy(struct ccp_device *ccp) +{ + struct ccp_cmd_queue *cmd_q; + struct ccp_cmd *cmd; + unsigned int qim, i; + + /* Remove general access to the device struct */ + ccp_del_device(ccp); + + /* Unregister the RNG */ + hwrng_unregister(&ccp->hwrng); + + /* Stop the queue kthreads */ + for (i = 0; i < ccp->cmd_q_count; i++) + if (ccp->cmd_q[i].kthread) + kthread_stop(ccp->cmd_q[i].kthread); + + /* Build queue interrupt mask (two interrupt masks per queue) */ + qim = 0; + for (i = 0; i < ccp->cmd_q_count; i++) { + cmd_q = &ccp->cmd_q[i]; + qim |= cmd_q->int_ok | cmd_q->int_err; + } + + /* Disable and clear interrupts */ + iowrite32(0x00, ccp->io_regs + IRQ_MASK_REG); + for (i = 0; i < ccp->cmd_q_count; i++) { + cmd_q = &ccp->cmd_q[i]; + + ioread32(cmd_q->reg_int_status); + ioread32(cmd_q->reg_status); + } + iowrite32(qim, ccp->io_regs + IRQ_STATUS_REG); + + ccp->free_irq(ccp); + + for (i = 0; i < ccp->cmd_q_count; i++) + dma_pool_destroy(ccp->cmd_q[i].dma_pool); + + /* Flush the cmd and backlog queue */ + while (!list_empty(&ccp->cmd)) { + /* Invoke the callback directly with an error code */ + cmd = list_first_entry(&ccp->cmd, struct ccp_cmd, entry); + list_del(&cmd->entry); + cmd->callback(cmd->data, -ENODEV); + } + while (!list_empty(&ccp->backlog)) { + /* Invoke the callback directly with an error code */ + cmd = list_first_entry(&ccp->backlog, struct ccp_cmd, entry); + list_del(&cmd->entry); + cmd->callback(cmd->data, -ENODEV); + } +} + +/** + * ccp_irq_handler - handle interrupts generated by the CCP device + * + * @irq: the irq associated with the interrupt + * @data: the data value supplied when the irq was created + */ +irqreturn_t ccp_irq_handler(int irq, void *data) +{ + struct device *dev = data; + struct ccp_device *ccp = dev_get_drvdata(dev); + struct ccp_cmd_queue *cmd_q; + u32 q_int, status; + unsigned int i; + + status = ioread32(ccp->io_regs + IRQ_STATUS_REG); + + for (i = 0; i < ccp->cmd_q_count; i++) { + cmd_q = &ccp->cmd_q[i]; + + q_int = status & (cmd_q->int_ok | cmd_q->int_err); + if (q_int) { + cmd_q->int_status = status; + cmd_q->q_status = ioread32(cmd_q->reg_status); + cmd_q->q_int_status = ioread32(cmd_q->reg_int_status); + + /* On error, only save the first error value */ + if ((q_int & cmd_q->int_err) && !cmd_q->cmd_error) + cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status); + + cmd_q->int_rcvd = 1; + + /* Acknowledge the interrupt and wake the kthread */ + iowrite32(q_int, ccp->io_regs + IRQ_STATUS_REG); + wake_up_interruptible(&cmd_q->int_queue); + } + } + + return IRQ_HANDLED; +} + +#ifdef CONFIG_PM +bool ccp_queues_suspended(struct ccp_device *ccp) +{ + unsigned int suspended = 0; + unsigned long flags; + unsigned int i; + + spin_lock_irqsave(&ccp->cmd_lock, flags); + + for (i = 0; i < ccp->cmd_q_count; i++) + if (ccp->cmd_q[i].suspended) + suspended++; + + spin_unlock_irqrestore(&ccp->cmd_lock, flags); + + return ccp->cmd_q_count == suspended; +} +#endif + +static const struct x86_cpu_id ccp_support[] = { + { X86_VENDOR_AMD, 22, }, +}; + +static int __init ccp_mod_init(void) +{ + struct cpuinfo_x86 *cpuinfo = &boot_cpu_data; + int ret; + + if (!x86_match_cpu(ccp_support)) + return -ENODEV; + + switch (cpuinfo->x86) { + case 22: + if ((cpuinfo->x86_model < 48) || (cpuinfo->x86_model > 63)) + return -ENODEV; + + ret = ccp_pci_init(); + if (ret) + return ret; + + /* Don't leave the driver loaded if init failed */ + if (!ccp_get_device()) { + ccp_pci_exit(); + return -ENODEV; + } + + return 0; + + break; + } + + return -ENODEV; +} + +static void __exit ccp_mod_exit(void) +{ + struct cpuinfo_x86 *cpuinfo = &boot_cpu_data; + + switch (cpuinfo->x86) { + case 22: + ccp_pci_exit(); + break; + } +} + +module_init(ccp_mod_init); +module_exit(ccp_mod_exit); diff --git a/drivers/crypto/ccp/ccp-dev.h b/drivers/crypto/ccp/ccp-dev.h new file mode 100644 index 000000000000..7ec536e702ec --- /dev/null +++ b/drivers/crypto/ccp/ccp-dev.h @@ -0,0 +1,272 @@ +/* + * AMD Cryptographic Coprocessor (CCP) driver + * + * Copyright (C) 2013 Advanced Micro Devices, Inc. + * + * Author: Tom Lendacky <thomas.lendacky@amd.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#ifndef __CCP_DEV_H__ +#define __CCP_DEV_H__ + +#include <linux/device.h> +#include <linux/pci.h> +#include <linux/spinlock.h> +#include <linux/mutex.h> +#include <linux/list.h> +#include <linux/wait.h> +#include <linux/dmapool.h> +#include <linux/hw_random.h> + + +#define IO_OFFSET 0x20000 + +#define MAX_DMAPOOL_NAME_LEN 32 + +#define MAX_HW_QUEUES 5 +#define MAX_CMD_QLEN 100 + +#define TRNG_RETRIES 10 + + +/****** Register Mappings ******/ +#define Q_MASK_REG 0x000 +#define TRNG_OUT_REG 0x00c +#define IRQ_MASK_REG 0x040 +#define IRQ_STATUS_REG 0x200 + +#define DEL_CMD_Q_JOB 0x124 +#define DEL_Q_ACTIVE 0x00000200 +#define DEL_Q_ID_SHIFT 6 + +#define CMD_REQ0 0x180 +#define CMD_REQ_INCR 0x04 + +#define CMD_Q_STATUS_BASE 0x210 +#define CMD_Q_INT_STATUS_BASE 0x214 +#define CMD_Q_STATUS_INCR 0x20 + +#define CMD_Q_CACHE 0x228 +#define CMD_Q_CACHE_INC 0x20 + +#define CMD_Q_ERROR(__qs) ((__qs) & 0x0000003f); +#define CMD_Q_DEPTH(__qs) (((__qs) >> 12) & 0x0000000f); + +/****** REQ0 Related Values ******/ +#define REQ0_WAIT_FOR_WRITE 0x00000004 +#define REQ0_INT_ON_COMPLETE 0x00000002 +#define REQ0_STOP_ON_COMPLETE 0x00000001 + +#define REQ0_CMD_Q_SHIFT 9 +#define REQ0_JOBID_SHIFT 3 + +/****** REQ1 Related Values ******/ +#define REQ1_PROTECT_SHIFT 27 +#define REQ1_ENGINE_SHIFT 23 +#define REQ1_KEY_KSB_SHIFT 2 + +#define REQ1_EOM 0x00000002 +#define REQ1_INIT 0x00000001 + +/* AES Related Values */ +#define REQ1_AES_TYPE_SHIFT 21 +#define REQ1_AES_MODE_SHIFT 18 +#define REQ1_AES_ACTION_SHIFT 17 +#define REQ1_AES_CFB_SIZE_SHIFT 10 + +/* XTS-AES Related Values */ +#define REQ1_XTS_AES_SIZE_SHIFT 10 + +/* SHA Related Values */ +#define REQ1_SHA_TYPE_SHIFT 21 + +/* RSA Related Values */ +#define REQ1_RSA_MOD_SIZE_SHIFT 10 + +/* Pass-Through Related Values */ +#define REQ1_PT_BW_SHIFT 12 +#define REQ1_PT_BS_SHIFT 10 + +/* ECC Related Values */ +#define REQ1_ECC_AFFINE_CONVERT 0x00200000 +#define REQ1_ECC_FUNCTION_SHIFT 18 + +/****** REQ4 Related Values ******/ +#define REQ4_KSB_SHIFT 18 +#define REQ4_MEMTYPE_SHIFT 16 + +/****** REQ6 Related Values ******/ +#define REQ6_MEMTYPE_SHIFT 16 + + +/****** Key Storage Block ******/ +#define KSB_START 77 +#define KSB_END 127 +#define KSB_COUNT (KSB_END - KSB_START + 1) +#define CCP_KSB_BITS 256 +#define CCP_KSB_BYTES 32 + +#define CCP_JOBID_MASK 0x0000003f + +#define CCP_DMAPOOL_MAX_SIZE 64 +#define CCP_DMAPOOL_ALIGN (1 << 5) + +#define CCP_REVERSE_BUF_SIZE 64 + +#define CCP_AES_KEY_KSB_COUNT 1 +#define CCP_AES_CTX_KSB_COUNT 1 + +#define CCP_XTS_AES_KEY_KSB_COUNT 1 +#define CCP_XTS_AES_CTX_KSB_COUNT 1 + +#define CCP_SHA_KSB_COUNT 1 + +#define CCP_RSA_MAX_WIDTH 4096 + +#define CCP_PASSTHRU_BLOCKSIZE 256 +#define CCP_PASSTHRU_MASKSIZE 32 +#define CCP_PASSTHRU_KSB_COUNT 1 + +#define CCP_ECC_MODULUS_BYTES 48 /* 384-bits */ +#define CCP_ECC_MAX_OPERANDS 6 +#define CCP_ECC_MAX_OUTPUTS 3 +#define CCP_ECC_SRC_BUF_SIZE 448 +#define CCP_ECC_DST_BUF_SIZE 192 +#define CCP_ECC_OPERAND_SIZE 64 +#define CCP_ECC_OUTPUT_SIZE 64 +#define CCP_ECC_RESULT_OFFSET 60 +#define CCP_ECC_RESULT_SUCCESS 0x0001 + + +struct ccp_device; +struct ccp_cmd; + +struct ccp_cmd_queue { + struct ccp_device *ccp; + + /* Queue identifier */ + u32 id; + + /* Queue dma pool */ + struct dma_pool *dma_pool; + + /* Queue reserved KSB regions */ + u32 ksb_key; + u32 ksb_ctx; + + /* Queue processing thread */ + struct task_struct *kthread; + unsigned int active; + unsigned int suspended; + + /* Number of free command slots available */ + unsigned int free_slots; + + /* Interrupt masks */ + u32 int_ok; + u32 int_err; + + /* Register addresses for queue */ + void __iomem *reg_status; + void __iomem *reg_int_status; + + /* Status values from job */ + u32 int_status; + u32 q_status; + u32 q_int_status; + u32 cmd_error; + + /* Interrupt wait queue */ + wait_queue_head_t int_queue; + unsigned int int_rcvd; +} ____cacheline_aligned; + +struct ccp_device { + struct device *dev; + + /* + * Bus specific device information + */ + void *dev_specific; + int (*get_irq)(struct ccp_device *ccp); + void (*free_irq)(struct ccp_device *ccp); + + /* + * I/O area used for device communication. The register mapping + * starts at an offset into the mapped bar. + * The CMD_REQx registers and the Delete_Cmd_Queue_Job register + * need to be protected while a command queue thread is accessing + * them. + */ + struct mutex req_mutex ____cacheline_aligned; + void __iomem *io_map; + void __iomem *io_regs; + + /* + * Master lists that all cmds are queued on. Because there can be + * more than one CCP command queue that can process a cmd a separate + * backlog list is neeeded so that the backlog completion call + * completes before the cmd is available for execution. + */ + spinlock_t cmd_lock ____cacheline_aligned; + unsigned int cmd_count; + struct list_head cmd; + struct list_head backlog; + + /* + * The command queues. These represent the queues available on the + * CCP that are available for processing cmds + */ + struct ccp_cmd_queue cmd_q[MAX_HW_QUEUES]; + unsigned int cmd_q_count; + + /* + * Support for the CCP True RNG + */ + struct hwrng hwrng; + unsigned int hwrng_retries; + + /* + * A counter used to generate job-ids for cmds submitted to the CCP + */ + atomic_t current_id ____cacheline_aligned; + + /* + * The CCP uses key storage blocks (KSB) to maintain context for certain + * operations. To prevent multiple cmds from using the same KSB range + * a command queue reserves a KSB range for the duration of the cmd. + * Each queue, will however, reserve 2 KSB blocks for operations that + * only require single KSB entries (eg. AES context/iv and key) in order + * to avoid allocation contention. This will reserve at most 10 KSB + * entries, leaving 40 KSB entries available for dynamic allocation. + */ + struct mutex ksb_mutex ____cacheline_aligned; + DECLARE_BITMAP(ksb, KSB_COUNT); + wait_queue_head_t ksb_queue; + unsigned int ksb_avail; + unsigned int ksb_count; + u32 ksb_start; + + /* Suspend support */ + unsigned int suspending; + wait_queue_head_t suspend_queue; +}; + + +int ccp_pci_init(void); +void ccp_pci_exit(void); + +struct ccp_device *ccp_alloc_struct(struct device *dev); +int ccp_init(struct ccp_device *ccp); +void ccp_destroy(struct ccp_device *ccp); +bool ccp_queues_suspended(struct ccp_device *ccp); + +irqreturn_t ccp_irq_handler(int irq, void *data); + +int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd); + +#endif diff --git a/drivers/crypto/ccp/ccp-ops.c b/drivers/crypto/ccp/ccp-ops.c new file mode 100644 index 000000000000..71ed3ade7e12 --- /dev/null +++ b/drivers/crypto/ccp/ccp-ops.c @@ -0,0 +1,2024 @@ +/* + * AMD Cryptographic Coprocessor (CCP) driver + * + * Copyright (C) 2013 Advanced Micro Devices, Inc. + * + * Author: Tom Lendacky <thomas.lendacky@amd.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/pci.h> +#include <linux/pci_ids.h> +#include <linux/kthread.h> +#include <linux/sched.h> +#include <linux/interrupt.h> +#include <linux/spinlock.h> +#include <linux/mutex.h> +#include <linux/delay.h> +#include <linux/ccp.h> +#include <linux/scatterlist.h> +#include <crypto/scatterwalk.h> + +#include "ccp-dev.h" + + +enum ccp_memtype { + CCP_MEMTYPE_SYSTEM = 0, + CCP_MEMTYPE_KSB, + CCP_MEMTYPE_LOCAL, + CCP_MEMTYPE__LAST, +}; + +struct ccp_dma_info { + dma_addr_t address; + unsigned int offset; + unsigned int length; + enum dma_data_direction dir; +}; + +struct ccp_dm_workarea { + struct device *dev; + struct dma_pool *dma_pool; + unsigned int length; + + u8 *address; + struct ccp_dma_info dma; +}; + +struct ccp_sg_workarea { + struct scatterlist *sg; + unsigned int nents; + unsigned int length; + + struct scatterlist *dma_sg; + struct device *dma_dev; + unsigned int dma_count; + enum dma_data_direction dma_dir; + + unsigned int sg_used; + + u64 bytes_left; +}; + +struct ccp_data { + struct ccp_sg_workarea sg_wa; + struct ccp_dm_workarea dm_wa; +}; + +struct ccp_mem { + enum ccp_memtype type; + union { + struct ccp_dma_info dma; + u32 ksb; + } u; +}; + +struct ccp_aes_op { + enum ccp_aes_type type; + enum ccp_aes_mode mode; + enum ccp_aes_action action; +}; + +struct ccp_xts_aes_op { + enum ccp_aes_action action; + enum ccp_xts_aes_unit_size unit_size; +}; + +struct ccp_sha_op { + enum ccp_sha_type type; + u64 msg_bits; +}; + +struct ccp_rsa_op { + u32 mod_size; + u32 input_len; +}; + +struct ccp_passthru_op { + enum ccp_passthru_bitwise bit_mod; + enum ccp_passthru_byteswap byte_swap; +}; + +struct ccp_ecc_op { + enum ccp_ecc_function function; +}; + +struct ccp_op { + struct ccp_cmd_queue *cmd_q; + + u32 jobid; + u32 ioc; + u32 soc; + u32 ksb_key; + u32 ksb_ctx; + u32 init; + u32 eom; + + struct ccp_mem src; + struct ccp_mem dst; + + union { + struct ccp_aes_op aes; + struct ccp_xts_aes_op xts; + struct ccp_sha_op sha; + struct ccp_rsa_op rsa; + struct ccp_passthru_op passthru; + struct ccp_ecc_op ecc; + } u; +}; + +/* The CCP cannot perform zero-length sha operations so the caller + * is required to buffer data for the final operation. However, a + * sha operation for a message with a total length of zero is valid + * so known values are required to supply the result. + */ +static const u8 ccp_sha1_zero[CCP_SHA_CTXSIZE] = { + 0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, + 0x32, 0x55, 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90, + 0xaf, 0xd8, 0x07, 0x09, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +}; + +static const u8 ccp_sha224_zero[CCP_SHA_CTXSIZE] = { + 0xd1, 0x4a, 0x02, 0x8c, 0x2a, 0x3a, 0x2b, 0xc9, + 0x47, 0x61, 0x02, 0xbb, 0x28, 0x82, 0x34, 0xc4, + 0x15, 0xa2, 0xb0, 0x1f, 0x82, 0x8e, 0xa6, 0x2a, + 0xc5, 0xb3, 0xe4, 0x2f, 0x00, 0x00, 0x00, 0x00, +}; + +static const u8 ccp_sha256_zero[CCP_SHA_CTXSIZE] = { + 0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, + 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, + 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, + 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55, +}; + +static u32 ccp_addr_lo(struct ccp_dma_info *info) +{ + return lower_32_bits(info->address + info->offset); +} + +static u32 ccp_addr_hi(struct ccp_dma_info *info) +{ + return upper_32_bits(info->address + info->offset) & 0x0000ffff; +} + +static int ccp_do_cmd(struct ccp_op *op, u32 *cr, unsigned int cr_count) +{ + struct ccp_cmd_queue *cmd_q = op->cmd_q; + struct ccp_device *ccp = cmd_q->ccp; + void __iomem *cr_addr; + u32 cr0, cmd; + unsigned int i; + int ret = 0; + + /* We could read a status register to see how many free slots + * are actually available, but reading that register resets it + * and you could lose some error information. + */ + cmd_q->free_slots--; + + cr0 = (cmd_q->id << REQ0_CMD_Q_SHIFT) + | (op->jobid << REQ0_JOBID_SHIFT) + | REQ0_WAIT_FOR_WRITE; + + if (op->soc) + cr0 |= REQ0_STOP_ON_COMPLETE + | REQ0_INT_ON_COMPLETE; + + if (op->ioc || !cmd_q->free_slots) + cr0 |= REQ0_INT_ON_COMPLETE; + + /* Start at CMD_REQ1 */ + cr_addr = ccp->io_regs + CMD_REQ0 + CMD_REQ_INCR; + + mutex_lock(&ccp->req_mutex); + + /* Write CMD_REQ1 through CMD_REQx first */ + for (i = 0; i < cr_count; i++, cr_addr += CMD_REQ_INCR) + iowrite32(*(cr + i), cr_addr); + + /* Tell the CCP to start */ + wmb(); + iowrite32(cr0, ccp->io_regs + CMD_REQ0); + + mutex_unlock(&ccp->req_mutex); + + if (cr0 & REQ0_INT_ON_COMPLETE) { + /* Wait for the job to complete */ + ret = wait_event_interruptible(cmd_q->int_queue, + cmd_q->int_rcvd); + if (ret || cmd_q->cmd_error) { + /* On error delete all related jobs from the queue */ + cmd = (cmd_q->id << DEL_Q_ID_SHIFT) + | op->jobid; + + iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB); + + if (!ret) + ret = -EIO; + } else if (op->soc) { + /* Delete just head job from the queue on SoC */ + cmd = DEL_Q_ACTIVE + | (cmd_q->id << DEL_Q_ID_SHIFT) + | op->jobid; + + iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB); + } + + cmd_q->free_slots = CMD_Q_DEPTH(cmd_q->q_status); + + cmd_q->int_rcvd = 0; + } + + return ret; +} + +static int ccp_perform_aes(struct ccp_op *op) +{ + u32 cr[6]; + + /* Fill out the register contents for REQ1 through REQ6 */ + cr[0] = (CCP_ENGINE_AES << REQ1_ENGINE_SHIFT) + | (op->u.aes.type << REQ1_AES_TYPE_SHIFT) + | (op->u.aes.mode << REQ1_AES_MODE_SHIFT) + | (op->u.aes.action << REQ1_AES_ACTION_SHIFT) + | (op->ksb_key << REQ1_KEY_KSB_SHIFT); + cr[1] = op->src.u.dma.length - 1; + cr[2] = ccp_addr_lo(&op->src.u.dma); + cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT) + | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) + | ccp_addr_hi(&op->src.u.dma); + cr[4] = ccp_addr_lo(&op->dst.u.dma); + cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) + | ccp_addr_hi(&op->dst.u.dma); + + if (op->u.aes.mode == CCP_AES_MODE_CFB) + cr[0] |= ((0x7f) << REQ1_AES_CFB_SIZE_SHIFT); + + if (op->eom) + cr[0] |= REQ1_EOM; + + if (op->init) + cr[0] |= REQ1_INIT; + + return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); +} + +static int ccp_perform_xts_aes(struct ccp_op *op) +{ + u32 cr[6]; + + /* Fill out the register contents for REQ1 through REQ6 */ + cr[0] = (CCP_ENGINE_XTS_AES_128 << REQ1_ENGINE_SHIFT) + | (op->u.xts.action << REQ1_AES_ACTION_SHIFT) + | (op->u.xts.unit_size << REQ1_XTS_AES_SIZE_SHIFT) + | (op->ksb_key << REQ1_KEY_KSB_SHIFT); + cr[1] = op->src.u.dma.length - 1; + cr[2] = ccp_addr_lo(&op->src.u.dma); + cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT) + | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) + | ccp_addr_hi(&op->src.u.dma); + cr[4] = ccp_addr_lo(&op->dst.u.dma); + cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) + | ccp_addr_hi(&op->dst.u.dma); + + if (op->eom) + cr[0] |= REQ1_EOM; + + if (op->init) + cr[0] |= REQ1_INIT; + + return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); +} + +static int ccp_perform_sha(struct ccp_op *op) +{ + u32 cr[6]; + + /* Fill out the register contents for REQ1 through REQ6 */ + cr[0] = (CCP_ENGINE_SHA << REQ1_ENGINE_SHIFT) + | (op->u.sha.type << REQ1_SHA_TYPE_SHIFT) + | REQ1_INIT; + cr[1] = op->src.u.dma.length - 1; + cr[2] = ccp_addr_lo(&op->src.u.dma); + cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT) + | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) + | ccp_addr_hi(&op->src.u.dma); + + if (op->eom) { + cr[0] |= REQ1_EOM; + cr[4] = lower_32_bits(op->u.sha.msg_bits); + cr[5] = upper_32_bits(op->u.sha.msg_bits); + } else { + cr[4] = 0; + cr[5] = 0; + } + + return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); +} + +static int ccp_perform_rsa(struct ccp_op *op) +{ + u32 cr[6]; + + /* Fill out the register contents for REQ1 through REQ6 */ + cr[0] = (CCP_ENGINE_RSA << REQ1_ENGINE_SHIFT) + | (op->u.rsa.mod_size << REQ1_RSA_MOD_SIZE_SHIFT) + | (op->ksb_key << REQ1_KEY_KSB_SHIFT) + | REQ1_EOM; + cr[1] = op->u.rsa.input_len - 1; + cr[2] = ccp_addr_lo(&op->src.u.dma); + cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT) + | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) + | ccp_addr_hi(&op->src.u.dma); + cr[4] = ccp_addr_lo(&op->dst.u.dma); + cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) + | ccp_addr_hi(&op->dst.u.dma); + + return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); +} + +static int ccp_perform_passthru(struct ccp_op *op) +{ + u32 cr[6]; + + /* Fill out the register contents for REQ1 through REQ6 */ + cr[0] = (CCP_ENGINE_PASSTHRU << REQ1_ENGINE_SHIFT) + | (op->u.passthru.bit_mod << REQ1_PT_BW_SHIFT) + | (op->u.passthru.byte_swap << REQ1_PT_BS_SHIFT); + + if (op->src.type == CCP_MEMTYPE_SYSTEM) + cr[1] = op->src.u.dma.length - 1; + else + cr[1] = op->dst.u.dma.length - 1; + + if (op->src.type == CCP_MEMTYPE_SYSTEM) { + cr[2] = ccp_addr_lo(&op->src.u.dma); + cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) + | ccp_addr_hi(&op->src.u.dma); + + if (op->u.passthru.bit_mod != CCP_PASSTHRU_BITWISE_NOOP) + cr[3] |= (op->ksb_key << REQ4_KSB_SHIFT); + } else { + cr[2] = op->src.u.ksb * CCP_KSB_BYTES; + cr[3] = (CCP_MEMTYPE_KSB << REQ4_MEMTYPE_SHIFT); + } + + if (op->dst.type == CCP_MEMTYPE_SYSTEM) { + cr[4] = ccp_addr_lo(&op->dst.u.dma); + cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) + | ccp_addr_hi(&op->dst.u.dma); + } else { + cr[4] = op->dst.u.ksb * CCP_KSB_BYTES; + cr[5] = (CCP_MEMTYPE_KSB << REQ6_MEMTYPE_SHIFT); + } + + if (op->eom) + cr[0] |= REQ1_EOM; + + return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); +} + +static int ccp_perform_ecc(struct ccp_op *op) +{ + u32 cr[6]; + + /* Fill out the register contents for REQ1 through REQ6 */ + cr[0] = REQ1_ECC_AFFINE_CONVERT + | (CCP_ENGINE_ECC << REQ1_ENGINE_SHIFT) + | (op->u.ecc.function << REQ1_ECC_FUNCTION_SHIFT) + | REQ1_EOM; + cr[1] = op->src.u.dma.length - 1; + cr[2] = ccp_addr_lo(&op->src.u.dma); + cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) + | ccp_addr_hi(&op->src.u.dma); + cr[4] = ccp_addr_lo(&op->dst.u.dma); + cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) + | ccp_addr_hi(&op->dst.u.dma); + + return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); +} + +static u32 ccp_alloc_ksb(struct ccp_device *ccp, unsigned int count) +{ + int start; + + for (;;) { + mutex_lock(&ccp->ksb_mutex); + + start = (u32)bitmap_find_next_zero_area(ccp->ksb, + ccp->ksb_count, + ccp->ksb_start, + count, 0); + if (start <= ccp->ksb_count) { + bitmap_set(ccp->ksb, start, count); + + mutex_unlock(&ccp->ksb_mutex); + break; + } + + ccp->ksb_avail = 0; + + mutex_unlock(&ccp->ksb_mutex); + + /* Wait for KSB entries to become available */ + if (wait_event_interruptible(ccp->ksb_queue, ccp->ksb_avail)) + return 0; + } + + return KSB_START + start; +} + +static void ccp_free_ksb(struct ccp_device *ccp, unsigned int start, + unsigned int count) +{ + if (!start) + return; + + mutex_lock(&ccp->ksb_mutex); + + bitmap_clear(ccp->ksb, start - KSB_START, count); + + ccp->ksb_avail = 1; + + mutex_unlock(&ccp->ksb_mutex); + + wake_up_interruptible_all(&ccp->ksb_queue); +} + +static u32 ccp_gen_jobid(struct ccp_device *ccp) +{ + return atomic_inc_return(&ccp->current_id) & CCP_JOBID_MASK; +} + +static void ccp_sg_free(struct ccp_sg_workarea *wa) +{ + if (wa->dma_count) + dma_unmap_sg(wa->dma_dev, wa->dma_sg, wa->nents, wa->dma_dir); + + wa->dma_count = 0; +} + +static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev, + struct scatterlist *sg, u64 len, + enum dma_data_direction dma_dir) +{ + memset(wa, 0, sizeof(*wa)); + + wa->sg = sg; + if (!sg) + return 0; + + wa->nents = sg_nents(sg); + wa->length = sg->length; + wa->bytes_left = len; + wa->sg_used = 0; + + if (len == 0) + return 0; + + if (dma_dir == DMA_NONE) + return 0; + + wa->dma_sg = sg; + wa->dma_dev = dev; + wa->dma_dir = dma_dir; + wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir); + if (!wa->dma_count) + return -ENOMEM; + + + return 0; +} + +static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len) +{ + unsigned int nbytes = min_t(u64, len, wa->bytes_left); + + if (!wa->sg) + return; + + wa->sg_used += nbytes; + wa->bytes_left -= nbytes; + if (wa->sg_used == wa->sg->length) { + wa->sg = sg_next(wa->sg); + wa->sg_used = 0; + } +} + +static void ccp_dm_free(struct ccp_dm_workarea *wa) +{ + if (wa->length <= CCP_DMAPOOL_MAX_SIZE) { + if (wa->address) + dma_pool_free(wa->dma_pool, wa->address, + wa->dma.address); + } else { + if (wa->dma.address) + dma_unmap_single(wa->dev, wa->dma.address, wa->length, + wa->dma.dir); + kfree(wa->address); + } + + wa->address = NULL; + wa->dma.address = 0; +} + +static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa, + struct ccp_cmd_queue *cmd_q, + unsigned int len, + enum dma_data_direction dir) +{ + memset(wa, 0, sizeof(*wa)); + + if (!len) + return 0; + + wa->dev = cmd_q->ccp->dev; + wa->length = len; + + if (len <= CCP_DMAPOOL_MAX_SIZE) { + wa->dma_pool = cmd_q->dma_pool; + + wa->address = dma_pool_alloc(wa->dma_pool, GFP_KERNEL, + &wa->dma.address); + if (!wa->address) + return -ENOMEM; + + wa->dma.length = CCP_DMAPOOL_MAX_SIZE; + + memset(wa->address, 0, CCP_DMAPOOL_MAX_SIZE); + } else { + wa->address = kzalloc(len, GFP_KERNEL); + if (!wa->address) + return -ENOMEM; + + wa->dma.address = dma_map_single(wa->dev, wa->address, len, + dir); + if (!wa->dma.address) + return -ENOMEM; + + wa->dma.length = len; + } + wa->dma.dir = dir; + + return 0; +} + +static void ccp_set_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset, + struct scatterlist *sg, unsigned int sg_offset, + unsigned int len) +{ + WARN_ON(!wa->address); + + scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len, + 0); +} + +static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset, + struct scatterlist *sg, unsigned int sg_offset, + unsigned int len) +{ + WARN_ON(!wa->address); + + scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len, + 1); +} + +static void ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa, + struct scatterlist *sg, + unsigned int len, unsigned int se_len, + bool sign_extend) +{ + unsigned int nbytes, sg_offset, dm_offset, ksb_len, i; + u8 buffer[CCP_REVERSE_BUF_SIZE]; + + BUG_ON(se_len > sizeof(buffer)); + + sg_offset = len; + dm_offset = 0; + nbytes = len; + while (nbytes) { + ksb_len = min_t(unsigned int, nbytes, se_len); + sg_offset -= ksb_len; + + scatterwalk_map_and_copy(buffer, sg, sg_offset, ksb_len, 0); + for (i = 0; i < ksb_len; i++) + wa->address[dm_offset + i] = buffer[ksb_len - i - 1]; + + dm_offset += ksb_len; + nbytes -= ksb_len; + + if ((ksb_len != se_len) && sign_extend) { + /* Must sign-extend to nearest sign-extend length */ + if (wa->address[dm_offset - 1] & 0x80) + memset(wa->address + dm_offset, 0xff, + se_len - ksb_len); + } + } +} + +static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa, + struct scatterlist *sg, + unsigned int len) +{ + unsigned int nbytes, sg_offset, dm_offset, ksb_len, i; + u8 buffer[CCP_REVERSE_BUF_SIZE]; + + sg_offset = 0; + dm_offset = len; + nbytes = len; + while (nbytes) { + ksb_len = min_t(unsigned int, nbytes, sizeof(buffer)); + dm_offset -= ksb_len; + + for (i = 0; i < ksb_len; i++) + buffer[ksb_len - i - 1] = wa->address[dm_offset + i]; + scatterwalk_map_and_copy(buffer, sg, sg_offset, ksb_len, 1); + + sg_offset += ksb_len; + nbytes -= ksb_len; + } +} + +static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q) +{ + ccp_dm_free(&data->dm_wa); + ccp_sg_free(&data->sg_wa); +} + +static int ccp_init_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q, + struct scatterlist *sg, u64 sg_len, + unsigned int dm_len, + enum dma_data_direction dir) +{ + int ret; + + memset(data, 0, sizeof(*data)); + + ret = ccp_init_sg_workarea(&data->sg_wa, cmd_q->ccp->dev, sg, sg_len, + dir); + if (ret) + goto e_err; + + ret = ccp_init_dm_workarea(&data->dm_wa, cmd_q, dm_len, dir); + if (ret) + goto e_err; + + return 0; + +e_err: + ccp_free_data(data, cmd_q); + + return ret; +} + +static unsigned int ccp_queue_buf(struct ccp_data *data, unsigned int from) +{ + struct ccp_sg_workarea *sg_wa = &data->sg_wa; + struct ccp_dm_workarea *dm_wa = &data->dm_wa; + unsigned int buf_count, nbytes; + + /* Clear the buffer if setting it */ + if (!from) + memset(dm_wa->address, 0, dm_wa->length); + + if (!sg_wa->sg) + return 0; + + /* Perform the copy operation + * nbytes will always be <= UINT_MAX because dm_wa->length is + * an unsigned int + */ + nbytes = min_t(u64, sg_wa->bytes_left, dm_wa->length); + scatterwalk_map_and_copy(dm_wa->address, sg_wa->sg, sg_wa->sg_used, + nbytes, from); + + /* Update the structures and generate the count */ + buf_count = 0; + while (sg_wa->bytes_left && (buf_count < dm_wa->length)) { + nbytes = min(sg_wa->sg->length - sg_wa->sg_used, + dm_wa->length - buf_count); + nbytes = min_t(u64, sg_wa->bytes_left, nbytes); + + buf_count += nbytes; + ccp_update_sg_workarea(sg_wa, nbytes); + } + + return buf_count; +} + +static unsigned int ccp_fill_queue_buf(struct ccp_data *data) +{ + return ccp_queue_buf(data, 0); +} + +static unsigned int ccp_empty_queue_buf(struct ccp_data *data) +{ + return ccp_queue_buf(data, 1); +} + +static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst, + struct ccp_op *op, unsigned int block_size, + bool blocksize_op) +{ + unsigned int sg_src_len, sg_dst_len, op_len; + + /* The CCP can only DMA from/to one address each per operation. This + * requires that we find the smallest DMA area between the source + * and destination. The resulting len values will always be <= UINT_MAX + * because the dma length is an unsigned int. + */ + sg_src_len = sg_dma_len(src->sg_wa.sg) - src->sg_wa.sg_used; + sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len); + + if (dst) { + sg_dst_len = sg_dma_len(dst->sg_wa.sg) - dst->sg_wa.sg_used; + sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len); + op_len = min(sg_src_len, sg_dst_len); + } else + op_len = sg_src_len; + + /* The data operation length will be at least block_size in length + * or the smaller of available sg room remaining for the source or + * the destination + */ + op_len = max(op_len, block_size); + + /* Unless we have to buffer data, there's no reason to wait */ + op->soc = 0; + + if (sg_src_len < block_size) { + /* Not enough data in the sg element, so it + * needs to be buffered into a blocksize chunk + */ + int cp_len = ccp_fill_queue_buf(src); + + op->soc = 1; + op->src.u.dma.address = src->dm_wa.dma.address; + op->src.u.dma.offset = 0; + op->src.u.dma.length = (blocksize_op) ? block_size : cp_len; + } else { + /* Enough data in the sg element, but we need to + * adjust for any previously copied data + */ + op->src.u.dma.address = sg_dma_address(src->sg_wa.sg); + op->src.u.dma.offset = src->sg_wa.sg_used; + op->src.u.dma.length = op_len & ~(block_size - 1); + + ccp_update_sg_workarea(&src->sg_wa, op->src.u.dma.length); + } + + if (dst) { + if (sg_dst_len < block_size) { + /* Not enough room in the sg element or we're on the + * last piece of data (when using padding), so the + * output needs to be buffered into a blocksize chunk + */ + op->soc = 1; + op->dst.u.dma.address = dst->dm_wa.dma.address; + op->dst.u.dma.offset = 0; + op->dst.u.dma.length = op->src.u.dma.length; + } else { + /* Enough room in the sg element, but we need to + * adjust for any previously used area + */ + op->dst.u.dma.address = sg_dma_address(dst->sg_wa.sg); + op->dst.u.dma.offset = dst->sg_wa.sg_used; + op->dst.u.dma.length = op->src.u.dma.length; + } + } +} + +static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst, + struct ccp_op *op) +{ + op->init = 0; + + if (dst) { + if (op->dst.u.dma.address == dst->dm_wa.dma.address) + ccp_empty_queue_buf(dst); + else + ccp_update_sg_workarea(&dst->sg_wa, + op->dst.u.dma.length); + } +} + +static int ccp_copy_to_from_ksb(struct ccp_cmd_queue *cmd_q, + struct ccp_dm_workarea *wa, u32 jobid, u32 ksb, + u32 byte_swap, bool from) +{ + struct ccp_op op; + + memset(&op, 0, sizeof(op)); + + op.cmd_q = cmd_q; + op.jobid = jobid; + op.eom = 1; + + if (from) { + op.soc = 1; + op.src.type = CCP_MEMTYPE_KSB; + op.src.u.ksb = ksb; + op.dst.type = CCP_MEMTYPE_SYSTEM; + op.dst.u.dma.address = wa->dma.address; + op.dst.u.dma.length = wa->length; + } else { + op.src.type = CCP_MEMTYPE_SYSTEM; + op.src.u.dma.address = wa->dma.address; + op.src.u.dma.length = wa->length; + op.dst.type = CCP_MEMTYPE_KSB; + op.dst.u.ksb = ksb; + } + + op.u.passthru.byte_swap = byte_swap; + + return ccp_perform_passthru(&op); +} + +static int ccp_copy_to_ksb(struct ccp_cmd_queue *cmd_q, + struct ccp_dm_workarea *wa, u32 jobid, u32 ksb, + u32 byte_swap) +{ + return ccp_copy_to_from_ksb(cmd_q, wa, jobid, ksb, byte_swap, false); +} + +static int ccp_copy_from_ksb(struct ccp_cmd_queue *cmd_q, + struct ccp_dm_workarea *wa, u32 jobid, u32 ksb, + u32 byte_swap) +{ + return ccp_copy_to_from_ksb(cmd_q, wa, jobid, ksb, byte_swap, true); +} + +static int ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q, + struct ccp_cmd *cmd) +{ + struct ccp_aes_engine *aes = &cmd->u.aes; + struct ccp_dm_workarea key, ctx; + struct ccp_data src; + struct ccp_op op; + unsigned int dm_offset; + int ret; + + if (!((aes->key_len == AES_KEYSIZE_128) || + (aes->key_len == AES_KEYSIZE_192) || + (aes->key_len == AES_KEYSIZE_256))) + return -EINVAL; + + if (aes->src_len & (AES_BLOCK_SIZE - 1)) + return -EINVAL; + + if (aes->iv_len != AES_BLOCK_SIZE) + return -EINVAL; + + if (!aes->key || !aes->iv || !aes->src) + return -EINVAL; + + if (aes->cmac_final) { + if (aes->cmac_key_len != AES_BLOCK_SIZE) + return -EINVAL; + + if (!aes->cmac_key) + return -EINVAL; + } + + BUILD_BUG_ON(CCP_AES_KEY_KSB_COUNT != 1); + BUILD_BUG_ON(CCP_AES_CTX_KSB_COUNT != 1); + + ret = -EIO; + memset(&op, 0, sizeof(op)); + op.cmd_q = cmd_q; + op.jobid = ccp_gen_jobid(cmd_q->ccp); + op.ksb_key = cmd_q->ksb_key; + op.ksb_ctx = cmd_q->ksb_ctx; + op.init = 1; + op.u.aes.type = aes->type; + op.u.aes.mode = aes->mode; + op.u.aes.action = aes->action; + + /* All supported key sizes fit in a single (32-byte) KSB entry + * and must be in little endian format. Use the 256-bit byte + * swap passthru option to convert from big endian to little + * endian. + */ + ret = ccp_init_dm_workarea(&key, cmd_q, + CCP_AES_KEY_KSB_COUNT * CCP_KSB_BYTES, + DMA_TO_DEVICE); + if (ret) + return ret; + + dm_offset = CCP_KSB_BYTES - aes->key_len; + ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len); + ret = ccp_copy_to_ksb(cmd_q, &key, op.jobid, op.ksb_key, + CCP_PASSTHRU_BYTESWAP_256BIT); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_key; + } + + /* The AES context fits in a single (32-byte) KSB entry and + * must be in little endian format. Use the 256-bit byte swap + * passthru option to convert from big endian to little endian. + */ + ret = ccp_init_dm_workarea(&ctx, cmd_q, + CCP_AES_CTX_KSB_COUNT * CCP_KSB_BYTES, + DMA_BIDIRECTIONAL); + if (ret) + goto e_key; + + dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE; + ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); + ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx, + CCP_PASSTHRU_BYTESWAP_256BIT); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_ctx; + } + + /* Send data to the CCP AES engine */ + ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len, + AES_BLOCK_SIZE, DMA_TO_DEVICE); + if (ret) + goto e_ctx; + + while (src.sg_wa.bytes_left) { + ccp_prepare_data(&src, NULL, &op, AES_BLOCK_SIZE, true); + if (aes->cmac_final && !src.sg_wa.bytes_left) { + op.eom = 1; + + /* Push the K1/K2 key to the CCP now */ + ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, + op.ksb_ctx, + CCP_PASSTHRU_BYTESWAP_256BIT); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_src; + } + + ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0, + aes->cmac_key_len); + ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx, + CCP_PASSTHRU_BYTESWAP_256BIT); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_src; + } + } + + ret = ccp_perform_aes(&op); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_src; + } + + ccp_process_data(&src, NULL, &op); + } + + /* Retrieve the AES context - convert from LE to BE using + * 32-byte (256-bit) byteswapping + */ + ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx, + CCP_PASSTHRU_BYTESWAP_256BIT); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_src; + } + + /* ...but we only need AES_BLOCK_SIZE bytes */ + dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE; + ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); + +e_src: + ccp_free_data(&src, cmd_q); + +e_ctx: + ccp_dm_free(&ctx); + +e_key: + ccp_dm_free(&key); + + return ret; +} + +static int ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) +{ + struct ccp_aes_engine *aes = &cmd->u.aes; + struct ccp_dm_workarea key, ctx; + struct ccp_data src, dst; + struct ccp_op op; + unsigned int dm_offset; + bool in_place = false; + int ret; + + if (aes->mode == CCP_AES_MODE_CMAC) + return ccp_run_aes_cmac_cmd(cmd_q, cmd); + + if (!((aes->key_len == AES_KEYSIZE_128) || + (aes->key_len == AES_KEYSIZE_192) || + (aes->key_len == AES_KEYSIZE_256))) + return -EINVAL; + + if (((aes->mode == CCP_AES_MODE_ECB) || + (aes->mode == CCP_AES_MODE_CBC) || + (aes->mode == CCP_AES_MODE_CFB)) && + (aes->src_len & (AES_BLOCK_SIZE - 1))) + return -EINVAL; + + if (!aes->key || !aes->src || !aes->dst) + return -EINVAL; + + if (aes->mode != CCP_AES_MODE_ECB) { + if (aes->iv_len != AES_BLOCK_SIZE) + return -EINVAL; + + if (!aes->iv) + return -EINVAL; + } + + BUILD_BUG_ON(CCP_AES_KEY_KSB_COUNT != 1); + BUILD_BUG_ON(CCP_AES_CTX_KSB_COUNT != 1); + + ret = -EIO; + memset(&op, 0, sizeof(op)); + op.cmd_q = cmd_q; + op.jobid = ccp_gen_jobid(cmd_q->ccp); + op.ksb_key = cmd_q->ksb_key; + op.ksb_ctx = cmd_q->ksb_ctx; + op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1; + op.u.aes.type = aes->type; + op.u.aes.mode = aes->mode; + op.u.aes.action = aes->action; + + /* All supported key sizes fit in a single (32-byte) KSB entry + * and must be in little endian format. Use the 256-bit byte + * swap passthru option to convert from big endian to little + * endian. + */ + ret = ccp_init_dm_workarea(&key, cmd_q, + CCP_AES_KEY_KSB_COUNT * CCP_KSB_BYTES, + DMA_TO_DEVICE); + if (ret) + return ret; + + dm_offset = CCP_KSB_BYTES - aes->key_len; + ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len); + ret = ccp_copy_to_ksb(cmd_q, &key, op.jobid, op.ksb_key, + CCP_PASSTHRU_BYTESWAP_256BIT); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_key; + } + + /* The AES context fits in a single (32-byte) KSB entry and + * must be in little endian format. Use the 256-bit byte swap + * passthru option to convert from big endian to little endian. + */ + ret = ccp_init_dm_workarea(&ctx, cmd_q, + CCP_AES_CTX_KSB_COUNT * CCP_KSB_BYTES, + DMA_BIDIRECTIONAL); + if (ret) + goto e_key; + + if (aes->mode != CCP_AES_MODE_ECB) { + /* Load the AES context - conver to LE */ + dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE; + ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); + ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx, + CCP_PASSTHRU_BYTESWAP_256BIT); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_ctx; + } + } + + /* Prepare the input and output data workareas. For in-place + * operations we need to set the dma direction to BIDIRECTIONAL + * and copy the src workarea to the dst workarea. + */ + if (sg_virt(aes->src) == sg_virt(aes->dst)) + in_place = true; + + ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len, + AES_BLOCK_SIZE, + in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); + if (ret) + goto e_ctx; + + if (in_place) + dst = src; + else { + ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len, + AES_BLOCK_SIZE, DMA_FROM_DEVICE); + if (ret) + goto e_src; + } + + /* Send data to the CCP AES engine */ + while (src.sg_wa.bytes_left) { + ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true); + if (!src.sg_wa.bytes_left) { + op.eom = 1; + + /* Since we don't retrieve the AES context in ECB + * mode we have to wait for the operation to complete + * on the last piece of data + */ + if (aes->mode == CCP_AES_MODE_ECB) + op.soc = 1; + } + + ret = ccp_perform_aes(&op); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_dst; + } + + ccp_process_data(&src, &dst, &op); + } + + if (aes->mode != CCP_AES_MODE_ECB) { + /* Retrieve the AES context - convert from LE to BE using + * 32-byte (256-bit) byteswapping + */ + ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx, + CCP_PASSTHRU_BYTESWAP_256BIT); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_dst; + } + + /* ...but we only need AES_BLOCK_SIZE bytes */ + dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE; + ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); + } + +e_dst: + if (!in_place) + ccp_free_data(&dst, cmd_q); + +e_src: + ccp_free_data(&src, cmd_q); + +e_ctx: + ccp_dm_free(&ctx); + +e_key: + ccp_dm_free(&key); + + return ret; +} + +static int ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q, + struct ccp_cmd *cmd) +{ + struct ccp_xts_aes_engine *xts = &cmd->u.xts; + struct ccp_dm_workarea key, ctx; + struct ccp_data src, dst; + struct ccp_op op; + unsigned int unit_size, dm_offset; + bool in_place = false; + int ret; + + switch (xts->unit_size) { + case CCP_XTS_AES_UNIT_SIZE_16: + unit_size = 16; + break; + case CCP_XTS_AES_UNIT_SIZE_512: + unit_size = 512; + break; + case CCP_XTS_AES_UNIT_SIZE_1024: + unit_size = 1024; + break; + case CCP_XTS_AES_UNIT_SIZE_2048: + unit_size = 2048; + break; + case CCP_XTS_AES_UNIT_SIZE_4096: + unit_size = 4096; + break; + + default: + return -EINVAL; + } + + if (xts->key_len != AES_KEYSIZE_128) + return -EINVAL; + + if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1))) + return -EINVAL; + + if (xts->iv_len != AES_BLOCK_SIZE) + return -EINVAL; + + if (!xts->key || !xts->iv || !xts->src || !xts->dst) + return -EINVAL; + + BUILD_BUG_ON(CCP_XTS_AES_KEY_KSB_COUNT != 1); + BUILD_BUG_ON(CCP_XTS_AES_CTX_KSB_COUNT != 1); + + ret = -EIO; + memset(&op, 0, sizeof(op)); + op.cmd_q = cmd_q; + op.jobid = ccp_gen_jobid(cmd_q->ccp); + op.ksb_key = cmd_q->ksb_key; + op.ksb_ctx = cmd_q->ksb_ctx; + op.init = 1; + op.u.xts.action = xts->action; + op.u.xts.unit_size = xts->unit_size; + + /* All supported key sizes fit in a single (32-byte) KSB entry + * and must be in little endian format. Use the 256-bit byte + * swap passthru option to convert from big endian to little + * endian. + */ + ret = ccp_init_dm_workarea(&key, cmd_q, + CCP_XTS_AES_KEY_KSB_COUNT * CCP_KSB_BYTES, + DMA_TO_DEVICE); + if (ret) + return ret; + + dm_offset = CCP_KSB_BYTES - AES_KEYSIZE_128; + ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len); + ccp_set_dm_area(&key, 0, xts->key, dm_offset, xts->key_len); + ret = ccp_copy_to_ksb(cmd_q, &key, op.jobid, op.ksb_key, + CCP_PASSTHRU_BYTESWAP_256BIT); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_key; + } + + /* The AES context fits in a single (32-byte) KSB entry and + * for XTS is already in little endian format so no byte swapping + * is needed. + */ + ret = ccp_init_dm_workarea(&ctx, cmd_q, + CCP_XTS_AES_CTX_KSB_COUNT * CCP_KSB_BYTES, + DMA_BIDIRECTIONAL); + if (ret) + goto e_key; + + ccp_set_dm_area(&ctx, 0, xts->iv, 0, xts->iv_len); + ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx, + CCP_PASSTHRU_BYTESWAP_NOOP); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_ctx; + } + + /* Prepare the input and output data workareas. For in-place + * operations we need to set the dma direction to BIDIRECTIONAL + * and copy the src workarea to the dst workarea. + */ + if (sg_virt(xts->src) == sg_virt(xts->dst)) + in_place = true; + + ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len, + unit_size, + in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); + if (ret) + goto e_ctx; + + if (in_place) + dst = src; + else { + ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len, + unit_size, DMA_FROM_DEVICE); + if (ret) + goto e_src; + } + + /* Send data to the CCP AES engine */ + while (src.sg_wa.bytes_left) { + ccp_prepare_data(&src, &dst, &op, unit_size, true); + if (!src.sg_wa.bytes_left) + op.eom = 1; + + ret = ccp_perform_xts_aes(&op); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_dst; + } + + ccp_process_data(&src, &dst, &op); + } + + /* Retrieve the AES context - convert from LE to BE using + * 32-byte (256-bit) byteswapping + */ + ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx, + CCP_PASSTHRU_BYTESWAP_256BIT); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_dst; + } + + /* ...but we only need AES_BLOCK_SIZE bytes */ + dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE; + ccp_get_dm_area(&ctx, dm_offset, xts->iv, 0, xts->iv_len); + +e_dst: + if (!in_place) + ccp_free_data(&dst, cmd_q); + +e_src: + ccp_free_data(&src, cmd_q); + +e_ctx: + ccp_dm_free(&ctx); + +e_key: + ccp_dm_free(&key); + + return ret; +} + +static int ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) +{ + struct ccp_sha_engine *sha = &cmd->u.sha; + struct ccp_dm_workarea ctx; + struct ccp_data src; + struct ccp_op op; + int ret; + + if (sha->ctx_len != CCP_SHA_CTXSIZE) + return -EINVAL; + + if (!sha->ctx) + return -EINVAL; + + if (!sha->final && (sha->src_len & (CCP_SHA_BLOCKSIZE - 1))) + return -EINVAL; + + if (!sha->src_len) { + const u8 *sha_zero; + + /* Not final, just return */ + if (!sha->final) + return 0; + + /* CCP can't do a zero length sha operation so the caller + * must buffer the data. + */ + if (sha->msg_bits) + return -EINVAL; + + /* A sha operation for a message with a total length of zero, + * return known result. + */ + switch (sha->type) { + case CCP_SHA_TYPE_1: + sha_zero = ccp_sha1_zero; + break; + case CCP_SHA_TYPE_224: + sha_zero = ccp_sha224_zero; + break; + case CCP_SHA_TYPE_256: + sha_zero = ccp_sha256_zero; + break; + default: + return -EINVAL; + } + + scatterwalk_map_and_copy((void *)sha_zero, sha->ctx, 0, + sha->ctx_len, 1); + + return 0; + } + + if (!sha->src) + return -EINVAL; + + BUILD_BUG_ON(CCP_SHA_KSB_COUNT != 1); + + memset(&op, 0, sizeof(op)); + op.cmd_q = cmd_q; + op.jobid = ccp_gen_jobid(cmd_q->ccp); + op.ksb_ctx = cmd_q->ksb_ctx; + op.u.sha.type = sha->type; + op.u.sha.msg_bits = sha->msg_bits; + + /* The SHA context fits in a single (32-byte) KSB entry and + * must be in little endian format. Use the 256-bit byte swap + * passthru option to convert from big endian to little endian. + */ + ret = ccp_init_dm_workarea(&ctx, cmd_q, + CCP_SHA_KSB_COUNT * CCP_KSB_BYTES, + DMA_BIDIRECTIONAL); + if (ret) + return ret; + + ccp_set_dm_area(&ctx, 0, sha->ctx, 0, sha->ctx_len); + ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx, + CCP_PASSTHRU_BYTESWAP_256BIT); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_ctx; + } + + /* Send data to the CCP SHA engine */ + ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len, + CCP_SHA_BLOCKSIZE, DMA_TO_DEVICE); + if (ret) + goto e_ctx; + + while (src.sg_wa.bytes_left) { + ccp_prepare_data(&src, NULL, &op, CCP_SHA_BLOCKSIZE, false); + if (sha->final && !src.sg_wa.bytes_left) + op.eom = 1; + + ret = ccp_perform_sha(&op); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_data; + } + + ccp_process_data(&src, NULL, &op); + } + + /* Retrieve the SHA context - convert from LE to BE using + * 32-byte (256-bit) byteswapping to BE + */ + ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx, + CCP_PASSTHRU_BYTESWAP_256BIT); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_data; + } + + ccp_get_dm_area(&ctx, 0, sha->ctx, 0, sha->ctx_len); + +e_data: + ccp_free_data(&src, cmd_q); + +e_ctx: + ccp_dm_free(&ctx); + + return ret; +} + +static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) +{ + struct ccp_rsa_engine *rsa = &cmd->u.rsa; + struct ccp_dm_workarea exp, src; + struct ccp_data dst; + struct ccp_op op; + unsigned int ksb_count, i_len, o_len; + int ret; + + if (rsa->key_size > CCP_RSA_MAX_WIDTH) + return -EINVAL; + + if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst) + return -EINVAL; + + /* The RSA modulus must precede the message being acted upon, so + * it must be copied to a DMA area where the message and the + * modulus can be concatenated. Therefore the input buffer + * length required is twice the output buffer length (which + * must be a multiple of 256-bits). + */ + o_len = ((rsa->key_size + 255) / 256) * 32; + i_len = o_len * 2; + + ksb_count = o_len / CCP_KSB_BYTES; + + memset(&op, 0, sizeof(op)); + op.cmd_q = cmd_q; + op.jobid = ccp_gen_jobid(cmd_q->ccp); + op.ksb_key = ccp_alloc_ksb(cmd_q->ccp, ksb_count); + if (!op.ksb_key) + return -EIO; + + /* The RSA exponent may span multiple (32-byte) KSB entries and must + * be in little endian format. Reverse copy each 32-byte chunk + * of the exponent (En chunk to E0 chunk, E(n-1) chunk to E1 chunk) + * and each byte within that chunk and do not perform any byte swap + * operations on the passthru operation. + */ + ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE); + if (ret) + goto e_ksb; + + ccp_reverse_set_dm_area(&exp, rsa->exp, rsa->exp_len, CCP_KSB_BYTES, + true); + ret = ccp_copy_to_ksb(cmd_q, &exp, op.jobid, op.ksb_key, + CCP_PASSTHRU_BYTESWAP_NOOP); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_exp; + } + + /* Concatenate the modulus and the message. Both the modulus and + * the operands must be in little endian format. Since the input + * is in big endian format it must be converted. + */ + ret = ccp_init_dm_workarea(&src, cmd_q, i_len, DMA_TO_DEVICE); + if (ret) + goto e_exp; + + ccp_reverse_set_dm_area(&src, rsa->mod, rsa->mod_len, CCP_KSB_BYTES, + true); + src.address += o_len; /* Adjust the address for the copy operation */ + ccp_reverse_set_dm_area(&src, rsa->src, rsa->src_len, CCP_KSB_BYTES, + true); + src.address -= o_len; /* Reset the address to original value */ + + /* Prepare the output area for the operation */ + ret = ccp_init_data(&dst, cmd_q, rsa->dst, rsa->mod_len, + o_len, DMA_FROM_DEVICE); + if (ret) + goto e_src; + + op.soc = 1; + op.src.u.dma.address = src.dma.address; + op.src.u.dma.offset = 0; + op.src.u.dma.length = i_len; + op.dst.u.dma.address = dst.dm_wa.dma.address; + op.dst.u.dma.offset = 0; + op.dst.u.dma.length = o_len; + + op.u.rsa.mod_size = rsa->key_size; + op.u.rsa.input_len = i_len; + + ret = ccp_perform_rsa(&op); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_dst; + } + + ccp_reverse_get_dm_area(&dst.dm_wa, rsa->dst, rsa->mod_len); + +e_dst: + ccp_free_data(&dst, cmd_q); + +e_src: + ccp_dm_free(&src); + +e_exp: + ccp_dm_free(&exp); + +e_ksb: + ccp_free_ksb(cmd_q->ccp, op.ksb_key, ksb_count); + + return ret; +} + +static int ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q, + struct ccp_cmd *cmd) +{ + struct ccp_passthru_engine *pt = &cmd->u.passthru; + struct ccp_dm_workarea mask; + struct ccp_data src, dst; + struct ccp_op op; + bool in_place = false; + unsigned int i; + int ret; + + if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1))) + return -EINVAL; + + if (!pt->src || !pt->dst) + return -EINVAL; + + if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) { + if (pt->mask_len != CCP_PASSTHRU_MASKSIZE) + return -EINVAL; + if (!pt->mask) + return -EINVAL; + } + + BUILD_BUG_ON(CCP_PASSTHRU_KSB_COUNT != 1); + + memset(&op, 0, sizeof(op)); + op.cmd_q = cmd_q; + op.jobid = ccp_gen_jobid(cmd_q->ccp); + + if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) { + /* Load the mask */ + op.ksb_key = cmd_q->ksb_key; + + ret = ccp_init_dm_workarea(&mask, cmd_q, + CCP_PASSTHRU_KSB_COUNT * + CCP_KSB_BYTES, + DMA_TO_DEVICE); + if (ret) + return ret; + + ccp_set_dm_area(&mask, 0, pt->mask, 0, pt->mask_len); + ret = ccp_copy_to_ksb(cmd_q, &mask, op.jobid, op.ksb_key, + CCP_PASSTHRU_BYTESWAP_NOOP); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_mask; + } + } + + /* Prepare the input and output data workareas. For in-place + * operations we need to set the dma direction to BIDIRECTIONAL + * and copy the src workarea to the dst workarea. + */ + if (sg_virt(pt->src) == sg_virt(pt->dst)) + in_place = true; + + ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len, + CCP_PASSTHRU_MASKSIZE, + in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); + if (ret) + goto e_mask; + + if (in_place) + dst = src; + else { + ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len, + CCP_PASSTHRU_MASKSIZE, DMA_FROM_DEVICE); + if (ret) + goto e_src; + } + + /* Send data to the CCP Passthru engine + * Because the CCP engine works on a single source and destination + * dma address at a time, each entry in the source scatterlist + * (after the dma_map_sg call) must be less than or equal to the + * (remaining) length in the destination scatterlist entry and the + * length must be a multiple of CCP_PASSTHRU_BLOCKSIZE + */ + dst.sg_wa.sg_used = 0; + for (i = 1; i <= src.sg_wa.dma_count; i++) { + if (!dst.sg_wa.sg || + (dst.sg_wa.sg->length < src.sg_wa.sg->length)) { + ret = -EINVAL; + goto e_dst; + } + + if (i == src.sg_wa.dma_count) { + op.eom = 1; + op.soc = 1; + } + + op.src.type = CCP_MEMTYPE_SYSTEM; + op.src.u.dma.address = sg_dma_address(src.sg_wa.sg); + op.src.u.dma.offset = 0; + op.src.u.dma.length = sg_dma_len(src.sg_wa.sg); + + op.dst.type = CCP_MEMTYPE_SYSTEM; + op.dst.u.dma.address = sg_dma_address(dst.sg_wa.sg); + op.src.u.dma.offset = dst.sg_wa.sg_used; + op.src.u.dma.length = op.src.u.dma.length; + + ret = ccp_perform_passthru(&op); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_dst; + } + + dst.sg_wa.sg_used += src.sg_wa.sg->length; + if (dst.sg_wa.sg_used == dst.sg_wa.sg->length) { + dst.sg_wa.sg = sg_next(dst.sg_wa.sg); + dst.sg_wa.sg_used = 0; + } + src.sg_wa.sg = sg_next(src.sg_wa.sg); + } + +e_dst: + if (!in_place) + ccp_free_data(&dst, cmd_q); + +e_src: + ccp_free_data(&src, cmd_q); + +e_mask: + if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) + ccp_dm_free(&mask); + + return ret; +} + +static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) +{ + struct ccp_ecc_engine *ecc = &cmd->u.ecc; + struct ccp_dm_workarea src, dst; + struct ccp_op op; + int ret; + u8 *save; + + if (!ecc->u.mm.operand_1 || + (ecc->u.mm.operand_1_len > CCP_ECC_MODULUS_BYTES)) + return -EINVAL; + + if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) + if (!ecc->u.mm.operand_2 || + (ecc->u.mm.operand_2_len > CCP_ECC_MODULUS_BYTES)) + return -EINVAL; + + if (!ecc->u.mm.result || + (ecc->u.mm.result_len < CCP_ECC_MODULUS_BYTES)) + return -EINVAL; + + memset(&op, 0, sizeof(op)); + op.cmd_q = cmd_q; + op.jobid = ccp_gen_jobid(cmd_q->ccp); + + /* Concatenate the modulus and the operands. Both the modulus and + * the operands must be in little endian format. Since the input + * is in big endian format it must be converted and placed in a + * fixed length buffer. + */ + ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE, + DMA_TO_DEVICE); + if (ret) + return ret; + + /* Save the workarea address since it is updated in order to perform + * the concatenation + */ + save = src.address; + + /* Copy the ECC modulus */ + ccp_reverse_set_dm_area(&src, ecc->mod, ecc->mod_len, + CCP_ECC_OPERAND_SIZE, true); + src.address += CCP_ECC_OPERAND_SIZE; + + /* Copy the first operand */ + ccp_reverse_set_dm_area(&src, ecc->u.mm.operand_1, + ecc->u.mm.operand_1_len, + CCP_ECC_OPERAND_SIZE, true); + src.address += CCP_ECC_OPERAND_SIZE; + + if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) { + /* Copy the second operand */ + ccp_reverse_set_dm_area(&src, ecc->u.mm.operand_2, + ecc->u.mm.operand_2_len, + CCP_ECC_OPERAND_SIZE, true); + src.address += CCP_ECC_OPERAND_SIZE; + } + + /* Restore the workarea address */ + src.address = save; + + /* Prepare the output area for the operation */ + ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE, + DMA_FROM_DEVICE); + if (ret) + goto e_src; + + op.soc = 1; + op.src.u.dma.address = src.dma.address; + op.src.u.dma.offset = 0; + op.src.u.dma.length = src.length; + op.dst.u.dma.address = dst.dma.address; + op.dst.u.dma.offset = 0; + op.dst.u.dma.length = dst.length; + + op.u.ecc.function = cmd->u.ecc.function; + + ret = ccp_perform_ecc(&op); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_dst; + } + + ecc->ecc_result = le16_to_cpup( + (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET)); + if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) { + ret = -EIO; + goto e_dst; + } + + /* Save the ECC result */ + ccp_reverse_get_dm_area(&dst, ecc->u.mm.result, CCP_ECC_MODULUS_BYTES); + +e_dst: + ccp_dm_free(&dst); + +e_src: + ccp_dm_free(&src); + + return ret; +} + +static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) +{ + struct ccp_ecc_engine *ecc = &cmd->u.ecc; + struct ccp_dm_workarea src, dst; + struct ccp_op op; + int ret; + u8 *save; + + if (!ecc->u.pm.point_1.x || + (ecc->u.pm.point_1.x_len > CCP_ECC_MODULUS_BYTES) || + !ecc->u.pm.point_1.y || + (ecc->u.pm.point_1.y_len > CCP_ECC_MODULUS_BYTES)) + return -EINVAL; + + if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) { + if (!ecc->u.pm.point_2.x || + (ecc->u.pm.point_2.x_len > CCP_ECC_MODULUS_BYTES) || + !ecc->u.pm.point_2.y || + (ecc->u.pm.point_2.y_len > CCP_ECC_MODULUS_BYTES)) + return -EINVAL; + } else { + if (!ecc->u.pm.domain_a || + (ecc->u.pm.domain_a_len > CCP_ECC_MODULUS_BYTES)) + return -EINVAL; + + if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) + if (!ecc->u.pm.scalar || + (ecc->u.pm.scalar_len > CCP_ECC_MODULUS_BYTES)) + return -EINVAL; + } + + if (!ecc->u.pm.result.x || + (ecc->u.pm.result.x_len < CCP_ECC_MODULUS_BYTES) || + !ecc->u.pm.result.y || + (ecc->u.pm.result.y_len < CCP_ECC_MODULUS_BYTES)) + return -EINVAL; + + memset(&op, 0, sizeof(op)); + op.cmd_q = cmd_q; + op.jobid = ccp_gen_jobid(cmd_q->ccp); + + /* Concatenate the modulus and the operands. Both the modulus and + * the operands must be in little endian format. Since the input + * is in big endian format it must be converted and placed in a + * fixed length buffer. + */ + ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE, + DMA_TO_DEVICE); + if (ret) + return ret; + + /* Save the workarea address since it is updated in order to perform + * the concatenation + */ + save = src.address; + + /* Copy the ECC modulus */ + ccp_reverse_set_dm_area(&src, ecc->mod, ecc->mod_len, + CCP_ECC_OPERAND_SIZE, true); + src.address += CCP_ECC_OPERAND_SIZE; + + /* Copy the first point X and Y coordinate */ + ccp_reverse_set_dm_area(&src, ecc->u.pm.point_1.x, + ecc->u.pm.point_1.x_len, + CCP_ECC_OPERAND_SIZE, true); + src.address += CCP_ECC_OPERAND_SIZE; + ccp_reverse_set_dm_area(&src, ecc->u.pm.point_1.y, + ecc->u.pm.point_1.y_len, + CCP_ECC_OPERAND_SIZE, true); + src.address += CCP_ECC_OPERAND_SIZE; + + /* Set the first point Z coordianate to 1 */ + *(src.address) = 0x01; + src.address += CCP_ECC_OPERAND_SIZE; + + if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) { + /* Copy the second point X and Y coordinate */ + ccp_reverse_set_dm_area(&src, ecc->u.pm.point_2.x, + ecc->u.pm.point_2.x_len, + CCP_ECC_OPERAND_SIZE, true); + src.address += CCP_ECC_OPERAND_SIZE; + ccp_reverse_set_dm_area(&src, ecc->u.pm.point_2.y, + ecc->u.pm.point_2.y_len, + CCP_ECC_OPERAND_SIZE, true); + src.address += CCP_ECC_OPERAND_SIZE; + + /* Set the second point Z coordianate to 1 */ + *(src.address) = 0x01; + src.address += CCP_ECC_OPERAND_SIZE; + } else { + /* Copy the Domain "a" parameter */ + ccp_reverse_set_dm_area(&src, ecc->u.pm.domain_a, + ecc->u.pm.domain_a_len, + CCP_ECC_OPERAND_SIZE, true); + src.address += CCP_ECC_OPERAND_SIZE; + + if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) { + /* Copy the scalar value */ + ccp_reverse_set_dm_area(&src, ecc->u.pm.scalar, + ecc->u.pm.scalar_len, + CCP_ECC_OPERAND_SIZE, true); + src.address += CCP_ECC_OPERAND_SIZE; + } + } + + /* Restore the workarea address */ + src.address = save; + + /* Prepare the output area for the operation */ + ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE, + DMA_FROM_DEVICE); + if (ret) + goto e_src; + + op.soc = 1; + op.src.u.dma.address = src.dma.address; + op.src.u.dma.offset = 0; + op.src.u.dma.length = src.length; + op.dst.u.dma.address = dst.dma.address; + op.dst.u.dma.offset = 0; + op.dst.u.dma.length = dst.length; + + op.u.ecc.function = cmd->u.ecc.function; + + ret = ccp_perform_ecc(&op); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_dst; + } + + ecc->ecc_result = le16_to_cpup( + (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET)); + if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) { + ret = -EIO; + goto e_dst; + } + + /* Save the workarea address since it is updated as we walk through + * to copy the point math result + */ + save = dst.address; + + /* Save the ECC result X and Y coordinates */ + ccp_reverse_get_dm_area(&dst, ecc->u.pm.result.x, + CCP_ECC_MODULUS_BYTES); + dst.address += CCP_ECC_OUTPUT_SIZE; + ccp_reverse_get_dm_area(&dst, ecc->u.pm.result.y, + CCP_ECC_MODULUS_BYTES); + dst.address += CCP_ECC_OUTPUT_SIZE; + + /* Restore the workarea address */ + dst.address = save; + +e_dst: + ccp_dm_free(&dst); + +e_src: + ccp_dm_free(&src); + + return ret; +} + +static int ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) +{ + struct ccp_ecc_engine *ecc = &cmd->u.ecc; + + ecc->ecc_result = 0; + + if (!ecc->mod || + (ecc->mod_len > CCP_ECC_MODULUS_BYTES)) + return -EINVAL; + + switch (ecc->function) { + case CCP_ECC_FUNCTION_MMUL_384BIT: + case CCP_ECC_FUNCTION_MADD_384BIT: + case CCP_ECC_FUNCTION_MINV_384BIT: + return ccp_run_ecc_mm_cmd(cmd_q, cmd); + + case CCP_ECC_FUNCTION_PADD_384BIT: + case CCP_ECC_FUNCTION_PMUL_384BIT: + case CCP_ECC_FUNCTION_PDBL_384BIT: + return ccp_run_ecc_pm_cmd(cmd_q, cmd); + + default: + return -EINVAL; + } +} + +int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) +{ + int ret; + + cmd->engine_error = 0; + cmd_q->cmd_error = 0; + cmd_q->int_rcvd = 0; + cmd_q->free_slots = CMD_Q_DEPTH(ioread32(cmd_q->reg_status)); + + switch (cmd->engine) { + case CCP_ENGINE_AES: + ret = ccp_run_aes_cmd(cmd_q, cmd); + break; + case CCP_ENGINE_XTS_AES_128: + ret = ccp_run_xts_aes_cmd(cmd_q, cmd); + break; + case CCP_ENGINE_SHA: + ret = ccp_run_sha_cmd(cmd_q, cmd); + break; + case CCP_ENGINE_RSA: + ret = ccp_run_rsa_cmd(cmd_q, cmd); + break; + case CCP_ENGINE_PASSTHRU: + ret = ccp_run_passthru_cmd(cmd_q, cmd); + break; + case CCP_ENGINE_ECC: + ret = ccp_run_ecc_cmd(cmd_q, cmd); + break; + default: + ret = -EINVAL; + } + + return ret; +} diff --git a/drivers/crypto/ccp/ccp-pci.c b/drivers/crypto/ccp/ccp-pci.c new file mode 100644 index 000000000000..93319f9db753 --- /dev/null +++ b/drivers/crypto/ccp/ccp-pci.c @@ -0,0 +1,361 @@ +/* + * AMD Cryptographic Coprocessor (CCP) driver + * + * Copyright (C) 2013 Advanced Micro Devices, Inc. + * + * Author: Tom Lendacky <thomas.lendacky@amd.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/pci.h> +#include <linux/pci_ids.h> +#include <linux/kthread.h> +#include <linux/sched.h> +#include <linux/interrupt.h> +#include <linux/spinlock.h> +#include <linux/delay.h> +#include <linux/ccp.h> + +#include "ccp-dev.h" + +#define IO_BAR 2 +#define MSIX_VECTORS 2 + +struct ccp_msix { + u32 vector; + char name[16]; +}; + +struct ccp_pci { + int msix_count; + struct ccp_msix msix[MSIX_VECTORS]; +}; + +static int ccp_get_msix_irqs(struct ccp_device *ccp) +{ + struct ccp_pci *ccp_pci = ccp->dev_specific; + struct device *dev = ccp->dev; + struct pci_dev *pdev = container_of(dev, struct pci_dev, dev); + struct msix_entry msix_entry[MSIX_VECTORS]; + unsigned int name_len = sizeof(ccp_pci->msix[0].name) - 1; + int v, ret; + + for (v = 0; v < ARRAY_SIZE(msix_entry); v++) + msix_entry[v].entry = v; + + while ((ret = pci_enable_msix(pdev, msix_entry, v)) > 0) + v = ret; + if (ret) + return ret; + + ccp_pci->msix_count = v; + for (v = 0; v < ccp_pci->msix_count; v++) { + /* Set the interrupt names and request the irqs */ + snprintf(ccp_pci->msix[v].name, name_len, "ccp-%u", v); + ccp_pci->msix[v].vector = msix_entry[v].vector; + ret = request_irq(ccp_pci->msix[v].vector, ccp_irq_handler, + 0, ccp_pci->msix[v].name, dev); + if (ret) { + dev_notice(dev, "unable to allocate MSI-X IRQ (%d)\n", + ret); + goto e_irq; + } + } + + return 0; + +e_irq: + while (v--) + free_irq(ccp_pci->msix[v].vector, dev); + + pci_disable_msix(pdev); + + ccp_pci->msix_count = 0; + + return ret; +} + +static int ccp_get_msi_irq(struct ccp_device *ccp) +{ + struct device *dev = ccp->dev; + struct pci_dev *pdev = container_of(dev, struct pci_dev, dev); + int ret; + + ret = pci_enable_msi(pdev); + if (ret) + return ret; + + ret = request_irq(pdev->irq, ccp_irq_handler, 0, "ccp", dev); + if (ret) { + dev_notice(dev, "unable to allocate MSI IRQ (%d)\n", ret); + goto e_msi; + } + + return 0; + +e_msi: + pci_disable_msi(pdev); + + return ret; +} + +static int ccp_get_irqs(struct ccp_device *ccp) +{ + struct device *dev = ccp->dev; + int ret; + + ret = ccp_get_msix_irqs(ccp); + if (!ret) + return 0; + + /* Couldn't get MSI-X vectors, try MSI */ + dev_notice(dev, "could not enable MSI-X (%d), trying MSI\n", ret); + ret = ccp_get_msi_irq(ccp); + if (!ret) + return 0; + + /* Couldn't get MSI interrupt */ + dev_notice(dev, "could not enable MSI (%d)\n", ret); + + return ret; +} + +static void ccp_free_irqs(struct ccp_device *ccp) +{ + struct ccp_pci *ccp_pci = ccp->dev_specific; + struct device *dev = ccp->dev; + struct pci_dev *pdev = container_of(dev, struct pci_dev, dev); + + if (ccp_pci->msix_count) { + while (ccp_pci->msix_count--) + free_irq(ccp_pci->msix[ccp_pci->msix_count].vector, + dev); + pci_disable_msix(pdev); + } else { + free_irq(pdev->irq, dev); + pci_disable_msi(pdev); + } +} + +static int ccp_find_mmio_area(struct ccp_device *ccp) +{ + struct device *dev = ccp->dev; + struct pci_dev *pdev = container_of(dev, struct pci_dev, dev); + resource_size_t io_len; + unsigned long io_flags; + int bar; + + io_flags = pci_resource_flags(pdev, IO_BAR); + io_len = pci_resource_len(pdev, IO_BAR); + if ((io_flags & IORESOURCE_MEM) && (io_len >= (IO_OFFSET + 0x800))) + return IO_BAR; + + for (bar = 0; bar < PCI_STD_RESOURCE_END; bar++) { + io_flags = pci_resource_flags(pdev, bar); + io_len = pci_resource_len(pdev, bar); + if ((io_flags & IORESOURCE_MEM) && + (io_len >= (IO_OFFSET + 0x800))) + return bar; + } + + return -EIO; +} + +static int ccp_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct ccp_device *ccp; + struct ccp_pci *ccp_pci; + struct device *dev = &pdev->dev; + unsigned int bar; + int ret; + + ret = -ENOMEM; + ccp = ccp_alloc_struct(dev); + if (!ccp) + goto e_err; + + ccp_pci = kzalloc(sizeof(*ccp_pci), GFP_KERNEL); + if (!ccp_pci) { + ret = -ENOMEM; + goto e_free1; + } + ccp->dev_specific = ccp_pci; + ccp->get_irq = ccp_get_irqs; + ccp->free_irq = ccp_free_irqs; + + ret = pci_request_regions(pdev, "ccp"); + if (ret) { + dev_err(dev, "pci_request_regions failed (%d)\n", ret); + goto e_free2; + } + + ret = pci_enable_device(pdev); + if (ret) { + dev_err(dev, "pci_enable_device failed (%d)\n", ret); + goto e_regions; + } + + pci_set_master(pdev); + + ret = ccp_find_mmio_area(ccp); + if (ret < 0) + goto e_device; + bar = ret; + + ret = -EIO; + ccp->io_map = pci_iomap(pdev, bar, 0); + if (ccp->io_map == NULL) { + dev_err(dev, "pci_iomap failed\n"); + goto e_device; + } + ccp->io_regs = ccp->io_map + IO_OFFSET; + + ret = dma_set_mask(dev, DMA_BIT_MASK(48)); + if (ret == 0) { + ret = dma_set_coherent_mask(dev, DMA_BIT_MASK(48)); + if (ret) { + dev_err(dev, + "pci_set_consistent_dma_mask failed (%d)\n", + ret); + goto e_bar0; + } + } else { + ret = dma_set_mask(dev, DMA_BIT_MASK(32)); + if (ret) { + dev_err(dev, "pci_set_dma_mask failed (%d)\n", ret); + goto e_bar0; + } + } + + dev_set_drvdata(dev, ccp); + + ret = ccp_init(ccp); + if (ret) + goto e_bar0; + + dev_notice(dev, "enabled\n"); + + return 0; + +e_bar0: + pci_iounmap(pdev, ccp->io_map); + +e_device: + pci_disable_device(pdev); + +e_regions: + pci_release_regions(pdev); + +e_free2: + kfree(ccp_pci); + +e_free1: + kfree(ccp); + +e_err: + dev_notice(dev, "initialization failed\n"); + return ret; +} + +static void ccp_pci_remove(struct pci_dev *pdev) +{ + struct device *dev = &pdev->dev; + struct ccp_device *ccp = dev_get_drvdata(dev); + + if (!ccp) + return; + + ccp_destroy(ccp); + + pci_iounmap(pdev, ccp->io_map); + + pci_disable_device(pdev); + + pci_release_regions(pdev); + + kfree(ccp); + + dev_notice(dev, "disabled\n"); +} + +#ifdef CONFIG_PM +static int ccp_pci_suspend(struct pci_dev *pdev, pm_message_t state) +{ + struct device *dev = &pdev->dev; + struct ccp_device *ccp = dev_get_drvdata(dev); + unsigned long flags; + unsigned int i; + + spin_lock_irqsave(&ccp->cmd_lock, flags); + + ccp->suspending = 1; + + /* Wake all the queue kthreads to prepare for suspend */ + for (i = 0; i < ccp->cmd_q_count; i++) + wake_up_process(ccp->cmd_q[i].kthread); + + spin_unlock_irqrestore(&ccp->cmd_lock, flags); + + /* Wait for all queue kthreads to say they're done */ + while (!ccp_queues_suspended(ccp)) + wait_event_interruptible(ccp->suspend_queue, + ccp_queues_suspended(ccp)); + + return 0; +} + +static int ccp_pci_resume(struct pci_dev *pdev) +{ + struct device *dev = &pdev->dev; + struct ccp_device *ccp = dev_get_drvdata(dev); + unsigned long flags; + unsigned int i; + + spin_lock_irqsave(&ccp->cmd_lock, flags); + + ccp->suspending = 0; + + /* Wake up all the kthreads */ + for (i = 0; i < ccp->cmd_q_count; i++) { + ccp->cmd_q[i].suspended = 0; + wake_up_process(ccp->cmd_q[i].kthread); + } + + spin_unlock_irqrestore(&ccp->cmd_lock, flags); + + return 0; +} +#endif + +static DEFINE_PCI_DEVICE_TABLE(ccp_pci_table) = { + { PCI_VDEVICE(AMD, 0x1537), }, + /* Last entry must be zero */ + { 0, } +}; +MODULE_DEVICE_TABLE(pci, ccp_pci_table); + +static struct pci_driver ccp_pci_driver = { + .name = "AMD Cryptographic Coprocessor", + .id_table = ccp_pci_table, + .probe = ccp_pci_probe, + .remove = ccp_pci_remove, +#ifdef CONFIG_PM + .suspend = ccp_pci_suspend, + .resume = ccp_pci_resume, +#endif +}; + +int ccp_pci_init(void) +{ + return pci_register_driver(&ccp_pci_driver); +} + +void ccp_pci_exit(void) +{ + pci_unregister_driver(&ccp_pci_driver); +} diff --git a/drivers/crypto/dcp.c b/drivers/crypto/dcp.c deleted file mode 100644 index a8a7dd4b0d25..000000000000 --- a/drivers/crypto/dcp.c +++ /dev/null @@ -1,912 +0,0 @@ -/* - * Cryptographic API. - * - * Support for DCP cryptographic accelerator. - * - * Copyright (c) 2013 - * Author: Tobias Rauter <tobias.rauter@gmail.com> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as published - * by the Free Software Foundation. - * - * Based on tegra-aes.c, dcp.c (from freescale SDK) and sahara.c - */ -#include <linux/module.h> -#include <linux/init.h> -#include <linux/errno.h> -#include <linux/kernel.h> -#include <linux/platform_device.h> -#include <linux/dma-mapping.h> -#include <linux/io.h> -#include <linux/mutex.h> -#include <linux/interrupt.h> -#include <linux/completion.h> -#include <linux/workqueue.h> -#include <linux/delay.h> -#include <linux/crypto.h> -#include <linux/miscdevice.h> - -#include <crypto/scatterwalk.h> -#include <crypto/aes.h> - - -/* IOCTL for DCP OTP Key AES - taken from Freescale's SDK*/ -#define DBS_IOCTL_BASE 'd' -#define DBS_ENC _IOW(DBS_IOCTL_BASE, 0x00, uint8_t[16]) -#define DBS_DEC _IOW(DBS_IOCTL_BASE, 0x01, uint8_t[16]) - -/* DCP channel used for AES */ -#define USED_CHANNEL 1 -/* Ring Buffers' maximum size */ -#define DCP_MAX_PKG 20 - -/* Control Register */ -#define DCP_REG_CTRL 0x000 -#define DCP_CTRL_SFRST (1<<31) -#define DCP_CTRL_CLKGATE (1<<30) -#define DCP_CTRL_CRYPTO_PRESENT (1<<29) -#define DCP_CTRL_SHA_PRESENT (1<<28) -#define DCP_CTRL_GATHER_RES_WRITE (1<<23) -#define DCP_CTRL_ENABLE_CONTEXT_CACHE (1<<22) -#define DCP_CTRL_ENABLE_CONTEXT_SWITCH (1<<21) -#define DCP_CTRL_CH_IRQ_E_0 0x01 -#define DCP_CTRL_CH_IRQ_E_1 0x02 -#define DCP_CTRL_CH_IRQ_E_2 0x04 -#define DCP_CTRL_CH_IRQ_E_3 0x08 - -/* Status register */ -#define DCP_REG_STAT 0x010 -#define DCP_STAT_OTP_KEY_READY (1<<28) -#define DCP_STAT_CUR_CHANNEL(stat) ((stat>>24)&0x0F) -#define DCP_STAT_READY_CHANNEL(stat) ((stat>>16)&0x0F) -#define DCP_STAT_IRQ(stat) (stat&0x0F) -#define DCP_STAT_CHAN_0 (0x01) -#define DCP_STAT_CHAN_1 (0x02) -#define DCP_STAT_CHAN_2 (0x04) -#define DCP_STAT_CHAN_3 (0x08) - -/* Channel Control Register */ -#define DCP_REG_CHAN_CTRL 0x020 -#define DCP_CHAN_CTRL_CH0_IRQ_MERGED (1<<16) -#define DCP_CHAN_CTRL_HIGH_PRIO_0 (0x0100) -#define DCP_CHAN_CTRL_HIGH_PRIO_1 (0x0200) -#define DCP_CHAN_CTRL_HIGH_PRIO_2 (0x0400) -#define DCP_CHAN_CTRL_HIGH_PRIO_3 (0x0800) -#define DCP_CHAN_CTRL_ENABLE_0 (0x01) -#define DCP_CHAN_CTRL_ENABLE_1 (0x02) -#define DCP_CHAN_CTRL_ENABLE_2 (0x04) -#define DCP_CHAN_CTRL_ENABLE_3 (0x08) - -/* - * Channel Registers: - * The DCP has 4 channels. Each of this channels - * has 4 registers (command pointer, semaphore, status and options). - * The address of register REG of channel CHAN is obtained by - * dcp_chan_reg(REG, CHAN) - */ -#define DCP_REG_CHAN_PTR 0x00000100 -#define DCP_REG_CHAN_SEMA 0x00000110 -#define DCP_REG_CHAN_STAT 0x00000120 -#define DCP_REG_CHAN_OPT 0x00000130 - -#define DCP_CHAN_STAT_NEXT_CHAIN_IS_0 0x010000 -#define DCP_CHAN_STAT_NO_CHAIN 0x020000 -#define DCP_CHAN_STAT_CONTEXT_ERROR 0x030000 -#define DCP_CHAN_STAT_PAYLOAD_ERROR 0x040000 -#define DCP_CHAN_STAT_INVALID_MODE 0x050000 -#define DCP_CHAN_STAT_PAGEFAULT 0x40 -#define DCP_CHAN_STAT_DST 0x20 -#define DCP_CHAN_STAT_SRC 0x10 -#define DCP_CHAN_STAT_PACKET 0x08 -#define DCP_CHAN_STAT_SETUP 0x04 -#define DCP_CHAN_STAT_MISMATCH 0x02 - -/* hw packet control*/ - -#define DCP_PKT_PAYLOAD_KEY (1<<11) -#define DCP_PKT_OTP_KEY (1<<10) -#define DCP_PKT_CIPHER_INIT (1<<9) -#define DCP_PKG_CIPHER_ENCRYPT (1<<8) -#define DCP_PKT_CIPHER_ENABLE (1<<5) -#define DCP_PKT_DECR_SEM (1<<1) -#define DCP_PKT_CHAIN (1<<2) -#define DCP_PKT_IRQ 1 - -#define DCP_PKT_MODE_CBC (1<<4) -#define DCP_PKT_KEYSELECT_OTP (0xFF<<8) - -/* cipher flags */ -#define DCP_ENC 0x0001 -#define DCP_DEC 0x0002 -#define DCP_ECB 0x0004 -#define DCP_CBC 0x0008 -#define DCP_CBC_INIT 0x0010 -#define DCP_NEW_KEY 0x0040 -#define DCP_OTP_KEY 0x0080 -#define DCP_AES 0x1000 - -/* DCP Flags */ -#define DCP_FLAG_BUSY 0x01 -#define DCP_FLAG_PRODUCING 0x02 - -/* clock defines */ -#define CLOCK_ON 1 -#define CLOCK_OFF 0 - -struct dcp_dev_req_ctx { - int mode; -}; - -struct dcp_op { - unsigned int flags; - u8 key[AES_KEYSIZE_128]; - int keylen; - - struct ablkcipher_request *req; - struct crypto_ablkcipher *fallback; - - uint32_t stat; - uint32_t pkt1; - uint32_t pkt2; - struct ablkcipher_walk walk; -}; - -struct dcp_dev { - struct device *dev; - void __iomem *dcp_regs_base; - - int dcp_vmi_irq; - int dcp_irq; - - spinlock_t queue_lock; - struct crypto_queue queue; - - uint32_t pkt_produced; - uint32_t pkt_consumed; - - struct dcp_hw_packet *hw_pkg[DCP_MAX_PKG]; - dma_addr_t hw_phys_pkg; - - /* [KEY][IV] Both with 16 Bytes */ - u8 *payload_base; - dma_addr_t payload_base_dma; - - - struct tasklet_struct done_task; - struct tasklet_struct queue_task; - struct timer_list watchdog; - - unsigned long flags; - - struct dcp_op *ctx; - - struct miscdevice dcp_bootstream_misc; -}; - -struct dcp_hw_packet { - uint32_t next; - uint32_t pkt1; - uint32_t pkt2; - uint32_t src; - uint32_t dst; - uint32_t size; - uint32_t payload; - uint32_t stat; -}; - -static struct dcp_dev *global_dev; - -static inline u32 dcp_chan_reg(u32 reg, int chan) -{ - return reg + (chan) * 0x40; -} - -static inline void dcp_write(struct dcp_dev *dev, u32 data, u32 reg) -{ - writel(data, dev->dcp_regs_base + reg); -} - -static inline void dcp_set(struct dcp_dev *dev, u32 data, u32 reg) -{ - writel(data, dev->dcp_regs_base + (reg | 0x04)); -} - -static inline void dcp_clear(struct dcp_dev *dev, u32 data, u32 reg) -{ - writel(data, dev->dcp_regs_base + (reg | 0x08)); -} - -static inline void dcp_toggle(struct dcp_dev *dev, u32 data, u32 reg) -{ - writel(data, dev->dcp_regs_base + (reg | 0x0C)); -} - -static inline unsigned int dcp_read(struct dcp_dev *dev, u32 reg) -{ - return readl(dev->dcp_regs_base + reg); -} - -static void dcp_dma_unmap(struct dcp_dev *dev, struct dcp_hw_packet *pkt) -{ - dma_unmap_page(dev->dev, pkt->src, pkt->size, DMA_TO_DEVICE); - dma_unmap_page(dev->dev, pkt->dst, pkt->size, DMA_FROM_DEVICE); - dev_dbg(dev->dev, "unmap packet %x", (unsigned int) pkt); -} - -static int dcp_dma_map(struct dcp_dev *dev, - struct ablkcipher_walk *walk, struct dcp_hw_packet *pkt) -{ - dev_dbg(dev->dev, "map packet %x", (unsigned int) pkt); - /* align to length = 16 */ - pkt->size = walk->nbytes - (walk->nbytes % 16); - - pkt->src = dma_map_page(dev->dev, walk->src.page, walk->src.offset, - pkt->size, DMA_TO_DEVICE); - - if (pkt->src == 0) { - dev_err(dev->dev, "Unable to map src"); - return -ENOMEM; - } - - pkt->dst = dma_map_page(dev->dev, walk->dst.page, walk->dst.offset, - pkt->size, DMA_FROM_DEVICE); - - if (pkt->dst == 0) { - dev_err(dev->dev, "Unable to map dst"); - dma_unmap_page(dev->dev, pkt->src, pkt->size, DMA_TO_DEVICE); - return -ENOMEM; - } - - return 0; -} - -static void dcp_op_one(struct dcp_dev *dev, struct dcp_hw_packet *pkt, - uint8_t last) -{ - struct dcp_op *ctx = dev->ctx; - pkt->pkt1 = ctx->pkt1; - pkt->pkt2 = ctx->pkt2; - - pkt->payload = (u32) dev->payload_base_dma; - pkt->stat = 0; - - if (ctx->flags & DCP_CBC_INIT) { - pkt->pkt1 |= DCP_PKT_CIPHER_INIT; - ctx->flags &= ~DCP_CBC_INIT; - } - - mod_timer(&dev->watchdog, jiffies + msecs_to_jiffies(500)); - pkt->pkt1 |= DCP_PKT_IRQ; - if (!last) - pkt->pkt1 |= DCP_PKT_CHAIN; - - dev->pkt_produced++; - - dcp_write(dev, 1, - dcp_chan_reg(DCP_REG_CHAN_SEMA, USED_CHANNEL)); -} - -static void dcp_op_proceed(struct dcp_dev *dev) -{ - struct dcp_op *ctx = dev->ctx; - struct dcp_hw_packet *pkt; - - while (ctx->walk.nbytes) { - int err = 0; - - pkt = dev->hw_pkg[dev->pkt_produced % DCP_MAX_PKG]; - err = dcp_dma_map(dev, &ctx->walk, pkt); - if (err) { - dev->ctx->stat |= err; - /* start timer to wait for already set up calls */ - mod_timer(&dev->watchdog, - jiffies + msecs_to_jiffies(500)); - break; - } - - - err = ctx->walk.nbytes - pkt->size; - ablkcipher_walk_done(dev->ctx->req, &dev->ctx->walk, err); - - dcp_op_one(dev, pkt, ctx->walk.nbytes == 0); - /* we have to wait if no space is left in buffer */ - if (dev->pkt_produced - dev->pkt_consumed == DCP_MAX_PKG) - break; - } - clear_bit(DCP_FLAG_PRODUCING, &dev->flags); -} - -static void dcp_op_start(struct dcp_dev *dev, uint8_t use_walk) -{ - struct dcp_op *ctx = dev->ctx; - - if (ctx->flags & DCP_NEW_KEY) { - memcpy(dev->payload_base, ctx->key, ctx->keylen); - ctx->flags &= ~DCP_NEW_KEY; - } - - ctx->pkt1 = 0; - ctx->pkt1 |= DCP_PKT_CIPHER_ENABLE; - ctx->pkt1 |= DCP_PKT_DECR_SEM; - - if (ctx->flags & DCP_OTP_KEY) - ctx->pkt1 |= DCP_PKT_OTP_KEY; - else - ctx->pkt1 |= DCP_PKT_PAYLOAD_KEY; - - if (ctx->flags & DCP_ENC) - ctx->pkt1 |= DCP_PKG_CIPHER_ENCRYPT; - - ctx->pkt2 = 0; - if (ctx->flags & DCP_CBC) - ctx->pkt2 |= DCP_PKT_MODE_CBC; - - dev->pkt_produced = 0; - dev->pkt_consumed = 0; - - ctx->stat = 0; - dcp_clear(dev, -1, dcp_chan_reg(DCP_REG_CHAN_STAT, USED_CHANNEL)); - dcp_write(dev, (u32) dev->hw_phys_pkg, - dcp_chan_reg(DCP_REG_CHAN_PTR, USED_CHANNEL)); - - set_bit(DCP_FLAG_PRODUCING, &dev->flags); - - if (use_walk) { - ablkcipher_walk_init(&ctx->walk, ctx->req->dst, - ctx->req->src, ctx->req->nbytes); - ablkcipher_walk_phys(ctx->req, &ctx->walk); - dcp_op_proceed(dev); - } else { - dcp_op_one(dev, dev->hw_pkg[0], 1); - clear_bit(DCP_FLAG_PRODUCING, &dev->flags); - } -} - -static void dcp_done_task(unsigned long data) -{ - struct dcp_dev *dev = (struct dcp_dev *)data; - struct dcp_hw_packet *last_packet; - int fin; - fin = 0; - - for (last_packet = dev->hw_pkg[(dev->pkt_consumed) % DCP_MAX_PKG]; - last_packet->stat == 1; - last_packet = - dev->hw_pkg[++(dev->pkt_consumed) % DCP_MAX_PKG]) { - - dcp_dma_unmap(dev, last_packet); - last_packet->stat = 0; - fin++; - } - /* the last call of this function already consumed this IRQ's packet */ - if (fin == 0) - return; - - dev_dbg(dev->dev, - "Packet(s) done with status %x; finished: %d, produced:%d, complete consumed: %d", - dev->ctx->stat, fin, dev->pkt_produced, dev->pkt_consumed); - - last_packet = dev->hw_pkg[(dev->pkt_consumed - 1) % DCP_MAX_PKG]; - if (!dev->ctx->stat && last_packet->pkt1 & DCP_PKT_CHAIN) { - if (!test_and_set_bit(DCP_FLAG_PRODUCING, &dev->flags)) - dcp_op_proceed(dev); - return; - } - - while (unlikely(dev->pkt_consumed < dev->pkt_produced)) { - dcp_dma_unmap(dev, - dev->hw_pkg[dev->pkt_consumed++ % DCP_MAX_PKG]); - } - - if (dev->ctx->flags & DCP_OTP_KEY) { - /* we used the miscdevice, no walk to finish */ - clear_bit(DCP_FLAG_BUSY, &dev->flags); - return; - } - - ablkcipher_walk_complete(&dev->ctx->walk); - dev->ctx->req->base.complete(&dev->ctx->req->base, - dev->ctx->stat); - dev->ctx->req = NULL; - /* in case there are other requests in the queue */ - tasklet_schedule(&dev->queue_task); -} - -static void dcp_watchdog(unsigned long data) -{ - struct dcp_dev *dev = (struct dcp_dev *)data; - dev->ctx->stat |= dcp_read(dev, - dcp_chan_reg(DCP_REG_CHAN_STAT, USED_CHANNEL)); - - dev_err(dev->dev, "Timeout, Channel status: %x", dev->ctx->stat); - - if (!dev->ctx->stat) - dev->ctx->stat = -ETIMEDOUT; - - dcp_done_task(data); -} - - -static irqreturn_t dcp_common_irq(int irq, void *context) -{ - u32 msk; - struct dcp_dev *dev = (struct dcp_dev *) context; - - del_timer(&dev->watchdog); - - msk = DCP_STAT_IRQ(dcp_read(dev, DCP_REG_STAT)); - dcp_clear(dev, msk, DCP_REG_STAT); - if (msk == 0) - return IRQ_NONE; - - dev->ctx->stat |= dcp_read(dev, - dcp_chan_reg(DCP_REG_CHAN_STAT, USED_CHANNEL)); - - if (msk & DCP_STAT_CHAN_1) - tasklet_schedule(&dev->done_task); - - return IRQ_HANDLED; -} - -static irqreturn_t dcp_vmi_irq(int irq, void *context) -{ - return dcp_common_irq(irq, context); -} - -static irqreturn_t dcp_irq(int irq, void *context) -{ - return dcp_common_irq(irq, context); -} - -static void dcp_crypt(struct dcp_dev *dev, struct dcp_op *ctx) -{ - dev->ctx = ctx; - - if ((ctx->flags & DCP_CBC) && ctx->req->info) { - ctx->flags |= DCP_CBC_INIT; - memcpy(dev->payload_base + AES_KEYSIZE_128, - ctx->req->info, AES_KEYSIZE_128); - } - - dcp_op_start(dev, 1); -} - -static void dcp_queue_task(unsigned long data) -{ - struct dcp_dev *dev = (struct dcp_dev *) data; - struct crypto_async_request *async_req, *backlog; - struct crypto_ablkcipher *tfm; - struct dcp_op *ctx; - struct dcp_dev_req_ctx *rctx; - struct ablkcipher_request *req; - unsigned long flags; - - spin_lock_irqsave(&dev->queue_lock, flags); - - backlog = crypto_get_backlog(&dev->queue); - async_req = crypto_dequeue_request(&dev->queue); - - spin_unlock_irqrestore(&dev->queue_lock, flags); - - if (!async_req) - goto ret_nothing_done; - - if (backlog) - backlog->complete(backlog, -EINPROGRESS); - - req = ablkcipher_request_cast(async_req); - tfm = crypto_ablkcipher_reqtfm(req); - rctx = ablkcipher_request_ctx(req); - ctx = crypto_ablkcipher_ctx(tfm); - - if (!req->src || !req->dst) - goto ret_nothing_done; - - ctx->flags |= rctx->mode; - ctx->req = req; - - dcp_crypt(dev, ctx); - - return; - -ret_nothing_done: - clear_bit(DCP_FLAG_BUSY, &dev->flags); -} - - -static int dcp_cra_init(struct crypto_tfm *tfm) -{ - const char *name = tfm->__crt_alg->cra_name; - struct dcp_op *ctx = crypto_tfm_ctx(tfm); - - tfm->crt_ablkcipher.reqsize = sizeof(struct dcp_dev_req_ctx); - - ctx->fallback = crypto_alloc_ablkcipher(name, 0, - CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK); - - if (IS_ERR(ctx->fallback)) { - dev_err(global_dev->dev, "Error allocating fallback algo %s\n", - name); - return PTR_ERR(ctx->fallback); - } - - return 0; -} - -static void dcp_cra_exit(struct crypto_tfm *tfm) -{ - struct dcp_op *ctx = crypto_tfm_ctx(tfm); - - if (ctx->fallback) - crypto_free_ablkcipher(ctx->fallback); - - ctx->fallback = NULL; -} - -/* async interface */ -static int dcp_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key, - unsigned int len) -{ - struct dcp_op *ctx = crypto_ablkcipher_ctx(tfm); - unsigned int ret = 0; - ctx->keylen = len; - ctx->flags = 0; - if (len == AES_KEYSIZE_128) { - if (memcmp(ctx->key, key, AES_KEYSIZE_128)) { - memcpy(ctx->key, key, len); - ctx->flags |= DCP_NEW_KEY; - } - return 0; - } - - ctx->fallback->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK; - ctx->fallback->base.crt_flags |= - (tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK); - - ret = crypto_ablkcipher_setkey(ctx->fallback, key, len); - if (ret) { - struct crypto_tfm *tfm_aux = crypto_ablkcipher_tfm(tfm); - - tfm_aux->crt_flags &= ~CRYPTO_TFM_RES_MASK; - tfm_aux->crt_flags |= - (ctx->fallback->base.crt_flags & CRYPTO_TFM_RES_MASK); - } - return ret; -} - -static int dcp_aes_cbc_crypt(struct ablkcipher_request *req, int mode) -{ - struct dcp_dev_req_ctx *rctx = ablkcipher_request_ctx(req); - struct dcp_dev *dev = global_dev; - unsigned long flags; - int err = 0; - - if (!IS_ALIGNED(req->nbytes, AES_BLOCK_SIZE)) - return -EINVAL; - - rctx->mode = mode; - - spin_lock_irqsave(&dev->queue_lock, flags); - err = ablkcipher_enqueue_request(&dev->queue, req); - spin_unlock_irqrestore(&dev->queue_lock, flags); - - flags = test_and_set_bit(DCP_FLAG_BUSY, &dev->flags); - - if (!(flags & DCP_FLAG_BUSY)) - tasklet_schedule(&dev->queue_task); - - return err; -} - -static int dcp_aes_cbc_encrypt(struct ablkcipher_request *req) -{ - struct crypto_tfm *tfm = - crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req)); - struct dcp_op *ctx = crypto_ablkcipher_ctx( - crypto_ablkcipher_reqtfm(req)); - - if (unlikely(ctx->keylen != AES_KEYSIZE_128)) { - int err = 0; - ablkcipher_request_set_tfm(req, ctx->fallback); - err = crypto_ablkcipher_encrypt(req); - ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm)); - return err; - } - - return dcp_aes_cbc_crypt(req, DCP_AES | DCP_ENC | DCP_CBC); -} - -static int dcp_aes_cbc_decrypt(struct ablkcipher_request *req) -{ - struct crypto_tfm *tfm = - crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req)); - struct dcp_op *ctx = crypto_ablkcipher_ctx( - crypto_ablkcipher_reqtfm(req)); - - if (unlikely(ctx->keylen != AES_KEYSIZE_128)) { - int err = 0; - ablkcipher_request_set_tfm(req, ctx->fallback); - err = crypto_ablkcipher_decrypt(req); - ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm)); - return err; - } - return dcp_aes_cbc_crypt(req, DCP_AES | DCP_DEC | DCP_CBC); -} - -static struct crypto_alg algs[] = { - { - .cra_name = "cbc(aes)", - .cra_driver_name = "dcp-cbc-aes", - .cra_alignmask = 3, - .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC | - CRYPTO_ALG_NEED_FALLBACK, - .cra_blocksize = AES_KEYSIZE_128, - .cra_type = &crypto_ablkcipher_type, - .cra_priority = 300, - .cra_u.ablkcipher = { - .min_keysize = AES_KEYSIZE_128, - .max_keysize = AES_KEYSIZE_128, - .setkey = dcp_aes_setkey, - .encrypt = dcp_aes_cbc_encrypt, - .decrypt = dcp_aes_cbc_decrypt, - .ivsize = AES_KEYSIZE_128, - } - - }, -}; - -/* DCP bootstream verification interface: uses OTP key for crypto */ -static int dcp_bootstream_open(struct inode *inode, struct file *file) -{ - file->private_data = container_of((file->private_data), - struct dcp_dev, dcp_bootstream_misc); - return 0; -} - -static long dcp_bootstream_ioctl(struct file *file, - unsigned int cmd, unsigned long arg) -{ - struct dcp_dev *dev = (struct dcp_dev *) file->private_data; - void __user *argp = (void __user *)arg; - int ret; - - if (dev == NULL) - return -EBADF; - - if (cmd != DBS_ENC && cmd != DBS_DEC) - return -EINVAL; - - if (copy_from_user(dev->payload_base, argp, 16)) - return -EFAULT; - - if (test_and_set_bit(DCP_FLAG_BUSY, &dev->flags)) - return -EAGAIN; - - dev->ctx = kzalloc(sizeof(struct dcp_op), GFP_KERNEL); - if (!dev->ctx) { - dev_err(dev->dev, - "cannot allocate context for OTP crypto"); - clear_bit(DCP_FLAG_BUSY, &dev->flags); - return -ENOMEM; - } - - dev->ctx->flags = DCP_AES | DCP_ECB | DCP_OTP_KEY | DCP_CBC_INIT; - dev->ctx->flags |= (cmd == DBS_ENC) ? DCP_ENC : DCP_DEC; - dev->hw_pkg[0]->src = dev->payload_base_dma; - dev->hw_pkg[0]->dst = dev->payload_base_dma; - dev->hw_pkg[0]->size = 16; - - dcp_op_start(dev, 0); - - while (test_bit(DCP_FLAG_BUSY, &dev->flags)) - cpu_relax(); - - ret = dev->ctx->stat; - if (!ret && copy_to_user(argp, dev->payload_base, 16)) - ret = -EFAULT; - - kfree(dev->ctx); - - return ret; -} - -static const struct file_operations dcp_bootstream_fops = { - .owner = THIS_MODULE, - .unlocked_ioctl = dcp_bootstream_ioctl, - .open = dcp_bootstream_open, -}; - -static int dcp_probe(struct platform_device *pdev) -{ - struct dcp_dev *dev = NULL; - struct resource *r; - int i, ret, j; - - dev = devm_kzalloc(&pdev->dev, sizeof(*dev), GFP_KERNEL); - if (!dev) - return -ENOMEM; - - global_dev = dev; - dev->dev = &pdev->dev; - - platform_set_drvdata(pdev, dev); - - r = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (!r) { - dev_err(&pdev->dev, "failed to get IORESOURCE_MEM\n"); - return -ENXIO; - } - dev->dcp_regs_base = devm_ioremap(&pdev->dev, r->start, - resource_size(r)); - - dcp_set(dev, DCP_CTRL_SFRST, DCP_REG_CTRL); - udelay(10); - dcp_clear(dev, DCP_CTRL_SFRST | DCP_CTRL_CLKGATE, DCP_REG_CTRL); - - dcp_write(dev, DCP_CTRL_GATHER_RES_WRITE | - DCP_CTRL_ENABLE_CONTEXT_CACHE | DCP_CTRL_CH_IRQ_E_1, - DCP_REG_CTRL); - - dcp_write(dev, DCP_CHAN_CTRL_ENABLE_1, DCP_REG_CHAN_CTRL); - - for (i = 0; i < 4; i++) - dcp_clear(dev, -1, dcp_chan_reg(DCP_REG_CHAN_STAT, i)); - - dcp_clear(dev, -1, DCP_REG_STAT); - - - r = platform_get_resource(pdev, IORESOURCE_IRQ, 0); - if (!r) { - dev_err(&pdev->dev, "can't get IRQ resource (0)\n"); - return -EIO; - } - dev->dcp_vmi_irq = r->start; - ret = request_irq(dev->dcp_vmi_irq, dcp_vmi_irq, 0, "dcp", dev); - if (ret != 0) { - dev_err(&pdev->dev, "can't request_irq (0)\n"); - return -EIO; - } - - r = platform_get_resource(pdev, IORESOURCE_IRQ, 1); - if (!r) { - dev_err(&pdev->dev, "can't get IRQ resource (1)\n"); - ret = -EIO; - goto err_free_irq0; - } - dev->dcp_irq = r->start; - ret = request_irq(dev->dcp_irq, dcp_irq, 0, "dcp", dev); - if (ret != 0) { - dev_err(&pdev->dev, "can't request_irq (1)\n"); - ret = -EIO; - goto err_free_irq0; - } - - dev->hw_pkg[0] = dma_alloc_coherent(&pdev->dev, - DCP_MAX_PKG * sizeof(struct dcp_hw_packet), - &dev->hw_phys_pkg, - GFP_KERNEL); - if (!dev->hw_pkg[0]) { - dev_err(&pdev->dev, "Could not allocate hw descriptors\n"); - ret = -ENOMEM; - goto err_free_irq1; - } - - for (i = 1; i < DCP_MAX_PKG; i++) { - dev->hw_pkg[i - 1]->next = dev->hw_phys_pkg - + i * sizeof(struct dcp_hw_packet); - dev->hw_pkg[i] = dev->hw_pkg[i - 1] + 1; - } - dev->hw_pkg[i - 1]->next = dev->hw_phys_pkg; - - - dev->payload_base = dma_alloc_coherent(&pdev->dev, 2 * AES_KEYSIZE_128, - &dev->payload_base_dma, GFP_KERNEL); - if (!dev->payload_base) { - dev_err(&pdev->dev, "Could not allocate memory for key\n"); - ret = -ENOMEM; - goto err_free_hw_packet; - } - tasklet_init(&dev->queue_task, dcp_queue_task, - (unsigned long) dev); - tasklet_init(&dev->done_task, dcp_done_task, - (unsigned long) dev); - spin_lock_init(&dev->queue_lock); - - crypto_init_queue(&dev->queue, 10); - - init_timer(&dev->watchdog); - dev->watchdog.function = &dcp_watchdog; - dev->watchdog.data = (unsigned long)dev; - - dev->dcp_bootstream_misc.minor = MISC_DYNAMIC_MINOR, - dev->dcp_bootstream_misc.name = "dcpboot", - dev->dcp_bootstream_misc.fops = &dcp_bootstream_fops, - ret = misc_register(&dev->dcp_bootstream_misc); - if (ret != 0) { - dev_err(dev->dev, "Unable to register misc device\n"); - goto err_free_key_iv; - } - - for (i = 0; i < ARRAY_SIZE(algs); i++) { - algs[i].cra_priority = 300; - algs[i].cra_ctxsize = sizeof(struct dcp_op); - algs[i].cra_module = THIS_MODULE; - algs[i].cra_init = dcp_cra_init; - algs[i].cra_exit = dcp_cra_exit; - if (crypto_register_alg(&algs[i])) { - dev_err(&pdev->dev, "register algorithm failed\n"); - ret = -ENOMEM; - goto err_unregister; - } - } - dev_notice(&pdev->dev, "DCP crypto enabled.!\n"); - - return 0; - -err_unregister: - for (j = 0; j < i; j++) - crypto_unregister_alg(&algs[j]); -err_free_key_iv: - dma_free_coherent(&pdev->dev, 2 * AES_KEYSIZE_128, dev->payload_base, - dev->payload_base_dma); -err_free_hw_packet: - dma_free_coherent(&pdev->dev, DCP_MAX_PKG * - sizeof(struct dcp_hw_packet), dev->hw_pkg[0], - dev->hw_phys_pkg); -err_free_irq1: - free_irq(dev->dcp_irq, dev); -err_free_irq0: - free_irq(dev->dcp_vmi_irq, dev); - - return ret; -} - -static int dcp_remove(struct platform_device *pdev) -{ - struct dcp_dev *dev; - int j; - dev = platform_get_drvdata(pdev); - - dma_free_coherent(&pdev->dev, - DCP_MAX_PKG * sizeof(struct dcp_hw_packet), - dev->hw_pkg[0], dev->hw_phys_pkg); - - dma_free_coherent(&pdev->dev, 2 * AES_KEYSIZE_128, dev->payload_base, - dev->payload_base_dma); - - free_irq(dev->dcp_irq, dev); - free_irq(dev->dcp_vmi_irq, dev); - - tasklet_kill(&dev->done_task); - tasklet_kill(&dev->queue_task); - - for (j = 0; j < ARRAY_SIZE(algs); j++) - crypto_unregister_alg(&algs[j]); - - misc_deregister(&dev->dcp_bootstream_misc); - - return 0; -} - -static struct of_device_id fs_dcp_of_match[] = { - { .compatible = "fsl-dcp"}, - {}, -}; - -static struct platform_driver fs_dcp_driver = { - .probe = dcp_probe, - .remove = dcp_remove, - .driver = { - .name = "fsl-dcp", - .owner = THIS_MODULE, - .of_match_table = fs_dcp_of_match - } -}; - -module_platform_driver(fs_dcp_driver); - - -MODULE_AUTHOR("Tobias Rauter <tobias.rauter@gmail.com>"); -MODULE_DESCRIPTION("Freescale DCP Crypto Driver"); -MODULE_LICENSE("GPL"); diff --git a/drivers/crypto/ixp4xx_crypto.c b/drivers/crypto/ixp4xx_crypto.c index 214357e12dc0..f757a0f428bd 100644 --- a/drivers/crypto/ixp4xx_crypto.c +++ b/drivers/crypto/ixp4xx_crypto.c @@ -1149,32 +1149,24 @@ static int aead_setkey(struct crypto_aead *tfm, const u8 *key, unsigned int keylen) { struct ixp_ctx *ctx = crypto_aead_ctx(tfm); - struct rtattr *rta = (struct rtattr *)key; - struct crypto_authenc_key_param *param; + struct crypto_authenc_keys keys; - if (!RTA_OK(rta, keylen)) + if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) goto badkey; - if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM) - goto badkey; - if (RTA_PAYLOAD(rta) < sizeof(*param)) - goto badkey; - - param = RTA_DATA(rta); - ctx->enckey_len = be32_to_cpu(param->enckeylen); - key += RTA_ALIGN(rta->rta_len); - keylen -= RTA_ALIGN(rta->rta_len); + if (keys.authkeylen > sizeof(ctx->authkey)) + goto badkey; - if (keylen < ctx->enckey_len) + if (keys.enckeylen > sizeof(ctx->enckey)) goto badkey; - ctx->authkey_len = keylen - ctx->enckey_len; - memcpy(ctx->enckey, key + ctx->authkey_len, ctx->enckey_len); - memcpy(ctx->authkey, key, ctx->authkey_len); + memcpy(ctx->authkey, keys.authkey, keys.authkeylen); + memcpy(ctx->enckey, keys.enckey, keys.enckeylen); + ctx->authkey_len = keys.authkeylen; + ctx->enckey_len = keys.enckeylen; return aead_setup(tfm, crypto_aead_authsize(tfm)); badkey: - ctx->enckey_len = 0; crypto_aead_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); return -EINVAL; } @@ -1418,14 +1410,12 @@ static const struct platform_device_info ixp_dev_info __initdata = { static int __init ixp_module_init(void) { int num = ARRAY_SIZE(ixp4xx_algos); - int i, err ; + int i, err; pdev = platform_device_register_full(&ixp_dev_info); if (IS_ERR(pdev)) return PTR_ERR(pdev); - dev = &pdev->dev; - spin_lock_init(&desc_lock); spin_lock_init(&emerg_lock); diff --git a/drivers/crypto/mv_cesa.c b/drivers/crypto/mv_cesa.c index 3374a3ebe4c7..8d1e6f8e9e9c 100644 --- a/drivers/crypto/mv_cesa.c +++ b/drivers/crypto/mv_cesa.c @@ -907,7 +907,7 @@ static int mv_cra_hash_hmac_sha1_init(struct crypto_tfm *tfm) return mv_cra_hash_init(tfm, "sha1", COP_HMAC_SHA1, SHA1_BLOCK_SIZE); } -irqreturn_t crypto_int(int irq, void *priv) +static irqreturn_t crypto_int(int irq, void *priv) { u32 val; @@ -928,7 +928,7 @@ irqreturn_t crypto_int(int irq, void *priv) return IRQ_HANDLED; } -struct crypto_alg mv_aes_alg_ecb = { +static struct crypto_alg mv_aes_alg_ecb = { .cra_name = "ecb(aes)", .cra_driver_name = "mv-ecb-aes", .cra_priority = 300, @@ -951,7 +951,7 @@ struct crypto_alg mv_aes_alg_ecb = { }, }; -struct crypto_alg mv_aes_alg_cbc = { +static struct crypto_alg mv_aes_alg_cbc = { .cra_name = "cbc(aes)", .cra_driver_name = "mv-cbc-aes", .cra_priority = 300, @@ -975,7 +975,7 @@ struct crypto_alg mv_aes_alg_cbc = { }, }; -struct ahash_alg mv_sha1_alg = { +static struct ahash_alg mv_sha1_alg = { .init = mv_hash_init, .update = mv_hash_update, .final = mv_hash_final, @@ -999,7 +999,7 @@ struct ahash_alg mv_sha1_alg = { } }; -struct ahash_alg mv_hmac_sha1_alg = { +static struct ahash_alg mv_hmac_sha1_alg = { .init = mv_hash_init, .update = mv_hash_update, .final = mv_hash_final, @@ -1084,7 +1084,7 @@ static int mv_probe(struct platform_device *pdev) goto err_unmap_sram; } - ret = request_irq(irq, crypto_int, IRQF_DISABLED, dev_name(&pdev->dev), + ret = request_irq(irq, crypto_int, 0, dev_name(&pdev->dev), cp); if (ret) goto err_thread; @@ -1187,7 +1187,7 @@ static struct platform_driver marvell_crypto = { .driver = { .owner = THIS_MODULE, .name = "mv_crypto", - .of_match_table = of_match_ptr(mv_cesa_of_match_table), + .of_match_table = mv_cesa_of_match_table, }, }; MODULE_ALIAS("platform:mv_crypto"); diff --git a/drivers/crypto/mxs-dcp.c b/drivers/crypto/mxs-dcp.c new file mode 100644 index 000000000000..a6db7fa6f891 --- /dev/null +++ b/drivers/crypto/mxs-dcp.c @@ -0,0 +1,1100 @@ +/* + * Freescale i.MX23/i.MX28 Data Co-Processor driver + * + * Copyright (C) 2013 Marek Vasut <marex@denx.de> + * + * The code contained herein is licensed under the GNU General Public + * License. You may obtain a copy of the GNU General Public License + * Version 2 or later at the following locations: + * + * http://www.opensource.org/licenses/gpl-license.html + * http://www.gnu.org/copyleft/gpl.html + */ + +#include <linux/crypto.h> +#include <linux/dma-mapping.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/kernel.h> +#include <linux/kthread.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/platform_device.h> +#include <linux/stmp_device.h> + +#include <crypto/aes.h> +#include <crypto/sha.h> +#include <crypto/internal/hash.h> + +#define DCP_MAX_CHANS 4 +#define DCP_BUF_SZ PAGE_SIZE + +/* DCP DMA descriptor. */ +struct dcp_dma_desc { + uint32_t next_cmd_addr; + uint32_t control0; + uint32_t control1; + uint32_t source; + uint32_t destination; + uint32_t size; + uint32_t payload; + uint32_t status; +}; + +/* Coherent aligned block for bounce buffering. */ +struct dcp_coherent_block { + uint8_t aes_in_buf[DCP_BUF_SZ]; + uint8_t aes_out_buf[DCP_BUF_SZ]; + uint8_t sha_in_buf[DCP_BUF_SZ]; + + uint8_t aes_key[2 * AES_KEYSIZE_128]; + uint8_t sha_digest[SHA256_DIGEST_SIZE]; + + struct dcp_dma_desc desc[DCP_MAX_CHANS]; +}; + +struct dcp { + struct device *dev; + void __iomem *base; + + uint32_t caps; + + struct dcp_coherent_block *coh; + + struct completion completion[DCP_MAX_CHANS]; + struct mutex mutex[DCP_MAX_CHANS]; + struct task_struct *thread[DCP_MAX_CHANS]; + struct crypto_queue queue[DCP_MAX_CHANS]; +}; + +enum dcp_chan { + DCP_CHAN_HASH_SHA = 0, + DCP_CHAN_CRYPTO = 2, +}; + +struct dcp_async_ctx { + /* Common context */ + enum dcp_chan chan; + uint32_t fill; + + /* SHA Hash-specific context */ + struct mutex mutex; + uint32_t alg; + unsigned int hot:1; + + /* Crypto-specific context */ + unsigned int enc:1; + unsigned int ecb:1; + struct crypto_ablkcipher *fallback; + unsigned int key_len; + uint8_t key[AES_KEYSIZE_128]; +}; + +struct dcp_sha_req_ctx { + unsigned int init:1; + unsigned int fini:1; +}; + +/* + * There can even be only one instance of the MXS DCP due to the + * design of Linux Crypto API. + */ +static struct dcp *global_sdcp; +static DEFINE_MUTEX(global_mutex); + +/* DCP register layout. */ +#define MXS_DCP_CTRL 0x00 +#define MXS_DCP_CTRL_GATHER_RESIDUAL_WRITES (1 << 23) +#define MXS_DCP_CTRL_ENABLE_CONTEXT_CACHING (1 << 22) + +#define MXS_DCP_STAT 0x10 +#define MXS_DCP_STAT_CLR 0x18 +#define MXS_DCP_STAT_IRQ_MASK 0xf + +#define MXS_DCP_CHANNELCTRL 0x20 +#define MXS_DCP_CHANNELCTRL_ENABLE_CHANNEL_MASK 0xff + +#define MXS_DCP_CAPABILITY1 0x40 +#define MXS_DCP_CAPABILITY1_SHA256 (4 << 16) +#define MXS_DCP_CAPABILITY1_SHA1 (1 << 16) +#define MXS_DCP_CAPABILITY1_AES128 (1 << 0) + +#define MXS_DCP_CONTEXT 0x50 + +#define MXS_DCP_CH_N_CMDPTR(n) (0x100 + ((n) * 0x40)) + +#define MXS_DCP_CH_N_SEMA(n) (0x110 + ((n) * 0x40)) + +#define MXS_DCP_CH_N_STAT(n) (0x120 + ((n) * 0x40)) +#define MXS_DCP_CH_N_STAT_CLR(n) (0x128 + ((n) * 0x40)) + +/* DMA descriptor bits. */ +#define MXS_DCP_CONTROL0_HASH_TERM (1 << 13) +#define MXS_DCP_CONTROL0_HASH_INIT (1 << 12) +#define MXS_DCP_CONTROL0_PAYLOAD_KEY (1 << 11) +#define MXS_DCP_CONTROL0_CIPHER_ENCRYPT (1 << 8) +#define MXS_DCP_CONTROL0_CIPHER_INIT (1 << 9) +#define MXS_DCP_CONTROL0_ENABLE_HASH (1 << 6) +#define MXS_DCP_CONTROL0_ENABLE_CIPHER (1 << 5) +#define MXS_DCP_CONTROL0_DECR_SEMAPHORE (1 << 1) +#define MXS_DCP_CONTROL0_INTERRUPT (1 << 0) + +#define MXS_DCP_CONTROL1_HASH_SELECT_SHA256 (2 << 16) +#define MXS_DCP_CONTROL1_HASH_SELECT_SHA1 (0 << 16) +#define MXS_DCP_CONTROL1_CIPHER_MODE_CBC (1 << 4) +#define MXS_DCP_CONTROL1_CIPHER_MODE_ECB (0 << 4) +#define MXS_DCP_CONTROL1_CIPHER_SELECT_AES128 (0 << 0) + +static int mxs_dcp_start_dma(struct dcp_async_ctx *actx) +{ + struct dcp *sdcp = global_sdcp; + const int chan = actx->chan; + uint32_t stat; + int ret; + struct dcp_dma_desc *desc = &sdcp->coh->desc[actx->chan]; + + dma_addr_t desc_phys = dma_map_single(sdcp->dev, desc, sizeof(*desc), + DMA_TO_DEVICE); + + reinit_completion(&sdcp->completion[chan]); + + /* Clear status register. */ + writel(0xffffffff, sdcp->base + MXS_DCP_CH_N_STAT_CLR(chan)); + + /* Load the DMA descriptor. */ + writel(desc_phys, sdcp->base + MXS_DCP_CH_N_CMDPTR(chan)); + + /* Increment the semaphore to start the DMA transfer. */ + writel(1, sdcp->base + MXS_DCP_CH_N_SEMA(chan)); + + ret = wait_for_completion_timeout(&sdcp->completion[chan], + msecs_to_jiffies(1000)); + if (!ret) { + dev_err(sdcp->dev, "Channel %i timeout (DCP_STAT=0x%08x)\n", + chan, readl(sdcp->base + MXS_DCP_STAT)); + return -ETIMEDOUT; + } + + stat = readl(sdcp->base + MXS_DCP_CH_N_STAT(chan)); + if (stat & 0xff) { + dev_err(sdcp->dev, "Channel %i error (CH_STAT=0x%08x)\n", + chan, stat); + return -EINVAL; + } + + dma_unmap_single(sdcp->dev, desc_phys, sizeof(*desc), DMA_TO_DEVICE); + + return 0; +} + +/* + * Encryption (AES128) + */ +static int mxs_dcp_run_aes(struct dcp_async_ctx *actx, int init) +{ + struct dcp *sdcp = global_sdcp; + struct dcp_dma_desc *desc = &sdcp->coh->desc[actx->chan]; + int ret; + + dma_addr_t key_phys = dma_map_single(sdcp->dev, sdcp->coh->aes_key, + 2 * AES_KEYSIZE_128, + DMA_TO_DEVICE); + dma_addr_t src_phys = dma_map_single(sdcp->dev, sdcp->coh->aes_in_buf, + DCP_BUF_SZ, DMA_TO_DEVICE); + dma_addr_t dst_phys = dma_map_single(sdcp->dev, sdcp->coh->aes_out_buf, + DCP_BUF_SZ, DMA_FROM_DEVICE); + + /* Fill in the DMA descriptor. */ + desc->control0 = MXS_DCP_CONTROL0_DECR_SEMAPHORE | + MXS_DCP_CONTROL0_INTERRUPT | + MXS_DCP_CONTROL0_ENABLE_CIPHER; + + /* Payload contains the key. */ + desc->control0 |= MXS_DCP_CONTROL0_PAYLOAD_KEY; + + if (actx->enc) + desc->control0 |= MXS_DCP_CONTROL0_CIPHER_ENCRYPT; + if (init) + desc->control0 |= MXS_DCP_CONTROL0_CIPHER_INIT; + + desc->control1 = MXS_DCP_CONTROL1_CIPHER_SELECT_AES128; + + if (actx->ecb) + desc->control1 |= MXS_DCP_CONTROL1_CIPHER_MODE_ECB; + else + desc->control1 |= MXS_DCP_CONTROL1_CIPHER_MODE_CBC; + + desc->next_cmd_addr = 0; + desc->source = src_phys; + desc->destination = dst_phys; + desc->size = actx->fill; + desc->payload = key_phys; + desc->status = 0; + + ret = mxs_dcp_start_dma(actx); + + dma_unmap_single(sdcp->dev, key_phys, 2 * AES_KEYSIZE_128, + DMA_TO_DEVICE); + dma_unmap_single(sdcp->dev, src_phys, DCP_BUF_SZ, DMA_TO_DEVICE); + dma_unmap_single(sdcp->dev, dst_phys, DCP_BUF_SZ, DMA_FROM_DEVICE); + + return ret; +} + +static int mxs_dcp_aes_block_crypt(struct crypto_async_request *arq) +{ + struct dcp *sdcp = global_sdcp; + + struct ablkcipher_request *req = ablkcipher_request_cast(arq); + struct dcp_async_ctx *actx = crypto_tfm_ctx(arq->tfm); + + struct scatterlist *dst = req->dst; + struct scatterlist *src = req->src; + const int nents = sg_nents(req->src); + + const int out_off = DCP_BUF_SZ; + uint8_t *in_buf = sdcp->coh->aes_in_buf; + uint8_t *out_buf = sdcp->coh->aes_out_buf; + + uint8_t *out_tmp, *src_buf, *dst_buf = NULL; + uint32_t dst_off = 0; + + uint8_t *key = sdcp->coh->aes_key; + + int ret = 0; + int split = 0; + unsigned int i, len, clen, rem = 0; + int init = 0; + + actx->fill = 0; + + /* Copy the key from the temporary location. */ + memcpy(key, actx->key, actx->key_len); + + if (!actx->ecb) { + /* Copy the CBC IV just past the key. */ + memcpy(key + AES_KEYSIZE_128, req->info, AES_KEYSIZE_128); + /* CBC needs the INIT set. */ + init = 1; + } else { + memset(key + AES_KEYSIZE_128, 0, AES_KEYSIZE_128); + } + + for_each_sg(req->src, src, nents, i) { + src_buf = sg_virt(src); + len = sg_dma_len(src); + + do { + if (actx->fill + len > out_off) + clen = out_off - actx->fill; + else + clen = len; + + memcpy(in_buf + actx->fill, src_buf, clen); + len -= clen; + src_buf += clen; + actx->fill += clen; + + /* + * If we filled the buffer or this is the last SG, + * submit the buffer. + */ + if (actx->fill == out_off || sg_is_last(src)) { + ret = mxs_dcp_run_aes(actx, init); + if (ret) + return ret; + init = 0; + + out_tmp = out_buf; + while (dst && actx->fill) { + if (!split) { + dst_buf = sg_virt(dst); + dst_off = 0; + } + rem = min(sg_dma_len(dst) - dst_off, + actx->fill); + + memcpy(dst_buf + dst_off, out_tmp, rem); + out_tmp += rem; + dst_off += rem; + actx->fill -= rem; + + if (dst_off == sg_dma_len(dst)) { + dst = sg_next(dst); + split = 0; + } else { + split = 1; + } + } + } + } while (len); + } + + return ret; +} + +static int dcp_chan_thread_aes(void *data) +{ + struct dcp *sdcp = global_sdcp; + const int chan = DCP_CHAN_CRYPTO; + + struct crypto_async_request *backlog; + struct crypto_async_request *arq; + + int ret; + + do { + __set_current_state(TASK_INTERRUPTIBLE); + + mutex_lock(&sdcp->mutex[chan]); + backlog = crypto_get_backlog(&sdcp->queue[chan]); + arq = crypto_dequeue_request(&sdcp->queue[chan]); + mutex_unlock(&sdcp->mutex[chan]); + + if (backlog) + backlog->complete(backlog, -EINPROGRESS); + + if (arq) { + ret = mxs_dcp_aes_block_crypt(arq); + arq->complete(arq, ret); + continue; + } + + schedule(); + } while (!kthread_should_stop()); + + return 0; +} + +static int mxs_dcp_block_fallback(struct ablkcipher_request *req, int enc) +{ + struct crypto_tfm *tfm = + crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req)); + struct dcp_async_ctx *ctx = crypto_ablkcipher_ctx( + crypto_ablkcipher_reqtfm(req)); + int ret; + + ablkcipher_request_set_tfm(req, ctx->fallback); + + if (enc) + ret = crypto_ablkcipher_encrypt(req); + else + ret = crypto_ablkcipher_decrypt(req); + + ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm)); + + return ret; +} + +static int mxs_dcp_aes_enqueue(struct ablkcipher_request *req, int enc, int ecb) +{ + struct dcp *sdcp = global_sdcp; + struct crypto_async_request *arq = &req->base; + struct dcp_async_ctx *actx = crypto_tfm_ctx(arq->tfm); + int ret; + + if (unlikely(actx->key_len != AES_KEYSIZE_128)) + return mxs_dcp_block_fallback(req, enc); + + actx->enc = enc; + actx->ecb = ecb; + actx->chan = DCP_CHAN_CRYPTO; + + mutex_lock(&sdcp->mutex[actx->chan]); + ret = crypto_enqueue_request(&sdcp->queue[actx->chan], &req->base); + mutex_unlock(&sdcp->mutex[actx->chan]); + + wake_up_process(sdcp->thread[actx->chan]); + + return -EINPROGRESS; +} + +static int mxs_dcp_aes_ecb_decrypt(struct ablkcipher_request *req) +{ + return mxs_dcp_aes_enqueue(req, 0, 1); +} + +static int mxs_dcp_aes_ecb_encrypt(struct ablkcipher_request *req) +{ + return mxs_dcp_aes_enqueue(req, 1, 1); +} + +static int mxs_dcp_aes_cbc_decrypt(struct ablkcipher_request *req) +{ + return mxs_dcp_aes_enqueue(req, 0, 0); +} + +static int mxs_dcp_aes_cbc_encrypt(struct ablkcipher_request *req) +{ + return mxs_dcp_aes_enqueue(req, 1, 0); +} + +static int mxs_dcp_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key, + unsigned int len) +{ + struct dcp_async_ctx *actx = crypto_ablkcipher_ctx(tfm); + unsigned int ret; + + /* + * AES 128 is supposed by the hardware, store key into temporary + * buffer and exit. We must use the temporary buffer here, since + * there can still be an operation in progress. + */ + actx->key_len = len; + if (len == AES_KEYSIZE_128) { + memcpy(actx->key, key, len); + return 0; + } + + /* Check if the key size is supported by kernel at all. */ + if (len != AES_KEYSIZE_192 && len != AES_KEYSIZE_256) { + tfm->base.crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; + return -EINVAL; + } + + /* + * If the requested AES key size is not supported by the hardware, + * but is supported by in-kernel software implementation, we use + * software fallback. + */ + actx->fallback->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK; + actx->fallback->base.crt_flags |= + tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK; + + ret = crypto_ablkcipher_setkey(actx->fallback, key, len); + if (!ret) + return 0; + + tfm->base.crt_flags &= ~CRYPTO_TFM_RES_MASK; + tfm->base.crt_flags |= + actx->fallback->base.crt_flags & CRYPTO_TFM_RES_MASK; + + return ret; +} + +static int mxs_dcp_aes_fallback_init(struct crypto_tfm *tfm) +{ + const char *name = tfm->__crt_alg->cra_name; + const uint32_t flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK; + struct dcp_async_ctx *actx = crypto_tfm_ctx(tfm); + struct crypto_ablkcipher *blk; + + blk = crypto_alloc_ablkcipher(name, 0, flags); + if (IS_ERR(blk)) + return PTR_ERR(blk); + + actx->fallback = blk; + tfm->crt_ablkcipher.reqsize = sizeof(struct dcp_async_ctx); + return 0; +} + +static void mxs_dcp_aes_fallback_exit(struct crypto_tfm *tfm) +{ + struct dcp_async_ctx *actx = crypto_tfm_ctx(tfm); + + crypto_free_ablkcipher(actx->fallback); + actx->fallback = NULL; +} + +/* + * Hashing (SHA1/SHA256) + */ +static int mxs_dcp_run_sha(struct ahash_request *req) +{ + struct dcp *sdcp = global_sdcp; + int ret; + + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct dcp_async_ctx *actx = crypto_ahash_ctx(tfm); + struct dcp_sha_req_ctx *rctx = ahash_request_ctx(req); + + struct dcp_dma_desc *desc = &sdcp->coh->desc[actx->chan]; + dma_addr_t digest_phys = dma_map_single(sdcp->dev, + sdcp->coh->sha_digest, + SHA256_DIGEST_SIZE, + DMA_FROM_DEVICE); + + dma_addr_t buf_phys = dma_map_single(sdcp->dev, sdcp->coh->sha_in_buf, + DCP_BUF_SZ, DMA_TO_DEVICE); + + /* Fill in the DMA descriptor. */ + desc->control0 = MXS_DCP_CONTROL0_DECR_SEMAPHORE | + MXS_DCP_CONTROL0_INTERRUPT | + MXS_DCP_CONTROL0_ENABLE_HASH; + if (rctx->init) + desc->control0 |= MXS_DCP_CONTROL0_HASH_INIT; + + desc->control1 = actx->alg; + desc->next_cmd_addr = 0; + desc->source = buf_phys; + desc->destination = 0; + desc->size = actx->fill; + desc->payload = 0; + desc->status = 0; + + /* Set HASH_TERM bit for last transfer block. */ + if (rctx->fini) { + desc->control0 |= MXS_DCP_CONTROL0_HASH_TERM; + desc->payload = digest_phys; + } + + ret = mxs_dcp_start_dma(actx); + + dma_unmap_single(sdcp->dev, digest_phys, SHA256_DIGEST_SIZE, + DMA_FROM_DEVICE); + dma_unmap_single(sdcp->dev, buf_phys, DCP_BUF_SZ, DMA_TO_DEVICE); + + return ret; +} + +static int dcp_sha_req_to_buf(struct crypto_async_request *arq) +{ + struct dcp *sdcp = global_sdcp; + + struct ahash_request *req = ahash_request_cast(arq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct dcp_async_ctx *actx = crypto_ahash_ctx(tfm); + struct dcp_sha_req_ctx *rctx = ahash_request_ctx(req); + struct hash_alg_common *halg = crypto_hash_alg_common(tfm); + const int nents = sg_nents(req->src); + + uint8_t *digest = sdcp->coh->sha_digest; + uint8_t *in_buf = sdcp->coh->sha_in_buf; + + uint8_t *src_buf; + + struct scatterlist *src; + + unsigned int i, len, clen; + int ret; + + int fin = rctx->fini; + if (fin) + rctx->fini = 0; + + for_each_sg(req->src, src, nents, i) { + src_buf = sg_virt(src); + len = sg_dma_len(src); + + do { + if (actx->fill + len > DCP_BUF_SZ) + clen = DCP_BUF_SZ - actx->fill; + else + clen = len; + + memcpy(in_buf + actx->fill, src_buf, clen); + len -= clen; + src_buf += clen; + actx->fill += clen; + + /* + * If we filled the buffer and still have some + * more data, submit the buffer. + */ + if (len && actx->fill == DCP_BUF_SZ) { + ret = mxs_dcp_run_sha(req); + if (ret) + return ret; + actx->fill = 0; + rctx->init = 0; + } + } while (len); + } + + if (fin) { + rctx->fini = 1; + + /* Submit whatever is left. */ + ret = mxs_dcp_run_sha(req); + if (ret || !req->result) + return ret; + actx->fill = 0; + + /* For some reason, the result is flipped. */ + for (i = 0; i < halg->digestsize; i++) + req->result[i] = digest[halg->digestsize - i - 1]; + } + + return 0; +} + +static int dcp_chan_thread_sha(void *data) +{ + struct dcp *sdcp = global_sdcp; + const int chan = DCP_CHAN_HASH_SHA; + + struct crypto_async_request *backlog; + struct crypto_async_request *arq; + + struct dcp_sha_req_ctx *rctx; + + struct ahash_request *req; + int ret, fini; + + do { + __set_current_state(TASK_INTERRUPTIBLE); + + mutex_lock(&sdcp->mutex[chan]); + backlog = crypto_get_backlog(&sdcp->queue[chan]); + arq = crypto_dequeue_request(&sdcp->queue[chan]); + mutex_unlock(&sdcp->mutex[chan]); + + if (backlog) + backlog->complete(backlog, -EINPROGRESS); + + if (arq) { + req = ahash_request_cast(arq); + rctx = ahash_request_ctx(req); + + ret = dcp_sha_req_to_buf(arq); + fini = rctx->fini; + arq->complete(arq, ret); + if (!fini) + continue; + } + + schedule(); + } while (!kthread_should_stop()); + + return 0; +} + +static int dcp_sha_init(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct dcp_async_ctx *actx = crypto_ahash_ctx(tfm); + + struct hash_alg_common *halg = crypto_hash_alg_common(tfm); + + /* + * Start hashing session. The code below only inits the + * hashing session context, nothing more. + */ + memset(actx, 0, sizeof(*actx)); + + if (strcmp(halg->base.cra_name, "sha1") == 0) + actx->alg = MXS_DCP_CONTROL1_HASH_SELECT_SHA1; + else + actx->alg = MXS_DCP_CONTROL1_HASH_SELECT_SHA256; + + actx->fill = 0; + actx->hot = 0; + actx->chan = DCP_CHAN_HASH_SHA; + + mutex_init(&actx->mutex); + + return 0; +} + +static int dcp_sha_update_fx(struct ahash_request *req, int fini) +{ + struct dcp *sdcp = global_sdcp; + + struct dcp_sha_req_ctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct dcp_async_ctx *actx = crypto_ahash_ctx(tfm); + + int ret; + + /* + * Ignore requests that have no data in them and are not + * the trailing requests in the stream of requests. + */ + if (!req->nbytes && !fini) + return 0; + + mutex_lock(&actx->mutex); + + rctx->fini = fini; + + if (!actx->hot) { + actx->hot = 1; + rctx->init = 1; + } + + mutex_lock(&sdcp->mutex[actx->chan]); + ret = crypto_enqueue_request(&sdcp->queue[actx->chan], &req->base); + mutex_unlock(&sdcp->mutex[actx->chan]); + + wake_up_process(sdcp->thread[actx->chan]); + mutex_unlock(&actx->mutex); + + return -EINPROGRESS; +} + +static int dcp_sha_update(struct ahash_request *req) +{ + return dcp_sha_update_fx(req, 0); +} + +static int dcp_sha_final(struct ahash_request *req) +{ + ahash_request_set_crypt(req, NULL, req->result, 0); + req->nbytes = 0; + return dcp_sha_update_fx(req, 1); +} + +static int dcp_sha_finup(struct ahash_request *req) +{ + return dcp_sha_update_fx(req, 1); +} + +static int dcp_sha_digest(struct ahash_request *req) +{ + int ret; + + ret = dcp_sha_init(req); + if (ret) + return ret; + + return dcp_sha_finup(req); +} + +static int dcp_sha_cra_init(struct crypto_tfm *tfm) +{ + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct dcp_sha_req_ctx)); + return 0; +} + +static void dcp_sha_cra_exit(struct crypto_tfm *tfm) +{ +} + +/* AES 128 ECB and AES 128 CBC */ +static struct crypto_alg dcp_aes_algs[] = { + { + .cra_name = "ecb(aes)", + .cra_driver_name = "ecb-aes-dcp", + .cra_priority = 400, + .cra_alignmask = 15, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_init = mxs_dcp_aes_fallback_init, + .cra_exit = mxs_dcp_aes_fallback_exit, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct dcp_async_ctx), + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_u = { + .ablkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = mxs_dcp_aes_setkey, + .encrypt = mxs_dcp_aes_ecb_encrypt, + .decrypt = mxs_dcp_aes_ecb_decrypt + }, + }, + }, { + .cra_name = "cbc(aes)", + .cra_driver_name = "cbc-aes-dcp", + .cra_priority = 400, + .cra_alignmask = 15, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_init = mxs_dcp_aes_fallback_init, + .cra_exit = mxs_dcp_aes_fallback_exit, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct dcp_async_ctx), + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_u = { + .ablkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = mxs_dcp_aes_setkey, + .encrypt = mxs_dcp_aes_cbc_encrypt, + .decrypt = mxs_dcp_aes_cbc_decrypt, + .ivsize = AES_BLOCK_SIZE, + }, + }, + }, +}; + +/* SHA1 */ +static struct ahash_alg dcp_sha1_alg = { + .init = dcp_sha_init, + .update = dcp_sha_update, + .final = dcp_sha_final, + .finup = dcp_sha_finup, + .digest = dcp_sha_digest, + .halg = { + .digestsize = SHA1_DIGEST_SIZE, + .base = { + .cra_name = "sha1", + .cra_driver_name = "sha1-dcp", + .cra_priority = 400, + .cra_alignmask = 63, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct dcp_async_ctx), + .cra_module = THIS_MODULE, + .cra_init = dcp_sha_cra_init, + .cra_exit = dcp_sha_cra_exit, + }, + }, +}; + +/* SHA256 */ +static struct ahash_alg dcp_sha256_alg = { + .init = dcp_sha_init, + .update = dcp_sha_update, + .final = dcp_sha_final, + .finup = dcp_sha_finup, + .digest = dcp_sha_digest, + .halg = { + .digestsize = SHA256_DIGEST_SIZE, + .base = { + .cra_name = "sha256", + .cra_driver_name = "sha256-dcp", + .cra_priority = 400, + .cra_alignmask = 63, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct dcp_async_ctx), + .cra_module = THIS_MODULE, + .cra_init = dcp_sha_cra_init, + .cra_exit = dcp_sha_cra_exit, + }, + }, +}; + +static irqreturn_t mxs_dcp_irq(int irq, void *context) +{ + struct dcp *sdcp = context; + uint32_t stat; + int i; + + stat = readl(sdcp->base + MXS_DCP_STAT); + stat &= MXS_DCP_STAT_IRQ_MASK; + if (!stat) + return IRQ_NONE; + + /* Clear the interrupts. */ + writel(stat, sdcp->base + MXS_DCP_STAT_CLR); + + /* Complete the DMA requests that finished. */ + for (i = 0; i < DCP_MAX_CHANS; i++) + if (stat & (1 << i)) + complete(&sdcp->completion[i]); + + return IRQ_HANDLED; +} + +static int mxs_dcp_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct dcp *sdcp = NULL; + int i, ret; + + struct resource *iores; + int dcp_vmi_irq, dcp_irq; + + mutex_lock(&global_mutex); + if (global_sdcp) { + dev_err(dev, "Only one DCP instance allowed!\n"); + ret = -ENODEV; + goto err_mutex; + } + + iores = platform_get_resource(pdev, IORESOURCE_MEM, 0); + dcp_vmi_irq = platform_get_irq(pdev, 0); + dcp_irq = platform_get_irq(pdev, 1); + if (dcp_vmi_irq < 0 || dcp_irq < 0) { + ret = -EINVAL; + goto err_mutex; + } + + sdcp = devm_kzalloc(dev, sizeof(*sdcp), GFP_KERNEL); + if (!sdcp) { + ret = -ENOMEM; + goto err_mutex; + } + + sdcp->dev = dev; + sdcp->base = devm_ioremap_resource(dev, iores); + if (IS_ERR(sdcp->base)) { + ret = PTR_ERR(sdcp->base); + goto err_mutex; + } + + ret = devm_request_irq(dev, dcp_vmi_irq, mxs_dcp_irq, 0, + "dcp-vmi-irq", sdcp); + if (ret) { + dev_err(dev, "Failed to claim DCP VMI IRQ!\n"); + goto err_mutex; + } + + ret = devm_request_irq(dev, dcp_irq, mxs_dcp_irq, 0, + "dcp-irq", sdcp); + if (ret) { + dev_err(dev, "Failed to claim DCP IRQ!\n"); + goto err_mutex; + } + + /* Allocate coherent helper block. */ + sdcp->coh = kzalloc(sizeof(struct dcp_coherent_block), GFP_KERNEL); + if (!sdcp->coh) { + dev_err(dev, "Error allocating coherent block\n"); + ret = -ENOMEM; + goto err_mutex; + } + + /* Restart the DCP block. */ + stmp_reset_block(sdcp->base); + + /* Initialize control register. */ + writel(MXS_DCP_CTRL_GATHER_RESIDUAL_WRITES | + MXS_DCP_CTRL_ENABLE_CONTEXT_CACHING | 0xf, + sdcp->base + MXS_DCP_CTRL); + + /* Enable all DCP DMA channels. */ + writel(MXS_DCP_CHANNELCTRL_ENABLE_CHANNEL_MASK, + sdcp->base + MXS_DCP_CHANNELCTRL); + + /* + * We do not enable context switching. Give the context buffer a + * pointer to an illegal address so if context switching is + * inadvertantly enabled, the DCP will return an error instead of + * trashing good memory. The DCP DMA cannot access ROM, so any ROM + * address will do. + */ + writel(0xffff0000, sdcp->base + MXS_DCP_CONTEXT); + for (i = 0; i < DCP_MAX_CHANS; i++) + writel(0xffffffff, sdcp->base + MXS_DCP_CH_N_STAT_CLR(i)); + writel(0xffffffff, sdcp->base + MXS_DCP_STAT_CLR); + + global_sdcp = sdcp; + + platform_set_drvdata(pdev, sdcp); + + for (i = 0; i < DCP_MAX_CHANS; i++) { + mutex_init(&sdcp->mutex[i]); + init_completion(&sdcp->completion[i]); + crypto_init_queue(&sdcp->queue[i], 50); + } + + /* Create the SHA and AES handler threads. */ + sdcp->thread[DCP_CHAN_HASH_SHA] = kthread_run(dcp_chan_thread_sha, + NULL, "mxs_dcp_chan/sha"); + if (IS_ERR(sdcp->thread[DCP_CHAN_HASH_SHA])) { + dev_err(dev, "Error starting SHA thread!\n"); + ret = PTR_ERR(sdcp->thread[DCP_CHAN_HASH_SHA]); + goto err_free_coherent; + } + + sdcp->thread[DCP_CHAN_CRYPTO] = kthread_run(dcp_chan_thread_aes, + NULL, "mxs_dcp_chan/aes"); + if (IS_ERR(sdcp->thread[DCP_CHAN_CRYPTO])) { + dev_err(dev, "Error starting SHA thread!\n"); + ret = PTR_ERR(sdcp->thread[DCP_CHAN_CRYPTO]); + goto err_destroy_sha_thread; + } + + /* Register the various crypto algorithms. */ + sdcp->caps = readl(sdcp->base + MXS_DCP_CAPABILITY1); + + if (sdcp->caps & MXS_DCP_CAPABILITY1_AES128) { + ret = crypto_register_algs(dcp_aes_algs, + ARRAY_SIZE(dcp_aes_algs)); + if (ret) { + /* Failed to register algorithm. */ + dev_err(dev, "Failed to register AES crypto!\n"); + goto err_destroy_aes_thread; + } + } + + if (sdcp->caps & MXS_DCP_CAPABILITY1_SHA1) { + ret = crypto_register_ahash(&dcp_sha1_alg); + if (ret) { + dev_err(dev, "Failed to register %s hash!\n", + dcp_sha1_alg.halg.base.cra_name); + goto err_unregister_aes; + } + } + + if (sdcp->caps & MXS_DCP_CAPABILITY1_SHA256) { + ret = crypto_register_ahash(&dcp_sha256_alg); + if (ret) { + dev_err(dev, "Failed to register %s hash!\n", + dcp_sha256_alg.halg.base.cra_name); + goto err_unregister_sha1; + } + } + + return 0; + +err_unregister_sha1: + if (sdcp->caps & MXS_DCP_CAPABILITY1_SHA1) + crypto_unregister_ahash(&dcp_sha1_alg); + +err_unregister_aes: + if (sdcp->caps & MXS_DCP_CAPABILITY1_AES128) + crypto_unregister_algs(dcp_aes_algs, ARRAY_SIZE(dcp_aes_algs)); + +err_destroy_aes_thread: + kthread_stop(sdcp->thread[DCP_CHAN_CRYPTO]); + +err_destroy_sha_thread: + kthread_stop(sdcp->thread[DCP_CHAN_HASH_SHA]); + +err_free_coherent: + kfree(sdcp->coh); +err_mutex: + mutex_unlock(&global_mutex); + return ret; +} + +static int mxs_dcp_remove(struct platform_device *pdev) +{ + struct dcp *sdcp = platform_get_drvdata(pdev); + + kfree(sdcp->coh); + + if (sdcp->caps & MXS_DCP_CAPABILITY1_SHA256) + crypto_unregister_ahash(&dcp_sha256_alg); + + if (sdcp->caps & MXS_DCP_CAPABILITY1_SHA1) + crypto_unregister_ahash(&dcp_sha1_alg); + + if (sdcp->caps & MXS_DCP_CAPABILITY1_AES128) + crypto_unregister_algs(dcp_aes_algs, ARRAY_SIZE(dcp_aes_algs)); + + kthread_stop(sdcp->thread[DCP_CHAN_HASH_SHA]); + kthread_stop(sdcp->thread[DCP_CHAN_CRYPTO]); + + platform_set_drvdata(pdev, NULL); + + mutex_lock(&global_mutex); + global_sdcp = NULL; + mutex_unlock(&global_mutex); + + return 0; +} + +static const struct of_device_id mxs_dcp_dt_ids[] = { + { .compatible = "fsl,imx23-dcp", .data = NULL, }, + { .compatible = "fsl,imx28-dcp", .data = NULL, }, + { /* sentinel */ } +}; + +MODULE_DEVICE_TABLE(of, mxs_dcp_dt_ids); + +static struct platform_driver mxs_dcp_driver = { + .probe = mxs_dcp_probe, + .remove = mxs_dcp_remove, + .driver = { + .name = "mxs-dcp", + .owner = THIS_MODULE, + .of_match_table = mxs_dcp_dt_ids, + }, +}; + +module_platform_driver(mxs_dcp_driver); + +MODULE_AUTHOR("Marek Vasut <marex@denx.de>"); +MODULE_DESCRIPTION("Freescale MXS DCP Driver"); +MODULE_LICENSE("GPL"); +MODULE_ALIAS("platform:mxs-dcp"); diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c index ce791c2f81f7..dde41f1df608 100644 --- a/drivers/crypto/omap-aes.c +++ b/drivers/crypto/omap-aes.c @@ -275,7 +275,7 @@ static int omap_aes_write_ctrl(struct omap_aes_dev *dd) if (dd->flags & FLAGS_CBC) val |= AES_REG_CTRL_CBC; if (dd->flags & FLAGS_CTR) { - val |= AES_REG_CTRL_CTR | AES_REG_CTRL_CTR_WIDTH_32; + val |= AES_REG_CTRL_CTR | AES_REG_CTRL_CTR_WIDTH_128; mask = AES_REG_CTRL_CTR | AES_REG_CTRL_CTR_WIDTH_MASK; } if (dd->flags & FLAGS_ENCRYPT) @@ -554,7 +554,7 @@ static int omap_aes_crypt_dma_stop(struct omap_aes_dev *dd) return err; } -int omap_aes_check_aligned(struct scatterlist *sg) +static int omap_aes_check_aligned(struct scatterlist *sg) { while (sg) { if (!IS_ALIGNED(sg->offset, 4)) @@ -566,7 +566,7 @@ int omap_aes_check_aligned(struct scatterlist *sg) return 0; } -int omap_aes_copy_sgs(struct omap_aes_dev *dd) +static int omap_aes_copy_sgs(struct omap_aes_dev *dd) { void *buf_in, *buf_out; int pages; @@ -784,6 +784,7 @@ static int omap_aes_ctr_decrypt(struct ablkcipher_request *req) static int omap_aes_cra_init(struct crypto_tfm *tfm) { struct omap_aes_dev *dd = NULL; + int err; /* Find AES device, currently picks the first device */ spin_lock_bh(&list_lock); @@ -792,7 +793,13 @@ static int omap_aes_cra_init(struct crypto_tfm *tfm) } spin_unlock_bh(&list_lock); - pm_runtime_get_sync(dd->dev); + err = pm_runtime_get_sync(dd->dev); + if (err < 0) { + dev_err(dd->dev, "%s: failed to get_sync(%d)\n", + __func__, err); + return err; + } + tfm->crt_ablkcipher.reqsize = sizeof(struct omap_aes_reqctx); return 0; @@ -1182,7 +1189,12 @@ static int omap_aes_probe(struct platform_device *pdev) dd->phys_base = res.start; pm_runtime_enable(dev); - pm_runtime_get_sync(dev); + err = pm_runtime_get_sync(dev); + if (err < 0) { + dev_err(dev, "%s: failed to get_sync(%d)\n", + __func__, err); + goto err_res; + } omap_aes_dma_stop(dd); diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c index e28104b4aab0..a727a6a59653 100644 --- a/drivers/crypto/omap-sham.c +++ b/drivers/crypto/omap-sham.c @@ -789,10 +789,13 @@ static int omap_sham_update_cpu(struct omap_sham_dev *dd) dev_dbg(dd->dev, "cpu: bufcnt: %u, digcnt: %d, final: %d\n", ctx->bufcnt, ctx->digcnt, final); - bufcnt = ctx->bufcnt; - ctx->bufcnt = 0; + if (final || (ctx->bufcnt == ctx->buflen && ctx->total)) { + bufcnt = ctx->bufcnt; + ctx->bufcnt = 0; + return omap_sham_xmit_cpu(dd, ctx->buffer, bufcnt, final); + } - return omap_sham_xmit_cpu(dd, ctx->buffer, bufcnt, final); + return 0; } static int omap_sham_update_dma_stop(struct omap_sham_dev *dd) @@ -1103,6 +1106,9 @@ static int omap_sham_update(struct ahash_request *req) return 0; } + if (dd->polling_mode) + ctx->flags |= BIT(FLAGS_CPU); + return omap_sham_enqueue(req, OP_UPDATE); } @@ -1970,7 +1976,8 @@ err_algs: crypto_unregister_ahash( &dd->pdata->algs_info[i].algs_list[j]); pm_runtime_disable(dev); - dma_release_channel(dd->dma_lch); + if (dd->dma_lch) + dma_release_channel(dd->dma_lch); data_err: dev_err(dev, "initialization failed.\n"); @@ -1994,7 +2001,9 @@ static int omap_sham_remove(struct platform_device *pdev) &dd->pdata->algs_info[i].algs_list[j]); tasklet_kill(&dd->done_task); pm_runtime_disable(&pdev->dev); - dma_release_channel(dd->dma_lch); + + if (dd->dma_lch) + dma_release_channel(dd->dma_lch); return 0; } @@ -2033,3 +2042,4 @@ module_platform_driver(omap_sham_driver); MODULE_DESCRIPTION("OMAP SHA1/MD5 hw acceleration support."); MODULE_LICENSE("GPL v2"); MODULE_AUTHOR("Dmitry Kasatkin"); +MODULE_ALIAS("platform:omap-sham"); diff --git a/drivers/crypto/picoxcell_crypto.c b/drivers/crypto/picoxcell_crypto.c index 888f7f4a6d3f..a6175ba6d238 100644 --- a/drivers/crypto/picoxcell_crypto.c +++ b/drivers/crypto/picoxcell_crypto.c @@ -495,45 +495,29 @@ static int spacc_aead_setkey(struct crypto_aead *tfm, const u8 *key, { struct spacc_aead_ctx *ctx = crypto_aead_ctx(tfm); struct spacc_alg *alg = to_spacc_alg(tfm->base.__crt_alg); - struct rtattr *rta = (void *)key; - struct crypto_authenc_key_param *param; - unsigned int authkeylen, enckeylen; + struct crypto_authenc_keys keys; int err = -EINVAL; - if (!RTA_OK(rta, keylen)) + if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) goto badkey; - if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM) + if (keys.enckeylen > AES_MAX_KEY_SIZE) goto badkey; - if (RTA_PAYLOAD(rta) < sizeof(*param)) - goto badkey; - - param = RTA_DATA(rta); - enckeylen = be32_to_cpu(param->enckeylen); - - key += RTA_ALIGN(rta->rta_len); - keylen -= RTA_ALIGN(rta->rta_len); - - if (keylen < enckeylen) - goto badkey; - - authkeylen = keylen - enckeylen; - - if (enckeylen > AES_MAX_KEY_SIZE) + if (keys.authkeylen > sizeof(ctx->hash_ctx)) goto badkey; if ((alg->ctrl_default & SPACC_CRYPTO_ALG_MASK) == SPA_CTRL_CIPH_ALG_AES) - err = spacc_aead_aes_setkey(tfm, key + authkeylen, enckeylen); + err = spacc_aead_aes_setkey(tfm, keys.enckey, keys.enckeylen); else - err = spacc_aead_des_setkey(tfm, key + authkeylen, enckeylen); + err = spacc_aead_des_setkey(tfm, keys.enckey, keys.enckeylen); if (err) goto badkey; - memcpy(ctx->hash_ctx, key, authkeylen); - ctx->hash_key_len = authkeylen; + memcpy(ctx->hash_ctx, keys.authkey, keys.authkeylen); + ctx->hash_key_len = keys.authkeylen; return 0; diff --git a/drivers/crypto/sahara.c b/drivers/crypto/sahara.c index d7bb8bac36e9..785a9ded7bdf 100644 --- a/drivers/crypto/sahara.c +++ b/drivers/crypto/sahara.c @@ -1058,7 +1058,7 @@ static struct platform_driver sahara_driver = { .driver = { .name = SAHARA_NAME, .owner = THIS_MODULE, - .of_match_table = of_match_ptr(sahara_dt_ids), + .of_match_table = sahara_dt_ids, }, .id_table = sahara_platform_ids, }; diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index 6cd0e6038583..5967667e1a8f 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -338,20 +338,29 @@ DEF_TALITOS_DONE(ch1_3, TALITOS_ISR_CH_1_3_DONE) static u32 current_desc_hdr(struct device *dev, int ch) { struct talitos_private *priv = dev_get_drvdata(dev); - int tail = priv->chan[ch].tail; + int tail, iter; dma_addr_t cur_desc; - cur_desc = in_be32(priv->chan[ch].reg + TALITOS_CDPR_LO); + cur_desc = ((u64)in_be32(priv->chan[ch].reg + TALITOS_CDPR)) << 32; + cur_desc |= in_be32(priv->chan[ch].reg + TALITOS_CDPR_LO); - while (priv->chan[ch].fifo[tail].dma_desc != cur_desc) { - tail = (tail + 1) & (priv->fifo_len - 1); - if (tail == priv->chan[ch].tail) { + if (!cur_desc) { + dev_err(dev, "CDPR is NULL, giving up search for offending descriptor\n"); + return 0; + } + + tail = priv->chan[ch].tail; + + iter = tail; + while (priv->chan[ch].fifo[iter].dma_desc != cur_desc) { + iter = (iter + 1) & (priv->fifo_len - 1); + if (iter == tail) { dev_err(dev, "couldn't locate current descriptor\n"); return 0; } } - return priv->chan[ch].fifo[tail].desc->hdr; + return priv->chan[ch].fifo[iter].desc->hdr; } /* @@ -673,39 +682,20 @@ static int aead_setkey(struct crypto_aead *authenc, const u8 *key, unsigned int keylen) { struct talitos_ctx *ctx = crypto_aead_ctx(authenc); - struct rtattr *rta = (void *)key; - struct crypto_authenc_key_param *param; - unsigned int authkeylen; - unsigned int enckeylen; + struct crypto_authenc_keys keys; - if (!RTA_OK(rta, keylen)) + if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) goto badkey; - if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM) + if (keys.authkeylen + keys.enckeylen > TALITOS_MAX_KEY_SIZE) goto badkey; - if (RTA_PAYLOAD(rta) < sizeof(*param)) - goto badkey; - - param = RTA_DATA(rta); - enckeylen = be32_to_cpu(param->enckeylen); + memcpy(ctx->key, keys.authkey, keys.authkeylen); + memcpy(&ctx->key[keys.authkeylen], keys.enckey, keys.enckeylen); - key += RTA_ALIGN(rta->rta_len); - keylen -= RTA_ALIGN(rta->rta_len); - - if (keylen < enckeylen) - goto badkey; - - authkeylen = keylen - enckeylen; - - if (keylen > TALITOS_MAX_KEY_SIZE) - goto badkey; - - memcpy(&ctx->key, key, keylen); - - ctx->keylen = keylen; - ctx->enckeylen = enckeylen; - ctx->authkeylen = authkeylen; + ctx->keylen = keys.authkeylen + keys.enckeylen; + ctx->enckeylen = keys.enckeylen; + ctx->authkeylen = keys.authkeylen; return 0; @@ -809,7 +799,7 @@ static void ipsec_esp_unmap(struct device *dev, if (edesc->assoc_chained) talitos_unmap_sg_chain(dev, areq->assoc, DMA_TO_DEVICE); - else + else if (areq->assoclen) /* assoc_nents counts also for IV in non-contiguous cases */ dma_unmap_sg(dev, areq->assoc, edesc->assoc_nents ? edesc->assoc_nents - 1 : 1, @@ -992,7 +982,11 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, dma_sync_single_for_device(dev, edesc->dma_link_tbl, edesc->dma_len, DMA_BIDIRECTIONAL); } else { - to_talitos_ptr(&desc->ptr[1], sg_dma_address(areq->assoc)); + if (areq->assoclen) + to_talitos_ptr(&desc->ptr[1], + sg_dma_address(areq->assoc)); + else + to_talitos_ptr(&desc->ptr[1], edesc->iv_dma); desc->ptr[1].j_extent = 0; } @@ -1127,7 +1121,8 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, unsigned int authsize, unsigned int ivsize, int icv_stashing, - u32 cryptoflags) + u32 cryptoflags, + bool encrypt) { struct talitos_edesc *edesc; int assoc_nents = 0, src_nents, dst_nents, alloc_len, dma_len; @@ -1141,10 +1136,10 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, return ERR_PTR(-EINVAL); } - if (iv) + if (ivsize) iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE); - if (assoc) { + if (assoclen) { /* * Currently it is assumed that iv is provided whenever assoc * is. @@ -1160,19 +1155,17 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, assoc_nents = assoc_nents ? assoc_nents + 1 : 2; } - src_nents = sg_count(src, cryptlen + authsize, &src_chained); - src_nents = (src_nents == 1) ? 0 : src_nents; - - if (!dst) { - dst_nents = 0; - } else { - if (dst == src) { - dst_nents = src_nents; - } else { - dst_nents = sg_count(dst, cryptlen + authsize, - &dst_chained); - dst_nents = (dst_nents == 1) ? 0 : dst_nents; - } + if (!dst || dst == src) { + src_nents = sg_count(src, cryptlen + authsize, &src_chained); + src_nents = (src_nents == 1) ? 0 : src_nents; + dst_nents = dst ? src_nents : 0; + } else { /* dst && dst != src*/ + src_nents = sg_count(src, cryptlen + (encrypt ? 0 : authsize), + &src_chained); + src_nents = (src_nents == 1) ? 0 : src_nents; + dst_nents = sg_count(dst, cryptlen + (encrypt ? authsize : 0), + &dst_chained); + dst_nents = (dst_nents == 1) ? 0 : dst_nents; } /* @@ -1192,9 +1185,16 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, edesc = kmalloc(alloc_len, GFP_DMA | flags); if (!edesc) { - talitos_unmap_sg_chain(dev, assoc, DMA_TO_DEVICE); + if (assoc_chained) + talitos_unmap_sg_chain(dev, assoc, DMA_TO_DEVICE); + else if (assoclen) + dma_unmap_sg(dev, assoc, + assoc_nents ? assoc_nents - 1 : 1, + DMA_TO_DEVICE); + if (iv_dma) dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE); + dev_err(dev, "could not allocate edescriptor\n"); return ERR_PTR(-ENOMEM); } @@ -1216,7 +1216,7 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, } static struct talitos_edesc *aead_edesc_alloc(struct aead_request *areq, u8 *iv, - int icv_stashing) + int icv_stashing, bool encrypt) { struct crypto_aead *authenc = crypto_aead_reqtfm(areq); struct talitos_ctx *ctx = crypto_aead_ctx(authenc); @@ -1225,7 +1225,7 @@ static struct talitos_edesc *aead_edesc_alloc(struct aead_request *areq, u8 *iv, return talitos_edesc_alloc(ctx->dev, areq->assoc, areq->src, areq->dst, iv, areq->assoclen, areq->cryptlen, ctx->authsize, ivsize, icv_stashing, - areq->base.flags); + areq->base.flags, encrypt); } static int aead_encrypt(struct aead_request *req) @@ -1235,7 +1235,7 @@ static int aead_encrypt(struct aead_request *req) struct talitos_edesc *edesc; /* allocate extended descriptor */ - edesc = aead_edesc_alloc(req, req->iv, 0); + edesc = aead_edesc_alloc(req, req->iv, 0, true); if (IS_ERR(edesc)) return PTR_ERR(edesc); @@ -1258,7 +1258,7 @@ static int aead_decrypt(struct aead_request *req) req->cryptlen -= authsize; /* allocate extended descriptor */ - edesc = aead_edesc_alloc(req, req->iv, 1); + edesc = aead_edesc_alloc(req, req->iv, 1, false); if (IS_ERR(edesc)) return PTR_ERR(edesc); @@ -1304,7 +1304,7 @@ static int aead_givencrypt(struct aead_givcrypt_request *req) struct talitos_edesc *edesc; /* allocate extended descriptor */ - edesc = aead_edesc_alloc(areq, req->giv, 0); + edesc = aead_edesc_alloc(areq, req->giv, 0, true); if (IS_ERR(edesc)) return PTR_ERR(edesc); @@ -1460,7 +1460,7 @@ static int common_nonsnoop(struct talitos_edesc *edesc, } static struct talitos_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request * - areq) + areq, bool encrypt) { struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq); struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher); @@ -1468,7 +1468,7 @@ static struct talitos_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request * return talitos_edesc_alloc(ctx->dev, NULL, areq->src, areq->dst, areq->info, 0, areq->nbytes, 0, ivsize, 0, - areq->base.flags); + areq->base.flags, encrypt); } static int ablkcipher_encrypt(struct ablkcipher_request *areq) @@ -1478,7 +1478,7 @@ static int ablkcipher_encrypt(struct ablkcipher_request *areq) struct talitos_edesc *edesc; /* allocate extended descriptor */ - edesc = ablkcipher_edesc_alloc(areq); + edesc = ablkcipher_edesc_alloc(areq, true); if (IS_ERR(edesc)) return PTR_ERR(edesc); @@ -1495,7 +1495,7 @@ static int ablkcipher_decrypt(struct ablkcipher_request *areq) struct talitos_edesc *edesc; /* allocate extended descriptor */ - edesc = ablkcipher_edesc_alloc(areq); + edesc = ablkcipher_edesc_alloc(areq, false); if (IS_ERR(edesc)) return PTR_ERR(edesc); @@ -1647,7 +1647,7 @@ static struct talitos_edesc *ahash_edesc_alloc(struct ahash_request *areq, struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); return talitos_edesc_alloc(ctx->dev, NULL, req_ctx->psrc, NULL, NULL, 0, - nbytes, 0, 0, 0, areq->base.flags); + nbytes, 0, 0, 0, areq->base.flags, false); } static int ahash_init(struct ahash_request *areq) @@ -2495,8 +2495,6 @@ static int talitos_remove(struct platform_device *ofdev) iounmap(priv->reg); - dev_set_drvdata(dev, NULL); - kfree(priv); return 0; diff --git a/drivers/crypto/tegra-aes.c b/drivers/crypto/tegra-aes.c index fa05e3c329bd..060eecc5dbc3 100644 --- a/drivers/crypto/tegra-aes.c +++ b/drivers/crypto/tegra-aes.c @@ -27,6 +27,8 @@ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + #include <linux/module.h> #include <linux/init.h> #include <linux/errno.h> @@ -199,8 +201,6 @@ static void aes_workqueue_handler(struct work_struct *work); static DECLARE_WORK(aes_work, aes_workqueue_handler); static struct workqueue_struct *aes_wq; -extern unsigned long long tegra_chip_uid(void); - static inline u32 aes_readl(struct tegra_aes_dev *dd, u32 offset) { return readl(dd->io_base + offset); @@ -713,13 +713,12 @@ static int tegra_aes_rng_reset(struct crypto_rng *tfm, u8 *seed, struct tegra_aes_dev *dd = aes_dev; struct tegra_aes_ctx *ctx = &rng_ctx; struct tegra_aes_slot *key_slot; - struct timespec ts; int ret = 0; - u64 nsec, tmp[2]; + u8 tmp[16]; /* 16 bytes = 128 bits of entropy */ u8 *dt; if (!ctx || !dd) { - dev_err(dd->dev, "ctx=0x%x, dd=0x%x\n", + pr_err("ctx=0x%x, dd=0x%x\n", (unsigned int)ctx, (unsigned int)dd); return -EINVAL; } @@ -778,14 +777,8 @@ static int tegra_aes_rng_reset(struct crypto_rng *tfm, u8 *seed, if (dd->ivlen >= (2 * DEFAULT_RNG_BLK_SZ + AES_KEYSIZE_128)) { dt = dd->iv + DEFAULT_RNG_BLK_SZ + AES_KEYSIZE_128; } else { - getnstimeofday(&ts); - nsec = timespec_to_ns(&ts); - do_div(nsec, 1000); - nsec ^= dd->ctr << 56; - dd->ctr++; - tmp[0] = nsec; - tmp[1] = tegra_chip_uid(); - dt = (u8 *)tmp; + get_random_bytes(tmp, sizeof(tmp)); + dt = tmp; } memcpy(dd->dt, dt, DEFAULT_RNG_BLK_SZ); @@ -804,7 +797,7 @@ static int tegra_aes_cra_init(struct crypto_tfm *tfm) return 0; } -void tegra_aes_cra_exit(struct crypto_tfm *tfm) +static void tegra_aes_cra_exit(struct crypto_tfm *tfm) { struct tegra_aes_ctx *ctx = crypto_ablkcipher_ctx((struct crypto_ablkcipher *)tfm); @@ -924,7 +917,7 @@ static int tegra_aes_probe(struct platform_device *pdev) } /* Initialize the vde clock */ - dd->aes_clk = clk_get(dev, "vde"); + dd->aes_clk = devm_clk_get(dev, "vde"); if (IS_ERR(dd->aes_clk)) { dev_err(dev, "iclock intialization failed.\n"); err = -ENODEV; @@ -1033,8 +1026,6 @@ out: if (dd->buf_out) dma_free_coherent(dev, AES_HW_DMA_BUFFER_SIZE_BYTES, dd->buf_out, dd->dma_buf_out); - if (!IS_ERR(dd->aes_clk)) - clk_put(dd->aes_clk); if (aes_wq) destroy_workqueue(aes_wq); spin_lock(&list_lock); @@ -1068,7 +1059,6 @@ static int tegra_aes_remove(struct platform_device *pdev) dd->buf_in, dd->dma_buf_in); dma_free_coherent(dev, AES_HW_DMA_BUFFER_SIZE_BYTES, dd->buf_out, dd->dma_buf_out); - clk_put(dd->aes_clk); aes_dev = NULL; return 0; |