summaryrefslogtreecommitdiffstats
path: root/drivers/crypto/hisilicon/qm.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/crypto/hisilicon/qm.c')
-rw-r--r--drivers/crypto/hisilicon/qm.c264
1 files changed, 130 insertions, 134 deletions
diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c
index 18599f3634c3..4b20b94e6371 100644
--- a/drivers/crypto/hisilicon/qm.c
+++ b/drivers/crypto/hisilicon/qm.c
@@ -129,16 +129,21 @@
#define QM_FIFO_OVERFLOW_TYPE 0xc0
#define QM_FIFO_OVERFLOW_TYPE_SHIFT 6
#define QM_FIFO_OVERFLOW_VF 0x3f
+#define QM_FIFO_OVERFLOW_QP_SHIFT 16
#define QM_ABNORMAL_INF01 0x100014
#define QM_DB_TIMEOUT_TYPE 0xc0
#define QM_DB_TIMEOUT_TYPE_SHIFT 6
#define QM_DB_TIMEOUT_VF 0x3f
+#define QM_DB_TIMEOUT_QP_SHIFT 16
+#define QM_ABNORMAL_INF02 0x100018
+#define QM_AXI_POISON_ERR BIT(22)
#define QM_RAS_CE_ENABLE 0x1000ec
#define QM_RAS_FE_ENABLE 0x1000f0
#define QM_RAS_NFE_ENABLE 0x1000f4
#define QM_RAS_CE_THRESHOLD 0x1000f8
#define QM_RAS_CE_TIMES_PER_IRQ 1
#define QM_OOO_SHUTDOWN_SEL 0x1040f8
+#define QM_AXI_RRESP_ERR BIT(0)
#define QM_ECC_MBIT BIT(2)
#define QM_DB_TIMEOUT BIT(10)
#define QM_OF_FIFO_OF BIT(11)
@@ -229,6 +234,8 @@
#define QM_QOS_MAX_CIR_U 6
#define QM_AUTOSUSPEND_DELAY 3000
+#define QM_DEV_ALG_MAX_LEN 256
+
#define QM_MK_CQC_DW3_V1(hop_num, pg_sz, buf_sz, cqe_sz) \
(((hop_num) << QM_CQ_HOP_NUM_SHIFT) | \
((pg_sz) << QM_CQ_PAGE_SIZE_SHIFT) | \
@@ -294,6 +301,13 @@ enum qm_basic_type {
QM_VF_IRQ_NUM_CAP,
};
+enum qm_pre_store_cap_idx {
+ QM_EQ_IRQ_TYPE_CAP_IDX = 0x0,
+ QM_AEQ_IRQ_TYPE_CAP_IDX,
+ QM_ABN_IRQ_TYPE_CAP_IDX,
+ QM_PF2VF_IRQ_TYPE_CAP_IDX,
+};
+
static const struct hisi_qm_cap_info qm_cap_info_comm[] = {
{QM_SUPPORT_DB_ISOLATION, 0x30, 0, BIT(0), 0x0, 0x0, 0x0},
{QM_SUPPORT_FUNC_QOS, 0x3100, 0, BIT(8), 0x0, 0x0, 0x1},
@@ -323,6 +337,13 @@ static const struct hisi_qm_cap_info qm_basic_info[] = {
{QM_VF_IRQ_NUM_CAP, 0x311c, 0, GENMASK(15, 0), 0x1, 0x2, 0x3},
};
+static const u32 qm_pre_store_caps[] = {
+ QM_EQ_IRQ_TYPE_CAP,
+ QM_AEQ_IRQ_TYPE_CAP,
+ QM_ABN_IRQ_TYPE_CAP,
+ QM_PF2VF_IRQ_TYPE_CAP,
+};
+
struct qm_mailbox {
__le16 w0;
__le16 queue_num;
@@ -386,7 +407,6 @@ static const struct hisi_qm_hw_error qm_hw_error[] = {
{ .int_msk = BIT(12), .msg = "qm_db_random_invalid" },
{ .int_msk = BIT(13), .msg = "qm_mailbox_timeout" },
{ .int_msk = BIT(14), .msg = "qm_flr_timeout" },
- { /* sentinel */ }
};
static const char * const qm_db_timeout[] = {
@@ -397,10 +417,6 @@ static const char * const qm_fifo_overflow[] = {
"cq", "eq", "aeq",
};
-static const char * const qp_s[] = {
- "none", "init", "start", "stop", "close",
-};
-
struct qm_typical_qos_table {
u32 start;
u32 end;
@@ -428,85 +444,6 @@ static struct qm_typical_qos_table shaper_cbs_s[] = {
static void qm_irqs_unregister(struct hisi_qm *qm);
-static bool qm_avail_state(struct hisi_qm *qm, enum qm_state new)
-{
- enum qm_state curr = atomic_read(&qm->status.flags);
- bool avail = false;
-
- switch (curr) {
- case QM_INIT:
- if (new == QM_START || new == QM_CLOSE)
- avail = true;
- break;
- case QM_START:
- if (new == QM_STOP)
- avail = true;
- break;
- case QM_STOP:
- if (new == QM_CLOSE || new == QM_START)
- avail = true;
- break;
- default:
- break;
- }
-
- dev_dbg(&qm->pdev->dev, "change qm state from %s to %s\n",
- qm_s[curr], qm_s[new]);
-
- if (!avail)
- dev_warn(&qm->pdev->dev, "Can not change qm state from %s to %s\n",
- qm_s[curr], qm_s[new]);
-
- return avail;
-}
-
-static bool qm_qp_avail_state(struct hisi_qm *qm, struct hisi_qp *qp,
- enum qp_state new)
-{
- enum qm_state qm_curr = atomic_read(&qm->status.flags);
- enum qp_state qp_curr = 0;
- bool avail = false;
-
- if (qp)
- qp_curr = atomic_read(&qp->qp_status.flags);
-
- switch (new) {
- case QP_INIT:
- if (qm_curr == QM_START || qm_curr == QM_INIT)
- avail = true;
- break;
- case QP_START:
- if ((qm_curr == QM_START && qp_curr == QP_INIT) ||
- (qm_curr == QM_START && qp_curr == QP_STOP))
- avail = true;
- break;
- case QP_STOP:
- if ((qm_curr == QM_START && qp_curr == QP_START) ||
- (qp_curr == QP_INIT))
- avail = true;
- break;
- case QP_CLOSE:
- if ((qm_curr == QM_START && qp_curr == QP_INIT) ||
- (qm_curr == QM_START && qp_curr == QP_STOP) ||
- (qm_curr == QM_STOP && qp_curr == QP_STOP) ||
- (qm_curr == QM_STOP && qp_curr == QP_INIT))
- avail = true;
- break;
- default:
- break;
- }
-
- dev_dbg(&qm->pdev->dev, "change qp state from %s to %s in QM %s\n",
- qp_s[qp_curr], qp_s[new], qm_s[qm_curr]);
-
- if (!avail)
- dev_warn(&qm->pdev->dev,
- "Can not change qp state from %s to %s in QM %s\n",
- qp_s[qp_curr], qp_s[new], qm_s[qm_curr]);
-
- return avail;
-}
-
static u32 qm_get_hw_error_status(struct hisi_qm *qm)
{
return readl(qm->io_base + QM_ABNORMAL_INT_STATUS);
@@ -660,9 +597,6 @@ int hisi_qm_mb(struct hisi_qm *qm, u8 cmd, dma_addr_t dma_addr, u16 queue,
struct qm_mailbox mailbox;
int ret;
- dev_dbg(&qm->pdev->dev, "QM mailbox request to q%u: %u-%llx\n",
- queue, cmd, (unsigned long long)dma_addr);
-
qm_mb_pre_init(&mailbox, cmd, dma_addr, queue, op);
mutex_lock(&qm->mailbox_lock);
@@ -828,6 +762,40 @@ static void qm_get_xqc_depth(struct hisi_qm *qm, u16 *low_bits,
*high_bits = (depth >> QM_XQ_DEPTH_SHIFT) & QM_XQ_DEPTH_MASK;
}
+int hisi_qm_set_algs(struct hisi_qm *qm, u64 alg_msk, const struct qm_dev_alg *dev_algs,
+ u32 dev_algs_size)
+{
+ struct device *dev = &qm->pdev->dev;
+ char *algs, *ptr;
+ int i;
+
+ if (!qm->uacce)
+ return 0;
+
+ if (dev_algs_size >= QM_DEV_ALG_MAX_LEN) {
+ dev_err(dev, "algs size %u is equal or larger than %d.\n",
+ dev_algs_size, QM_DEV_ALG_MAX_LEN);
+ return -EINVAL;
+ }
+
+ algs = devm_kzalloc(dev, QM_DEV_ALG_MAX_LEN * sizeof(char), GFP_KERNEL);
+ if (!algs)
+ return -ENOMEM;
+
+ for (i = 0; i < dev_algs_size; i++)
+ if (alg_msk & dev_algs[i].alg_msk)
+ strcat(algs, dev_algs[i].alg);
+
+ ptr = strrchr(algs, '\n');
+ if (ptr) {
+ *ptr = '\0';
+ qm->uacce->algs = algs;
+ }
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(hisi_qm_set_algs);
+
static u32 qm_get_irq_num(struct hisi_qm *qm)
{
if (qm->fun_type == QM_HW_PF)
@@ -1406,7 +1374,7 @@ static void qm_log_hw_error(struct hisi_qm *qm, u32 error_status)
{
const struct hisi_qm_hw_error *err;
struct device *dev = &qm->pdev->dev;
- u32 reg_val, type, vf_num;
+ u32 reg_val, type, vf_num, qp_id;
int i;
for (i = 0; i < ARRAY_SIZE(qm_hw_error); i++) {
@@ -1422,19 +1390,24 @@ static void qm_log_hw_error(struct hisi_qm *qm, u32 error_status)
type = (reg_val & QM_DB_TIMEOUT_TYPE) >>
QM_DB_TIMEOUT_TYPE_SHIFT;
vf_num = reg_val & QM_DB_TIMEOUT_VF;
- dev_err(dev, "qm %s doorbell timeout in function %u\n",
- qm_db_timeout[type], vf_num);
+ qp_id = reg_val >> QM_DB_TIMEOUT_QP_SHIFT;
+ dev_err(dev, "qm %s doorbell timeout in function %u qp %u\n",
+ qm_db_timeout[type], vf_num, qp_id);
} else if (err->int_msk & QM_OF_FIFO_OF) {
reg_val = readl(qm->io_base + QM_ABNORMAL_INF00);
type = (reg_val & QM_FIFO_OVERFLOW_TYPE) >>
QM_FIFO_OVERFLOW_TYPE_SHIFT;
vf_num = reg_val & QM_FIFO_OVERFLOW_VF;
-
+ qp_id = reg_val >> QM_FIFO_OVERFLOW_QP_SHIFT;
if (type < ARRAY_SIZE(qm_fifo_overflow))
- dev_err(dev, "qm %s fifo overflow in function %u\n",
- qm_fifo_overflow[type], vf_num);
+ dev_err(dev, "qm %s fifo overflow in function %u qp %u\n",
+ qm_fifo_overflow[type], vf_num, qp_id);
else
dev_err(dev, "unknown error type\n");
+ } else if (err->int_msk & QM_AXI_RRESP_ERR) {
+ reg_val = readl(qm->io_base + QM_ABNORMAL_INF02);
+ if (reg_val & QM_AXI_POISON_ERR)
+ dev_err(dev, "qm axi poison error happened\n");
}
}
}
@@ -1843,8 +1816,10 @@ static struct hisi_qp *qm_create_qp_nolock(struct hisi_qm *qm, u8 alg_type)
struct hisi_qp *qp;
int qp_id;
- if (!qm_qp_avail_state(qm, NULL, QP_INIT))
+ if (atomic_read(&qm->status.flags) == QM_STOP) {
+ dev_info_ratelimited(dev, "failed to create qp as qm is stop!\n");
return ERR_PTR(-EPERM);
+ }
if (qm->qp_in_used == qm->qp_num) {
dev_info_ratelimited(dev, "All %u queues of QM are busy!\n",
@@ -1871,7 +1846,6 @@ static struct hisi_qp *qm_create_qp_nolock(struct hisi_qm *qm, u8 alg_type)
qp->alg_type = alg_type;
qp->is_in_kernel = true;
qm->qp_in_used++;
- atomic_set(&qp->qp_status.flags, QP_INIT);
return qp;
}
@@ -1914,11 +1888,6 @@ static void hisi_qm_release_qp(struct hisi_qp *qp)
down_write(&qm->qps_lock);
- if (!qm_qp_avail_state(qm, qp, QP_CLOSE)) {
- up_write(&qm->qps_lock);
- return;
- }
-
qm->qp_in_used--;
idr_remove(&qm->qp_idr, qp->qp_id);
@@ -1966,6 +1935,11 @@ static int qm_cq_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid)
cqc.dw3 = cpu_to_le32(QM_MK_CQC_DW3_V2(QM_QC_CQE_SIZE, qp->cq_depth));
cqc.w8 = 0; /* rand_qc */
}
+ /*
+ * Enable request finishing interrupts defaultly.
+ * So, there will be some interrupts until disabling
+ * this.
+ */
cqc.dw6 = cpu_to_le32(1 << QM_CQ_PHASE_SHIFT | 1 << QM_CQ_FLAG_SHIFT);
cqc.base_l = cpu_to_le32(lower_32_bits(qp->cqe_dma));
cqc.base_h = cpu_to_le32(upper_32_bits(qp->cqe_dma));
@@ -1998,8 +1972,10 @@ static int qm_start_qp_nolock(struct hisi_qp *qp, unsigned long arg)
u32 pasid = arg;
int ret;
- if (!qm_qp_avail_state(qm, qp, QP_START))
+ if (atomic_read(&qm->status.flags) == QM_STOP) {
+ dev_info_ratelimited(dev, "failed to start qp as qm is stop!\n");
return -EPERM;
+ }
ret = qm_qp_ctx_cfg(qp, qp_id, pasid);
if (ret)
@@ -2121,21 +2097,17 @@ static int qm_stop_qp_nolock(struct hisi_qp *qp)
* is_resetting flag should be set negative so that this qp will not
* be restarted after reset.
*/
- if (atomic_read(&qp->qp_status.flags) == QP_STOP) {
+ if (atomic_read(&qp->qp_status.flags) != QP_START) {
qp->is_resetting = false;
return 0;
}
- if (!qm_qp_avail_state(qp->qm, qp, QP_STOP))
- return -EPERM;
-
atomic_set(&qp->qp_status.flags, QP_STOP);
ret = qm_drain_qp(qp);
if (ret)
dev_err(dev, "Failed to drain out data for stopping!\n");
-
flush_workqueue(qp->qm->wq);
if (unlikely(qp->is_resetting && atomic_read(&qp->qp_status.used)))
qp_stop_fail_cb(qp);
@@ -2855,13 +2827,8 @@ void hisi_qm_uninit(struct hisi_qm *qm)
{
qm_cmd_uninit(qm);
hisi_qm_unint_work(qm);
- down_write(&qm->qps_lock);
-
- if (!qm_avail_state(qm, QM_CLOSE)) {
- up_write(&qm->qps_lock);
- return;
- }
+ down_write(&qm->qps_lock);
hisi_qm_memory_uninit(qm);
hisi_qm_set_state(qm, QM_NOT_READY);
up_write(&qm->qps_lock);
@@ -3035,11 +3002,6 @@ int hisi_qm_start(struct hisi_qm *qm)
down_write(&qm->qps_lock);
- if (!qm_avail_state(qm, QM_START)) {
- up_write(&qm->qps_lock);
- return -EPERM;
- }
-
dev_dbg(dev, "qm start with %u queue pairs\n", qm->qp_num);
if (!qm->qp_num) {
@@ -3049,10 +3011,12 @@ int hisi_qm_start(struct hisi_qm *qm)
}
ret = __hisi_qm_start(qm);
- if (!ret)
- atomic_set(&qm->status.flags, QM_START);
+ if (ret)
+ goto err_unlock;
+ atomic_set(&qm->status.flags, QM_WORK);
hisi_qm_set_state(qm, QM_READY);
+
err_unlock:
up_write(&qm->qps_lock);
return ret;
@@ -3149,10 +3113,11 @@ int hisi_qm_stop(struct hisi_qm *qm, enum qm_stop_reason r)
down_write(&qm->qps_lock);
qm->status.stop_reason = r;
- if (!qm_avail_state(qm, QM_STOP)) {
- ret = -EPERM;
+ if (atomic_read(&qm->status.flags) == QM_STOP)
goto err_unlock;
- }
+
+ /* Stop all the request sending at first. */
+ atomic_set(&qm->status.flags, QM_STOP);
if (qm->status.stop_reason == QM_SOFT_RESET ||
qm->status.stop_reason == QM_DOWN) {
@@ -3176,7 +3141,6 @@ int hisi_qm_stop(struct hisi_qm *qm, enum qm_stop_reason r)
}
qm_clear_queues(qm);
- atomic_set(&qm->status.flags, QM_STOP);
err_unlock:
up_write(&qm->qps_lock);
@@ -3966,6 +3930,11 @@ static int qm_set_vf_mse(struct hisi_qm *qm, bool set)
int pos;
int i;
+ /*
+ * Since function qm_set_vf_mse is called only after SRIOV is enabled,
+ * pci_find_ext_capability cannot return 0, pos does not need to be
+ * checked.
+ */
pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_SRIOV);
pci_read_config_word(pdev, pos + PCI_SRIOV_CTRL, &sriov_ctrl);
if (set)
@@ -4816,7 +4785,7 @@ static void qm_unregister_abnormal_irq(struct hisi_qm *qm)
if (qm->fun_type == QM_HW_VF)
return;
- val = hisi_qm_get_hw_info(qm, qm_basic_info, QM_ABN_IRQ_TYPE_CAP, qm->cap_ver);
+ val = qm->cap_tables.qm_cap_table[QM_ABN_IRQ_TYPE_CAP_IDX].cap_val;
if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_ABN_IRQ_TYPE_MASK))
return;
@@ -4833,7 +4802,7 @@ static int qm_register_abnormal_irq(struct hisi_qm *qm)
if (qm->fun_type == QM_HW_VF)
return 0;
- val = hisi_qm_get_hw_info(qm, qm_basic_info, QM_ABN_IRQ_TYPE_CAP, qm->cap_ver);
+ val = qm->cap_tables.qm_cap_table[QM_ABN_IRQ_TYPE_CAP_IDX].cap_val;
if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_ABN_IRQ_TYPE_MASK))
return 0;
@@ -4850,7 +4819,7 @@ static void qm_unregister_mb_cmd_irq(struct hisi_qm *qm)
struct pci_dev *pdev = qm->pdev;
u32 irq_vector, val;
- val = hisi_qm_get_hw_info(qm, qm_basic_info, QM_PF2VF_IRQ_TYPE_CAP, qm->cap_ver);
+ val = qm->cap_tables.qm_cap_table[QM_PF2VF_IRQ_TYPE_CAP_IDX].cap_val;
if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK))
return;
@@ -4864,7 +4833,7 @@ static int qm_register_mb_cmd_irq(struct hisi_qm *qm)
u32 irq_vector, val;
int ret;
- val = hisi_qm_get_hw_info(qm, qm_basic_info, QM_PF2VF_IRQ_TYPE_CAP, qm->cap_ver);
+ val = qm->cap_tables.qm_cap_table[QM_PF2VF_IRQ_TYPE_CAP_IDX].cap_val;
if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK))
return 0;
@@ -4881,7 +4850,7 @@ static void qm_unregister_aeq_irq(struct hisi_qm *qm)
struct pci_dev *pdev = qm->pdev;
u32 irq_vector, val;
- val = hisi_qm_get_hw_info(qm, qm_basic_info, QM_AEQ_IRQ_TYPE_CAP, qm->cap_ver);
+ val = qm->cap_tables.qm_cap_table[QM_AEQ_IRQ_TYPE_CAP_IDX].cap_val;
if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK))
return;
@@ -4895,7 +4864,7 @@ static int qm_register_aeq_irq(struct hisi_qm *qm)
u32 irq_vector, val;
int ret;
- val = hisi_qm_get_hw_info(qm, qm_basic_info, QM_AEQ_IRQ_TYPE_CAP, qm->cap_ver);
+ val = qm->cap_tables.qm_cap_table[QM_AEQ_IRQ_TYPE_CAP_IDX].cap_val;
if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK))
return 0;
@@ -4913,7 +4882,7 @@ static void qm_unregister_eq_irq(struct hisi_qm *qm)
struct pci_dev *pdev = qm->pdev;
u32 irq_vector, val;
- val = hisi_qm_get_hw_info(qm, qm_basic_info, QM_EQ_IRQ_TYPE_CAP, qm->cap_ver);
+ val = qm->cap_tables.qm_cap_table[QM_EQ_IRQ_TYPE_CAP_IDX].cap_val;
if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK))
return;
@@ -4927,7 +4896,7 @@ static int qm_register_eq_irq(struct hisi_qm *qm)
u32 irq_vector, val;
int ret;
- val = hisi_qm_get_hw_info(qm, qm_basic_info, QM_EQ_IRQ_TYPE_CAP, qm->cap_ver);
+ val = qm->cap_tables.qm_cap_table[QM_EQ_IRQ_TYPE_CAP_IDX].cap_val;
if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK))
return 0;
@@ -5015,7 +4984,29 @@ static int qm_get_qp_num(struct hisi_qm *qm)
return 0;
}
-static void qm_get_hw_caps(struct hisi_qm *qm)
+static int qm_pre_store_irq_type_caps(struct hisi_qm *qm)
+{
+ struct hisi_qm_cap_record *qm_cap;
+ struct pci_dev *pdev = qm->pdev;
+ size_t i, size;
+
+ size = ARRAY_SIZE(qm_pre_store_caps);
+ qm_cap = devm_kzalloc(&pdev->dev, sizeof(*qm_cap) * size, GFP_KERNEL);
+ if (!qm_cap)
+ return -ENOMEM;
+
+ for (i = 0; i < size; i++) {
+ qm_cap[i].type = qm_pre_store_caps[i];
+ qm_cap[i].cap_val = hisi_qm_get_hw_info(qm, qm_basic_info,
+ qm_pre_store_caps[i], qm->cap_ver);
+ }
+
+ qm->cap_tables.qm_cap_table = qm_cap;
+
+ return 0;
+}
+
+static int qm_get_hw_caps(struct hisi_qm *qm)
{
const struct hisi_qm_cap_info *cap_info = qm->fun_type == QM_HW_PF ?
qm_cap_info_pf : qm_cap_info_vf;
@@ -5046,6 +5037,9 @@ static void qm_get_hw_caps(struct hisi_qm *qm)
if (val)
set_bit(cap_info[i].type, &qm->caps);
}
+
+ /* Fetch and save the value of irq type related capability registers */
+ return qm_pre_store_irq_type_caps(qm);
}
static int qm_get_pci_res(struct hisi_qm *qm)
@@ -5067,7 +5061,10 @@ static int qm_get_pci_res(struct hisi_qm *qm)
goto err_request_mem_regions;
}
- qm_get_hw_caps(qm);
+ ret = qm_get_hw_caps(qm);
+ if (ret)
+ goto err_ioremap;
+
if (test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps)) {
qm->db_interval = QM_QP_DB_INTERVAL;
qm->db_phys_base = pci_resource_start(pdev, PCI_BAR_4);
@@ -5340,7 +5337,6 @@ int hisi_qm_init(struct hisi_qm *qm)
goto err_free_qm_memory;
qm_cmd_init(qm);
- atomic_set(&qm->status.flags, QM_INIT);
return 0;