From 0794d601d1748b007dc338127232e74c19fd1337 Mon Sep 17 00:00:00 2001 From: James Smart Date: Mon, 28 Jan 2019 11:14:19 -0800 Subject: scsi: lpfc: Implement common IO buffers between NVME and SCSI Currently, both NVME and SCSI get their IO buffers from separate pools. XRI's are associated 1:1 with IO buffers, so XRI's are also split between protocols. Eliminate the independent pools and use a single pool. Each buffer structure now has a common section and a protocol section. Per protocol routines for SGL initialization are removed and replaced by common routines. Initialization of the buffers is only done on the common area. All other fields, which are protocol specific, are initialized when the buffer is allocated for use in the per-protocol allocation routine. In the past, the SCSI side allocated IO buffers as part of slave_alloc calls until the maximum XRIs for SCSI was reached. As all XRIs are now common and may be used for either protocol, allocation for everything is done as part of adapter initialization and the scsi side has no action in slave alloc. As XRI's are no longer split, the lpfc_xri_split module parameter is removed. Adapters based on SLI3 will continue to use the older scsi_buf_list_get/put routines. All SLI4 adapters utilize the new IO buffer scheme Signed-off-by: Dick Kennedy Signed-off-by: James Smart Reviewed-by: Hannes Reinecke Signed-off-by: Martin K. Petersen --- drivers/scsi/lpfc/lpfc.h | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) (limited to 'drivers/scsi/lpfc/lpfc.h') diff --git a/drivers/scsi/lpfc/lpfc.h b/drivers/scsi/lpfc/lpfc.h index ebdfe5b26937..858a9a50f94d 100644 --- a/drivers/scsi/lpfc/lpfc.h +++ b/drivers/scsi/lpfc/lpfc.h @@ -617,8 +617,6 @@ struct lpfc_ras_fwlog { struct lpfc_hba { /* SCSI interface function jump table entries */ - int (*lpfc_new_scsi_buf) - (struct lpfc_vport *, int); struct lpfc_scsi_buf * (*lpfc_get_scsi_buf) (struct lpfc_hba *, struct lpfc_nodelist *); int (*lpfc_scsi_prep_dma_buf) @@ -875,7 +873,6 @@ struct lpfc_hba { uint32_t cfg_enable_fc4_type; uint32_t cfg_enable_bbcr; /* Enable BB Credit Recovery */ uint32_t cfg_enable_dpp; /* Enable Direct Packet Push */ - uint32_t cfg_xri_split; #define LPFC_ENABLE_FCP 1 #define LPFC_ENABLE_NVME 2 #define LPFC_ENABLE_BOTH 3 @@ -970,13 +967,13 @@ struct lpfc_hba { struct list_head lpfc_scsi_buf_list_get; struct list_head lpfc_scsi_buf_list_put; uint32_t total_scsi_bufs; - spinlock_t nvme_buf_list_get_lock; /* NVME buf alloc list lock */ - spinlock_t nvme_buf_list_put_lock; /* NVME buf free list lock */ - struct list_head lpfc_nvme_buf_list_get; - struct list_head lpfc_nvme_buf_list_put; - uint32_t total_nvme_bufs; - uint32_t get_nvme_bufs; - uint32_t put_nvme_bufs; + spinlock_t common_buf_list_get_lock; /* Common buf alloc list lock */ + spinlock_t common_buf_list_put_lock; /* Common buf free list lock */ + struct list_head lpfc_common_buf_list_get; + struct list_head lpfc_common_buf_list_put; + uint32_t total_common_bufs; + uint32_t get_common_bufs; + uint32_t put_common_bufs; struct list_head lpfc_iocb_list; uint32_t total_iocbq_bufs; struct list_head active_rrq_list; -- cgit v1.2.3 From cdb42becdd40eeb320af3f21ac9a34e9d7517516 Mon Sep 17 00:00:00 2001 From: James Smart Date: Mon, 28 Jan 2019 11:14:21 -0800 Subject: scsi: lpfc: Replace io_channels for nvme and fcp with general hdw_queues per cpu Currently, both nvme and fcp each have their own concept of an io_channel, which is a combination wq/cq and associated msix. Different cpus would share an io_channel. The driver is now moving to per-cpu wq/cq pairs and msix vectors. The driver will still use separate wq/cq pairs per protocol on each cpu, but the protocols will share the msix vector. Given the elimination of the nvme and fcp io channels, the module parameters will be removed. A new parameter, lpfc_hdw_queue is added which allows the wq/cq pair allocation per cpu to be overridden and allocated to lesser value. If lpfc_hdw_queue is zero, the number of pairs allocated will be based on the number of cpus. If non-zero, the parameter specifies the number of queues to allocate. At this time, the maximum non-zero value is 64. To manage this new paradigm, a new hardware queue structure is created to track queue activity and relationships. As MSIX vector allocation must be known before setting up the relationships, msix allocation now occurs before queue datastructures are allocated. If the number of vectors allocated is less than the desired hardware queues, the hardware queue counts will be reduced to the number of vectors Signed-off-by: Dick Kennedy Signed-off-by: James Smart Reviewed-by: Hannes Reinecke Signed-off-by: Martin K. Petersen --- drivers/scsi/lpfc/lpfc.h | 4 +- drivers/scsi/lpfc/lpfc_attr.c | 84 ++----- drivers/scsi/lpfc/lpfc_debugfs.c | 152 ++++++------ drivers/scsi/lpfc/lpfc_debugfs.h | 65 +++--- drivers/scsi/lpfc/lpfc_init.c | 489 ++++++++++++++------------------------- drivers/scsi/lpfc/lpfc_nvme.c | 16 +- drivers/scsi/lpfc/lpfc_nvmet.c | 10 +- drivers/scsi/lpfc/lpfc_scsi.c | 8 +- drivers/scsi/lpfc/lpfc_sli.c | 159 ++++++------- drivers/scsi/lpfc/lpfc_sli4.h | 36 +-- 10 files changed, 417 insertions(+), 606 deletions(-) (limited to 'drivers/scsi/lpfc/lpfc.h') diff --git a/drivers/scsi/lpfc/lpfc.h b/drivers/scsi/lpfc/lpfc.h index 858a9a50f94d..da12476dd933 100644 --- a/drivers/scsi/lpfc/lpfc.h +++ b/drivers/scsi/lpfc/lpfc.h @@ -810,11 +810,10 @@ struct lpfc_hba { uint32_t cfg_auto_imax; uint32_t cfg_fcp_imax; uint32_t cfg_fcp_cpu_map; - uint32_t cfg_fcp_io_channel; + uint32_t cfg_hdw_queue; uint32_t cfg_suppress_rsp; uint32_t cfg_nvme_oas; uint32_t cfg_nvme_embed_cmd; - uint32_t cfg_nvme_io_channel; uint32_t cfg_nvmet_mrq_post; uint32_t cfg_nvmet_mrq; uint32_t cfg_enable_nvmet; @@ -877,7 +876,6 @@ struct lpfc_hba { #define LPFC_ENABLE_NVME 2 #define LPFC_ENABLE_BOTH 3 uint32_t cfg_enable_pbde; - uint32_t io_channel_irqs; /* number of irqs for io channels */ struct nvmet_fc_target_port *targetport; lpfc_vpd_t vpd; /* vital product data */ diff --git a/drivers/scsi/lpfc/lpfc_attr.c b/drivers/scsi/lpfc/lpfc_attr.c index 0980e1b67b83..c6b1d432dd07 100644 --- a/drivers/scsi/lpfc/lpfc_attr.c +++ b/drivers/scsi/lpfc/lpfc_attr.c @@ -456,7 +456,7 @@ lpfc_nvme_info_show(struct device *dev, struct device_attribute *attr, totin = 0; totout = 0; - for (i = 0; i < phba->cfg_nvme_io_channel; i++) { + for (i = 0; i < phba->cfg_hdw_queue; i++) { cstat = &lport->cstat[i]; tot = atomic_read(&cstat->fc4NvmeIoCmpls); totin += tot; @@ -4909,7 +4909,7 @@ lpfc_fcp_imax_store(struct device *dev, struct device_attribute *attr, phba->cfg_fcp_imax = (uint32_t)val; phba->initial_imax = phba->cfg_fcp_imax; - for (i = 0; i < phba->io_channel_irqs; i += LPFC_MAX_EQ_DELAY_EQID_CNT) + for (i = 0; i < phba->cfg_hdw_queue; i += LPFC_MAX_EQ_DELAY_EQID_CNT) lpfc_modify_hba_eq_delay(phba, i, LPFC_MAX_EQ_DELAY_EQID_CNT, val); @@ -5398,41 +5398,23 @@ LPFC_ATTR_RW(nvme_embed_cmd, 1, 0, 2, "Embed NVME Command in WQE"); /* - * lpfc_fcp_io_channel: Set the number of FCP IO channels the driver - * will advertise it supports to the SCSI layer. This also will map to - * the number of WQs the driver will create. - * - * 0 = Configure the number of io channels to the number of active CPUs. - * 1,32 = Manually specify how many io channels to use. - * - * Value range is [0,32]. Default value is 4. - */ -LPFC_ATTR_R(fcp_io_channel, - LPFC_FCP_IO_CHAN_DEF, - LPFC_HBA_IO_CHAN_MIN, LPFC_HBA_IO_CHAN_MAX, - "Set the number of FCP I/O channels"); - -/* - * lpfc_nvme_io_channel: Set the number of IO hardware queues the driver - * will advertise it supports to the NVME layer. This also will map to - * the number of WQs the driver will create. - * - * This module parameter is valid when lpfc_enable_fc4_type is set - * to support NVME. + * lpfc_hdw_queue: Set the number of IO channels the driver + * will advertise it supports to the NVME and SCSI layers. This also + * will map to the number of EQ/CQ/WQs the driver will create. * * The NVME Layer will try to create this many, plus 1 administrative * hardware queue. The administrative queue will always map to WQ 0 * A hardware IO queue maps (qidx) to a specific driver WQ. * - * 0 = Configure the number of io channels to the number of active CPUs. - * 1,32 = Manually specify how many io channels to use. + * 0 = Configure the number of hdw queues to the number of active CPUs. + * 1,64 = Manually specify how many hdw queues to use. * - * Value range is [0,32]. Default value is 0. + * Value range is [0,64]. Default value is 0. */ -LPFC_ATTR_R(nvme_io_channel, - LPFC_NVME_IO_CHAN_DEF, - LPFC_HBA_IO_CHAN_MIN, LPFC_HBA_IO_CHAN_MAX, - "Set the number of NVME I/O channels"); +LPFC_ATTR_R(hdw_queue, + LPFC_HBA_HDWQ_DEF, + LPFC_HBA_HDWQ_MIN, LPFC_HBA_HDWQ_MAX, + "Set the number of I/O Hardware Queues"); /* # lpfc_enable_hba_reset: Allow or prevent HBA resets to the hardware. @@ -5727,9 +5709,8 @@ struct device_attribute *lpfc_hba_attrs[] = { &dev_attr_lpfc_auto_imax, &dev_attr_lpfc_fcp_imax, &dev_attr_lpfc_fcp_cpu_map, - &dev_attr_lpfc_fcp_io_channel, + &dev_attr_lpfc_hdw_queue, &dev_attr_lpfc_suppress_rsp, - &dev_attr_lpfc_nvme_io_channel, &dev_attr_lpfc_nvmet_mrq, &dev_attr_lpfc_nvmet_mrq_post, &dev_attr_lpfc_nvme_enable_fb, @@ -6806,8 +6787,7 @@ lpfc_get_cfgparam(struct lpfc_hba *phba) /* Initialize first burst. Target vs Initiator are different. */ lpfc_nvme_enable_fb_init(phba, lpfc_nvme_enable_fb); lpfc_nvmet_fb_size_init(phba, lpfc_nvmet_fb_size); - lpfc_fcp_io_channel_init(phba, lpfc_fcp_io_channel); - lpfc_nvme_io_channel_init(phba, lpfc_nvme_io_channel); + lpfc_hdw_queue_init(phba, lpfc_hdw_queue); lpfc_enable_bbcr_init(phba, lpfc_enable_bbcr); lpfc_enable_dpp_init(phba, lpfc_enable_dpp); @@ -6829,21 +6809,8 @@ lpfc_get_cfgparam(struct lpfc_hba *phba) phba->cfg_enable_pbde = 0; /* A value of 0 means use the number of CPUs found in the system */ - if (phba->cfg_fcp_io_channel == 0) - phba->cfg_fcp_io_channel = phba->sli4_hba.num_present_cpu; - if (phba->cfg_nvme_io_channel == 0) - phba->cfg_nvme_io_channel = phba->sli4_hba.num_present_cpu; - - if (phba->cfg_enable_fc4_type == LPFC_ENABLE_NVME) - phba->cfg_fcp_io_channel = 0; - - if (phba->cfg_enable_fc4_type == LPFC_ENABLE_FCP) - phba->cfg_nvme_io_channel = 0; - - if (phba->cfg_fcp_io_channel > phba->cfg_nvme_io_channel) - phba->io_channel_irqs = phba->cfg_fcp_io_channel; - else - phba->io_channel_irqs = phba->cfg_nvme_io_channel; + if (phba->cfg_hdw_queue == 0) + phba->cfg_hdw_queue = phba->sli4_hba.num_present_cpu; phba->cfg_soft_wwnn = 0L; phba->cfg_soft_wwpn = 0L; @@ -6884,16 +6851,12 @@ lpfc_get_cfgparam(struct lpfc_hba *phba) void lpfc_nvme_mod_param_dep(struct lpfc_hba *phba) { - if (phba->cfg_nvme_io_channel > phba->sli4_hba.num_present_cpu) - phba->cfg_nvme_io_channel = phba->sli4_hba.num_present_cpu; - - if (phba->cfg_fcp_io_channel > phba->sli4_hba.num_present_cpu) - phba->cfg_fcp_io_channel = phba->sli4_hba.num_present_cpu; + if (phba->cfg_hdw_queue > phba->sli4_hba.num_present_cpu) + phba->cfg_hdw_queue = phba->sli4_hba.num_present_cpu; if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME && phba->nvmet_support) { phba->cfg_enable_fc4_type &= ~LPFC_ENABLE_FCP; - phba->cfg_fcp_io_channel = 0; lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC, "6013 %s x%x fb_size x%x, fb_max x%x\n", @@ -6910,11 +6873,11 @@ lpfc_nvme_mod_param_dep(struct lpfc_hba *phba) } if (!phba->cfg_nvmet_mrq) - phba->cfg_nvmet_mrq = phba->cfg_nvme_io_channel; + phba->cfg_nvmet_mrq = phba->cfg_hdw_queue; /* Adjust lpfc_nvmet_mrq to avoid running out of WQE slots */ - if (phba->cfg_nvmet_mrq > phba->cfg_nvme_io_channel) { - phba->cfg_nvmet_mrq = phba->cfg_nvme_io_channel; + if (phba->cfg_nvmet_mrq > phba->cfg_hdw_queue) { + phba->cfg_nvmet_mrq = phba->cfg_hdw_queue; lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC, "6018 Adjust lpfc_nvmet_mrq to %d\n", phba->cfg_nvmet_mrq); @@ -6928,11 +6891,6 @@ lpfc_nvme_mod_param_dep(struct lpfc_hba *phba) phba->cfg_nvmet_mrq = LPFC_NVMET_MRQ_OFF; phba->cfg_nvmet_fb_size = 0; } - - if (phba->cfg_fcp_io_channel > phba->cfg_nvme_io_channel) - phba->io_channel_irqs = phba->cfg_fcp_io_channel; - else - phba->io_channel_irqs = phba->cfg_nvme_io_channel; } /** diff --git a/drivers/scsi/lpfc/lpfc_debugfs.c b/drivers/scsi/lpfc/lpfc_debugfs.c index 355857e1ef7d..fc5ec99bfde2 100644 --- a/drivers/scsi/lpfc/lpfc_debugfs.c +++ b/drivers/scsi/lpfc/lpfc_debugfs.c @@ -919,13 +919,13 @@ lpfc_debugfs_nvmestat_data(struct lpfc_vport *vport, char *buf, int size) atomic_read(&lport->fc4NvmeLsRequests), atomic_read(&lport->fc4NvmeLsCmpls)); - if (phba->cfg_nvme_io_channel < 32) - maxch = phba->cfg_nvme_io_channel; + if (phba->cfg_hdw_queue < LPFC_HBA_HDWQ_MAX) + maxch = phba->cfg_hdw_queue; else - maxch = 32; + maxch = LPFC_HBA_HDWQ_MAX; totin = 0; totout = 0; - for (i = 0; i < phba->cfg_nvme_io_channel; i++) { + for (i = 0; i < phba->cfg_hdw_queue; i++) { cstat = &lport->cstat[i]; tot = atomic_read(&cstat->fc4NvmeIoCmpls); totin += tot; @@ -3182,21 +3182,23 @@ lpfc_idiag_wqs_for_cq(struct lpfc_hba *phba, char *wqtype, char *pbuffer, struct lpfc_queue *qp; int qidx; - for (qidx = 0; qidx < phba->cfg_fcp_io_channel; qidx++) { - qp = phba->sli4_hba.fcp_wq[qidx]; + for (qidx = 0; qidx < phba->cfg_hdw_queue; qidx++) { + qp = phba->sli4_hba.hdwq[qidx].fcp_wq; if (qp->assoc_qid != cq_id) continue; *len = __lpfc_idiag_print_wq(qp, wqtype, pbuffer, *len); if (*len >= max_cnt) return 1; } - for (qidx = 0; qidx < phba->cfg_nvme_io_channel; qidx++) { - qp = phba->sli4_hba.nvme_wq[qidx]; - if (qp->assoc_qid != cq_id) - continue; - *len = __lpfc_idiag_print_wq(qp, wqtype, pbuffer, *len); - if (*len >= max_cnt) - return 1; + if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { + for (qidx = 0; qidx < phba->cfg_hdw_queue; qidx++) { + qp = phba->sli4_hba.hdwq[qidx].nvme_wq; + if (qp->assoc_qid != cq_id) + continue; + *len = __lpfc_idiag_print_wq(qp, wqtype, pbuffer, *len); + if (*len >= max_cnt) + return 1; + } } return 0; } @@ -3262,8 +3264,8 @@ lpfc_idiag_cqs_for_eq(struct lpfc_hba *phba, char *pbuffer, struct lpfc_queue *qp; int qidx, rc; - for (qidx = 0; qidx < phba->cfg_fcp_io_channel; qidx++) { - qp = phba->sli4_hba.fcp_cq[qidx]; + for (qidx = 0; qidx < phba->cfg_hdw_queue; qidx++) { + qp = phba->sli4_hba.hdwq[qidx].fcp_cq; if (qp->assoc_qid != eq_id) continue; @@ -3281,23 +3283,25 @@ lpfc_idiag_cqs_for_eq(struct lpfc_hba *phba, char *pbuffer, return 1; } - for (qidx = 0; qidx < phba->cfg_nvme_io_channel; qidx++) { - qp = phba->sli4_hba.nvme_cq[qidx]; - if (qp->assoc_qid != eq_id) - continue; + if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { + for (qidx = 0; qidx < phba->cfg_hdw_queue; qidx++) { + qp = phba->sli4_hba.hdwq[qidx].nvme_cq; + if (qp->assoc_qid != eq_id) + continue; - *len = __lpfc_idiag_print_cq(qp, "NVME", pbuffer, *len); + *len = __lpfc_idiag_print_cq(qp, "NVME", pbuffer, *len); - /* Reset max counter */ - qp->CQ_max_cqe = 0; + /* Reset max counter */ + qp->CQ_max_cqe = 0; - if (*len >= max_cnt) - return 1; + if (*len >= max_cnt) + return 1; - rc = lpfc_idiag_wqs_for_cq(phba, "NVME", pbuffer, len, - max_cnt, qp->queue_id); - if (rc) - return 1; + rc = lpfc_idiag_wqs_for_cq(phba, "NVME", pbuffer, len, + max_cnt, qp->queue_id); + if (rc) + return 1; + } } if ((eqidx < phba->cfg_nvmet_mrq) && phba->nvmet_support) { @@ -3387,19 +3391,19 @@ lpfc_idiag_queinfo_read(struct file *file, char __user *buf, size_t nbytes, spin_lock_irq(&phba->hbalock); /* Fast-path event queue */ - if (phba->sli4_hba.hba_eq && phba->io_channel_irqs) { + if (phba->sli4_hba.hdwq && phba->cfg_hdw_queue) { x = phba->lpfc_idiag_last_eq; phba->lpfc_idiag_last_eq++; - if (phba->lpfc_idiag_last_eq >= phba->io_channel_irqs) + if (phba->lpfc_idiag_last_eq >= phba->cfg_hdw_queue) phba->lpfc_idiag_last_eq = 0; len += snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len, "EQ %d out of %d HBA EQs\n", - x, phba->io_channel_irqs); + x, phba->cfg_hdw_queue); /* Fast-path EQ */ - qp = phba->sli4_hba.hba_eq[x]; + qp = phba->sli4_hba.hdwq[x].hba_eq; if (!qp) goto out; @@ -3691,9 +3695,9 @@ lpfc_idiag_queacc_write(struct file *file, const char __user *buf, switch (quetp) { case LPFC_IDIAG_EQ: /* HBA event queue */ - if (phba->sli4_hba.hba_eq) { - for (qidx = 0; qidx < phba->io_channel_irqs; qidx++) { - qp = phba->sli4_hba.hba_eq[qidx]; + if (phba->sli4_hba.hdwq) { + for (qidx = 0; qidx < phba->cfg_hdw_queue; qidx++) { + qp = phba->sli4_hba.hdwq[qidx].hba_eq; if (qp && qp->queue_id == queid) { /* Sanity check */ rc = lpfc_idiag_que_param_check(qp, @@ -3742,10 +3746,10 @@ lpfc_idiag_queacc_write(struct file *file, const char __user *buf, goto pass_check; } /* FCP complete queue */ - if (phba->sli4_hba.fcp_cq) { - for (qidx = 0; qidx < phba->cfg_fcp_io_channel; + if (phba->sli4_hba.hdwq) { + for (qidx = 0; qidx < phba->cfg_hdw_queue; qidx++) { - qp = phba->sli4_hba.fcp_cq[qidx]; + qp = phba->sli4_hba.hdwq[qidx].fcp_cq; if (qp && qp->queue_id == queid) { /* Sanity check */ rc = lpfc_idiag_que_param_check( @@ -3758,23 +3762,20 @@ lpfc_idiag_queacc_write(struct file *file, const char __user *buf, } } /* NVME complete queue */ - if (phba->sli4_hba.nvme_cq) { + if (phba->sli4_hba.hdwq) { qidx = 0; do { - if (phba->sli4_hba.nvme_cq[qidx] && - phba->sli4_hba.nvme_cq[qidx]->queue_id == - queid) { + qp = phba->sli4_hba.hdwq[qidx].nvme_cq; + if (qp && qp->queue_id == queid) { /* Sanity check */ rc = lpfc_idiag_que_param_check( - phba->sli4_hba.nvme_cq[qidx], - index, count); + qp, index, count); if (rc) goto error_out; - idiag.ptr_private = - phba->sli4_hba.nvme_cq[qidx]; + idiag.ptr_private = qp; goto pass_check; } - } while (++qidx < phba->cfg_nvme_io_channel); + } while (++qidx < phba->cfg_hdw_queue); } goto error_out; break; @@ -3815,11 +3816,11 @@ lpfc_idiag_queacc_write(struct file *file, const char __user *buf, idiag.ptr_private = phba->sli4_hba.nvmels_wq; goto pass_check; } - /* FCP work queue */ - if (phba->sli4_hba.fcp_wq) { - for (qidx = 0; qidx < phba->cfg_fcp_io_channel; - qidx++) { - qp = phba->sli4_hba.fcp_wq[qidx]; + + if (phba->sli4_hba.hdwq) { + /* FCP/SCSI work queue */ + for (qidx = 0; qidx < phba->cfg_hdw_queue; qidx++) { + qp = phba->sli4_hba.hdwq[qidx].fcp_wq; if (qp && qp->queue_id == queid) { /* Sanity check */ rc = lpfc_idiag_que_param_check( @@ -3830,12 +3831,9 @@ lpfc_idiag_queacc_write(struct file *file, const char __user *buf, goto pass_check; } } - } - /* NVME work queue */ - if (phba->sli4_hba.nvme_wq) { - for (qidx = 0; qidx < phba->cfg_nvme_io_channel; - qidx++) { - qp = phba->sli4_hba.nvme_wq[qidx]; + /* NVME work queue */ + for (qidx = 0; qidx < phba->cfg_hdw_queue; qidx++) { + qp = phba->sli4_hba.hdwq[qidx].nvme_wq; if (qp && qp->queue_id == queid) { /* Sanity check */ rc = lpfc_idiag_que_param_check( @@ -3848,26 +3846,6 @@ lpfc_idiag_queacc_write(struct file *file, const char __user *buf, } } - /* NVME work queues */ - if (phba->sli4_hba.nvme_wq) { - for (qidx = 0; qidx < phba->cfg_nvme_io_channel; - qidx++) { - if (!phba->sli4_hba.nvme_wq[qidx]) - continue; - if (phba->sli4_hba.nvme_wq[qidx]->queue_id == - queid) { - /* Sanity check */ - rc = lpfc_idiag_que_param_check( - phba->sli4_hba.nvme_wq[qidx], - index, count); - if (rc) - goto error_out; - idiag.ptr_private = - phba->sli4_hba.nvme_wq[qidx]; - goto pass_check; - } - } - } goto error_out; break; case LPFC_IDIAG_RQ: @@ -5784,11 +5762,13 @@ lpfc_debug_dump_all_queues(struct lpfc_hba *phba) lpfc_debug_dump_wq(phba, DUMP_ELS, 0); lpfc_debug_dump_wq(phba, DUMP_NVMELS, 0); - for (idx = 0; idx < phba->cfg_fcp_io_channel; idx++) + for (idx = 0; idx < phba->cfg_hdw_queue; idx++) lpfc_debug_dump_wq(phba, DUMP_FCP, idx); - for (idx = 0; idx < phba->cfg_nvme_io_channel; idx++) - lpfc_debug_dump_wq(phba, DUMP_NVME, idx); + if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { + for (idx = 0; idx < phba->cfg_hdw_queue; idx++) + lpfc_debug_dump_wq(phba, DUMP_NVME, idx); + } lpfc_debug_dump_hdr_rq(phba); lpfc_debug_dump_dat_rq(phba); @@ -5799,15 +5779,17 @@ lpfc_debug_dump_all_queues(struct lpfc_hba *phba) lpfc_debug_dump_cq(phba, DUMP_ELS, 0); lpfc_debug_dump_cq(phba, DUMP_NVMELS, 0); - for (idx = 0; idx < phba->cfg_fcp_io_channel; idx++) + for (idx = 0; idx < phba->cfg_hdw_queue; idx++) lpfc_debug_dump_cq(phba, DUMP_FCP, idx); - for (idx = 0; idx < phba->cfg_nvme_io_channel; idx++) - lpfc_debug_dump_cq(phba, DUMP_NVME, idx); + if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { + for (idx = 0; idx < phba->cfg_hdw_queue; idx++) + lpfc_debug_dump_cq(phba, DUMP_NVME, idx); + } /* * Dump Event Queues (EQs) */ - for (idx = 0; idx < phba->io_channel_irqs; idx++) + for (idx = 0; idx < phba->cfg_hdw_queue; idx++) lpfc_debug_dump_hba_eq(phba, idx); } diff --git a/drivers/scsi/lpfc/lpfc_debugfs.h b/drivers/scsi/lpfc/lpfc_debugfs.h index 30efc7bf91bd..2c5bc494b247 100644 --- a/drivers/scsi/lpfc/lpfc_debugfs.h +++ b/drivers/scsi/lpfc/lpfc_debugfs.h @@ -410,10 +410,10 @@ lpfc_debug_dump_wq(struct lpfc_hba *phba, int qtype, int wqidx) char *qtypestr; if (qtype == DUMP_FCP) { - wq = phba->sli4_hba.fcp_wq[wqidx]; + wq = phba->sli4_hba.hdwq[wqidx].fcp_wq; qtypestr = "FCP"; } else if (qtype == DUMP_NVME) { - wq = phba->sli4_hba.nvme_wq[wqidx]; + wq = phba->sli4_hba.hdwq[wqidx].nvme_wq; qtypestr = "NVME"; } else if (qtype == DUMP_MBX) { wq = phba->sli4_hba.mbx_wq; @@ -454,14 +454,15 @@ lpfc_debug_dump_cq(struct lpfc_hba *phba, int qtype, int wqidx) int eqidx; /* fcp/nvme wq and cq are 1:1, thus same indexes */ + eq = NULL; if (qtype == DUMP_FCP) { - wq = phba->sli4_hba.fcp_wq[wqidx]; - cq = phba->sli4_hba.fcp_cq[wqidx]; + wq = phba->sli4_hba.hdwq[wqidx].fcp_wq; + cq = phba->sli4_hba.hdwq[wqidx].fcp_cq; qtypestr = "FCP"; } else if (qtype == DUMP_NVME) { - wq = phba->sli4_hba.nvme_wq[wqidx]; - cq = phba->sli4_hba.nvme_cq[wqidx]; + wq = phba->sli4_hba.hdwq[wqidx].nvme_wq; + cq = phba->sli4_hba.hdwq[wqidx].nvme_cq; qtypestr = "NVME"; } else if (qtype == DUMP_MBX) { wq = phba->sli4_hba.mbx_wq; @@ -478,17 +479,17 @@ lpfc_debug_dump_cq(struct lpfc_hba *phba, int qtype, int wqidx) } else return; - for (eqidx = 0; eqidx < phba->io_channel_irqs; eqidx++) { - if (cq->assoc_qid == phba->sli4_hba.hba_eq[eqidx]->queue_id) + for (eqidx = 0; eqidx < phba->cfg_hdw_queue; eqidx++) { + eq = phba->sli4_hba.hdwq[eqidx].hba_eq; + if (cq->assoc_qid == eq->queue_id) break; } - if (eqidx == phba->io_channel_irqs) { + if (eqidx == phba->cfg_hdw_queue) { pr_err("Couldn't find EQ for CQ. Using EQ[0]\n"); eqidx = 0; + eq = phba->sli4_hba.hdwq[0].hba_eq; } - eq = phba->sli4_hba.hba_eq[eqidx]; - if (qtype == DUMP_FCP || qtype == DUMP_NVME) pr_err("%s CQ: WQ[Idx:%d|Qid%d]->CQ[Idx%d|Qid%d]" "->EQ[Idx:%d|Qid:%d]:\n", @@ -516,7 +517,7 @@ lpfc_debug_dump_hba_eq(struct lpfc_hba *phba, int qidx) { struct lpfc_queue *qp; - qp = phba->sli4_hba.hba_eq[qidx]; + qp = phba->sli4_hba.hdwq[qidx].hba_eq; pr_err("EQ[Idx:%d|Qid:%d]\n", qidx, qp->queue_id); @@ -564,21 +565,21 @@ lpfc_debug_dump_wq_by_id(struct lpfc_hba *phba, int qid) { int wq_idx; - for (wq_idx = 0; wq_idx < phba->cfg_fcp_io_channel; wq_idx++) - if (phba->sli4_hba.fcp_wq[wq_idx]->queue_id == qid) + for (wq_idx = 0; wq_idx < phba->cfg_hdw_queue; wq_idx++) + if (phba->sli4_hba.hdwq[wq_idx].fcp_wq->queue_id == qid) break; - if (wq_idx < phba->cfg_fcp_io_channel) { + if (wq_idx < phba->cfg_hdw_queue) { pr_err("FCP WQ[Idx:%d|Qid:%d]\n", wq_idx, qid); - lpfc_debug_dump_q(phba->sli4_hba.fcp_wq[wq_idx]); + lpfc_debug_dump_q(phba->sli4_hba.hdwq[wq_idx].fcp_wq); return; } - for (wq_idx = 0; wq_idx < phba->cfg_nvme_io_channel; wq_idx++) - if (phba->sli4_hba.nvme_wq[wq_idx]->queue_id == qid) + for (wq_idx = 0; wq_idx < phba->cfg_hdw_queue; wq_idx++) + if (phba->sli4_hba.hdwq[wq_idx].nvme_wq->queue_id == qid) break; - if (wq_idx < phba->cfg_nvme_io_channel) { + if (wq_idx < phba->cfg_hdw_queue) { pr_err("NVME WQ[Idx:%d|Qid:%d]\n", wq_idx, qid); - lpfc_debug_dump_q(phba->sli4_hba.nvme_wq[wq_idx]); + lpfc_debug_dump_q(phba->sli4_hba.hdwq[wq_idx].nvme_wq); return; } @@ -646,23 +647,23 @@ lpfc_debug_dump_cq_by_id(struct lpfc_hba *phba, int qid) { int cq_idx; - for (cq_idx = 0; cq_idx < phba->cfg_fcp_io_channel; cq_idx++) - if (phba->sli4_hba.fcp_cq[cq_idx]->queue_id == qid) + for (cq_idx = 0; cq_idx < phba->cfg_hdw_queue; cq_idx++) + if (phba->sli4_hba.hdwq[cq_idx].fcp_cq->queue_id == qid) break; - if (cq_idx < phba->cfg_fcp_io_channel) { + if (cq_idx < phba->cfg_hdw_queue) { pr_err("FCP CQ[Idx:%d|Qid:%d]\n", cq_idx, qid); - lpfc_debug_dump_q(phba->sli4_hba.fcp_cq[cq_idx]); + lpfc_debug_dump_q(phba->sli4_hba.hdwq[cq_idx].fcp_cq); return; } - for (cq_idx = 0; cq_idx < phba->cfg_nvme_io_channel; cq_idx++) - if (phba->sli4_hba.nvme_cq[cq_idx]->queue_id == qid) + for (cq_idx = 0; cq_idx < phba->cfg_hdw_queue; cq_idx++) + if (phba->sli4_hba.hdwq[cq_idx].nvme_cq->queue_id == qid) break; - if (cq_idx < phba->cfg_nvme_io_channel) { + if (cq_idx < phba->cfg_hdw_queue) { pr_err("NVME CQ[Idx:%d|Qid:%d]\n", cq_idx, qid); - lpfc_debug_dump_q(phba->sli4_hba.nvme_cq[cq_idx]); + lpfc_debug_dump_q(phba->sli4_hba.hdwq[cq_idx].nvme_cq); return; } @@ -697,13 +698,13 @@ lpfc_debug_dump_eq_by_id(struct lpfc_hba *phba, int qid) { int eq_idx; - for (eq_idx = 0; eq_idx < phba->io_channel_irqs; eq_idx++) - if (phba->sli4_hba.hba_eq[eq_idx]->queue_id == qid) + for (eq_idx = 0; eq_idx < phba->cfg_hdw_queue; eq_idx++) + if (phba->sli4_hba.hdwq[eq_idx].hba_eq->queue_id == qid) break; - if (eq_idx < phba->io_channel_irqs) { + if (eq_idx < phba->cfg_hdw_queue) { printk(KERN_ERR "FCP EQ[Idx:%d|Qid:%d]\n", eq_idx, qid); - lpfc_debug_dump_q(phba->sli4_hba.hba_eq[eq_idx]); + lpfc_debug_dump_q(phba->sli4_hba.hdwq[eq_idx].hba_eq); return; } } diff --git a/drivers/scsi/lpfc/lpfc_init.c b/drivers/scsi/lpfc/lpfc_init.c index 9d9b965f796d..c4acd9ee55b1 100644 --- a/drivers/scsi/lpfc/lpfc_init.c +++ b/drivers/scsi/lpfc/lpfc_init.c @@ -1315,7 +1315,7 @@ lpfc_hb_timeout_handler(struct lpfc_hba *phba) localport->private; tot = 0; for (i = 0; - i < phba->cfg_nvme_io_channel; i++) { + i < phba->cfg_hdw_queue; i++) { cstat = &lport->cstat[i]; data1 = atomic_read( &cstat->fc4NvmeInputRequests); @@ -1331,15 +1331,15 @@ lpfc_hb_timeout_handler(struct lpfc_hba *phba) } /* Interrupts per sec per EQ */ - val = phba->cfg_fcp_imax / phba->io_channel_irqs; + val = phba->cfg_fcp_imax / phba->cfg_hdw_queue; tick_cqe = val / CONFIG_HZ; /* Per tick per EQ */ /* Assume 1 CQE/ISR, calc max CQEs allowed for time duration */ max_cqe = time_elapsed * tick_cqe; - for (i = 0; i < phba->io_channel_irqs; i++) { + for (i = 0; i < phba->cfg_hdw_queue; i++) { /* Fast-path EQ */ - qp = phba->sli4_hba.hba_eq[i]; + qp = phba->sli4_hba.hdwq[i].hba_eq; if (!qp) continue; @@ -1361,7 +1361,7 @@ lpfc_hb_timeout_handler(struct lpfc_hba *phba) if (val) { /* First, interrupts per sec per EQ */ val = phba->cfg_fcp_imax / - phba->io_channel_irqs; + phba->cfg_hdw_queue; /* us delay between each interrupt */ val = LPFC_SEC_TO_USEC / val; @@ -2945,7 +2945,8 @@ lpfc_sli4_stop_fcf_redisc_wait_timer(struct lpfc_hba *phba) void lpfc_stop_hba_timers(struct lpfc_hba *phba) { - lpfc_stop_vport_timers(phba->pport); + if (phba->pport) + lpfc_stop_vport_timers(phba->pport); del_timer_sync(&phba->sli.mbox_tmo); del_timer_sync(&phba->fabric_block_timer); del_timer_sync(&phba->eratt_poll); @@ -3989,7 +3990,7 @@ lpfc_create_port(struct lpfc_hba *phba, int instance, struct device *dev) shost->max_lun = vport->cfg_max_luns; shost->this_id = -1; shost->max_cmd_len = 16; - shost->nr_hw_queues = phba->cfg_fcp_io_channel; + shost->nr_hw_queues = phba->cfg_hdw_queue; if (phba->sli_rev == LPFC_SLI_REV4) { shost->dma_boundary = phba->sli4_hba.pc_sli4_params.sge_supp_len-1; @@ -4248,7 +4249,8 @@ lpfc_stop_port_s4(struct lpfc_hba *phba) { /* Reset some HBA SLI4 setup states */ lpfc_stop_hba_timers(phba); - phba->pport->work_port_events = 0; + if (phba->pport) + phba->pport->work_port_events = 0; phba->sli4_hba.intr_enable = 0; } @@ -6475,9 +6477,9 @@ lpfc_sli4_driver_resource_setup(struct lpfc_hba *phba) goto out_remove_rpi_hdrs; } - phba->sli4_hba.hba_eq_hdl = kcalloc(phba->io_channel_irqs, - sizeof(struct lpfc_hba_eq_hdl), - GFP_KERNEL); + phba->sli4_hba.hba_eq_hdl = kcalloc(phba->cfg_hdw_queue, + sizeof(struct lpfc_hba_eq_hdl), + GFP_KERNEL); if (!phba->sli4_hba.hba_eq_hdl) { lpfc_printf_log(phba, KERN_ERR, LOG_INIT, "2572 Failed allocate memory for " @@ -8049,21 +8051,23 @@ lpfc_sli4_read_config(struct lpfc_hba *phba) * NVMET, FCP io channel WQs are not created. */ length -= 6; + + /* Take off FCP queues */ if (!phba->nvmet_support) - length -= phba->cfg_fcp_io_channel; + length -= phba->cfg_hdw_queue; - if (phba->cfg_nvme_io_channel > length) { + /* Check to see if there is enough for NVME */ + if (phba->cfg_hdw_queue > length) { lpfc_printf_log( phba, KERN_ERR, LOG_SLI, "2005 Reducing NVME IO channel to %d: " - "WQ %d CQ %d NVMEIO %d FCPIO %d\n", + "WQ %d CQ %d CommonIO %d\n", length, phba->sli4_hba.max_cfg_param.max_wq, phba->sli4_hba.max_cfg_param.max_cq, - phba->cfg_nvme_io_channel, - phba->cfg_fcp_io_channel); + phba->cfg_hdw_queue); - phba->cfg_nvme_io_channel = length; + phba->cfg_hdw_queue = length; } } } @@ -8276,52 +8280,30 @@ lpfc_setup_endian_order(struct lpfc_hba *phba) static int lpfc_sli4_queue_verify(struct lpfc_hba *phba) { - int io_channel; - /* * Sanity check for configured queue parameters against the run-time * device parameters */ - /* Sanity check on HBA EQ parameters */ - io_channel = phba->io_channel_irqs; - - if (phba->sli4_hba.num_online_cpu < io_channel) { - lpfc_printf_log(phba, - KERN_ERR, LOG_INIT, - "3188 Reducing IO channels to match number of " - "online CPUs: from %d to %d\n", - io_channel, phba->sli4_hba.num_online_cpu); - io_channel = phba->sli4_hba.num_online_cpu; - } - - if (io_channel > phba->sli4_hba.max_cfg_param.max_eq) { + if (phba->cfg_hdw_queue > phba->sli4_hba.max_cfg_param.max_eq) { lpfc_printf_log(phba, KERN_ERR, LOG_INIT, "2575 Reducing IO channels to match number of " "available EQs: from %d to %d\n", - io_channel, + phba->cfg_hdw_queue, phba->sli4_hba.max_cfg_param.max_eq); - io_channel = phba->sli4_hba.max_cfg_param.max_eq; + phba->cfg_hdw_queue = phba->sli4_hba.max_cfg_param.max_eq; } - /* The actual number of FCP / NVME event queues adopted */ - if (io_channel != phba->io_channel_irqs) - phba->io_channel_irqs = io_channel; - if (phba->cfg_fcp_io_channel > io_channel) - phba->cfg_fcp_io_channel = io_channel; - if (phba->cfg_nvme_io_channel > io_channel) - phba->cfg_nvme_io_channel = io_channel; if (phba->nvmet_support) { - if (phba->cfg_nvme_io_channel < phba->cfg_nvmet_mrq) - phba->cfg_nvmet_mrq = phba->cfg_nvme_io_channel; + if (phba->cfg_hdw_queue < phba->cfg_nvmet_mrq) + phba->cfg_nvmet_mrq = phba->cfg_hdw_queue; } if (phba->cfg_nvmet_mrq > LPFC_NVMET_MRQ_MAX) phba->cfg_nvmet_mrq = LPFC_NVMET_MRQ_MAX; lpfc_printf_log(phba, KERN_ERR, LOG_INIT, - "2574 IO channels: irqs %d fcp %d nvme %d MRQ: %d\n", - phba->io_channel_irqs, phba->cfg_fcp_io_channel, - phba->cfg_nvme_io_channel, phba->cfg_nvmet_mrq); + "2574 IO channels: hdwQ %d MRQ: %d\n", + phba->cfg_hdw_queue, phba->cfg_nvmet_mrq); /* Get EQ depth from module parameter, fake the default for now */ phba->sli4_hba.eq_esize = LPFC_EQE_SIZE_4B; @@ -8348,7 +8330,7 @@ lpfc_alloc_nvme_wq_cq(struct lpfc_hba *phba, int wqidx) return 1; } qdesc->qe_valid = 1; - phba->sli4_hba.nvme_cq[wqidx] = qdesc; + phba->sli4_hba.hdwq[wqidx].nvme_cq = qdesc; qdesc = lpfc_sli4_queue_alloc(phba, LPFC_EXPANDED_PAGE_SIZE, LPFC_WQE128_SIZE, LPFC_WQE_EXP_COUNT); @@ -8358,7 +8340,7 @@ lpfc_alloc_nvme_wq_cq(struct lpfc_hba *phba, int wqidx) wqidx); return 1; } - phba->sli4_hba.nvme_wq[wqidx] = qdesc; + phba->sli4_hba.hdwq[wqidx].nvme_wq = qdesc; list_add_tail(&qdesc->wq_list, &phba->sli4_hba.lpfc_wq_list); return 0; } @@ -8386,7 +8368,7 @@ lpfc_alloc_fcp_wq_cq(struct lpfc_hba *phba, int wqidx) return 1; } qdesc->qe_valid = 1; - phba->sli4_hba.fcp_cq[wqidx] = qdesc; + phba->sli4_hba.hdwq[wqidx].fcp_cq = qdesc; /* Create Fast Path FCP WQs */ if (phba->enab_exp_wqcq_pages) { @@ -8407,7 +8389,7 @@ lpfc_alloc_fcp_wq_cq(struct lpfc_hba *phba, int wqidx) wqidx); return 1; } - phba->sli4_hba.fcp_wq[wqidx] = qdesc; + phba->sli4_hba.hdwq[wqidx].fcp_wq = qdesc; list_add_tail(&qdesc->wq_list, &phba->sli4_hba.lpfc_wq_list); return 0; } @@ -8430,16 +8412,12 @@ int lpfc_sli4_queue_create(struct lpfc_hba *phba) { struct lpfc_queue *qdesc; - int idx, io_channel; + int idx; /* * Create HBA Record arrays. * Both NVME and FCP will share that same vectors / EQs */ - io_channel = phba->io_channel_irqs; - if (!io_channel) - return -ERANGE; - phba->sli4_hba.mq_esize = LPFC_MQE_SIZE; phba->sli4_hba.mq_ecount = LPFC_MQE_DEF_COUNT; phba->sli4_hba.wq_esize = LPFC_WQE_SIZE; @@ -8451,87 +8429,17 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba) phba->sli4_hba.cq_esize = LPFC_CQE_SIZE; phba->sli4_hba.cq_ecount = LPFC_CQE_DEF_COUNT; - phba->sli4_hba.hba_eq = kcalloc(io_channel, - sizeof(struct lpfc_queue *), - GFP_KERNEL); - if (!phba->sli4_hba.hba_eq) { + phba->sli4_hba.hdwq = kcalloc(phba->cfg_hdw_queue, + sizeof(struct lpfc_sli4_hdw_queue), + GFP_KERNEL); + if (!phba->sli4_hba.hdwq) { lpfc_printf_log(phba, KERN_ERR, LOG_INIT, - "2576 Failed allocate memory for " - "fast-path EQ record array\n"); + "6427 Failed allocate memory for " + "fast-path Hardware Queue array\n"); goto out_error; } - if (phba->cfg_fcp_io_channel) { - phba->sli4_hba.fcp_cq = kcalloc(phba->cfg_fcp_io_channel, - sizeof(struct lpfc_queue *), - GFP_KERNEL); - if (!phba->sli4_hba.fcp_cq) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, - "2577 Failed allocate memory for " - "fast-path CQ record array\n"); - goto out_error; - } - phba->sli4_hba.fcp_wq = kcalloc(phba->cfg_fcp_io_channel, - sizeof(struct lpfc_queue *), - GFP_KERNEL); - if (!phba->sli4_hba.fcp_wq) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, - "2578 Failed allocate memory for " - "fast-path FCP WQ record array\n"); - goto out_error; - } - /* - * Since the first EQ can have multiple CQs associated with it, - * this array is used to quickly see if we have a FCP fast-path - * CQ match. - */ - phba->sli4_hba.fcp_cq_map = kcalloc(phba->cfg_fcp_io_channel, - sizeof(uint16_t), - GFP_KERNEL); - if (!phba->sli4_hba.fcp_cq_map) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, - "2545 Failed allocate memory for " - "fast-path CQ map\n"); - goto out_error; - } - } - - if (phba->cfg_nvme_io_channel) { - phba->sli4_hba.nvme_cq = kcalloc(phba->cfg_nvme_io_channel, - sizeof(struct lpfc_queue *), - GFP_KERNEL); - if (!phba->sli4_hba.nvme_cq) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, - "6077 Failed allocate memory for " - "fast-path CQ record array\n"); - goto out_error; - } - - phba->sli4_hba.nvme_wq = kcalloc(phba->cfg_nvme_io_channel, - sizeof(struct lpfc_queue *), - GFP_KERNEL); - if (!phba->sli4_hba.nvme_wq) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, - "2581 Failed allocate memory for " - "fast-path NVME WQ record array\n"); - goto out_error; - } - - /* - * Since the first EQ can have multiple CQs associated with it, - * this array is used to quickly see if we have a NVME fast-path - * CQ match. - */ - phba->sli4_hba.nvme_cq_map = kcalloc(phba->cfg_nvme_io_channel, - sizeof(uint16_t), - GFP_KERNEL); - if (!phba->sli4_hba.nvme_cq_map) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, - "6078 Failed allocate memory for " - "fast-path CQ map\n"); - goto out_error; - } - + if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { if (phba->nvmet_support) { phba->sli4_hba.nvmet_cqset = kcalloc( phba->cfg_nvmet_mrq, @@ -8569,7 +8477,7 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba) INIT_LIST_HEAD(&phba->sli4_hba.lpfc_wq_list); /* Create HBA Event Queues (EQs) */ - for (idx = 0; idx < io_channel; idx++) { + for (idx = 0; idx < phba->cfg_hdw_queue; idx++) { /* Create EQs */ qdesc = lpfc_sli4_queue_alloc(phba, LPFC_DEFAULT_PAGE_SIZE, phba->sli4_hba.eq_esize, @@ -8580,33 +8488,38 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba) goto out_error; } qdesc->qe_valid = 1; - phba->sli4_hba.hba_eq[idx] = qdesc; + phba->sli4_hba.hdwq[idx].hba_eq = qdesc; } - /* FCP and NVME io channels are not required to be balanced */ - for (idx = 0; idx < phba->cfg_fcp_io_channel; idx++) + /* Allocate SCSI SLI4 CQ/WQs */ + for (idx = 0; idx < phba->cfg_hdw_queue; idx++) if (lpfc_alloc_fcp_wq_cq(phba, idx)) goto out_error; - for (idx = 0; idx < phba->cfg_nvme_io_channel; idx++) - if (lpfc_alloc_nvme_wq_cq(phba, idx)) - goto out_error; + /* Allocate NVME SLI4 CQ/WQs */ + if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { + for (idx = 0; idx < phba->cfg_hdw_queue; idx++) + if (lpfc_alloc_nvme_wq_cq(phba, idx)) + goto out_error; - if (phba->nvmet_support) { - for (idx = 0; idx < phba->cfg_nvmet_mrq; idx++) { - qdesc = lpfc_sli4_queue_alloc(phba, + if (phba->nvmet_support) { + for (idx = 0; idx < phba->cfg_nvmet_mrq; idx++) { + qdesc = lpfc_sli4_queue_alloc( + phba, LPFC_DEFAULT_PAGE_SIZE, phba->sli4_hba.cq_esize, phba->sli4_hba.cq_ecount); - if (!qdesc) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, - "3142 Failed allocate NVME " - "CQ Set (%d)\n", idx); - goto out_error; + if (!qdesc) { + lpfc_printf_log( + phba, KERN_ERR, LOG_INIT, + "3142 Failed allocate NVME " + "CQ Set (%d)\n", idx); + goto out_error; + } + qdesc->qe_valid = 1; + phba->sli4_hba.nvmet_cqset[idx] = qdesc; } - qdesc->qe_valid = 1; - phba->sli4_hba.nvmet_cqset[idx] = qdesc; } } @@ -8723,7 +8636,8 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba) } phba->sli4_hba.dat_rq = qdesc; - if (phba->nvmet_support) { + if ((phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) && + phba->nvmet_support) { for (idx = 0; idx < phba->cfg_nvmet_mrq; idx++) { /* Create NVMET Receive Queue for header */ qdesc = lpfc_sli4_queue_alloc(phba, @@ -8797,12 +8711,23 @@ lpfc_sli4_release_queues(struct lpfc_queue ***qs, int max) } static inline void -lpfc_sli4_release_queue_map(uint16_t **qmap) +lpfc_sli4_release_hdwq(struct lpfc_sli4_hdw_queue *hdwq, int max) { - if (*qmap != NULL) { - kfree(*qmap); - *qmap = NULL; + uint32_t idx; + + for (idx = 0; idx < max; idx++) { + lpfc_sli4_queue_free(hdwq[idx].hba_eq); + lpfc_sli4_queue_free(hdwq[idx].fcp_cq); + lpfc_sli4_queue_free(hdwq[idx].nvme_cq); + lpfc_sli4_queue_free(hdwq[idx].fcp_wq); + lpfc_sli4_queue_free(hdwq[idx].nvme_wq); + hdwq[idx].hba_eq = NULL; + hdwq[idx].fcp_cq = NULL; + hdwq[idx].nvme_cq = NULL; + hdwq[idx].fcp_wq = NULL; + hdwq[idx].nvme_wq = NULL; } + kfree(hdwq); } /** @@ -8821,29 +8746,10 @@ void lpfc_sli4_queue_destroy(struct lpfc_hba *phba) { /* Release HBA eqs */ - lpfc_sli4_release_queues(&phba->sli4_hba.hba_eq, phba->io_channel_irqs); - - /* Release FCP cqs */ - lpfc_sli4_release_queues(&phba->sli4_hba.fcp_cq, - phba->cfg_fcp_io_channel); - - /* Release FCP wqs */ - lpfc_sli4_release_queues(&phba->sli4_hba.fcp_wq, - phba->cfg_fcp_io_channel); - - /* Release FCP CQ mapping array */ - lpfc_sli4_release_queue_map(&phba->sli4_hba.fcp_cq_map); - - /* Release NVME cqs */ - lpfc_sli4_release_queues(&phba->sli4_hba.nvme_cq, - phba->cfg_nvme_io_channel); - - /* Release NVME wqs */ - lpfc_sli4_release_queues(&phba->sli4_hba.nvme_wq, - phba->cfg_nvme_io_channel); - - /* Release NVME CQ mapping array */ - lpfc_sli4_release_queue_map(&phba->sli4_hba.nvme_cq_map); + if (phba->sli4_hba.hdwq) + lpfc_sli4_release_hdwq(phba->sli4_hba.hdwq, + phba->cfg_hdw_queue); + phba->sli4_hba.hdwq = NULL; if (phba->nvmet_support) { lpfc_sli4_release_queues(&phba->sli4_hba.nvmet_cqset, @@ -8927,7 +8833,7 @@ lpfc_create_wq_cq(struct lpfc_hba *phba, struct lpfc_queue *eq, cq->chann = qidx; if (qtype != LPFC_MBOX) { - /* Setup nvme_cq_map for fast lookup */ + /* Setup cq_map for fast lookup */ if (cq_map) *cq_map = cq->queue_id; @@ -8990,9 +8896,10 @@ lpfc_sli4_queue_setup(struct lpfc_hba *phba) { uint32_t shdr_status, shdr_add_status; union lpfc_sli4_cfg_shdr *shdr; + struct lpfc_sli4_hdw_queue *qp; LPFC_MBOXQ_t *mboxq; int qidx; - uint32_t length, io_channel; + uint32_t length; int rc = -ENOMEM; /* Check for dual-ULP support */ @@ -9043,25 +8950,25 @@ lpfc_sli4_queue_setup(struct lpfc_hba *phba) /* * Set up HBA Event Queues (EQs) */ - io_channel = phba->io_channel_irqs; + qp = phba->sli4_hba.hdwq; /* Set up HBA event queue */ - if (io_channel && !phba->sli4_hba.hba_eq) { + if (!qp) { lpfc_printf_log(phba, KERN_ERR, LOG_INIT, "3147 Fast-path EQs not allocated\n"); rc = -ENOMEM; goto out_error; } - for (qidx = 0; qidx < io_channel; qidx++) { - if (!phba->sli4_hba.hba_eq[qidx]) { + for (qidx = 0; qidx < phba->cfg_hdw_queue; qidx++) { + if (!qp[qidx].hba_eq) { lpfc_printf_log(phba, KERN_ERR, LOG_INIT, "0522 Fast-path EQ (%d) not " "allocated\n", qidx); rc = -ENOMEM; goto out_destroy; } - rc = lpfc_eq_create(phba, phba->sli4_hba.hba_eq[qidx], - phba->cfg_fcp_imax); + rc = lpfc_eq_create(phba, qp[qidx].hba_eq, + phba->cfg_fcp_imax); if (rc) { lpfc_printf_log(phba, KERN_ERR, LOG_INIT, "0523 Failed setup of fast-path EQ " @@ -9070,26 +8977,17 @@ lpfc_sli4_queue_setup(struct lpfc_hba *phba) goto out_destroy; } lpfc_printf_log(phba, KERN_INFO, LOG_INIT, - "2584 HBA EQ setup: queue[%d]-id=%d\n", - qidx, phba->sli4_hba.hba_eq[qidx]->queue_id); + "2584 HBA EQ setup: queue[%d]-id=%d\n", qidx, + qp[qidx].hba_eq->queue_id); } - if (phba->cfg_nvme_io_channel) { - if (!phba->sli4_hba.nvme_cq || !phba->sli4_hba.nvme_wq) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, - "6084 Fast-path NVME %s array not allocated\n", - (phba->sli4_hba.nvme_cq) ? "CQ" : "WQ"); - rc = -ENOMEM; - goto out_destroy; - } - - for (qidx = 0; qidx < phba->cfg_nvme_io_channel; qidx++) { + if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { + for (qidx = 0; qidx < phba->cfg_hdw_queue; qidx++) { rc = lpfc_create_wq_cq(phba, - phba->sli4_hba.hba_eq[ - qidx % io_channel], - phba->sli4_hba.nvme_cq[qidx], - phba->sli4_hba.nvme_wq[qidx], - &phba->sli4_hba.nvme_cq_map[qidx], + qp[qidx].hba_eq, + qp[qidx].nvme_cq, + qp[qidx].nvme_wq, + &phba->sli4_hba.hdwq[qidx].nvme_cq_map, qidx, LPFC_NVME); if (rc) { lpfc_printf_log(phba, KERN_ERR, LOG_INIT, @@ -9101,31 +8999,19 @@ lpfc_sli4_queue_setup(struct lpfc_hba *phba) } } - if (phba->cfg_fcp_io_channel) { - /* Set up fast-path FCP Response Complete Queue */ - if (!phba->sli4_hba.fcp_cq || !phba->sli4_hba.fcp_wq) { + for (qidx = 0; qidx < phba->cfg_hdw_queue; qidx++) { + rc = lpfc_create_wq_cq(phba, + qp[qidx].hba_eq, + qp[qidx].fcp_cq, + qp[qidx].fcp_wq, + &phba->sli4_hba.hdwq[qidx].fcp_cq_map, + qidx, LPFC_FCP); + if (rc) { lpfc_printf_log(phba, KERN_ERR, LOG_INIT, - "3148 Fast-path FCP %s array not allocated\n", - phba->sli4_hba.fcp_cq ? "WQ" : "CQ"); - rc = -ENOMEM; - goto out_destroy; - } - - for (qidx = 0; qidx < phba->cfg_fcp_io_channel; qidx++) { - rc = lpfc_create_wq_cq(phba, - phba->sli4_hba.hba_eq[ - qidx % io_channel], - phba->sli4_hba.fcp_cq[qidx], - phba->sli4_hba.fcp_wq[qidx], - &phba->sli4_hba.fcp_cq_map[qidx], - qidx, LPFC_FCP); - if (rc) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, "0535 Failed to setup fastpath " "FCP WQ/CQ (%d), rc = 0x%x\n", qidx, (uint32_t)rc); - goto out_destroy; - } + goto out_destroy; } } @@ -9144,7 +9030,7 @@ lpfc_sli4_queue_setup(struct lpfc_hba *phba) goto out_destroy; } - rc = lpfc_create_wq_cq(phba, phba->sli4_hba.hba_eq[0], + rc = lpfc_create_wq_cq(phba, qp[0].hba_eq, phba->sli4_hba.mbx_cq, phba->sli4_hba.mbx_wq, NULL, 0, LPFC_MBOX); @@ -9165,7 +9051,7 @@ lpfc_sli4_queue_setup(struct lpfc_hba *phba) if (phba->cfg_nvmet_mrq > 1) { rc = lpfc_cq_create_set(phba, phba->sli4_hba.nvmet_cqset, - phba->sli4_hba.hba_eq, + qp, LPFC_WCQ, LPFC_NVMET); if (rc) { lpfc_printf_log(phba, KERN_ERR, LOG_INIT, @@ -9177,7 +9063,7 @@ lpfc_sli4_queue_setup(struct lpfc_hba *phba) } else { /* Set up NVMET Receive Complete Queue */ rc = lpfc_cq_create(phba, phba->sli4_hba.nvmet_cqset[0], - phba->sli4_hba.hba_eq[0], + qp[0].hba_eq, LPFC_WCQ, LPFC_NVMET); if (rc) { lpfc_printf_log(phba, KERN_ERR, LOG_INIT, @@ -9191,7 +9077,7 @@ lpfc_sli4_queue_setup(struct lpfc_hba *phba) "6090 NVMET CQ setup: cq-id=%d, " "parent eq-id=%d\n", phba->sli4_hba.nvmet_cqset[0]->queue_id, - phba->sli4_hba.hba_eq[0]->queue_id); + qp[0].hba_eq->queue_id); } } @@ -9203,14 +9089,14 @@ lpfc_sli4_queue_setup(struct lpfc_hba *phba) rc = -ENOMEM; goto out_destroy; } - rc = lpfc_create_wq_cq(phba, phba->sli4_hba.hba_eq[0], - phba->sli4_hba.els_cq, - phba->sli4_hba.els_wq, - NULL, 0, LPFC_ELS); + rc = lpfc_create_wq_cq(phba, qp[0].hba_eq, + phba->sli4_hba.els_cq, + phba->sli4_hba.els_wq, + NULL, 0, LPFC_ELS); if (rc) { lpfc_printf_log(phba, KERN_ERR, LOG_INIT, - "0529 Failed setup of ELS WQ/CQ: rc = 0x%x\n", - (uint32_t)rc); + "0525 Failed setup of ELS WQ/CQ: rc = 0x%x\n", + (uint32_t)rc); goto out_destroy; } lpfc_printf_log(phba, KERN_INFO, LOG_INIT, @@ -9218,7 +9104,7 @@ lpfc_sli4_queue_setup(struct lpfc_hba *phba) phba->sli4_hba.els_wq->queue_id, phba->sli4_hba.els_cq->queue_id); - if (phba->cfg_nvme_io_channel) { + if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { /* Set up NVME LS Complete Queue */ if (!phba->sli4_hba.nvmels_cq || !phba->sli4_hba.nvmels_wq) { lpfc_printf_log(phba, KERN_ERR, LOG_INIT, @@ -9227,14 +9113,14 @@ lpfc_sli4_queue_setup(struct lpfc_hba *phba) rc = -ENOMEM; goto out_destroy; } - rc = lpfc_create_wq_cq(phba, phba->sli4_hba.hba_eq[0], - phba->sli4_hba.nvmels_cq, - phba->sli4_hba.nvmels_wq, - NULL, 0, LPFC_NVME_LS); + rc = lpfc_create_wq_cq(phba, qp[0].hba_eq, + phba->sli4_hba.nvmels_cq, + phba->sli4_hba.nvmels_wq, + NULL, 0, LPFC_NVME_LS); if (rc) { lpfc_printf_log(phba, KERN_ERR, LOG_INIT, - "0529 Failed setup of NVVME LS WQ/CQ: " - "rc = 0x%x\n", (uint32_t)rc); + "0526 Failed setup of NVVME LS WQ/CQ: " + "rc = 0x%x\n", (uint32_t)rc); goto out_destroy; } @@ -9320,7 +9206,8 @@ lpfc_sli4_queue_setup(struct lpfc_hba *phba) phba->sli4_hba.dat_rq->queue_id, phba->sli4_hba.els_cq->queue_id); - for (qidx = 0; qidx < io_channel; qidx += LPFC_MAX_EQ_DELAY_EQID_CNT) + for (qidx = 0; qidx < phba->cfg_hdw_queue; + qidx += LPFC_MAX_EQ_DELAY_EQID_CNT) lpfc_modify_hba_eq_delay(phba, qidx, LPFC_MAX_EQ_DELAY_EQID_CNT, phba->cfg_fcp_imax); @@ -9347,6 +9234,7 @@ out_error: void lpfc_sli4_queue_unset(struct lpfc_hba *phba) { + struct lpfc_sli4_hdw_queue *qp; int qidx; /* Unset mailbox command work queue */ @@ -9366,17 +9254,6 @@ lpfc_sli4_queue_unset(struct lpfc_hba *phba) lpfc_rq_destroy(phba, phba->sli4_hba.hdr_rq, phba->sli4_hba.dat_rq); - /* Unset FCP work queue */ - if (phba->sli4_hba.fcp_wq) - for (qidx = 0; qidx < phba->cfg_fcp_io_channel; qidx++) - lpfc_wq_destroy(phba, phba->sli4_hba.fcp_wq[qidx]); - - /* Unset NVME work queue */ - if (phba->sli4_hba.nvme_wq) { - for (qidx = 0; qidx < phba->cfg_nvme_io_channel; qidx++) - lpfc_wq_destroy(phba, phba->sli4_hba.nvme_wq[qidx]); - } - /* Unset mailbox command complete queue */ if (phba->sli4_hba.mbx_cq) lpfc_cq_destroy(phba, phba->sli4_hba.mbx_cq); @@ -9389,11 +9266,6 @@ lpfc_sli4_queue_unset(struct lpfc_hba *phba) if (phba->sli4_hba.nvmels_cq) lpfc_cq_destroy(phba, phba->sli4_hba.nvmels_cq); - /* Unset NVME response complete queue */ - if (phba->sli4_hba.nvme_cq) - for (qidx = 0; qidx < phba->cfg_nvme_io_channel; qidx++) - lpfc_cq_destroy(phba, phba->sli4_hba.nvme_cq[qidx]); - if (phba->nvmet_support) { /* Unset NVMET MRQ queue */ if (phba->sli4_hba.nvmet_mrq_hdr) { @@ -9412,15 +9284,17 @@ lpfc_sli4_queue_unset(struct lpfc_hba *phba) } } - /* Unset FCP response complete queue */ - if (phba->sli4_hba.fcp_cq) - for (qidx = 0; qidx < phba->cfg_fcp_io_channel; qidx++) - lpfc_cq_destroy(phba, phba->sli4_hba.fcp_cq[qidx]); - - /* Unset fast-path event queue */ - if (phba->sli4_hba.hba_eq) - for (qidx = 0; qidx < phba->io_channel_irqs; qidx++) - lpfc_eq_destroy(phba, phba->sli4_hba.hba_eq[qidx]); + /* Unset fast-path SLI4 queues */ + if (phba->sli4_hba.hdwq) { + for (qidx = 0; qidx < phba->cfg_hdw_queue; qidx++) { + qp = &phba->sli4_hba.hdwq[qidx]; + lpfc_wq_destroy(phba, qp->fcp_wq); + lpfc_wq_destroy(phba, qp->nvme_wq); + lpfc_cq_destroy(phba, qp->fcp_cq); + lpfc_cq_destroy(phba, qp->nvme_cq); + lpfc_eq_destroy(phba, qp->hba_eq); + } + } } /** @@ -10246,7 +10120,7 @@ lpfc_cpu_affinity_check(struct lpfc_hba *phba, int vectors) if (vec >= vectors) vec = 0; index++; - if (index >= phba->cfg_fcp_io_channel) + if (index >= phba->cfg_hdw_queue) index = 0; cpup++; } @@ -10271,7 +10145,7 @@ lpfc_sli4_enable_msix(struct lpfc_hba *phba) char *name; /* Set up MSI-X multi-message vectors */ - vectors = phba->io_channel_irqs; + vectors = phba->cfg_hdw_queue; rc = pci_alloc_irq_vectors(phba->pcidev, (phba->nvmet_support) ? 1 : 2, @@ -10305,19 +10179,15 @@ lpfc_sli4_enable_msix(struct lpfc_hba *phba) } } - if (vectors != phba->io_channel_irqs) { + if (vectors != phba->cfg_hdw_queue) { lpfc_printf_log(phba, KERN_ERR, LOG_INIT, "3238 Reducing IO channels to match number of " "MSI-X vectors, requested %d got %d\n", - phba->io_channel_irqs, vectors); - if (phba->cfg_fcp_io_channel > vectors) - phba->cfg_fcp_io_channel = vectors; - if (phba->cfg_nvme_io_channel > vectors) - phba->cfg_nvme_io_channel = vectors; - if (phba->cfg_fcp_io_channel > phba->cfg_nvme_io_channel) - phba->io_channel_irqs = phba->cfg_fcp_io_channel; - else - phba->io_channel_irqs = phba->cfg_nvme_io_channel; + phba->cfg_hdw_queue, vectors); + if (phba->cfg_hdw_queue > vectors) + phba->cfg_hdw_queue = vectors; + if (phba->cfg_nvmet_mrq > vectors) + phba->cfg_nvmet_mrq = vectors; } lpfc_cpu_affinity_check(phba, vectors); @@ -10374,7 +10244,7 @@ lpfc_sli4_enable_msi(struct lpfc_hba *phba) return rc; } - for (index = 0; index < phba->io_channel_irqs; index++) { + for (index = 0; index < phba->cfg_hdw_queue; index++) { phba->sli4_hba.hba_eq_hdl[index].idx = index; phba->sli4_hba.hba_eq_hdl[index].phba = phba; } @@ -10439,7 +10309,7 @@ lpfc_sli4_enable_intr(struct lpfc_hba *phba, uint32_t cfg_mode) phba->intr_type = INTx; intr_mode = 0; - for (idx = 0; idx < phba->io_channel_irqs; idx++) { + for (idx = 0; idx < phba->cfg_hdw_queue; idx++) { eqhdl = &phba->sli4_hba.hba_eq_hdl[idx]; eqhdl->idx = idx; eqhdl->phba = phba; @@ -10467,7 +10337,7 @@ lpfc_sli4_disable_intr(struct lpfc_hba *phba) int index; /* Free up MSI-X multi-message vectors */ - for (index = 0; index < phba->io_channel_irqs; index++) + for (index = 0; index < phba->cfg_hdw_queue; index++) free_irq(pci_irq_vector(phba->pcidev, index), &phba->sli4_hba.hba_eq_hdl[index]); } else { @@ -10620,7 +10490,8 @@ lpfc_sli4_hba_unset(struct lpfc_hba *phba) struct pci_dev *pdev = phba->pcidev; lpfc_stop_hba_timers(phba); - phba->sli4_hba.intr_enable = 0; + if (phba->pport) + phba->sli4_hba.intr_enable = 0; /* * Gracefully wait out the potential current outstanding asynchronous @@ -10839,8 +10710,6 @@ lpfc_get_sli4_parameters(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq) phba->nvme_support = 0; phba->nvmet_support = 0; phba->cfg_nvmet_mrq = LPFC_NVMET_MRQ_OFF; - phba->cfg_nvme_io_channel = 0; - phba->io_channel_irqs = phba->cfg_fcp_io_channel; lpfc_printf_log(phba, KERN_ERR, LOG_INIT | LOG_NVME, "6101 Disabling NVME support: " "Not supported by firmware: %d %d\n", @@ -11792,28 +11661,11 @@ lpfc_pci_probe_one_s4(struct pci_dev *pdev, const struct pci_device_id *pid) /* Get the default values for Model Name and Description */ lpfc_get_hba_model_desc(phba, phba->ModelName, phba->ModelDesc); - /* Create SCSI host to the physical port */ - error = lpfc_create_shost(phba); - if (error) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, - "1415 Failed to create scsi host.\n"); - goto out_unset_driver_resource; - } - - /* Configure sysfs attributes */ - vport = phba->pport; - error = lpfc_alloc_sysfs_attr(vport); - if (error) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, - "1416 Failed to allocate sysfs attr\n"); - goto out_destroy_shost; - } - - shost = lpfc_shost_from_vport(vport); /* save shost for error cleanup */ /* Now, trying to enable interrupt and bring up the device */ cfg_mode = phba->cfg_use_msi; /* Put device to a known state before enabling interrupt */ + phba->pport = NULL; lpfc_stop_port(phba); /* Configure and enable interrupt */ @@ -11822,18 +11674,33 @@ lpfc_pci_probe_one_s4(struct pci_dev *pdev, const struct pci_device_id *pid) lpfc_printf_log(phba, KERN_ERR, LOG_INIT, "0426 Failed to enable interrupt.\n"); error = -ENODEV; - goto out_free_sysfs_attr; + goto out_unset_driver_resource; } /* Default to single EQ for non-MSI-X */ if (phba->intr_type != MSIX) { - if (phba->cfg_enable_fc4_type & LPFC_ENABLE_FCP) - phba->cfg_fcp_io_channel = 1; + phba->cfg_hdw_queue = 1; if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { - phba->cfg_nvme_io_channel = 1; if (phba->nvmet_support) phba->cfg_nvmet_mrq = 1; } - phba->io_channel_irqs = 1; + } + + /* Create SCSI host to the physical port */ + error = lpfc_create_shost(phba); + if (error) { + lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + "1415 Failed to create scsi host.\n"); + goto out_disable_intr; + } + vport = phba->pport; + shost = lpfc_shost_from_vport(vport); /* save shost for error cleanup */ + + /* Configure sysfs attributes */ + error = lpfc_alloc_sysfs_attr(vport); + if (error) { + lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + "1416 Failed to allocate sysfs attr\n"); + goto out_destroy_shost; } /* Set up SLI-4 HBA */ @@ -11841,7 +11708,7 @@ lpfc_pci_probe_one_s4(struct pci_dev *pdev, const struct pci_device_id *pid) lpfc_printf_log(phba, KERN_ERR, LOG_INIT, "1421 Failed to set up hba\n"); error = -ENODEV; - goto out_disable_intr; + goto out_free_sysfs_attr; } /* Log the current active interrupt mode */ @@ -11877,7 +11744,7 @@ lpfc_pci_probe_one_s4(struct pci_dev *pdev, const struct pci_device_id *pid) phba, phba->sli4_hba.common_xri_max); if (len == 0) { error = -ENOMEM; - goto out_disable_intr; + goto out_free_sysfs_attr; } phba->total_common_bufs += len; } @@ -11895,12 +11762,12 @@ lpfc_pci_probe_one_s4(struct pci_dev *pdev, const struct pci_device_id *pid) return 0; -out_disable_intr: - lpfc_sli4_disable_intr(phba); out_free_sysfs_attr: lpfc_free_sysfs_attr(vport); out_destroy_shost: lpfc_destroy_shost(phba); +out_disable_intr: + lpfc_sli4_disable_intr(phba); out_unset_driver_resource: lpfc_unset_driver_resource_phase2(phba); out_unset_driver_resource_s4: diff --git a/drivers/scsi/lpfc/lpfc_nvme.c b/drivers/scsi/lpfc/lpfc_nvme.c index 1068a5a7b126..c0f59fbaa173 100644 --- a/drivers/scsi/lpfc/lpfc_nvme.c +++ b/drivers/scsi/lpfc/lpfc_nvme.c @@ -239,7 +239,7 @@ lpfc_nvme_create_queue(struct nvme_fc_local_port *pnvme_lport, if (qidx) { str = "IO "; /* IO queue */ qhandle->index = ((qidx - 1) % - vport->phba->cfg_nvme_io_channel); + vport->phba->cfg_hdw_queue); } else { str = "ADM"; /* Admin queue */ qhandle->index = qidx; @@ -247,7 +247,7 @@ lpfc_nvme_create_queue(struct nvme_fc_local_port *pnvme_lport, lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME, "6073 Binding %s HdwQueue %d (cpu %d) to " - "io_channel %d qhandle %p\n", str, + "hdw_queue %d qhandle %p\n", str, qidx, qhandle->cpu_id, qhandle->index, qhandle); *handle = (void *)qhandle; return 0; @@ -2083,10 +2083,10 @@ lpfc_nvme_create_localport(struct lpfc_vport *vport) * allocate + 3, one for cmd, one for rsp and one for this alignment */ lpfc_nvme_template.max_sgl_segments = phba->cfg_nvme_seg_cnt + 1; - lpfc_nvme_template.max_hw_queues = phba->cfg_nvme_io_channel; + lpfc_nvme_template.max_hw_queues = phba->cfg_hdw_queue; cstat = kmalloc((sizeof(struct lpfc_nvme_ctrl_stat) * - phba->cfg_nvme_io_channel), GFP_KERNEL); + phba->cfg_hdw_queue), GFP_KERNEL); if (!cstat) return -ENOMEM; @@ -2130,7 +2130,7 @@ lpfc_nvme_create_localport(struct lpfc_vport *vport) atomic_set(&lport->fc4NvmeLsRequests, 0); atomic_set(&lport->fc4NvmeLsCmpls, 0); - for (i = 0; i < phba->cfg_nvme_io_channel; i++) { + for (i = 0; i < phba->cfg_hdw_queue; i++) { cstat = &lport->cstat[i]; atomic_set(&cstat->fc4NvmeInputRequests, 0); atomic_set(&cstat->fc4NvmeOutputRequests, 0); @@ -2587,14 +2587,14 @@ lpfc_nvme_wait_for_io_drain(struct lpfc_hba *phba) struct lpfc_sli_ring *pring; u32 i, wait_cnt = 0; - if (phba->sli_rev < LPFC_SLI_REV4 || !phba->sli4_hba.nvme_wq) + if (phba->sli_rev < LPFC_SLI_REV4 || !phba->sli4_hba.hdwq) return; /* Cycle through all NVME rings and make sure all outstanding * WQEs have been removed from the txcmplqs. */ - for (i = 0; i < phba->cfg_nvme_io_channel; i++) { - pring = phba->sli4_hba.nvme_wq[i]->pring; + for (i = 0; i < phba->cfg_hdw_queue; i++) { + pring = phba->sli4_hba.hdwq[i].nvme_wq->pring; if (!pring) continue; diff --git a/drivers/scsi/lpfc/lpfc_nvmet.c b/drivers/scsi/lpfc/lpfc_nvmet.c index 6245f442d784..c64a8234d5bd 100644 --- a/drivers/scsi/lpfc/lpfc_nvmet.c +++ b/drivers/scsi/lpfc/lpfc_nvmet.c @@ -973,7 +973,7 @@ lpfc_nvmet_xmt_fcp_op(struct nvmet_fc_target_port *tgtport, * WQE release CQE */ ctxp->flag |= LPFC_NVMET_DEFER_WQFULL; - wq = phba->sli4_hba.nvme_wq[rsp->hwqid]; + wq = phba->sli4_hba.hdwq[rsp->hwqid].nvme_wq; pring = wq->pring; spin_lock_irqsave(&pring->ring_lock, iflags); list_add_tail(&nvmewqeq->list, &wq->wqfull_list); @@ -1047,7 +1047,7 @@ lpfc_nvmet_xmt_fcp_abort(struct nvmet_fc_target_port *tgtport, if (ctxp->flag & LPFC_NVMET_DEFER_WQFULL) { lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, ctxp->sid, ctxp->oxid); - wq = phba->sli4_hba.nvme_wq[ctxp->wqeq->hba_wqidx]; + wq = phba->sli4_hba.hdwq[ctxp->wqeq->hba_wqidx].nvme_wq; spin_unlock_irqrestore(&ctxp->ctxlock, flags); lpfc_nvmet_wqfull_flush(phba, wq, ctxp); return; @@ -1377,7 +1377,7 @@ lpfc_nvmet_create_targetport(struct lpfc_hba *phba) * allocate + 3, one for cmd, one for rsp and one for this alignment */ lpfc_tgttemplate.max_sgl_segments = phba->cfg_nvme_seg_cnt + 1; - lpfc_tgttemplate.max_hw_queues = phba->cfg_nvme_io_channel; + lpfc_tgttemplate.max_hw_queues = phba->cfg_hdw_queue; lpfc_tgttemplate.target_features = NVMET_FCTGTFEAT_READDATA_RSP; #if (IS_ENABLED(CONFIG_NVME_TARGET_FC)) @@ -1697,8 +1697,8 @@ lpfc_nvmet_destroy_targetport(struct lpfc_hba *phba) return; if (phba->targetport) { tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; - for (qidx = 0; qidx < phba->cfg_nvme_io_channel; qidx++) { - wq = phba->sli4_hba.nvme_wq[qidx]; + for (qidx = 0; qidx < phba->cfg_hdw_queue; qidx++) { + wq = phba->sli4_hba.hdwq[qidx].nvme_wq; lpfc_nvmet_wqfull_flush(phba, wq, NULL); } init_completion(&tgtp->tport_unreg_done); diff --git a/drivers/scsi/lpfc/lpfc_scsi.c b/drivers/scsi/lpfc/lpfc_scsi.c index 9793016046de..63841f49affb 100644 --- a/drivers/scsi/lpfc/lpfc_scsi.c +++ b/drivers/scsi/lpfc/lpfc_scsi.c @@ -3661,8 +3661,8 @@ int lpfc_sli4_scmd_to_wqidx_distr(struct lpfc_hba *phba, return hwq; } - if (phba->cfg_fcp_io_sched == LPFC_FCP_SCHED_BY_CPU - && phba->cfg_fcp_io_channel > 1) { + if (phba->cfg_fcp_io_sched == LPFC_FCP_SCHED_BY_CPU && + phba->cfg_hdw_queue > 1) { cpu = lpfc_cmd->cpu; if (cpu < phba->sli4_hba.num_present_cpu) { cpup = phba->sli4_hba.cpu_map; @@ -3671,7 +3671,7 @@ int lpfc_sli4_scmd_to_wqidx_distr(struct lpfc_hba *phba, } } chann = atomic_add_return(1, &phba->fcp_qidx); - chann = chann % phba->cfg_fcp_io_channel; + chann = chann % phba->cfg_hdw_queue; return chann; } @@ -4598,7 +4598,7 @@ lpfc_abort_handler(struct scsi_cmnd *cmnd) iocb = &lpfc_cmd->cur_iocbq; if (phba->sli_rev == LPFC_SLI_REV4) { - pring_s4 = phba->sli4_hba.fcp_wq[iocb->hba_wqidx]->pring; + pring_s4 = phba->sli4_hba.hdwq[iocb->hba_wqidx].fcp_wq->pring; if (!pring_s4) { ret = FAILED; goto out_unlock; diff --git a/drivers/scsi/lpfc/lpfc_sli.c b/drivers/scsi/lpfc/lpfc_sli.c index 30a786304848..d41bfa4569a4 100644 --- a/drivers/scsi/lpfc/lpfc_sli.c +++ b/drivers/scsi/lpfc/lpfc_sli.c @@ -3981,8 +3981,8 @@ lpfc_sli_abort_fcp_rings(struct lpfc_hba *phba) /* Look on all the FCP Rings for the iotag */ if (phba->sli_rev >= LPFC_SLI_REV4) { - for (i = 0; i < phba->cfg_fcp_io_channel; i++) { - pring = phba->sli4_hba.fcp_wq[i]->pring; + for (i = 0; i < phba->cfg_hdw_queue; i++) { + pring = phba->sli4_hba.hdwq[i].fcp_wq->pring; lpfc_sli_abort_iocb_ring(phba, pring); } } else { @@ -4006,12 +4006,13 @@ lpfc_sli_abort_nvme_rings(struct lpfc_hba *phba) struct lpfc_sli_ring *pring; uint32_t i; - if (phba->sli_rev < LPFC_SLI_REV4) + if ((phba->sli_rev < LPFC_SLI_REV4) || + !(phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME)) return; /* Abort all IO on each NVME ring. */ - for (i = 0; i < phba->cfg_nvme_io_channel; i++) { - pring = phba->sli4_hba.nvme_wq[i]->pring; + for (i = 0; i < phba->cfg_hdw_queue; i++) { + pring = phba->sli4_hba.hdwq[i].nvme_wq->pring; lpfc_sli_abort_wqe_ring(phba, pring); } } @@ -4044,8 +4045,8 @@ lpfc_sli_flush_fcp_rings(struct lpfc_hba *phba) /* Look on all the FCP Rings for the iotag */ if (phba->sli_rev >= LPFC_SLI_REV4) { - for (i = 0; i < phba->cfg_fcp_io_channel; i++) { - pring = phba->sli4_hba.fcp_wq[i]->pring; + for (i = 0; i < phba->cfg_hdw_queue; i++) { + pring = phba->sli4_hba.hdwq[i].fcp_wq->pring; spin_lock_irq(&pring->ring_lock); /* Retrieve everything on txq */ @@ -4110,7 +4111,8 @@ lpfc_sli_flush_nvme_rings(struct lpfc_hba *phba) uint32_t i; struct lpfc_iocbq *piocb, *next_iocb; - if (phba->sli_rev < LPFC_SLI_REV4) + if ((phba->sli_rev < LPFC_SLI_REV4) || + !(phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME)) return; /* Hint to other driver operations that a flush is in progress. */ @@ -4122,8 +4124,8 @@ lpfc_sli_flush_nvme_rings(struct lpfc_hba *phba) * a local driver reason code. This is a flush so no * abort exchange to FW. */ - for (i = 0; i < phba->cfg_nvme_io_channel; i++) { - pring = phba->sli4_hba.nvme_wq[i]->pring; + for (i = 0; i < phba->cfg_hdw_queue; i++) { + pring = phba->sli4_hba.hdwq[i].nvme_wq->pring; spin_lock_irq(&pring->ring_lock); list_for_each_entry_safe(piocb, next_iocb, @@ -5564,6 +5566,7 @@ lpfc_sli4_arm_cqeq_intr(struct lpfc_hba *phba) { int qidx; struct lpfc_sli4_hba *sli4_hba = &phba->sli4_hba; + struct lpfc_sli4_hdw_queue *qp; sli4_hba->sli4_cq_release(sli4_hba->mbx_cq, LPFC_QUEUE_REARM); sli4_hba->sli4_cq_release(sli4_hba->els_cq, LPFC_QUEUE_REARM); @@ -5571,20 +5574,19 @@ lpfc_sli4_arm_cqeq_intr(struct lpfc_hba *phba) sli4_hba->sli4_cq_release(sli4_hba->nvmels_cq, LPFC_QUEUE_REARM); - if (sli4_hba->fcp_cq) - for (qidx = 0; qidx < phba->cfg_fcp_io_channel; qidx++) - sli4_hba->sli4_cq_release(sli4_hba->fcp_cq[qidx], + qp = sli4_hba->hdwq; + if (sli4_hba->hdwq) { + for (qidx = 0; qidx < phba->cfg_hdw_queue; qidx++) { + sli4_hba->sli4_cq_release(qp[qidx].fcp_cq, LPFC_QUEUE_REARM); - - if (sli4_hba->nvme_cq) - for (qidx = 0; qidx < phba->cfg_nvme_io_channel; qidx++) - sli4_hba->sli4_cq_release(sli4_hba->nvme_cq[qidx], + sli4_hba->sli4_cq_release(qp[qidx].nvme_cq, LPFC_QUEUE_REARM); + } - if (sli4_hba->hba_eq) - for (qidx = 0; qidx < phba->io_channel_irqs; qidx++) - sli4_hba->sli4_eq_release(sli4_hba->hba_eq[qidx], - LPFC_QUEUE_REARM); + for (qidx = 0; qidx < phba->cfg_hdw_queue; qidx++) + sli4_hba->sli4_eq_release(qp[qidx].hba_eq, + LPFC_QUEUE_REARM); + } if (phba->nvmet_support) { for (qidx = 0; qidx < phba->cfg_nvmet_mrq; qidx++) { @@ -7857,11 +7859,11 @@ lpfc_sli4_process_missed_mbox_completions(struct lpfc_hba *phba) /* Find the eq associated with the mcq */ - if (sli4_hba->hba_eq) - for (eqidx = 0; eqidx < phba->io_channel_irqs; eqidx++) - if (sli4_hba->hba_eq[eqidx]->queue_id == + if (sli4_hba->hdwq) + for (eqidx = 0; eqidx < phba->cfg_hdw_queue; eqidx++) + if (sli4_hba->hdwq[eqidx].hba_eq->queue_id == sli4_hba->mbx_cq->assoc_qid) { - fpeq = sli4_hba->hba_eq[eqidx]; + fpeq = sli4_hba->hdwq[eqidx].hba_eq; break; } if (!fpeq) @@ -9866,7 +9868,7 @@ __lpfc_sli_issue_iocb_s4(struct lpfc_hba *phba, uint32_t ring_number, /* Get the WQ */ if ((piocb->iocb_flag & LPFC_IO_FCP) || (piocb->iocb_flag & LPFC_USE_FCPWQIDX)) { - wq = phba->sli4_hba.fcp_wq[piocb->hba_wqidx]; + wq = phba->sli4_hba.hdwq[piocb->hba_wqidx].fcp_wq; } else { wq = phba->sli4_hba.els_wq; } @@ -10001,7 +10003,7 @@ struct lpfc_sli_ring * lpfc_sli4_calc_ring(struct lpfc_hba *phba, struct lpfc_iocbq *piocb) { if (piocb->iocb_flag & (LPFC_IO_FCP | LPFC_USE_FCPWQIDX)) { - if (unlikely(!phba->sli4_hba.fcp_wq)) + if (unlikely(!phba->sli4_hba.hdwq)) return NULL; /* * for abort iocb hba_wqidx should already @@ -10012,9 +10014,9 @@ lpfc_sli4_calc_ring(struct lpfc_hba *phba, struct lpfc_iocbq *piocb) lpfc_sli4_scmd_to_wqidx_distr( phba, piocb->context1); piocb->hba_wqidx = piocb->hba_wqidx % - phba->cfg_fcp_io_channel; + phba->cfg_hdw_queue; } - return phba->sli4_hba.fcp_wq[piocb->hba_wqidx]->pring; + return phba->sli4_hba.hdwq[piocb->hba_wqidx].fcp_wq->pring; } else { if (unlikely(!phba->sli4_hba.els_wq)) return NULL; @@ -10063,7 +10065,7 @@ lpfc_sli_issue_iocb(struct lpfc_hba *phba, uint32_t ring_number, if (atomic_dec_and_test(&hba_eq_hdl->hba_eq_in_use)) { /* Get associated EQ with this index */ - fpeq = phba->sli4_hba.hba_eq[idx]; + fpeq = phba->sli4_hba.hdwq[idx].hba_eq; /* Turn off interrupts from this EQ */ phba->sli4_hba.sli4_eq_clr_intr(fpeq); @@ -10497,17 +10499,8 @@ lpfc_sli4_queue_init(struct lpfc_hba *phba) INIT_LIST_HEAD(&psli->mboxq); INIT_LIST_HEAD(&psli->mboxq_cmpl); /* Initialize list headers for txq and txcmplq as double linked lists */ - for (i = 0; i < phba->cfg_fcp_io_channel; i++) { - pring = phba->sli4_hba.fcp_wq[i]->pring; - pring->flag = 0; - pring->ringno = LPFC_FCP_RING; - INIT_LIST_HEAD(&pring->txq); - INIT_LIST_HEAD(&pring->txcmplq); - INIT_LIST_HEAD(&pring->iocb_continueq); - spin_lock_init(&pring->ring_lock); - } - for (i = 0; i < phba->cfg_nvme_io_channel; i++) { - pring = phba->sli4_hba.nvme_wq[i]->pring; + for (i = 0; i < phba->cfg_hdw_queue; i++) { + pring = phba->sli4_hba.hdwq[i].fcp_wq->pring; pring->flag = 0; pring->ringno = LPFC_FCP_RING; INIT_LIST_HEAD(&pring->txq); @@ -10523,7 +10516,16 @@ lpfc_sli4_queue_init(struct lpfc_hba *phba) INIT_LIST_HEAD(&pring->iocb_continueq); spin_lock_init(&pring->ring_lock); - if (phba->cfg_nvme_io_channel) { + if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { + for (i = 0; i < phba->cfg_hdw_queue; i++) { + pring = phba->sli4_hba.hdwq[i].nvme_wq->pring; + pring->flag = 0; + pring->ringno = LPFC_FCP_RING; + INIT_LIST_HEAD(&pring->txq); + INIT_LIST_HEAD(&pring->txcmplq); + INIT_LIST_HEAD(&pring->iocb_continueq); + spin_lock_init(&pring->ring_lock); + } pring = phba->sli4_hba.nvmels_wq->pring; pring->flag = 0; pring->ringno = LPFC_ELS_RING; @@ -14070,6 +14072,20 @@ lpfc_sli4_hba_handle_eqe(struct lpfc_hba *phba, struct lpfc_eqe *eqe, /* Get the reference to the corresponding CQ */ cqid = bf_get_le32(lpfc_eqe_resource_id, eqe); + /* First check for NVME/SCSI completion */ + if (cqid == phba->sli4_hba.hdwq[qidx].nvme_cq_map) { + /* Process NVME / NVMET command completion */ + cq = phba->sli4_hba.hdwq[qidx].nvme_cq; + goto process_cq; + } + + if (cqid == phba->sli4_hba.hdwq[qidx].fcp_cq_map) { + /* Process FCP command completion */ + cq = phba->sli4_hba.hdwq[qidx].fcp_cq; + goto process_cq; + } + + /* Next check for NVMET completion */ if (phba->cfg_nvmet_mrq && phba->sli4_hba.nvmet_cqset) { id = phba->sli4_hba.nvmet_cqset[0]->queue_id; if ((cqid >= id) && (cqid < (id + phba->cfg_nvmet_mrq))) { @@ -14079,20 +14095,6 @@ lpfc_sli4_hba_handle_eqe(struct lpfc_hba *phba, struct lpfc_eqe *eqe, } } - if (phba->sli4_hba.nvme_cq_map && - (cqid == phba->sli4_hba.nvme_cq_map[qidx])) { - /* Process NVME / NVMET command completion */ - cq = phba->sli4_hba.nvme_cq[qidx]; - goto process_cq; - } - - if (phba->sli4_hba.fcp_cq_map && - (cqid == phba->sli4_hba.fcp_cq_map[qidx])) { - /* Process FCP command completion */ - cq = phba->sli4_hba.fcp_cq[qidx]; - goto process_cq; - } - if (phba->sli4_hba.nvmels_cq && (cqid == phba->sli4_hba.nvmels_cq->queue_id)) { /* Process NVME unsol rcv */ @@ -14101,7 +14103,8 @@ lpfc_sli4_hba_handle_eqe(struct lpfc_hba *phba, struct lpfc_eqe *eqe, /* Otherwise this is a Slow path event */ if (cq == NULL) { - lpfc_sli4_sp_handle_eqe(phba, eqe, phba->sli4_hba.hba_eq[qidx]); + lpfc_sli4_sp_handle_eqe(phba, eqe, + phba->sli4_hba.hdwq[qidx].hba_eq); return; } @@ -14115,7 +14118,7 @@ process_cq: } /* Save EQ associated with this CQ */ - cq->assoc_qp = phba->sli4_hba.hba_eq[qidx]; + cq->assoc_qp = phba->sli4_hba.hdwq[qidx].hba_eq; if (!queue_work(phba->wq, &cq->irqwork)) lpfc_printf_log(phba, KERN_ERR, LOG_SLI, @@ -14236,11 +14239,11 @@ lpfc_sli4_hba_intr_handler(int irq, void *dev_id) if (unlikely(!phba)) return IRQ_NONE; - if (unlikely(!phba->sli4_hba.hba_eq)) + if (unlikely(!phba->sli4_hba.hdwq)) return IRQ_NONE; /* Get to the EQ struct associated with this vector */ - fpeq = phba->sli4_hba.hba_eq[hba_eqidx]; + fpeq = phba->sli4_hba.hdwq[hba_eqidx].hba_eq; if (unlikely(!fpeq)) return IRQ_NONE; @@ -14340,7 +14343,7 @@ lpfc_sli4_intr_handler(int irq, void *dev_id) /* * Invoke fast-path host attention interrupt handling as appropriate. */ - for (qidx = 0; qidx < phba->io_channel_irqs; qidx++) { + for (qidx = 0; qidx < phba->cfg_hdw_queue; qidx++) { hba_irq_rc = lpfc_sli4_hba_intr_handler(irq, &phba->sli4_hba.hba_eq_hdl[qidx]); if (hba_irq_rc == IRQ_HANDLED) @@ -14527,7 +14530,7 @@ lpfc_modify_hba_eq_delay(struct lpfc_hba *phba, uint32_t startq, union lpfc_sli4_cfg_shdr *shdr; uint16_t dmult; - if (startq >= phba->io_channel_irqs) + if (startq >= phba->cfg_hdw_queue) return 0; mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL); @@ -14541,7 +14544,7 @@ lpfc_modify_hba_eq_delay(struct lpfc_hba *phba, uint32_t startq, eq_delay = &mbox->u.mqe.un.eq_delay; /* Calculate delay multiper from maximum interrupt per second */ - result = imax / phba->io_channel_irqs; + result = imax / phba->cfg_hdw_queue; if (result > LPFC_DMULT_CONST || result == 0) dmult = 0; else @@ -14550,8 +14553,8 @@ lpfc_modify_hba_eq_delay(struct lpfc_hba *phba, uint32_t startq, dmult = LPFC_DMULT_MAX; cnt = 0; - for (qidx = startq; qidx < phba->io_channel_irqs; qidx++) { - eq = phba->sli4_hba.hba_eq[qidx]; + for (qidx = startq; qidx < phba->cfg_hdw_queue; qidx++) { + eq = phba->sli4_hba.hdwq[qidx].hba_eq; if (!eq) continue; eq->q_mode = imax; @@ -14568,8 +14571,7 @@ lpfc_modify_hba_eq_delay(struct lpfc_hba *phba, uint32_t startq, val = phba->cfg_fcp_imax; if (val) { /* First, interrupts per sec per EQ */ - val = phba->cfg_fcp_imax / - phba->io_channel_irqs; + val = phba->cfg_fcp_imax / phba->cfg_hdw_queue; /* us delay between each interrupt */ val = LPFC_SEC_TO_USEC / val; @@ -14877,7 +14879,7 @@ out: * lpfc_cq_create_set - Create a set of Completion Queues on the HBA for MRQ * @phba: HBA structure that indicates port to create a queue on. * @cqp: The queue structure array to use to create the completion queues. - * @eqp: The event queue array to bind these completion queues to. + * @hdwq: The hardware queue array with the EQ to bind completion queues to. * * This function creates a set of completion queue, s to support MRQ * as detailed in @cqp, on a port, @@ -14897,7 +14899,8 @@ out: **/ int lpfc_cq_create_set(struct lpfc_hba *phba, struct lpfc_queue **cqp, - struct lpfc_queue **eqp, uint32_t type, uint32_t subtype) + struct lpfc_sli4_hdw_queue *hdwq, uint32_t type, + uint32_t subtype) { struct lpfc_queue *cq; struct lpfc_queue *eq; @@ -14912,7 +14915,7 @@ lpfc_cq_create_set(struct lpfc_hba *phba, struct lpfc_queue **cqp, /* sanity check on queue memory */ numcq = phba->cfg_nvmet_mrq; - if (!cqp || !eqp || !numcq) + if (!cqp || !hdwq || !numcq) return -ENODEV; mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL); @@ -14939,7 +14942,7 @@ lpfc_cq_create_set(struct lpfc_hba *phba, struct lpfc_queue **cqp, for (idx = 0; idx < numcq; idx++) { cq = cqp[idx]; - eq = eqp[idx]; + eq = hdwq[idx].hba_eq; if (!cq || !eq) { status = -ENOMEM; goto out; @@ -19462,7 +19465,7 @@ lpfc_drain_txq(struct lpfc_hba *phba) if (phba->link_flag & LS_MDS_LOOPBACK) { /* MDS WQE are posted only to first WQ*/ - wq = phba->sli4_hba.fcp_wq[0]; + wq = phba->sli4_hba.hdwq[0].fcp_wq; if (unlikely(!wq)) return 0; pring = wq->pring; @@ -19712,12 +19715,12 @@ lpfc_sli4_issue_wqe(struct lpfc_hba *phba, uint32_t ring_number, /* NVME_FCREQ and NVME_ABTS requests */ if (pwqe->iocb_flag & LPFC_IO_NVME) { /* Get the IO distribution (hba_wqidx) for WQ assignment. */ - pring = phba->sli4_hba.nvme_wq[pwqe->hba_wqidx]->pring; + pring = phba->sli4_hba.hdwq[pwqe->hba_wqidx].nvme_wq->pring; spin_lock_irqsave(&pring->ring_lock, iflags); - wq = phba->sli4_hba.nvme_wq[pwqe->hba_wqidx]; + wq = phba->sli4_hba.hdwq[pwqe->hba_wqidx].nvme_wq; bf_set(wqe_cqid, &wqe->generic.wqe_com, - phba->sli4_hba.nvme_cq[pwqe->hba_wqidx]->queue_id); + phba->sli4_hba.hdwq[pwqe->hba_wqidx].nvme_cq->queue_id); ret = lpfc_sli4_wq_put(wq, wqe); if (ret) { spin_unlock_irqrestore(&pring->ring_lock, iflags); @@ -19731,7 +19734,7 @@ lpfc_sli4_issue_wqe(struct lpfc_hba *phba, uint32_t ring_number, /* NVMET requests */ if (pwqe->iocb_flag & LPFC_IO_NVMET) { /* Get the IO distribution (hba_wqidx) for WQ assignment. */ - pring = phba->sli4_hba.nvme_wq[pwqe->hba_wqidx]->pring; + pring = phba->sli4_hba.hdwq[pwqe->hba_wqidx].nvme_wq->pring; spin_lock_irqsave(&pring->ring_lock, iflags); ctxp = pwqe->context2; @@ -19742,9 +19745,9 @@ lpfc_sli4_issue_wqe(struct lpfc_hba *phba, uint32_t ring_number, } bf_set(wqe_xri_tag, &pwqe->wqe.xmit_bls_rsp.wqe_com, pwqe->sli4_xritag); - wq = phba->sli4_hba.nvme_wq[pwqe->hba_wqidx]; + wq = phba->sli4_hba.hdwq[pwqe->hba_wqidx].nvme_wq; bf_set(wqe_cqid, &wqe->generic.wqe_com, - phba->sli4_hba.nvme_cq[pwqe->hba_wqidx]->queue_id); + phba->sli4_hba.hdwq[pwqe->hba_wqidx].nvme_cq->queue_id); ret = lpfc_sli4_wq_put(wq, wqe); if (ret) { spin_unlock_irqrestore(&pring->ring_lock, iflags); diff --git a/drivers/scsi/lpfc/lpfc_sli4.h b/drivers/scsi/lpfc/lpfc_sli4.h index 645620175453..6e1e958df10a 100644 --- a/drivers/scsi/lpfc/lpfc_sli4.h +++ b/drivers/scsi/lpfc/lpfc_sli4.h @@ -36,18 +36,13 @@ #define LPFC_NEMBED_MBOX_SGL_CNT 254 /* Multi-queue arrangement for FCP EQ/CQ/WQ tuples */ -#define LPFC_HBA_IO_CHAN_MIN 0 -#define LPFC_HBA_IO_CHAN_MAX 32 -#define LPFC_FCP_IO_CHAN_DEF 4 -#define LPFC_NVME_IO_CHAN_DEF 0 +#define LPFC_HBA_HDWQ_MIN 0 +#define LPFC_HBA_HDWQ_MAX 64 +#define LPFC_HBA_HDWQ_DEF 0 /* Common buffer size to accomidate SCSI and NVME IO buffers */ #define LPFC_COMMON_IO_BUF_SZ 768 -/* Number of channels used for Flash Optimized Fabric (FOF) operations */ - -#define LPFC_FOF_IO_CHAN_NUM 1 - /* * Provide the default FCF Record attributes used by the driver * when nonFIP mode is configured and there is no other default @@ -534,6 +529,17 @@ struct lpfc_vector_map_info { #define LPFC_VECTOR_MAP_EMPTY 0xffff /* SLI4 HBA data structure entries */ +struct lpfc_sli4_hdw_queue { + /* Pointers to the constructed SLI4 queues */ + struct lpfc_queue *hba_eq; /* Event queues for HBA */ + struct lpfc_queue *fcp_cq; /* Fast-path FCP compl queue */ + struct lpfc_queue *nvme_cq; /* Fast-path NVME compl queue */ + struct lpfc_queue *fcp_wq; /* Fast-path FCP work queue */ + struct lpfc_queue *nvme_wq; /* Fast-path NVME work queue */ + uint16_t fcp_cq_map; + uint16_t nvme_cq_map; +}; + struct lpfc_sli4_hba { void __iomem *conf_regs_memmap_p; /* Kernel memory mapped address for * config space registers @@ -606,17 +612,13 @@ struct lpfc_sli4_hba { uint32_t (*sli4_cq_release)(struct lpfc_queue *q, bool arm); /* Pointers to the constructed SLI4 queues */ - struct lpfc_queue **hba_eq; /* Event queues for HBA */ - struct lpfc_queue **fcp_cq; /* Fast-path FCP compl queue */ - struct lpfc_queue **nvme_cq; /* Fast-path NVME compl queue */ + struct lpfc_sli4_hdw_queue *hdwq; + struct list_head lpfc_wq_list; + + /* Pointers to the constructed SLI4 queues for NVMET */ struct lpfc_queue **nvmet_cqset; /* Fast-path NVMET CQ Set queues */ struct lpfc_queue **nvmet_mrq_hdr; /* Fast-path NVMET hdr MRQs */ struct lpfc_queue **nvmet_mrq_data; /* Fast-path NVMET data MRQs */ - struct lpfc_queue **fcp_wq; /* Fast-path FCP work queue */ - struct lpfc_queue **nvme_wq; /* Fast-path NVME work queue */ - uint16_t *fcp_cq_map; - uint16_t *nvme_cq_map; - struct list_head lpfc_wq_list; struct lpfc_queue *mbx_cq; /* Slow-path mailbox complete queue */ struct lpfc_queue *els_cq; /* Slow-path ELS response complete queue */ @@ -817,7 +819,7 @@ int lpfc_modify_hba_eq_delay(struct lpfc_hba *phba, uint32_t startq, int lpfc_cq_create(struct lpfc_hba *, struct lpfc_queue *, struct lpfc_queue *, uint32_t, uint32_t); int lpfc_cq_create_set(struct lpfc_hba *phba, struct lpfc_queue **cqp, - struct lpfc_queue **eqp, uint32_t type, + struct lpfc_sli4_hdw_queue *hdwq, uint32_t type, uint32_t subtype); int32_t lpfc_mq_create(struct lpfc_hba *, struct lpfc_queue *, struct lpfc_queue *, uint32_t); -- cgit v1.2.3 From 5e5b511d8bfaf765cb92a695cda336c936cb86dc Mon Sep 17 00:00:00 2001 From: James Smart Date: Mon, 28 Jan 2019 11:14:22 -0800 Subject: scsi: lpfc: Partition XRI buffer list across Hardware Queues Once the IO buff allocations were made shared, there was a single XRI buffer list shared by all hardware queues. A single list isn't great for performance when shared across the per-cpu hardware queues. Create a separate XRI IO buffer get/put list for each Hardware Queue. As SGLs and associated IO buffers get allocated/posted to the firmware; round robin their assignment across all available hardware Queues so that there is an equitable assignment. Modify SCSI and NVME IO submit code paths to use the Hardware Queue logic for XRI allocation. Add a debugfs interface to display hardware queue statistics Added new empty_io_bufs counter to track if a cpu runs out of XRIs. Replace common_ variables/names with io_ to make meanings clearer. Signed-off-by: Dick Kennedy Signed-off-by: James Smart Reviewed-by: Hannes Reinecke Signed-off-by: Martin K. Petersen --- drivers/scsi/lpfc/lpfc.h | 8 +- drivers/scsi/lpfc/lpfc_attr.c | 2 +- drivers/scsi/lpfc/lpfc_crtn.h | 10 +- drivers/scsi/lpfc/lpfc_debugfs.c | 141 +++++++++++- drivers/scsi/lpfc/lpfc_debugfs.h | 3 + drivers/scsi/lpfc/lpfc_init.c | 447 +++++++++++++++++++++++++-------------- drivers/scsi/lpfc/lpfc_nvme.c | 90 ++++---- drivers/scsi/lpfc/lpfc_nvme.h | 3 +- drivers/scsi/lpfc/lpfc_nvmet.c | 22 +- drivers/scsi/lpfc/lpfc_scsi.c | 107 ++++++---- drivers/scsi/lpfc/lpfc_scsi.h | 3 +- drivers/scsi/lpfc/lpfc_sli.c | 88 +++----- drivers/scsi/lpfc/lpfc_sli.h | 1 + drivers/scsi/lpfc/lpfc_sli4.h | 36 +++- 14 files changed, 623 insertions(+), 338 deletions(-) (limited to 'drivers/scsi/lpfc/lpfc.h') diff --git a/drivers/scsi/lpfc/lpfc.h b/drivers/scsi/lpfc/lpfc.h index da12476dd933..19827ce7a4d9 100644 --- a/drivers/scsi/lpfc/lpfc.h +++ b/drivers/scsi/lpfc/lpfc.h @@ -965,13 +965,6 @@ struct lpfc_hba { struct list_head lpfc_scsi_buf_list_get; struct list_head lpfc_scsi_buf_list_put; uint32_t total_scsi_bufs; - spinlock_t common_buf_list_get_lock; /* Common buf alloc list lock */ - spinlock_t common_buf_list_put_lock; /* Common buf free list lock */ - struct list_head lpfc_common_buf_list_get; - struct list_head lpfc_common_buf_list_put; - uint32_t total_common_bufs; - uint32_t get_common_bufs; - uint32_t put_common_bufs; struct list_head lpfc_iocb_list; uint32_t total_iocbq_bufs; struct list_head active_rrq_list; @@ -1045,6 +1038,7 @@ struct lpfc_hba { struct dentry *debug_nvmeio_trc; struct lpfc_debugfs_nvmeio_trc *nvmeio_trc; + struct dentry *debug_hdwqinfo; atomic_t nvmeio_trc_cnt; uint32_t nvmeio_trc_size; uint32_t nvmeio_trc_output_idx; diff --git a/drivers/scsi/lpfc/lpfc_attr.c b/drivers/scsi/lpfc/lpfc_attr.c index c6b1d432dd07..1671d9371d3b 100644 --- a/drivers/scsi/lpfc/lpfc_attr.c +++ b/drivers/scsi/lpfc/lpfc_attr.c @@ -337,7 +337,7 @@ lpfc_nvme_info_show(struct device *dev, struct device_attribute *attr, "XRI Dist lpfc%d Total %d IO %d ELS %d\n", phba->brd_no, phba->sli4_hba.max_cfg_param.max_xri, - phba->sli4_hba.common_xri_max, + phba->sli4_hba.io_xri_max, lpfc_sli4_get_els_iocb_cnt(phba)); if (strlcat(buf, tmp, PAGE_SIZE) >= PAGE_SIZE) goto buffer_done; diff --git a/drivers/scsi/lpfc/lpfc_crtn.h b/drivers/scsi/lpfc/lpfc_crtn.h index 6dc427d4228c..a623f6f619cc 100644 --- a/drivers/scsi/lpfc/lpfc_crtn.h +++ b/drivers/scsi/lpfc/lpfc_crtn.h @@ -515,10 +515,12 @@ int lpfc_sli4_read_config(struct lpfc_hba *); void lpfc_sli4_node_prep(struct lpfc_hba *); int lpfc_sli4_els_sgl_update(struct lpfc_hba *phba); int lpfc_sli4_nvmet_sgl_update(struct lpfc_hba *phba); -int lpfc_sli4_common_sgl_update(struct lpfc_hba *phba); -int lpfc_sli4_post_common_sgl_list(struct lpfc_hba *phba, - struct list_head *blist, int xricnt); -int lpfc_new_common_buf(struct lpfc_hba *phba, int num_to_alloc); +int lpfc_io_buf_flush(struct lpfc_hba *phba, struct list_head *sglist); +int lpfc_io_buf_replenish(struct lpfc_hba *phba, struct list_head *cbuf); +int lpfc_sli4_io_sgl_update(struct lpfc_hba *phba); +int lpfc_sli4_post_io_sgl_list(struct lpfc_hba *phba, + struct list_head *blist, int xricnt); +int lpfc_new_io_buf(struct lpfc_hba *phba, int num_to_alloc); void lpfc_free_sgl_list(struct lpfc_hba *, struct list_head *); uint32_t lpfc_sli_port_speed_get(struct lpfc_hba *); int lpfc_sli4_request_firmware_update(struct lpfc_hba *, uint8_t); diff --git a/drivers/scsi/lpfc/lpfc_debugfs.c b/drivers/scsi/lpfc/lpfc_debugfs.c index fc5ec99bfde2..a1b7263bfe2a 100644 --- a/drivers/scsi/lpfc/lpfc_debugfs.c +++ b/drivers/scsi/lpfc/lpfc_debugfs.c @@ -378,6 +378,73 @@ skipit: return len; } +static int lpfc_debugfs_last_hdwq; + +/** + * lpfc_debugfs_hdwqinfo_data - Dump Hardware Queue info to a buffer + * @phba: The HBA to gather host buffer info from. + * @buf: The buffer to dump log into. + * @size: The maximum amount of data to process. + * + * Description: + * This routine dumps the Hardware Queue info from the @phba to @buf up to + * @size number of bytes. A header that describes the current hdwq state will be + * dumped to @buf first and then info on each hdwq entry will be dumped to @buf + * until @size bytes have been dumped or all the hdwq info has been dumped. + * + * Notes: + * This routine will rotate through each configured Hardware Queue each + * time called. + * + * Return Value: + * This routine returns the amount of bytes that were dumped into @buf and will + * not exceed @size. + **/ +static int +lpfc_debugfs_hdwqinfo_data(struct lpfc_hba *phba, char *buf, int size) +{ + struct lpfc_sli4_hdw_queue *qp; + int len = 0; + int i, out; + unsigned long iflag; + + if (phba->sli_rev != LPFC_SLI_REV4) + return 0; + + if (!phba->sli4_hba.hdwq) + return 0; + + for (i = 0; i < phba->cfg_hdw_queue; i++) { + if (len > (LPFC_HDWQINFO_SIZE - 80)) + break; + qp = &phba->sli4_hba.hdwq[lpfc_debugfs_last_hdwq]; + + len += snprintf(buf + len, size - len, "HdwQ %d Info ", i); + spin_lock_irqsave(&qp->abts_scsi_buf_list_lock, iflag); + spin_lock(&qp->abts_nvme_buf_list_lock); + spin_lock(&qp->io_buf_list_get_lock); + spin_lock(&qp->io_buf_list_put_lock); + out = qp->total_io_bufs - (qp->get_io_bufs + qp->put_io_bufs + + qp->abts_scsi_io_bufs + qp->abts_nvme_io_bufs); + len += snprintf(buf + len, size - len, + "tot:%d get:%d put:%d mt:%d " + "ABTS scsi:%d nvme:%d Out:%d\n", + qp->total_io_bufs, qp->get_io_bufs, qp->put_io_bufs, + qp->empty_io_bufs, qp->abts_scsi_io_bufs, + qp->abts_nvme_io_bufs, out); + spin_unlock(&qp->io_buf_list_put_lock); + spin_unlock(&qp->io_buf_list_get_lock); + spin_unlock(&qp->abts_nvme_buf_list_lock); + spin_unlock_irqrestore(&qp->abts_scsi_buf_list_lock, iflag); + + lpfc_debugfs_last_hdwq++; + if (lpfc_debugfs_last_hdwq >= phba->cfg_hdw_queue) + lpfc_debugfs_last_hdwq = 0; + } + + return len; +} + static int lpfc_debugfs_last_hba_slim_off; /** @@ -863,17 +930,17 @@ lpfc_debugfs_nvmestat_data(struct lpfc_vport *vport, char *buf, int size) len += snprintf(buf + len, size - len, "\n"); cnt = 0; - spin_lock(&phba->sli4_hba.abts_nvme_buf_list_lock); + spin_lock(&phba->sli4_hba.abts_nvmet_buf_list_lock); list_for_each_entry_safe(ctxp, next_ctxp, &phba->sli4_hba.lpfc_abts_nvmet_ctx_list, list) { cnt++; } - spin_unlock(&phba->sli4_hba.abts_nvme_buf_list_lock); + spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock); if (cnt) { len += snprintf(buf + len, size - len, "ABORT: %d ctx entries\n", cnt); - spin_lock(&phba->sli4_hba.abts_nvme_buf_list_lock); + spin_lock(&phba->sli4_hba.abts_nvmet_buf_list_lock); list_for_each_entry_safe(ctxp, next_ctxp, &phba->sli4_hba.lpfc_abts_nvmet_ctx_list, list) { @@ -885,7 +952,7 @@ lpfc_debugfs_nvmestat_data(struct lpfc_vport *vport, char *buf, int size) ctxp->oxid, ctxp->state, ctxp->flag); } - spin_unlock(&phba->sli4_hba.abts_nvme_buf_list_lock); + spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock); } /* Calculate outstanding IOs */ @@ -1619,6 +1686,48 @@ out: return rc; } +/** + * lpfc_debugfs_hdwqinfo_open - Open the hdwqinfo debugfs buffer + * @inode: The inode pointer that contains a vport pointer. + * @file: The file pointer to attach the log output. + * + * Description: + * This routine is the entry point for the debugfs open file operation. It gets + * the vport from the i_private field in @inode, allocates the necessary buffer + * for the log, fills the buffer from the in-memory log for this vport, and then + * returns a pointer to that log in the private_data field in @file. + * + * Returns: + * This function returns zero if successful. On error it will return a negative + * error value. + **/ +static int +lpfc_debugfs_hdwqinfo_open(struct inode *inode, struct file *file) +{ + struct lpfc_hba *phba = inode->i_private; + struct lpfc_debug *debug; + int rc = -ENOMEM; + + debug = kmalloc(sizeof(*debug), GFP_KERNEL); + if (!debug) + goto out; + + /* Round to page boundary */ + debug->buffer = kmalloc(LPFC_HDWQINFO_SIZE, GFP_KERNEL); + if (!debug->buffer) { + kfree(debug); + goto out; + } + + debug->len = lpfc_debugfs_hdwqinfo_data(phba, debug->buffer, + LPFC_HBQINFO_SIZE); + file->private_data = debug; + + rc = 0; +out: + return rc; +} + /** * lpfc_debugfs_dumpHBASlim_open - Open the Dump HBA SLIM debugfs buffer * @inode: The inode pointer that contains a vport pointer. @@ -4819,6 +4928,15 @@ static const struct file_operations lpfc_debugfs_op_hbqinfo = { .release = lpfc_debugfs_release, }; +#undef lpfc_debugfs_op_hdwqinfo +static const struct file_operations lpfc_debugfs_op_hdwqinfo = { + .owner = THIS_MODULE, + .open = lpfc_debugfs_hdwqinfo_open, + .llseek = lpfc_debugfs_lseek, + .read = lpfc_debugfs_read, + .release = lpfc_debugfs_release, +}; + #undef lpfc_debugfs_op_dumpHBASlim static const struct file_operations lpfc_debugfs_op_dumpHBASlim = { .owner = THIS_MODULE, @@ -5244,6 +5362,18 @@ lpfc_debugfs_initialize(struct lpfc_vport *vport) phba->hba_debugfs_root, phba, &lpfc_debugfs_op_hbqinfo); + /* Setup hdwqinfo */ + snprintf(name, sizeof(name), "hdwqinfo"); + phba->debug_hdwqinfo = + debugfs_create_file(name, S_IFREG | 0644, + phba->hba_debugfs_root, + phba, &lpfc_debugfs_op_hdwqinfo); + if (!phba->debug_hdwqinfo) { + lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT, + "0411 Cant create debugfs hdwqinfo\n"); + goto debug_failed; + } + /* Setup dumpHBASlim */ if (phba->sli_rev < LPFC_SLI_REV4) { snprintf(name, sizeof(name), "dumpHBASlim"); @@ -5630,6 +5760,9 @@ lpfc_debugfs_terminate(struct lpfc_vport *vport) debugfs_remove(phba->debug_hbqinfo); /* hbqinfo */ phba->debug_hbqinfo = NULL; + debugfs_remove(phba->debug_hdwqinfo); /* hdwqinfo */ + phba->debug_hdwqinfo = NULL; + debugfs_remove(phba->debug_dumpHBASlim); /* HBASlim */ phba->debug_dumpHBASlim = NULL; diff --git a/drivers/scsi/lpfc/lpfc_debugfs.h b/drivers/scsi/lpfc/lpfc_debugfs.h index 2c5bc494b247..c904fa74dfbc 100644 --- a/drivers/scsi/lpfc/lpfc_debugfs.h +++ b/drivers/scsi/lpfc/lpfc_debugfs.h @@ -284,6 +284,9 @@ struct lpfc_idiag { #endif +/* hdwqinfo output buffer size */ +#define LPFC_HDWQINFO_SIZE 8192 + enum { DUMP_FCP, DUMP_NVME, diff --git a/drivers/scsi/lpfc/lpfc_init.c b/drivers/scsi/lpfc/lpfc_init.c index c4acd9ee55b1..ed0318b9003e 100644 --- a/drivers/scsi/lpfc/lpfc_init.c +++ b/drivers/scsi/lpfc/lpfc_init.c @@ -1039,12 +1039,12 @@ lpfc_hba_down_post_s4(struct lpfc_hba *phba) { struct lpfc_scsi_buf *psb, *psb_next; struct lpfc_nvmet_rcv_ctx *ctxp, *ctxp_next; + struct lpfc_sli4_hdw_queue *qp; LIST_HEAD(aborts); LIST_HEAD(nvme_aborts); LIST_HEAD(nvmet_aborts); - unsigned long iflag = 0; struct lpfc_sglq *sglq_entry = NULL; - int cnt; + int cnt, idx; lpfc_sli_hbqbuf_free_all(phba); @@ -1071,57 +1071,65 @@ lpfc_hba_down_post_s4(struct lpfc_hba *phba) spin_unlock(&phba->sli4_hba.sgl_list_lock); - /* abts_scsi_buf_list_lock required because worker thread uses this + + /* abts_xxxx_buf_list_lock required because worker thread uses this * list. */ - if (phba->cfg_enable_fc4_type & LPFC_ENABLE_FCP) { - spin_lock(&phba->sli4_hba.abts_scsi_buf_list_lock); - list_splice_init(&phba->sli4_hba.lpfc_abts_scsi_buf_list, - &aborts); - spin_unlock(&phba->sli4_hba.abts_scsi_buf_list_lock); - } - - if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { - spin_lock(&phba->sli4_hba.abts_nvme_buf_list_lock); - list_splice_init(&phba->sli4_hba.lpfc_abts_nvme_buf_list, - &nvme_aborts); - list_splice_init(&phba->sli4_hba.lpfc_abts_nvmet_ctx_list, - &nvmet_aborts); - spin_unlock(&phba->sli4_hba.abts_nvme_buf_list_lock); - } + cnt = 0; + for (idx = 0; idx < phba->cfg_hdw_queue; idx++) { + qp = &phba->sli4_hba.hdwq[idx]; - spin_unlock_irq(&phba->hbalock); + spin_lock(&qp->abts_scsi_buf_list_lock); + list_splice_init(&qp->lpfc_abts_scsi_buf_list, + &aborts); - if (phba->cfg_enable_fc4_type & LPFC_ENABLE_FCP) { list_for_each_entry_safe(psb, psb_next, &aborts, list) { psb->pCmd = NULL; psb->status = IOSTAT_SUCCESS; + cnt++; } - spin_lock_irqsave(&phba->common_buf_list_put_lock, iflag); - list_splice(&aborts, &phba->lpfc_common_buf_list_put); - spin_unlock_irqrestore(&phba->common_buf_list_put_lock, iflag); - } + spin_lock(&qp->io_buf_list_put_lock); + list_splice_init(&aborts, &qp->lpfc_io_buf_list_put); + qp->put_io_bufs += qp->abts_scsi_io_bufs; + qp->abts_scsi_io_bufs = 0; + spin_unlock(&qp->io_buf_list_put_lock); + spin_unlock(&qp->abts_scsi_buf_list_lock); + + if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { + spin_lock(&qp->abts_nvme_buf_list_lock); + list_splice_init(&qp->lpfc_abts_nvme_buf_list, + &nvme_aborts); + list_for_each_entry_safe(psb, psb_next, &nvme_aborts, + list) { + psb->pCmd = NULL; + psb->status = IOSTAT_SUCCESS; + cnt++; + } + spin_lock(&qp->io_buf_list_put_lock); + qp->put_io_bufs += qp->abts_nvme_io_bufs; + qp->abts_nvme_io_bufs = 0; + list_splice_init(&nvme_aborts, + &qp->lpfc_io_buf_list_put); + spin_unlock(&qp->io_buf_list_put_lock); + spin_unlock(&qp->abts_nvme_buf_list_lock); - if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { - cnt = 0; - list_for_each_entry_safe(psb, psb_next, &nvme_aborts, list) { - psb->pCmd = NULL; - psb->status = IOSTAT_SUCCESS; - cnt++; } - spin_lock_irqsave(&phba->common_buf_list_put_lock, iflag); - phba->put_common_bufs += cnt; - list_splice(&nvme_aborts, &phba->lpfc_common_buf_list_put); - spin_unlock_irqrestore(&phba->common_buf_list_put_lock, iflag); + } + if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { + spin_lock(&phba->sli4_hba.abts_nvmet_buf_list_lock); + list_splice_init(&phba->sli4_hba.lpfc_abts_nvmet_ctx_list, + &nvmet_aborts); + spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock); list_for_each_entry_safe(ctxp, ctxp_next, &nvmet_aborts, list) { ctxp->flag &= ~(LPFC_NVMET_XBUSY | LPFC_NVMET_ABORT_OP); lpfc_nvmet_ctxbuf_post(phba, ctxp->ctxbuf); } } + spin_unlock_irq(&phba->hbalock); lpfc_sli4_free_sp_events(phba); - return 0; + return cnt; } /** @@ -3126,18 +3134,6 @@ lpfc_online(struct lpfc_hba *phba) "6132 NVME restore reg failed " "on nvmei error x%x\n", error); } - /* Don't post more new bufs if repost already recovered - * the nvme sgls. - */ - if (phba->sli4_hba.common_xri_cnt == 0) { - i = lpfc_new_common_buf(phba, - phba->sli4_hba.common_xri_max); - if (i == 0) { - lpfc_unblock_mgmt_io(phba); - return 1; - } - phba->total_common_bufs += i; - } } else { lpfc_sli_queue_init(phba); if (lpfc_sli_hba_setup(phba)) { /* Initialize SLI2/SLI3 HBA */ @@ -3372,7 +3368,7 @@ lpfc_scsi_free(struct lpfc_hba *phba) } /** - * lpfc_common_free - Free all the IO buffers and IOCBs from driver lists + * lpfc_io_free - Free all the IO buffers and IOCBs from driver lists * @phba: pointer to lpfc hba data structure. * * This routine is to free all the IO buffers and IOCBs from the driver @@ -3380,36 +3376,44 @@ lpfc_scsi_free(struct lpfc_hba *phba) * the internal resources before the device is removed from the system. **/ static void -lpfc_common_free(struct lpfc_hba *phba) +lpfc_io_free(struct lpfc_hba *phba) { struct lpfc_nvme_buf *lpfc_ncmd, *lpfc_ncmd_next; + struct lpfc_sli4_hdw_queue *qp; + int idx; spin_lock_irq(&phba->hbalock); - /* Release all the lpfc_nvme_bufs maintained by this host. */ - spin_lock(&phba->common_buf_list_put_lock); - list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next, - &phba->lpfc_common_buf_list_put, list) { - list_del(&lpfc_ncmd->list); - phba->put_common_bufs--; - dma_pool_free(phba->lpfc_sg_dma_buf_pool, lpfc_ncmd->data, - lpfc_ncmd->dma_handle); - kfree(lpfc_ncmd); - phba->total_common_bufs--; + for (idx = 0; idx < phba->cfg_hdw_queue; idx++) { + qp = &phba->sli4_hba.hdwq[idx]; + /* Release all the lpfc_nvme_bufs maintained by this host. */ + spin_lock(&qp->io_buf_list_put_lock); + list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next, + &qp->lpfc_io_buf_list_put, + list) { + list_del(&lpfc_ncmd->list); + qp->put_io_bufs--; + dma_pool_free(phba->lpfc_sg_dma_buf_pool, + lpfc_ncmd->data, lpfc_ncmd->dma_handle); + kfree(lpfc_ncmd); + qp->total_io_bufs--; + } + spin_unlock(&qp->io_buf_list_put_lock); + + spin_lock(&qp->io_buf_list_get_lock); + list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next, + &qp->lpfc_io_buf_list_get, + list) { + list_del(&lpfc_ncmd->list); + qp->get_io_bufs--; + dma_pool_free(phba->lpfc_sg_dma_buf_pool, + lpfc_ncmd->data, lpfc_ncmd->dma_handle); + kfree(lpfc_ncmd); + qp->total_io_bufs--; + } + spin_unlock(&qp->io_buf_list_get_lock); } - spin_unlock(&phba->common_buf_list_put_lock); - spin_lock(&phba->common_buf_list_get_lock); - list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next, - &phba->lpfc_common_buf_list_get, list) { - list_del(&lpfc_ncmd->list); - phba->get_common_bufs--; - dma_pool_free(phba->lpfc_sg_dma_buf_pool, lpfc_ncmd->data, - lpfc_ncmd->dma_handle); - kfree(lpfc_ncmd); - phba->total_common_bufs--; - } - spin_unlock(&phba->common_buf_list_get_lock); spin_unlock_irq(&phba->hbalock); } @@ -3654,8 +3658,101 @@ out_free_mem: return rc; } +int +lpfc_io_buf_flush(struct lpfc_hba *phba, struct list_head *cbuf) +{ + LIST_HEAD(blist); + struct lpfc_sli4_hdw_queue *qp; + struct lpfc_scsi_buf *lpfc_cmd; + struct lpfc_scsi_buf *iobufp, *prev_iobufp; + int idx, cnt, xri, inserted; + + cnt = 0; + for (idx = 0; idx < phba->cfg_hdw_queue; idx++) { + qp = &phba->sli4_hba.hdwq[idx]; + spin_lock_irq(&qp->io_buf_list_get_lock); + spin_lock(&qp->io_buf_list_put_lock); + + /* Take everything off the get and put lists */ + list_splice_init(&qp->lpfc_io_buf_list_get, &blist); + list_splice(&qp->lpfc_io_buf_list_put, &blist); + INIT_LIST_HEAD(&qp->lpfc_io_buf_list_get); + INIT_LIST_HEAD(&qp->lpfc_io_buf_list_put); + cnt += qp->get_io_bufs + qp->put_io_bufs; + qp->get_io_bufs = 0; + qp->put_io_bufs = 0; + qp->total_io_bufs = 0; + spin_unlock(&qp->io_buf_list_put_lock); + spin_unlock_irq(&qp->io_buf_list_get_lock); + } + + /* + * Take IO buffers off blist and put on cbuf sorted by XRI. + * This is because POST_SGL takes a sequential range of XRIs + * to post to the firmware. + */ + for (idx = 0; idx < cnt; idx++) { + list_remove_head(&blist, lpfc_cmd, struct lpfc_scsi_buf, list); + if (!lpfc_cmd) + return cnt; + if (idx == 0) { + list_add_tail(&lpfc_cmd->list, cbuf); + continue; + } + xri = lpfc_cmd->cur_iocbq.sli4_xritag; + inserted = 0; + prev_iobufp = NULL; + list_for_each_entry(iobufp, cbuf, list) { + if (xri < iobufp->cur_iocbq.sli4_xritag) { + if (prev_iobufp) + list_add(&lpfc_cmd->list, + &prev_iobufp->list); + else + list_add(&lpfc_cmd->list, cbuf); + inserted = 1; + break; + } + prev_iobufp = iobufp; + } + if (!inserted) + list_add_tail(&lpfc_cmd->list, cbuf); + } + return cnt; +} + +int +lpfc_io_buf_replenish(struct lpfc_hba *phba, struct list_head *cbuf) +{ + struct lpfc_sli4_hdw_queue *qp; + struct lpfc_scsi_buf *lpfc_cmd; + int idx, cnt; + + qp = phba->sli4_hba.hdwq; + cnt = 0; + while (!list_empty(cbuf)) { + for (idx = 0; idx < phba->cfg_hdw_queue; idx++) { + list_remove_head(cbuf, lpfc_cmd, + struct lpfc_scsi_buf, list); + if (!lpfc_cmd) + return cnt; + cnt++; + qp = &phba->sli4_hba.hdwq[idx]; + lpfc_cmd->hdwq = idx; + lpfc_cmd->cur_iocbq.wqe_cmpl = NULL; + lpfc_cmd->cur_iocbq.iocb_cmpl = NULL; + spin_lock(&qp->io_buf_list_put_lock); + list_add_tail(&lpfc_cmd->list, + &qp->lpfc_io_buf_list_put); + qp->put_io_bufs++; + qp->total_io_bufs++; + spin_unlock(&qp->io_buf_list_put_lock); + } + } + return cnt; +} + /** - * lpfc_sli4_common_sgl_update - update xri-sgl sizing and mapping + * lpfc_sli4_io_sgl_update - update xri-sgl sizing and mapping * @phba: pointer to lpfc hba data structure. * * This routine first calculates the sizes of the current els and allocated @@ -3667,52 +3764,38 @@ out_free_mem: * 0 - successful (for now, it always returns 0) **/ int -lpfc_sli4_common_sgl_update(struct lpfc_hba *phba) +lpfc_sli4_io_sgl_update(struct lpfc_hba *phba) { struct lpfc_nvme_buf *lpfc_ncmd = NULL, *lpfc_ncmd_next = NULL; uint16_t i, lxri, els_xri_cnt; - uint16_t common_xri_cnt, common_xri_max; - LIST_HEAD(common_sgl_list); + uint16_t io_xri_cnt, io_xri_max; + LIST_HEAD(io_sgl_list); int rc, cnt; - phba->total_common_bufs = 0; - phba->get_common_bufs = 0; - phba->put_common_bufs = 0; - /* * update on pci function's allocated nvme xri-sgl list */ /* maximum number of xris available for nvme buffers */ els_xri_cnt = lpfc_sli4_get_els_iocb_cnt(phba); - common_xri_max = phba->sli4_hba.max_cfg_param.max_xri - els_xri_cnt; - phba->sli4_hba.common_xri_max = common_xri_max; + io_xri_max = phba->sli4_hba.max_cfg_param.max_xri - els_xri_cnt; + phba->sli4_hba.io_xri_max = io_xri_max; lpfc_printf_log(phba, KERN_INFO, LOG_SLI, "6074 Current allocated XRI sgl count:%d, " "maximum XRI count:%d\n", - phba->sli4_hba.common_xri_cnt, - phba->sli4_hba.common_xri_max); - - spin_lock_irq(&phba->common_buf_list_get_lock); - spin_lock(&phba->common_buf_list_put_lock); - list_splice_init(&phba->lpfc_common_buf_list_get, &common_sgl_list); - list_splice(&phba->lpfc_common_buf_list_put, &common_sgl_list); - cnt = phba->get_common_bufs + phba->put_common_bufs; - phba->get_common_bufs = 0; - phba->put_common_bufs = 0; - spin_unlock(&phba->common_buf_list_put_lock); - spin_unlock_irq(&phba->common_buf_list_get_lock); - - if (phba->sli4_hba.common_xri_cnt > phba->sli4_hba.common_xri_max) { + phba->sli4_hba.io_xri_cnt, + phba->sli4_hba.io_xri_max); + + cnt = lpfc_io_buf_flush(phba, &io_sgl_list); + + if (phba->sli4_hba.io_xri_cnt > phba->sli4_hba.io_xri_max) { /* max nvme xri shrunk below the allocated nvme buffers */ - spin_lock_irq(&phba->common_buf_list_get_lock); - common_xri_cnt = phba->sli4_hba.common_xri_cnt - - phba->sli4_hba.common_xri_max; - spin_unlock_irq(&phba->common_buf_list_get_lock); + io_xri_cnt = phba->sli4_hba.io_xri_cnt - + phba->sli4_hba.io_xri_max; /* release the extra allocated nvme buffers */ - for (i = 0; i < common_xri_cnt; i++) { - list_remove_head(&common_sgl_list, lpfc_ncmd, + for (i = 0; i < io_xri_cnt; i++) { + list_remove_head(&io_sgl_list, lpfc_ncmd, struct lpfc_nvme_buf, list); if (lpfc_ncmd) { dma_pool_free(phba->lpfc_sg_dma_buf_pool, @@ -3721,16 +3804,15 @@ lpfc_sli4_common_sgl_update(struct lpfc_hba *phba) kfree(lpfc_ncmd); } } - spin_lock_irq(&phba->common_buf_list_get_lock); - phba->sli4_hba.common_xri_cnt -= common_xri_cnt; - spin_unlock_irq(&phba->common_buf_list_get_lock); + phba->sli4_hba.io_xri_cnt -= io_xri_cnt; } /* update xris associated to remaining allocated nvme buffers */ lpfc_ncmd = NULL; lpfc_ncmd_next = NULL; + phba->sli4_hba.io_xri_cnt = cnt; list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next, - &common_sgl_list, list) { + &io_sgl_list, list) { lxri = lpfc_sli4_next_xritag(phba); if (lxri == NO_XRI) { lpfc_printf_log(phba, KERN_ERR, LOG_SLI, @@ -3742,22 +3824,16 @@ lpfc_sli4_common_sgl_update(struct lpfc_hba *phba) lpfc_ncmd->cur_iocbq.sli4_lxritag = lxri; lpfc_ncmd->cur_iocbq.sli4_xritag = phba->sli4_hba.xri_ids[lxri]; } - spin_lock_irq(&phba->common_buf_list_get_lock); - spin_lock(&phba->common_buf_list_put_lock); - list_splice_init(&common_sgl_list, &phba->lpfc_common_buf_list_get); - phba->get_common_bufs = cnt; - INIT_LIST_HEAD(&phba->lpfc_common_buf_list_put); - spin_unlock(&phba->common_buf_list_put_lock); - spin_unlock_irq(&phba->common_buf_list_get_lock); + cnt = lpfc_io_buf_replenish(phba, &io_sgl_list); return 0; out_free_mem: - lpfc_common_free(phba); + lpfc_io_free(phba); return rc; } /** - * lpfc_new_common_buf - IO buffer allocator for HBA with SLI4 IF spec + * lpfc_new_io_buf - IO buffer allocator for HBA with SLI4 IF spec * @vport: The virtual port for which this call being executed. * @num_to_allocate: The requested number of buffers to allocate. * @@ -3767,11 +3843,11 @@ out_free_mem: * them on a list, it post them to the port by using SGL block post. * * Return codes: - * int - number of nvme buffers that were allocated and posted. + * int - number of IO buffers that were allocated and posted. * 0 = failure, less than num_to_alloc is a partial failure. **/ int -lpfc_new_common_buf(struct lpfc_hba *phba, int num_to_alloc) +lpfc_new_io_buf(struct lpfc_hba *phba, int num_to_alloc) { struct lpfc_nvme_buf *lpfc_ncmd; struct lpfc_iocbq *pwqeq; @@ -3791,6 +3867,7 @@ lpfc_new_common_buf(struct lpfc_hba *phba, int num_to_alloc) return 0; } + phba->sli4_hba.io_xri_cnt = 0; for (bcnt = 0; bcnt < num_to_alloc; bcnt++) { lpfc_ncmd = kzalloc(LPFC_COMMON_IO_BUF_SZ, GFP_KERNEL); if (!lpfc_ncmd) @@ -3857,9 +3934,7 @@ lpfc_new_common_buf(struct lpfc_hba *phba, int num_to_alloc) /* add the nvme buffer to a post list */ list_add_tail(&lpfc_ncmd->list, &post_nblist); - spin_lock_irq(&phba->common_buf_list_get_lock); - phba->sli4_hba.common_xri_cnt++; - spin_unlock_irq(&phba->common_buf_list_get_lock); + phba->sli4_hba.io_xri_cnt++; } lpfc_printf_log(phba, KERN_INFO, LOG_NVME, "6114 Allocate %d out of %d requested new NVME " @@ -3867,7 +3942,7 @@ lpfc_new_common_buf(struct lpfc_hba *phba, int num_to_alloc) /* post the list of nvme buffer sgls to port if available */ if (!list_empty(&post_nblist)) - num_posted = lpfc_sli4_post_common_sgl_list( + num_posted = lpfc_sli4_post_io_sgl_list( phba, &post_nblist, bcnt); else num_posted = 0; @@ -5855,14 +5930,6 @@ lpfc_setup_driver_resource_phase1(struct lpfc_hba *phba) spin_lock_init(&phba->scsi_buf_list_put_lock); INIT_LIST_HEAD(&phba->lpfc_scsi_buf_list_put); - /* Initialize the IO buffer list used by driver for SLI4 SCSI/NVME */ - spin_lock_init(&phba->common_buf_list_get_lock); - INIT_LIST_HEAD(&phba->lpfc_common_buf_list_get); - phba->get_common_bufs = 0; - spin_lock_init(&phba->common_buf_list_put_lock); - INIT_LIST_HEAD(&phba->lpfc_common_buf_list_put); - phba->put_common_bufs = 0; - /* Initialize the fabric iocb list */ INIT_LIST_HEAD(&phba->fabric_iocb_list); @@ -6229,8 +6296,7 @@ lpfc_sli4_driver_resource_setup(struct lpfc_hba *phba) if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { /* Initialize the Abort nvme buffer list used by driver */ - spin_lock_init(&phba->sli4_hba.abts_nvme_buf_list_lock); - INIT_LIST_HEAD(&phba->sli4_hba.lpfc_abts_nvme_buf_list); + spin_lock_init(&phba->sli4_hba.abts_nvmet_buf_list_lock); INIT_LIST_HEAD(&phba->sli4_hba.lpfc_abts_nvmet_ctx_list); INIT_LIST_HEAD(&phba->sli4_hba.lpfc_nvmet_io_wait_list); } @@ -6903,7 +6969,7 @@ lpfc_init_sgl_list(struct lpfc_hba *phba) phba->sli4_hba.els_xri_cnt = 0; /* nvme xri-buffer book keeping */ - phba->sli4_hba.common_xri_cnt = 0; + phba->sli4_hba.io_xri_cnt = 0; } /** @@ -7117,6 +7183,9 @@ lpfc_hba_alloc(struct pci_dev *pdev) static void lpfc_hba_free(struct lpfc_hba *phba) { + if (phba->sli_rev == LPFC_SLI_REV4) + kfree(phba->sli4_hba.hdwq); + /* Release the driver assigned board number */ idr_remove(&lpfc_hba_index, phba->brd_no); @@ -8330,6 +8399,7 @@ lpfc_alloc_nvme_wq_cq(struct lpfc_hba *phba, int wqidx) return 1; } qdesc->qe_valid = 1; + qdesc->hdwq = wqidx; phba->sli4_hba.hdwq[wqidx].nvme_cq = qdesc; qdesc = lpfc_sli4_queue_alloc(phba, LPFC_EXPANDED_PAGE_SIZE, @@ -8340,6 +8410,7 @@ lpfc_alloc_nvme_wq_cq(struct lpfc_hba *phba, int wqidx) wqidx); return 1; } + qdesc->hdwq = wqidx; phba->sli4_hba.hdwq[wqidx].nvme_wq = qdesc; list_add_tail(&qdesc->wq_list, &phba->sli4_hba.lpfc_wq_list); return 0; @@ -8368,6 +8439,7 @@ lpfc_alloc_fcp_wq_cq(struct lpfc_hba *phba, int wqidx) return 1; } qdesc->qe_valid = 1; + qdesc->hdwq = wqidx; phba->sli4_hba.hdwq[wqidx].fcp_cq = qdesc; /* Create Fast Path FCP WQs */ @@ -8389,6 +8461,7 @@ lpfc_alloc_fcp_wq_cq(struct lpfc_hba *phba, int wqidx) wqidx); return 1; } + qdesc->hdwq = wqidx; phba->sli4_hba.hdwq[wqidx].fcp_wq = qdesc; list_add_tail(&qdesc->wq_list, &phba->sli4_hba.lpfc_wq_list); return 0; @@ -8413,6 +8486,7 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba) { struct lpfc_queue *qdesc; int idx; + struct lpfc_sli4_hdw_queue *qp; /* * Create HBA Record arrays. @@ -8429,14 +8503,33 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba) phba->sli4_hba.cq_esize = LPFC_CQE_SIZE; phba->sli4_hba.cq_ecount = LPFC_CQE_DEF_COUNT; - phba->sli4_hba.hdwq = kcalloc(phba->cfg_hdw_queue, - sizeof(struct lpfc_sli4_hdw_queue), - GFP_KERNEL); if (!phba->sli4_hba.hdwq) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, - "6427 Failed allocate memory for " - "fast-path Hardware Queue array\n"); - goto out_error; + phba->sli4_hba.hdwq = kcalloc( + phba->cfg_hdw_queue, sizeof(struct lpfc_sli4_hdw_queue), + GFP_KERNEL); + if (!phba->sli4_hba.hdwq) { + lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + "6427 Failed allocate memory for " + "fast-path Hardware Queue array\n"); + goto out_error; + } + /* Prepare hardware queues to take IO buffers */ + for (idx = 0; idx < phba->cfg_hdw_queue; idx++) { + qp = &phba->sli4_hba.hdwq[idx]; + spin_lock_init(&qp->io_buf_list_get_lock); + spin_lock_init(&qp->io_buf_list_put_lock); + INIT_LIST_HEAD(&qp->lpfc_io_buf_list_get); + INIT_LIST_HEAD(&qp->lpfc_io_buf_list_put); + qp->get_io_bufs = 0; + qp->put_io_bufs = 0; + qp->total_io_bufs = 0; + spin_lock_init(&qp->abts_scsi_buf_list_lock); + INIT_LIST_HEAD(&qp->lpfc_abts_scsi_buf_list); + qp->abts_scsi_io_bufs = 0; + spin_lock_init(&qp->abts_nvme_buf_list_lock); + INIT_LIST_HEAD(&qp->lpfc_abts_nvme_buf_list); + qp->abts_nvme_io_bufs = 0; + } } if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { @@ -8488,6 +8581,7 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba) goto out_error; } qdesc->qe_valid = 1; + qdesc->hdwq = idx; phba->sli4_hba.hdwq[idx].hba_eq = qdesc; } @@ -8518,6 +8612,7 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba) goto out_error; } qdesc->qe_valid = 1; + qdesc->hdwq = idx; phba->sli4_hba.nvmet_cqset[idx] = qdesc; } } @@ -8650,6 +8745,7 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba) "receive HRQ\n"); goto out_error; } + qdesc->hdwq = idx; phba->sli4_hba.nvmet_mrq_hdr[idx] = qdesc; /* Only needed for header of RQ pair */ @@ -8676,6 +8772,7 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba) "receive DRQ\n"); goto out_error; } + qdesc->hdwq = idx; phba->sli4_hba.nvmet_mrq_data[idx] = qdesc; } } @@ -8727,7 +8824,6 @@ lpfc_sli4_release_hdwq(struct lpfc_sli4_hdw_queue *hdwq, int max) hdwq[idx].fcp_wq = NULL; hdwq[idx].nvme_wq = NULL; } - kfree(hdwq); } /** @@ -8749,7 +8845,6 @@ lpfc_sli4_queue_destroy(struct lpfc_hba *phba) if (phba->sli4_hba.hdwq) lpfc_sli4_release_hdwq(phba->sli4_hba.hdwq, phba->cfg_hdw_queue); - phba->sli4_hba.hdwq = NULL; if (phba->nvmet_support) { lpfc_sli4_release_queues(&phba->sli4_hba.nvmet_cqset, @@ -10400,8 +10495,10 @@ lpfc_unset_hba(struct lpfc_hba *phba) static void lpfc_sli4_xri_exchange_busy_wait(struct lpfc_hba *phba) { + struct lpfc_sli4_hdw_queue *qp; + int idx, ccnt, fcnt; int wait_time = 0; - int common_xri_cmpl = 1; + int io_xri_cmpl = 1; int nvmet_xri_cmpl = 1; int fcp_xri_cmpl = 1; int els_xri_cmpl = list_empty(&phba->sli4_hba.lpfc_abts_els_sgl_list); @@ -10416,17 +10513,32 @@ lpfc_sli4_xri_exchange_busy_wait(struct lpfc_hba *phba) if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) lpfc_nvme_wait_for_io_drain(phba); - if (phba->cfg_enable_fc4_type & LPFC_ENABLE_FCP) - fcp_xri_cmpl = - list_empty(&phba->sli4_hba.lpfc_abts_scsi_buf_list); + ccnt = 0; + fcnt = 0; + for (idx = 0; idx < phba->cfg_hdw_queue; idx++) { + qp = &phba->sli4_hba.hdwq[idx]; + fcp_xri_cmpl = list_empty( + &qp->lpfc_abts_scsi_buf_list); + if (!fcp_xri_cmpl) /* if list is NOT empty */ + fcnt++; + if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { + io_xri_cmpl = list_empty( + &qp->lpfc_abts_nvme_buf_list); + if (!io_xri_cmpl) /* if list is NOT empty */ + ccnt++; + } + } + if (ccnt) + io_xri_cmpl = 0; + if (fcnt) + fcp_xri_cmpl = 0; + if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { - common_xri_cmpl = - list_empty(&phba->sli4_hba.lpfc_abts_nvme_buf_list); nvmet_xri_cmpl = list_empty(&phba->sli4_hba.lpfc_abts_nvmet_ctx_list); } - while (!fcp_xri_cmpl || !els_xri_cmpl || !common_xri_cmpl || + while (!fcp_xri_cmpl || !els_xri_cmpl || !io_xri_cmpl || !nvmet_xri_cmpl) { if (wait_time > LPFC_XRI_EXCH_BUSY_WAIT_TMO) { if (!nvmet_xri_cmpl) @@ -10434,7 +10546,7 @@ lpfc_sli4_xri_exchange_busy_wait(struct lpfc_hba *phba) "6424 NVMET XRI exchange busy " "wait time: %d seconds.\n", wait_time/1000); - if (!common_xri_cmpl) + if (!io_xri_cmpl) lpfc_printf_log(phba, KERN_ERR, LOG_INIT, "6100 NVME XRI exchange busy " "wait time: %d seconds.\n", @@ -10455,17 +10567,31 @@ lpfc_sli4_xri_exchange_busy_wait(struct lpfc_hba *phba) msleep(LPFC_XRI_EXCH_BUSY_WAIT_T1); wait_time += LPFC_XRI_EXCH_BUSY_WAIT_T1; } + + ccnt = 0; + fcnt = 0; + for (idx = 0; idx < phba->cfg_hdw_queue; idx++) { + qp = &phba->sli4_hba.hdwq[idx]; + fcp_xri_cmpl = list_empty( + &qp->lpfc_abts_scsi_buf_list); + if (!fcp_xri_cmpl) /* if list is NOT empty */ + fcnt++; + if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { + io_xri_cmpl = list_empty( + &qp->lpfc_abts_nvme_buf_list); + if (!io_xri_cmpl) /* if list is NOT empty */ + ccnt++; + } + } + if (ccnt) + io_xri_cmpl = 0; + if (fcnt) + fcp_xri_cmpl = 0; + if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { - common_xri_cmpl = list_empty( - &phba->sli4_hba.lpfc_abts_nvme_buf_list); nvmet_xri_cmpl = list_empty( &phba->sli4_hba.lpfc_abts_nvmet_ctx_list); } - - if (phba->cfg_enable_fc4_type & LPFC_ENABLE_FCP) - fcp_xri_cmpl = list_empty( - &phba->sli4_hba.lpfc_abts_scsi_buf_list); - els_xri_cmpl = list_empty(&phba->sli4_hba.lpfc_abts_els_sgl_list); @@ -11739,14 +11865,13 @@ lpfc_pci_probe_one_s4(struct pci_dev *pdev, const struct pci_device_id *pid) /* Don't post more new bufs if repost already recovered * the nvme sgls. */ - if (phba->sli4_hba.common_xri_cnt == 0) { - len = lpfc_new_common_buf( - phba, phba->sli4_hba.common_xri_max); + if (phba->sli4_hba.io_xri_cnt == 0) { + len = lpfc_new_io_buf( + phba, phba->sli4_hba.io_xri_max); if (len == 0) { error = -ENOMEM; goto out_free_sysfs_attr; } - phba->total_common_bufs += len; } } @@ -11836,7 +11961,6 @@ lpfc_pci_remove_one_s4(struct pci_dev *pdev) * the HBA FCoE function. */ lpfc_debugfs_terminate(vport); - lpfc_sli4_hba_unset(phba); lpfc_stop_hba_timers(phba); spin_lock_irq(&phba->port_list_lock); @@ -11846,8 +11970,9 @@ lpfc_pci_remove_one_s4(struct pci_dev *pdev) /* Perform scsi free before driver resource_unset since scsi * buffers are released to their corresponding pools here. */ - lpfc_common_free(phba); + lpfc_io_free(phba); lpfc_free_iocb_list(phba); + lpfc_sli4_hba_unset(phba); lpfc_unset_driver_resource_phase2(phba); lpfc_sli4_driver_resource_unset(phba); diff --git a/drivers/scsi/lpfc/lpfc_nvme.c b/drivers/scsi/lpfc/lpfc_nvme.c index c0f59fbaa173..0ecc73a6634f 100644 --- a/drivers/scsi/lpfc/lpfc_nvme.c +++ b/drivers/scsi/lpfc/lpfc_nvme.c @@ -58,7 +58,7 @@ static struct lpfc_nvme_buf * lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, - int expedite); + int idx, int expedite); static void lpfc_release_nvme_buf(struct lpfc_hba *, struct lpfc_nvme_buf *); @@ -1545,7 +1545,8 @@ lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport, } } - lpfc_ncmd = lpfc_get_nvme_buf(phba, ndlp, expedite); + lpfc_ncmd = lpfc_get_nvme_buf(phba, ndlp, + lpfc_queue_info->index, expedite); if (lpfc_ncmd == NULL) { atomic_inc(&lport->xmt_fcp_noxri); lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, @@ -1913,24 +1914,26 @@ static struct nvme_fc_port_template lpfc_nvme_template = { }; static inline struct lpfc_nvme_buf * -lpfc_nvme_buf(struct lpfc_hba *phba) +lpfc_nvme_buf(struct lpfc_hba *phba, int idx) { + struct lpfc_sli4_hdw_queue *qp; struct lpfc_nvme_buf *lpfc_ncmd, *lpfc_ncmd_next; + qp = &phba->sli4_hba.hdwq[idx]; list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next, - &phba->lpfc_common_buf_list_get, list) { + &qp->lpfc_io_buf_list_get, list) { list_del_init(&lpfc_ncmd->list); - phba->get_common_bufs--; + qp->get_io_bufs--; return lpfc_ncmd; } return NULL; } /** - * lpfc_get_nvme_buf - Get a nvme buffer from lpfc_common_buf_list of the HBA + * lpfc_get_nvme_buf - Get a nvme buffer from io_buf_list of the HBA * @phba: The HBA for which this call is being executed. * - * This routine removes a nvme buffer from head of @phba lpfc_common_buf_list + * This routine removes a nvme buffer from head of @hdwq io_buf_list * and returns to caller. * * Return codes: @@ -1939,30 +1942,32 @@ lpfc_nvme_buf(struct lpfc_hba *phba) **/ static struct lpfc_nvme_buf * lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, - int expedite) + int idx, int expedite) { struct lpfc_nvme_buf *lpfc_ncmd = NULL; + struct lpfc_sli4_hdw_queue *qp; struct sli4_sge *sgl; struct lpfc_iocbq *pwqeq; union lpfc_wqe128 *wqe; unsigned long iflag = 0; - spin_lock_irqsave(&phba->common_buf_list_get_lock, iflag); - if (phba->get_common_bufs > LPFC_NVME_EXPEDITE_XRICNT || expedite) - lpfc_ncmd = lpfc_nvme_buf(phba); + qp = &phba->sli4_hba.hdwq[idx]; + spin_lock_irqsave(&qp->io_buf_list_get_lock, iflag); + if (qp->get_io_bufs > LPFC_NVME_EXPEDITE_XRICNT || expedite) + lpfc_ncmd = lpfc_nvme_buf(phba, idx); if (!lpfc_ncmd) { - spin_lock(&phba->common_buf_list_put_lock); - list_splice(&phba->lpfc_common_buf_list_put, - &phba->lpfc_common_buf_list_get); - phba->get_common_bufs += phba->put_common_bufs; - INIT_LIST_HEAD(&phba->lpfc_common_buf_list_put); - phba->put_common_bufs = 0; - spin_unlock(&phba->common_buf_list_put_lock); - if (phba->get_common_bufs > LPFC_NVME_EXPEDITE_XRICNT || + spin_lock(&qp->io_buf_list_put_lock); + list_splice(&qp->lpfc_io_buf_list_put, + &qp->lpfc_io_buf_list_get); + qp->get_io_bufs += qp->put_io_bufs; + INIT_LIST_HEAD(&qp->lpfc_io_buf_list_put); + qp->put_io_bufs = 0; + spin_unlock(&qp->io_buf_list_put_lock); + if (qp->get_io_bufs > LPFC_NVME_EXPEDITE_XRICNT || expedite) - lpfc_ncmd = lpfc_nvme_buf(phba); + lpfc_ncmd = lpfc_nvme_buf(phba, idx); } - spin_unlock_irqrestore(&phba->common_buf_list_get_lock, iflag); + spin_unlock_irqrestore(&qp->io_buf_list_get_lock, iflag); if (lpfc_ncmd) { pwqeq = &(lpfc_ncmd->cur_iocbq); @@ -1975,6 +1980,7 @@ lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, pwqeq->wqe_cmpl = lpfc_nvme_io_cmd_wqe_cmpl; lpfc_ncmd->start_time = jiffies; lpfc_ncmd->flags = 0; + lpfc_ncmd->hdwq = idx; /* Rsp SGE will be filled in when we rcv an IO * from the NVME Layer to be sent. @@ -1993,7 +1999,10 @@ lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, atomic_inc(&ndlp->cmd_pending); lpfc_ncmd->flags |= LPFC_BUMP_QDEPTH; } - } + + } else + qp->empty_io_bufs++; + return lpfc_ncmd; } @@ -2003,13 +2012,14 @@ lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, * @lpfc_ncmd: The nvme buffer which is being released. * * This routine releases @lpfc_ncmd nvme buffer by adding it to tail of @phba - * lpfc_common_buf_list list. For SLI4 XRI's are tied to the nvme buffer + * lpfc_io_buf_list list. For SLI4 XRI's are tied to the nvme buffer * and cannot be reused for at least RA_TOV amount of time if it was * aborted. **/ static void lpfc_release_nvme_buf(struct lpfc_hba *phba, struct lpfc_nvme_buf *lpfc_ncmd) { + struct lpfc_sli4_hdw_queue *qp; unsigned long iflag = 0; if ((lpfc_ncmd->flags & LPFC_BUMP_QDEPTH) && lpfc_ncmd->ndlp) @@ -2018,6 +2028,7 @@ lpfc_release_nvme_buf(struct lpfc_hba *phba, struct lpfc_nvme_buf *lpfc_ncmd) lpfc_ncmd->ndlp = NULL; lpfc_ncmd->flags &= ~LPFC_BUMP_QDEPTH; + qp = &phba->sli4_hba.hdwq[lpfc_ncmd->hdwq]; if (lpfc_ncmd->flags & LPFC_SBUF_XBUSY) { lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, "6310 XB release deferred for " @@ -2025,21 +2036,21 @@ lpfc_release_nvme_buf(struct lpfc_hba *phba, struct lpfc_nvme_buf *lpfc_ncmd) lpfc_ncmd->cur_iocbq.sli4_xritag, lpfc_ncmd->cur_iocbq.iotag); - spin_lock_irqsave(&phba->sli4_hba.abts_nvme_buf_list_lock, - iflag); + spin_lock_irqsave(&qp->abts_nvme_buf_list_lock, iflag); list_add_tail(&lpfc_ncmd->list, - &phba->sli4_hba.lpfc_abts_nvme_buf_list); - spin_unlock_irqrestore(&phba->sli4_hba.abts_nvme_buf_list_lock, - iflag); + &qp->lpfc_abts_nvme_buf_list); + qp->abts_nvme_io_bufs++; + spin_unlock_irqrestore(&qp->abts_nvme_buf_list_lock, iflag); } else { /* MUST zero fields if buffer is reused by another protocol */ lpfc_ncmd->nvmeCmd = NULL; lpfc_ncmd->cur_iocbq.wqe_cmpl = NULL; - spin_lock_irqsave(&phba->common_buf_list_put_lock, iflag); + + spin_lock_irqsave(&qp->io_buf_list_put_lock, iflag); list_add_tail(&lpfc_ncmd->list, - &phba->lpfc_common_buf_list_put); - phba->put_common_bufs++; - spin_unlock_irqrestore(&phba->common_buf_list_put_lock, iflag); + &qp->lpfc_io_buf_list_put); + qp->put_io_bufs++; + spin_unlock_irqrestore(&qp->io_buf_list_put_lock, iflag); } } @@ -2517,27 +2528,28 @@ lpfc_nvme_unregister_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp) **/ void lpfc_sli4_nvme_xri_aborted(struct lpfc_hba *phba, - struct sli4_wcqe_xri_aborted *axri) + struct sli4_wcqe_xri_aborted *axri, int idx) { uint16_t xri = bf_get(lpfc_wcqe_xa_xri, axri); struct lpfc_nvme_buf *lpfc_ncmd, *next_lpfc_ncmd; struct nvmefc_fcp_req *nvme_cmd = NULL; struct lpfc_nodelist *ndlp; + struct lpfc_sli4_hdw_queue *qp; unsigned long iflag = 0; if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME)) return; + qp = &phba->sli4_hba.hdwq[idx]; spin_lock_irqsave(&phba->hbalock, iflag); - spin_lock(&phba->sli4_hba.abts_nvme_buf_list_lock); + spin_lock(&qp->abts_nvme_buf_list_lock); list_for_each_entry_safe(lpfc_ncmd, next_lpfc_ncmd, - &phba->sli4_hba.lpfc_abts_nvme_buf_list, - list) { + &qp->lpfc_abts_nvme_buf_list, list) { if (lpfc_ncmd->cur_iocbq.sli4_xritag == xri) { list_del_init(&lpfc_ncmd->list); + qp->abts_nvme_io_bufs--; lpfc_ncmd->flags &= ~LPFC_SBUF_XBUSY; lpfc_ncmd->status = IOSTAT_SUCCESS; - spin_unlock( - &phba->sli4_hba.abts_nvme_buf_list_lock); + spin_unlock(&qp->abts_nvme_buf_list_lock); spin_unlock_irqrestore(&phba->hbalock, iflag); ndlp = lpfc_ncmd->ndlp; @@ -2563,7 +2575,7 @@ lpfc_sli4_nvme_xri_aborted(struct lpfc_hba *phba, return; } } - spin_unlock(&phba->sli4_hba.abts_nvme_buf_list_lock); + spin_unlock(&qp->abts_nvme_buf_list_lock); spin_unlock_irqrestore(&phba->hbalock, iflag); lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, diff --git a/drivers/scsi/lpfc/lpfc_nvme.h b/drivers/scsi/lpfc/lpfc_nvme.h index e3a554417e98..4a020b9c8fbf 100644 --- a/drivers/scsi/lpfc/lpfc_nvme.h +++ b/drivers/scsi/lpfc/lpfc_nvme.h @@ -84,6 +84,8 @@ struct lpfc_nvme_buf { dma_addr_t dma_phys_sgl; struct sli4_sge *dma_sgl; struct lpfc_iocbq cur_iocbq; + uint16_t hdwq; + uint16_t cpu; /* NVME specific fields */ struct nvmefc_fcp_req *nvmeCmd; @@ -95,7 +97,6 @@ struct lpfc_nvme_buf { #define LPFC_SBUF_XBUSY 0x1 /* SLI4 hba reported XB on WCQE cmpl */ #define LPFC_BUMP_QDEPTH 0x2 /* bumped queue depth counter */ uint16_t exch_busy; /* SLI4 hba reported XB on complete WCQE */ - uint16_t cpu; uint16_t status; /* From IOCB Word 7- ulpStatus */ uint32_t result; /* From IOCB Word 4. */ diff --git a/drivers/scsi/lpfc/lpfc_nvmet.c b/drivers/scsi/lpfc/lpfc_nvmet.c index c64a8234d5bd..f2a30ee9702b 100644 --- a/drivers/scsi/lpfc/lpfc_nvmet.c +++ b/drivers/scsi/lpfc/lpfc_nvmet.c @@ -226,15 +226,15 @@ lpfc_nvmet_defer_release(struct lpfc_hba *phba, struct lpfc_nvmet_rcv_ctx *ctxp) "6313 NVMET Defer ctx release xri x%x flg x%x\n", ctxp->oxid, ctxp->flag); - spin_lock_irqsave(&phba->sli4_hba.abts_nvme_buf_list_lock, iflag); + spin_lock_irqsave(&phba->sli4_hba.abts_nvmet_buf_list_lock, iflag); if (ctxp->flag & LPFC_NVMET_CTX_RLS) { - spin_unlock_irqrestore(&phba->sli4_hba.abts_nvme_buf_list_lock, + spin_unlock_irqrestore(&phba->sli4_hba.abts_nvmet_buf_list_lock, iflag); return; } ctxp->flag |= LPFC_NVMET_CTX_RLS; list_add_tail(&ctxp->list, &phba->sli4_hba.lpfc_abts_nvmet_ctx_list); - spin_unlock_irqrestore(&phba->sli4_hba.abts_nvme_buf_list_lock, iflag); + spin_unlock_irqrestore(&phba->sli4_hba.abts_nvmet_buf_list_lock, iflag); } /** @@ -1162,9 +1162,9 @@ __lpfc_nvmet_clean_io_for_cpu(struct lpfc_hba *phba, spin_lock_irqsave(&infop->nvmet_ctx_list_lock, flags); list_for_each_entry_safe(ctx_buf, next_ctx_buf, &infop->nvmet_ctx_list, list) { - spin_lock(&phba->sli4_hba.abts_nvme_buf_list_lock); + spin_lock(&phba->sli4_hba.abts_nvmet_buf_list_lock); list_del_init(&ctx_buf->list); - spin_unlock(&phba->sli4_hba.abts_nvme_buf_list_lock); + spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock); __lpfc_clear_active_sglq(phba, ctx_buf->sglq->sli4_lxritag); ctx_buf->sglq->state = SGL_FREED; @@ -1502,7 +1502,7 @@ lpfc_sli4_nvmet_xri_aborted(struct lpfc_hba *phba, } spin_lock_irqsave(&phba->hbalock, iflag); - spin_lock(&phba->sli4_hba.abts_nvme_buf_list_lock); + spin_lock(&phba->sli4_hba.abts_nvmet_buf_list_lock); list_for_each_entry_safe(ctxp, next_ctxp, &phba->sli4_hba.lpfc_abts_nvmet_ctx_list, list) { @@ -1518,7 +1518,7 @@ lpfc_sli4_nvmet_xri_aborted(struct lpfc_hba *phba, released = true; } ctxp->flag &= ~LPFC_NVMET_XBUSY; - spin_unlock(&phba->sli4_hba.abts_nvme_buf_list_lock); + spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock); rrq_empty = list_empty(&phba->active_rrq_list); spin_unlock_irqrestore(&phba->hbalock, iflag); @@ -1542,7 +1542,7 @@ lpfc_sli4_nvmet_xri_aborted(struct lpfc_hba *phba, lpfc_worker_wake_up(phba); return; } - spin_unlock(&phba->sli4_hba.abts_nvme_buf_list_lock); + spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock); spin_unlock_irqrestore(&phba->hbalock, iflag); } @@ -1561,14 +1561,14 @@ lpfc_nvmet_rcv_unsol_abort(struct lpfc_vport *vport, xri = be16_to_cpu(fc_hdr->fh_ox_id); spin_lock_irqsave(&phba->hbalock, iflag); - spin_lock(&phba->sli4_hba.abts_nvme_buf_list_lock); + spin_lock(&phba->sli4_hba.abts_nvmet_buf_list_lock); list_for_each_entry_safe(ctxp, next_ctxp, &phba->sli4_hba.lpfc_abts_nvmet_ctx_list, list) { if (ctxp->ctxbuf->sglq->sli4_xritag != xri) continue; - spin_unlock(&phba->sli4_hba.abts_nvme_buf_list_lock); + spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock); spin_unlock_irqrestore(&phba->hbalock, iflag); spin_lock_irqsave(&ctxp->ctxlock, iflag); @@ -1589,7 +1589,7 @@ lpfc_nvmet_rcv_unsol_abort(struct lpfc_vport *vport, lpfc_sli4_seq_abort_rsp(vport, fc_hdr, 1); return 0; } - spin_unlock(&phba->sli4_hba.abts_nvme_buf_list_lock); + spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock); spin_unlock_irqrestore(&phba->hbalock, iflag); lpfc_nvmeio_data(phba, "NVMET ABTS RCV: xri x%x CPU %02x rjt %d\n", diff --git a/drivers/scsi/lpfc/lpfc_scsi.c b/drivers/scsi/lpfc/lpfc_scsi.c index 63841f49affb..261d0ebf0093 100644 --- a/drivers/scsi/lpfc/lpfc_scsi.c +++ b/drivers/scsi/lpfc/lpfc_scsi.c @@ -525,19 +525,26 @@ lpfc_sli4_vport_delete_fcp_xri_aborted(struct lpfc_vport *vport) { struct lpfc_hba *phba = vport->phba; struct lpfc_scsi_buf *psb, *next_psb; + struct lpfc_sli4_hdw_queue *qp; unsigned long iflag = 0; + int idx; if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_FCP)) return; + spin_lock_irqsave(&phba->hbalock, iflag); - spin_lock(&phba->sli4_hba.abts_scsi_buf_list_lock); - list_for_each_entry_safe(psb, next_psb, - &phba->sli4_hba.lpfc_abts_scsi_buf_list, list) { - if (psb->rdata && psb->rdata->pnode - && psb->rdata->pnode->vport == vport) - psb->rdata = NULL; + for (idx = 0; idx < phba->cfg_hdw_queue; idx++) { + qp = &phba->sli4_hba.hdwq[idx]; + + spin_lock(&qp->abts_scsi_buf_list_lock); + list_for_each_entry_safe(psb, next_psb, + &qp->lpfc_abts_scsi_buf_list, list) { + if (psb->rdata && psb->rdata->pnode && + psb->rdata->pnode->vport == vport) + psb->rdata = NULL; + } + spin_unlock(&qp->abts_scsi_buf_list_lock); } - spin_unlock(&phba->sli4_hba.abts_scsi_buf_list_lock); spin_unlock_irqrestore(&phba->hbalock, iflag); } @@ -551,11 +558,12 @@ lpfc_sli4_vport_delete_fcp_xri_aborted(struct lpfc_vport *vport) **/ void lpfc_sli4_fcp_xri_aborted(struct lpfc_hba *phba, - struct sli4_wcqe_xri_aborted *axri) + struct sli4_wcqe_xri_aborted *axri, int idx) { uint16_t xri = bf_get(lpfc_wcqe_xa_xri, axri); uint16_t rxid = bf_get(lpfc_wcqe_xa_remote_xid, axri); struct lpfc_scsi_buf *psb, *next_psb; + struct lpfc_sli4_hdw_queue *qp; unsigned long iflag = 0; struct lpfc_iocbq *iocbq; int i; @@ -565,16 +573,19 @@ lpfc_sli4_fcp_xri_aborted(struct lpfc_hba *phba, if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_FCP)) return; + + qp = &phba->sli4_hba.hdwq[idx]; spin_lock_irqsave(&phba->hbalock, iflag); - spin_lock(&phba->sli4_hba.abts_scsi_buf_list_lock); + spin_lock(&qp->abts_scsi_buf_list_lock); list_for_each_entry_safe(psb, next_psb, - &phba->sli4_hba.lpfc_abts_scsi_buf_list, list) { + &qp->lpfc_abts_scsi_buf_list, list) { if (psb->cur_iocbq.sli4_xritag == xri) { list_del(&psb->list); + qp->abts_scsi_io_bufs--; psb->exch_busy = 0; psb->status = IOSTAT_SUCCESS; spin_unlock( - &phba->sli4_hba.abts_scsi_buf_list_lock); + &qp->abts_scsi_buf_list_lock); if (psb->rdata && psb->rdata->pnode) ndlp = psb->rdata->pnode; else @@ -593,7 +604,7 @@ lpfc_sli4_fcp_xri_aborted(struct lpfc_hba *phba, return; } } - spin_unlock(&phba->sli4_hba.abts_scsi_buf_list_lock); + spin_unlock(&qp->abts_scsi_buf_list_lock); for (i = 1; i <= phba->sli.last_iotag; i++) { iocbq = phba->sli.iocbq_lookup[i]; @@ -652,10 +663,10 @@ lpfc_get_scsi_buf_s3(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp) return lpfc_cmd; } /** - * lpfc_get_scsi_buf_s4 - Get a scsi buffer from lpfc_common_buf_list of the HBA + * lpfc_get_scsi_buf_s4 - Get a scsi buffer from io_buf_list of the HBA * @phba: The HBA for which this call is being executed. * - * This routine removes a scsi buffer from head of @phba lpfc_common_buf_list + * This routine removes a scsi buffer from head of @hdwq io_buf_list * and returns to caller. * * Return codes: @@ -666,48 +677,58 @@ static struct lpfc_scsi_buf* lpfc_get_scsi_buf_s4(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp) { struct lpfc_scsi_buf *lpfc_cmd, *lpfc_cmd_next; + struct lpfc_sli4_hdw_queue *qp; unsigned long iflag = 0; struct sli4_sge *sgl; IOCB_t *iocb; dma_addr_t pdma_phys_fcp_rsp; dma_addr_t pdma_phys_fcp_cmd; - uint32_t sgl_size; + uint32_t sgl_size, cpu, idx; int found = 0; - spin_lock_irqsave(&phba->common_buf_list_get_lock, iflag); + cpu = smp_processor_id(); + if (cpu < phba->cfg_hdw_queue) + idx = cpu; + else + idx = cpu % phba->cfg_hdw_queue; + + qp = &phba->sli4_hba.hdwq[idx]; + spin_lock_irqsave(&qp->io_buf_list_get_lock, iflag); list_for_each_entry_safe(lpfc_cmd, lpfc_cmd_next, - &phba->lpfc_common_buf_list_get, list) { + &qp->lpfc_io_buf_list_get, list) { if (lpfc_test_rrq_active(phba, ndlp, lpfc_cmd->cur_iocbq.sli4_lxritag)) continue; list_del_init(&lpfc_cmd->list); - phba->get_common_bufs--; + qp->get_io_bufs--; found = 1; break; } if (!found) { - spin_lock(&phba->common_buf_list_put_lock); - list_splice(&phba->lpfc_common_buf_list_put, - &phba->lpfc_common_buf_list_get); - phba->get_common_bufs += phba->put_common_bufs; - INIT_LIST_HEAD(&phba->lpfc_common_buf_list_put); - phba->put_common_bufs = 0; - spin_unlock(&phba->common_buf_list_put_lock); + spin_lock(&qp->io_buf_list_put_lock); + list_splice(&qp->lpfc_io_buf_list_put, + &qp->lpfc_io_buf_list_get); + qp->get_io_bufs += qp->put_io_bufs; + INIT_LIST_HEAD(&qp->lpfc_io_buf_list_put); + qp->put_io_bufs = 0; + spin_unlock(&qp->io_buf_list_put_lock); list_for_each_entry_safe(lpfc_cmd, lpfc_cmd_next, - &phba->lpfc_common_buf_list_get, + &qp->lpfc_io_buf_list_get, list) { if (lpfc_test_rrq_active( phba, ndlp, lpfc_cmd->cur_iocbq.sli4_lxritag)) continue; list_del_init(&lpfc_cmd->list); - phba->get_common_bufs--; + qp->get_io_bufs--; found = 1; break; } } - spin_unlock_irqrestore(&phba->common_buf_list_get_lock, iflag); - if (!found) + spin_unlock_irqrestore(&qp->io_buf_list_get_lock, iflag); + if (!found) { + qp->empty_io_bufs++; return NULL; + } sgl_size = phba->cfg_sg_dma_buf_size - (sizeof(struct fcp_cmnd) + sizeof(struct fcp_rsp)); @@ -723,10 +744,11 @@ lpfc_get_scsi_buf_s4(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp) lpfc_cmd->flags = 0; lpfc_cmd->start_time = jiffies; lpfc_cmd->waitq = NULL; - lpfc_cmd->cpu = smp_processor_id(); + lpfc_cmd->cpu = cpu; #ifdef CONFIG_SCSI_LPFC_DEBUG_FS lpfc_cmd->prot_data_type = 0; #endif + lpfc_cmd->hdwq = idx; lpfc_cmd->fcp_cmnd = (lpfc_cmd->data + sgl_size); lpfc_cmd->fcp_rsp = (struct fcp_rsp *)((uint8_t *)lpfc_cmd->fcp_cmnd + @@ -825,35 +847,36 @@ lpfc_release_scsi_buf_s3(struct lpfc_hba *phba, struct lpfc_scsi_buf *psb) * @phba: The Hba for which this call is being executed. * @psb: The scsi buffer which is being released. * - * This routine releases @psb scsi buffer by adding it to tail of @phba - * lpfc_common_buf_list list. For SLI4 XRI's are tied to the scsi buffer + * This routine releases @psb scsi buffer by adding it to tail of @hdwq + * io_buf_list list. For SLI4 XRI's are tied to the scsi buffer * and cannot be reused for at least RA_TOV amount of time if it was * aborted. **/ static void lpfc_release_scsi_buf_s4(struct lpfc_hba *phba, struct lpfc_scsi_buf *psb) { + struct lpfc_sli4_hdw_queue *qp; unsigned long iflag = 0; psb->seg_cnt = 0; psb->prot_seg_cnt = 0; + qp = &phba->sli4_hba.hdwq[psb->hdwq]; if (psb->exch_busy) { - spin_lock_irqsave(&phba->sli4_hba.abts_scsi_buf_list_lock, - iflag); + spin_lock_irqsave(&qp->abts_scsi_buf_list_lock, iflag); psb->pCmd = NULL; - list_add_tail(&psb->list, - &phba->sli4_hba.lpfc_abts_scsi_buf_list); - spin_unlock_irqrestore(&phba->sli4_hba.abts_scsi_buf_list_lock, - iflag); + list_add_tail(&psb->list, &qp->lpfc_abts_scsi_buf_list); + qp->abts_scsi_io_bufs++; + spin_unlock_irqrestore(&qp->abts_scsi_buf_list_lock, iflag); } else { /* MUST zero fields if buffer is reused by another protocol */ psb->pCmd = NULL; psb->cur_iocbq.iocb_cmpl = NULL; - spin_lock_irqsave(&phba->common_buf_list_put_lock, iflag); - list_add_tail(&psb->list, &phba->lpfc_common_buf_list_put); - phba->put_common_bufs++; - spin_unlock_irqrestore(&phba->common_buf_list_put_lock, iflag); + + spin_lock_irqsave(&qp->io_buf_list_put_lock, iflag); + list_add_tail(&psb->list, &qp->lpfc_io_buf_list_put); + qp->put_io_bufs++; + spin_unlock_irqrestore(&qp->io_buf_list_put_lock, iflag); } } diff --git a/drivers/scsi/lpfc/lpfc_scsi.h b/drivers/scsi/lpfc/lpfc_scsi.h index d784de3da0c0..ebb5d5860cdb 100644 --- a/drivers/scsi/lpfc/lpfc_scsi.h +++ b/drivers/scsi/lpfc/lpfc_scsi.h @@ -138,6 +138,8 @@ struct lpfc_scsi_buf { dma_addr_t dma_phys_sgl; struct ulp_bde64 *dma_sgl; struct lpfc_iocbq cur_iocbq; + uint16_t hdwq; + uint16_t cpu; /* SCSI specific fields */ struct scsi_cmnd *pCmd; @@ -150,7 +152,6 @@ struct lpfc_scsi_buf { #define LPFC_SBUF_XBUSY 0x1 /* SLI4 hba reported XB on WCQE cmpl */ #define LPFC_SBUF_BUMP_QDEPTH 0x2 /* bumped queue depth counter */ uint16_t exch_busy; /* SLI4 hba reported XB on complete WCQE */ - uint16_t cpu; uint16_t status; /* From IOCB Word 7- ulpStatus */ uint32_t result; /* From IOCB Word 4. */ diff --git a/drivers/scsi/lpfc/lpfc_sli.c b/drivers/scsi/lpfc/lpfc_sli.c index d41bfa4569a4..ab1b9d9123b6 100644 --- a/drivers/scsi/lpfc/lpfc_sli.c +++ b/drivers/scsi/lpfc/lpfc_sli.c @@ -6023,7 +6023,7 @@ lpfc_sli4_alloc_extent(struct lpfc_hba *phba, uint16_t type) list_add_tail(&rsrc_blks->list, ext_blk_list); rsrc_start = rsrc_id; if ((type == LPFC_RSC_TYPE_FCOE_XRI) && (j == 0)) { - phba->sli4_hba.common_xri_start = rsrc_start + + phba->sli4_hba.io_xri_start = rsrc_start + lpfc_sli4_get_iocb_cnt(phba); } @@ -7051,37 +7051,30 @@ lpfc_sli4_repost_sgl_list(struct lpfc_hba *phba, } /** - * lpfc_sli4_repost_common_sgl_list - Repost all the allocated nvme buffer sgls + * lpfc_sli4_repost_io_sgl_list - Repost all the allocated nvme buffer sgls * @phba: pointer to lpfc hba data structure. * * This routine walks the list of nvme buffers that have been allocated and * repost them to the port by using SGL block post. This is needed after a * pci_function_reset/warm_start or start. The lpfc_hba_down_post_s4 routine * is responsible for moving all nvme buffers on the lpfc_abts_nvme_sgl_list - * to the lpfc_common_buf_list. If the repost fails, reject all nvme buffers. + * to the lpfc_io_buf_list. If the repost fails, reject all nvme buffers. * * Returns: 0 = success, non-zero failure. **/ int -lpfc_sli4_repost_common_sgl_list(struct lpfc_hba *phba) +lpfc_sli4_repost_io_sgl_list(struct lpfc_hba *phba) { LIST_HEAD(post_nblist); int num_posted, rc = 0; /* get all NVME buffers need to repost to a local list */ - spin_lock_irq(&phba->common_buf_list_get_lock); - spin_lock(&phba->common_buf_list_put_lock); - list_splice_init(&phba->lpfc_common_buf_list_get, &post_nblist); - list_splice(&phba->lpfc_common_buf_list_put, &post_nblist); - phba->get_common_bufs = 0; - phba->put_common_bufs = 0; - spin_unlock(&phba->common_buf_list_put_lock); - spin_unlock_irq(&phba->common_buf_list_get_lock); + lpfc_io_buf_flush(phba, &post_nblist); /* post the list of nvme buffer sgls to port if available */ if (!list_empty(&post_nblist)) { - num_posted = lpfc_sli4_post_common_sgl_list( - phba, &post_nblist, phba->sli4_hba.common_xri_cnt); + num_posted = lpfc_sli4_post_io_sgl_list( + phba, &post_nblist, phba->sli4_hba.io_xri_cnt); /* failed to post any nvme buffer, return error */ if (num_posted == 0) rc = -EIO; @@ -7551,7 +7544,7 @@ lpfc_sli4_hba_setup(struct lpfc_hba *phba) cnt += phba->sli4_hba.nvmet_xri_cnt; } else { /* update host common xri-sgl sizes and mappings */ - rc = lpfc_sli4_common_sgl_update(phba); + rc = lpfc_sli4_io_sgl_update(phba); if (unlikely(rc)) { lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI, "6082 Failed to update nvme-sgl size " @@ -7560,7 +7553,7 @@ lpfc_sli4_hba_setup(struct lpfc_hba *phba) } /* register the allocated common sgl pool to the port */ - rc = lpfc_sli4_repost_common_sgl_list(phba); + rc = lpfc_sli4_repost_io_sgl_list(phba); if (unlikely(rc)) { lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI, "6116 Error %d during nvme sgl post " @@ -8562,7 +8555,6 @@ lpfc_sli4_post_sync_mbox(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq) rc = lpfc_sli4_wait_bmbx_ready(phba, mboxq); if (rc) goto exit; - /* * Initialize the bootstrap memory region to avoid stale data areas * in the mailbox post. Then copy the caller's mailbox contents to @@ -10002,6 +9994,8 @@ lpfc_sli_api_table_setup(struct lpfc_hba *phba, uint8_t dev_grp) struct lpfc_sli_ring * lpfc_sli4_calc_ring(struct lpfc_hba *phba, struct lpfc_iocbq *piocb) { + struct lpfc_scsi_buf *lpfc_cmd; + if (piocb->iocb_flag & (LPFC_IO_FCP | LPFC_USE_FCPWQIDX)) { if (unlikely(!phba->sli4_hba.hdwq)) return NULL; @@ -10010,11 +10004,8 @@ lpfc_sli4_calc_ring(struct lpfc_hba *phba, struct lpfc_iocbq *piocb) * be setup based on what work queue we used. */ if (!(piocb->iocb_flag & LPFC_USE_FCPWQIDX)) { - piocb->hba_wqidx = - lpfc_sli4_scmd_to_wqidx_distr( - phba, piocb->context1); - piocb->hba_wqidx = piocb->hba_wqidx % - phba->cfg_hdw_queue; + lpfc_cmd = (struct lpfc_scsi_buf *)piocb->context1; + piocb->hba_wqidx = lpfc_cmd->hdwq; } return phba->sli4_hba.hdwq[piocb->hba_wqidx].fcp_wq->pring; } else { @@ -12924,7 +12915,8 @@ void lpfc_sli4_fcp_xri_abort_event_proc(struct lpfc_hba *phba) cq_event, struct lpfc_cq_event, list); spin_unlock_irq(&phba->hbalock); /* Notify aborted XRI for FCP work queue */ - lpfc_sli4_fcp_xri_aborted(phba, &cq_event->cqe.wcqe_axri); + lpfc_sli4_fcp_xri_aborted(phba, &cq_event->cqe.wcqe_axri, + cq_event->hdwq); /* Free the event processed back to the free pool */ lpfc_sli4_cq_event_release(phba, cq_event); } @@ -13426,17 +13418,8 @@ lpfc_sli4_sp_handle_abort_xri_wcqe(struct lpfc_hba *phba, switch (cq->subtype) { case LPFC_FCP: - cq_event = lpfc_cq_event_setup( - phba, wcqe, sizeof(struct sli4_wcqe_xri_aborted)); - if (!cq_event) - return false; - spin_lock_irqsave(&phba->hbalock, iflags); - list_add_tail(&cq_event->list, - &phba->sli4_hba.sp_fcp_xri_aborted_work_queue); - /* Set the fcp xri abort event flag */ - phba->hba_flag |= FCP_XRI_ABORT_EVENT; - spin_unlock_irqrestore(&phba->hbalock, iflags); - workposted = true; + lpfc_sli4_fcp_xri_aborted(phba, wcqe, cq->hdwq); + workposted = false; break; case LPFC_NVME_LS: /* NVME LS uses ELS resources */ case LPFC_ELS: @@ -13444,6 +13427,7 @@ lpfc_sli4_sp_handle_abort_xri_wcqe(struct lpfc_hba *phba, phba, wcqe, sizeof(struct sli4_wcqe_xri_aborted)); if (!cq_event) return false; + cq_event->hdwq = cq->hdwq; spin_lock_irqsave(&phba->hbalock, iflags); list_add_tail(&cq_event->list, &phba->sli4_hba.sp_els_xri_aborted_work_queue); @@ -13457,7 +13441,7 @@ lpfc_sli4_sp_handle_abort_xri_wcqe(struct lpfc_hba *phba, if (phba->nvmet_support) lpfc_sli4_nvmet_xri_aborted(phba, wcqe); else - lpfc_sli4_nvme_xri_aborted(phba, wcqe); + lpfc_sli4_nvme_xri_aborted(phba, wcqe, cq->hdwq); workposted = false; break; @@ -14073,7 +14057,8 @@ lpfc_sli4_hba_handle_eqe(struct lpfc_hba *phba, struct lpfc_eqe *eqe, cqid = bf_get_le32(lpfc_eqe_resource_id, eqe); /* First check for NVME/SCSI completion */ - if (cqid == phba->sli4_hba.hdwq[qidx].nvme_cq_map) { + if ((phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) && + (cqid == phba->sli4_hba.hdwq[qidx].nvme_cq_map)) { /* Process NVME / NVMET command completion */ cq = phba->sli4_hba.hdwq[qidx].nvme_cq; goto process_cq; @@ -16656,7 +16641,7 @@ lpfc_sli4_post_sgl_list(struct lpfc_hba *phba, } /** - * lpfc_sli4_post_common_sgl_block - post a block of nvme sgl list to firmware + * lpfc_sli4_post_io_sgl_block - post a block of nvme sgl list to firmware * @phba: pointer to lpfc hba data structure. * @nblist: pointer to nvme buffer list. * @count: number of scsi buffers on the list. @@ -16667,9 +16652,8 @@ lpfc_sli4_post_sgl_list(struct lpfc_hba *phba, * **/ static int -lpfc_sli4_post_common_sgl_block(struct lpfc_hba *phba, - struct list_head *nblist, - int count) +lpfc_sli4_post_io_sgl_block(struct lpfc_hba *phba, struct list_head *nblist, + int count) { struct lpfc_nvme_buf *lpfc_ncmd; struct lpfc_mbx_post_uembed_sgl_page1 *sgl; @@ -16770,7 +16754,7 @@ lpfc_sli4_post_common_sgl_block(struct lpfc_hba *phba, } /** - * lpfc_sli4_post_common_sgl_list - Post blocks of nvme buffer sgls from a list + * lpfc_sli4_post_io_sgl_list - Post blocks of nvme buffer sgls from a list * @phba: pointer to lpfc hba data structure. * @post_nblist: pointer to the nvme buffer list. * @@ -16784,8 +16768,8 @@ lpfc_sli4_post_common_sgl_block(struct lpfc_hba *phba, * Returns: 0 = failure, non-zero number of successfully posted buffers. **/ int -lpfc_sli4_post_common_sgl_list(struct lpfc_hba *phba, - struct list_head *post_nblist, int sb_count) +lpfc_sli4_post_io_sgl_list(struct lpfc_hba *phba, + struct list_head *post_nblist, int sb_count) { struct lpfc_nvme_buf *lpfc_ncmd, *lpfc_ncmd_next; int status, sgl_size; @@ -16793,7 +16777,6 @@ lpfc_sli4_post_common_sgl_list(struct lpfc_hba *phba, dma_addr_t pdma_phys_sgl1; int last_xritag = NO_XRI; int cur_xritag; - unsigned long iflag; LIST_HEAD(prep_nblist); LIST_HEAD(blck_nblist); LIST_HEAD(nvme_nblist); @@ -16864,8 +16847,8 @@ lpfc_sli4_post_common_sgl_list(struct lpfc_hba *phba, continue; /* post block of NVME buffer list sgls */ - status = lpfc_sli4_post_common_sgl_block(phba, &blck_nblist, - post_cnt); + status = lpfc_sli4_post_io_sgl_block(phba, &blck_nblist, + post_cnt); /* don't reset xirtag due to hole in xri block */ if (block_cnt == 0) @@ -16891,17 +16874,8 @@ lpfc_sli4_post_common_sgl_list(struct lpfc_hba *phba, } } /* Push NVME buffers with sgl posted to the available list */ - while (!list_empty(&nvme_nblist)) { - list_remove_head(&nvme_nblist, lpfc_ncmd, - struct lpfc_nvme_buf, list); - lpfc_ncmd->cur_iocbq.wqe_cmpl = NULL; - lpfc_ncmd->cur_iocbq.iocb_cmpl = NULL; - spin_lock_irqsave(&phba->common_buf_list_put_lock, iflag); - list_add_tail(&lpfc_ncmd->list, - &phba->lpfc_common_buf_list_put); - phba->put_common_bufs++; - spin_unlock_irqrestore(&phba->common_buf_list_put_lock, iflag); - } + lpfc_io_buf_replenish(phba, &nvme_nblist); + return num_posted; } diff --git a/drivers/scsi/lpfc/lpfc_sli.h b/drivers/scsi/lpfc/lpfc_sli.h index 7abb395bb64a..3922e762ed31 100644 --- a/drivers/scsi/lpfc/lpfc_sli.h +++ b/drivers/scsi/lpfc/lpfc_sli.h @@ -33,6 +33,7 @@ typedef enum _lpfc_ctx_cmd { struct lpfc_cq_event { struct list_head list; + uint16_t hdwq; union { struct lpfc_mcqe mcqe_cmpl; struct lpfc_acqe_link acqe_link; diff --git a/drivers/scsi/lpfc/lpfc_sli4.h b/drivers/scsi/lpfc/lpfc_sli4.h index 6e1e958df10a..4862249732dd 100644 --- a/drivers/scsi/lpfc/lpfc_sli4.h +++ b/drivers/scsi/lpfc/lpfc_sli4.h @@ -214,6 +214,7 @@ struct lpfc_queue { struct work_struct spwork; uint64_t isr_timestamp; + uint16_t hdwq; uint8_t qe_valid; struct lpfc_queue *assoc_qp; union sli4_qe qe[1]; /* array to index entries (must be last) */ @@ -538,6 +539,22 @@ struct lpfc_sli4_hdw_queue { struct lpfc_queue *nvme_wq; /* Fast-path NVME work queue */ uint16_t fcp_cq_map; uint16_t nvme_cq_map; + + /* Keep track of IO buffers for this hardware queue */ + spinlock_t io_buf_list_get_lock; /* Common buf alloc list lock */ + struct list_head lpfc_io_buf_list_get; + spinlock_t io_buf_list_put_lock; /* Common buf free list lock */ + struct list_head lpfc_io_buf_list_put; + spinlock_t abts_scsi_buf_list_lock; /* list of aborted SCSI IOs */ + struct list_head lpfc_abts_scsi_buf_list; + spinlock_t abts_nvme_buf_list_lock; /* list of aborted NVME IOs */ + struct list_head lpfc_abts_nvme_buf_list; + uint32_t total_io_bufs; + uint32_t get_io_bufs; + uint32_t put_io_bufs; + uint32_t empty_io_bufs; + uint32_t abts_scsi_io_bufs; + uint32_t abts_nvme_io_bufs; }; struct lpfc_sli4_hba { @@ -662,19 +679,20 @@ struct lpfc_sli4_hba { uint16_t rpi_hdrs_in_use; /* must post rpi hdrs if set. */ uint16_t next_xri; /* last_xri - max_cfg_param.xri_base = used */ uint16_t next_rpi; - uint16_t common_xri_max; - uint16_t common_xri_cnt; - uint16_t common_xri_start; + uint16_t io_xri_max; + uint16_t io_xri_cnt; + uint16_t io_xri_start; uint16_t els_xri_cnt; uint16_t nvmet_xri_cnt; uint16_t nvmet_io_wait_cnt; uint16_t nvmet_io_wait_total; struct list_head lpfc_els_sgl_list; struct list_head lpfc_abts_els_sgl_list; + spinlock_t abts_scsi_buf_list_lock; /* list of aborted SCSI IOs */ + struct list_head lpfc_abts_scsi_buf_list; struct list_head lpfc_nvmet_sgl_list; + spinlock_t abts_nvmet_buf_list_lock; /* list of aborted NVMET IOs */ struct list_head lpfc_abts_nvmet_ctx_list; - struct list_head lpfc_abts_scsi_buf_list; - struct list_head lpfc_abts_nvme_buf_list; struct list_head lpfc_nvmet_io_wait_list; struct lpfc_nvmet_ctx_info *nvmet_ctx_info; struct lpfc_sglq **lpfc_sglq_active_list; @@ -703,8 +721,6 @@ struct lpfc_sli4_hba { #define LPFC_SLI4_PPNAME_NON 0 #define LPFC_SLI4_PPNAME_GET 1 struct lpfc_iov iov; - spinlock_t abts_nvme_buf_list_lock; /* list of aborted SCSI IOs */ - spinlock_t abts_scsi_buf_list_lock; /* list of aborted SCSI IOs */ spinlock_t sgl_list_lock; /* list of aborted els IOs */ spinlock_t nvmet_io_wait_lock; /* IOs waiting for ctx resources */ uint32_t physical_port; @@ -839,7 +855,7 @@ int lpfc_rq_destroy(struct lpfc_hba *, struct lpfc_queue *, int lpfc_sli4_queue_setup(struct lpfc_hba *); void lpfc_sli4_queue_unset(struct lpfc_hba *); int lpfc_sli4_post_sgl(struct lpfc_hba *, dma_addr_t, dma_addr_t, uint16_t); -int lpfc_repost_common_sgl_list(struct lpfc_hba *phba); +int lpfc_repost_io_sgl_list(struct lpfc_hba *phba); uint16_t lpfc_sli4_next_xritag(struct lpfc_hba *); void lpfc_sli4_free_xri(struct lpfc_hba *, int); int lpfc_sli4_post_async_mbox(struct lpfc_hba *); @@ -862,9 +878,9 @@ int lpfc_sli4_resume_rpi(struct lpfc_nodelist *, void lpfc_sli4_fcp_xri_abort_event_proc(struct lpfc_hba *); void lpfc_sli4_els_xri_abort_event_proc(struct lpfc_hba *); void lpfc_sli4_fcp_xri_aborted(struct lpfc_hba *, - struct sli4_wcqe_xri_aborted *); + struct sli4_wcqe_xri_aborted *, int); void lpfc_sli4_nvme_xri_aborted(struct lpfc_hba *phba, - struct sli4_wcqe_xri_aborted *axri); + struct sli4_wcqe_xri_aborted *axri, int idx); void lpfc_sli4_nvmet_xri_aborted(struct lpfc_hba *phba, struct sli4_wcqe_xri_aborted *axri); void lpfc_sli4_els_xri_aborted(struct lpfc_hba *, -- cgit v1.2.3 From 18c27a621614716485b2e881d7e29599f063bb03 Mon Sep 17 00:00:00 2001 From: James Smart Date: Mon, 28 Jan 2019 11:14:23 -0800 Subject: scsi: lpfc: cleanup: Remove unused FCP_XRI_ABORT_EVENT slowpath event Both NVME and SCSI aborts are now processed off the CQ workqueue and do not generate events for the slowpath any more. Remove the unused event code. Signed-off-by: Dick Kennedy Signed-off-by: James Smart Reviewed-by: Hannes Reinecke Signed-off-by: Martin K. Petersen --- drivers/scsi/lpfc/lpfc.h | 1 - drivers/scsi/lpfc/lpfc_hbadisc.c | 2 -- drivers/scsi/lpfc/lpfc_sli.c | 30 ------------------------------ 3 files changed, 33 deletions(-) (limited to 'drivers/scsi/lpfc/lpfc.h') diff --git a/drivers/scsi/lpfc/lpfc.h b/drivers/scsi/lpfc/lpfc.h index 19827ce7a4d9..feae8fb57623 100644 --- a/drivers/scsi/lpfc/lpfc.h +++ b/drivers/scsi/lpfc/lpfc.h @@ -711,7 +711,6 @@ struct lpfc_hba { #define HBA_FCOE_MODE 0x4 /* HBA function in FCoE Mode */ #define HBA_SP_QUEUE_EVT 0x8 /* Slow-path qevt posted to worker thread*/ #define HBA_POST_RECEIVE_BUFFER 0x10 /* Rcv buffers need to be posted */ -#define FCP_XRI_ABORT_EVENT 0x20 #define ELS_XRI_ABORT_EVENT 0x40 #define ASYNC_EVENT 0x80 #define LINK_DISABLED 0x100 /* Link disabled by user */ diff --git a/drivers/scsi/lpfc/lpfc_hbadisc.c b/drivers/scsi/lpfc/lpfc_hbadisc.c index b183b882d506..62689a06c188 100644 --- a/drivers/scsi/lpfc/lpfc_hbadisc.c +++ b/drivers/scsi/lpfc/lpfc_hbadisc.c @@ -638,8 +638,6 @@ lpfc_work_done(struct lpfc_hba *phba) if (phba->pci_dev_grp == LPFC_PCI_DEV_OC) { if (phba->hba_flag & HBA_RRQ_ACTIVE) lpfc_handle_rrq_active(phba); - if (phba->hba_flag & FCP_XRI_ABORT_EVENT) - lpfc_sli4_fcp_xri_abort_event_proc(phba); if (phba->hba_flag & ELS_XRI_ABORT_EVENT) lpfc_sli4_els_xri_abort_event_proc(phba); if (phba->hba_flag & ASYNC_EVENT) diff --git a/drivers/scsi/lpfc/lpfc_sli.c b/drivers/scsi/lpfc/lpfc_sli.c index ab1b9d9123b6..7847ce2a9409 100644 --- a/drivers/scsi/lpfc/lpfc_sli.c +++ b/drivers/scsi/lpfc/lpfc_sli.c @@ -12892,36 +12892,6 @@ lpfc_sli_intr_handler(int irq, void *dev_id) return (sp_irq_rc == IRQ_HANDLED) ? sp_irq_rc : fp_irq_rc; } /* lpfc_sli_intr_handler */ -/** - * lpfc_sli4_fcp_xri_abort_event_proc - Process fcp xri abort event - * @phba: pointer to lpfc hba data structure. - * - * This routine is invoked by the worker thread to process all the pending - * SLI4 FCP abort XRI events. - **/ -void lpfc_sli4_fcp_xri_abort_event_proc(struct lpfc_hba *phba) -{ - struct lpfc_cq_event *cq_event; - - /* First, declare the fcp xri abort event has been handled */ - spin_lock_irq(&phba->hbalock); - phba->hba_flag &= ~FCP_XRI_ABORT_EVENT; - spin_unlock_irq(&phba->hbalock); - /* Now, handle all the fcp xri abort events */ - while (!list_empty(&phba->sli4_hba.sp_fcp_xri_aborted_work_queue)) { - /* Get the first event from the head of the event queue */ - spin_lock_irq(&phba->hbalock); - list_remove_head(&phba->sli4_hba.sp_fcp_xri_aborted_work_queue, - cq_event, struct lpfc_cq_event, list); - spin_unlock_irq(&phba->hbalock); - /* Notify aborted XRI for FCP work queue */ - lpfc_sli4_fcp_xri_aborted(phba, &cq_event->cqe.wcqe_axri, - cq_event->hdwq); - /* Free the event processed back to the free pool */ - lpfc_sli4_cq_event_release(phba, cq_event); - } -} - /** * lpfc_sli4_els_xri_abort_event_proc - Process els xri abort event * @phba: pointer to lpfc hba data structure. -- cgit v1.2.3 From 63df6d637e3358e64b43e7a774939f8f963926cb Mon Sep 17 00:00:00 2001 From: James Smart Date: Mon, 28 Jan 2019 11:14:24 -0800 Subject: scsi: lpfc: Adapt cpucheck debugfs logic to Hardware Queues Similar to the io execution path that reports cpu context information, the debugfs routines for cpu information needs to be aligned with new hardware queue implementation. Convert debugfs cnd nvme cpucheck statistics to report information per Hardware Queue. Signed-off-by: Dick Kennedy Signed-off-by: James Smart Reviewed-by: Hannes Reinecke Signed-off-by: Martin K. Petersen --- drivers/scsi/lpfc/lpfc.h | 5 -- drivers/scsi/lpfc/lpfc_debugfs.c | 129 +++++++++++++++++++++------------------ drivers/scsi/lpfc/lpfc_nvme.c | 37 +++++------ drivers/scsi/lpfc/lpfc_nvmet.c | 58 ++++++++---------- drivers/scsi/lpfc/lpfc_sli4.h | 11 ++++ 5 files changed, 124 insertions(+), 116 deletions(-) (limited to 'drivers/scsi/lpfc/lpfc.h') diff --git a/drivers/scsi/lpfc/lpfc.h b/drivers/scsi/lpfc/lpfc.h index feae8fb57623..310437b6b51a 100644 --- a/drivers/scsi/lpfc/lpfc.h +++ b/drivers/scsi/lpfc/lpfc.h @@ -1152,11 +1152,6 @@ struct lpfc_hba { uint16_t sfp_warning; #ifdef CONFIG_SCSI_LPFC_DEBUG_FS -#define LPFC_CHECK_CPU_CNT 32 - uint32_t cpucheck_rcv_io[LPFC_CHECK_CPU_CNT]; - uint32_t cpucheck_xmt_io[LPFC_CHECK_CPU_CNT]; - uint32_t cpucheck_cmpl_io[LPFC_CHECK_CPU_CNT]; - uint32_t cpucheck_ccmpl_io[LPFC_CHECK_CPU_CNT]; uint16_t cpucheck_on; #define LPFC_CHECK_OFF 0 #define LPFC_CHECK_NVME_IO 1 diff --git a/drivers/scsi/lpfc/lpfc_debugfs.c b/drivers/scsi/lpfc/lpfc_debugfs.c index a1b7263bfe2a..d84fb8c088b1 100644 --- a/drivers/scsi/lpfc/lpfc_debugfs.c +++ b/drivers/scsi/lpfc/lpfc_debugfs.c @@ -1366,62 +1366,67 @@ static int lpfc_debugfs_cpucheck_data(struct lpfc_vport *vport, char *buf, int size) { struct lpfc_hba *phba = vport->phba; - int i; + struct lpfc_sli4_hdw_queue *qp; + int i, j; int len = 0; - uint32_t tot_xmt = 0; - uint32_t tot_rcv = 0; - uint32_t tot_cmpl = 0; - uint32_t tot_ccmpl = 0; + uint32_t tot_xmt; + uint32_t tot_rcv; + uint32_t tot_cmpl; - if (phba->nvmet_support == 0) { - /* NVME Initiator */ - len += snprintf(buf + len, PAGE_SIZE - len, - "CPUcheck %s\n", - (phba->cpucheck_on & LPFC_CHECK_NVME_IO ? - "Enabled" : "Disabled")); - for (i = 0; i < phba->sli4_hba.num_present_cpu; i++) { - if (i >= LPFC_CHECK_CPU_CNT) - break; - len += snprintf(buf + len, PAGE_SIZE - len, - "%02d: xmit x%08x cmpl x%08x\n", - i, phba->cpucheck_xmt_io[i], - phba->cpucheck_cmpl_io[i]); - tot_xmt += phba->cpucheck_xmt_io[i]; - tot_cmpl += phba->cpucheck_cmpl_io[i]; - } + len += snprintf(buf + len, PAGE_SIZE - len, + "CPUcheck %s ", + (phba->cpucheck_on & LPFC_CHECK_NVME_IO ? + "Enabled" : "Disabled")); + if (phba->nvmet_support) { len += snprintf(buf + len, PAGE_SIZE - len, - "tot:xmit x%08x cmpl x%08x\n", - tot_xmt, tot_cmpl); - return len; + "%s\n", + (phba->cpucheck_on & LPFC_CHECK_NVMET_RCV ? + "Rcv Enabled\n" : "Rcv Disabled\n")); + } else { + len += snprintf(buf + len, PAGE_SIZE - len, "\n"); } - /* NVME Target */ - len += snprintf(buf + len, PAGE_SIZE - len, - "CPUcheck %s ", - (phba->cpucheck_on & LPFC_CHECK_NVMET_IO ? - "IO Enabled - " : "IO Disabled - ")); - len += snprintf(buf + len, PAGE_SIZE - len, - "%s\n", - (phba->cpucheck_on & LPFC_CHECK_NVMET_RCV ? - "Rcv Enabled\n" : "Rcv Disabled\n")); - for (i = 0; i < phba->sli4_hba.num_present_cpu; i++) { - if (i >= LPFC_CHECK_CPU_CNT) - break; + for (i = 0; i < phba->cfg_hdw_queue; i++) { + qp = &phba->sli4_hba.hdwq[i]; + + tot_rcv = 0; + tot_xmt = 0; + tot_cmpl = 0; + for (j = 0; j < LPFC_CHECK_CPU_CNT; j++) { + tot_xmt += qp->cpucheck_xmt_io[j]; + tot_cmpl += qp->cpucheck_cmpl_io[j]; + if (phba->nvmet_support) + tot_rcv += qp->cpucheck_rcv_io[j]; + } + + /* Only display Hardware Qs with something */ + if (!tot_xmt && !tot_cmpl && !tot_rcv) + continue; + + len += snprintf(buf + len, PAGE_SIZE - len, + "HDWQ %03d: ", i); + for (j = 0; j < LPFC_CHECK_CPU_CNT; j++) { + /* Only display non-zero counters */ + if (!qp->cpucheck_xmt_io[j] && + !qp->cpucheck_cmpl_io[j] && + !qp->cpucheck_rcv_io[j]) + continue; + if (phba->nvmet_support) { + len += snprintf(buf + len, PAGE_SIZE - len, + "CPU %03d: %x/%x/%x ", j, + qp->cpucheck_rcv_io[j], + qp->cpucheck_xmt_io[j], + qp->cpucheck_cmpl_io[j]); + } else { + len += snprintf(buf + len, PAGE_SIZE - len, + "CPU %03d: %x/%x ", j, + qp->cpucheck_xmt_io[j], + qp->cpucheck_cmpl_io[j]); + } + } len += snprintf(buf + len, PAGE_SIZE - len, - "%02d: xmit x%08x ccmpl x%08x " - "cmpl x%08x rcv x%08x\n", - i, phba->cpucheck_xmt_io[i], - phba->cpucheck_ccmpl_io[i], - phba->cpucheck_cmpl_io[i], - phba->cpucheck_rcv_io[i]); - tot_xmt += phba->cpucheck_xmt_io[i]; - tot_rcv += phba->cpucheck_rcv_io[i]; - tot_cmpl += phba->cpucheck_cmpl_io[i]; - tot_ccmpl += phba->cpucheck_ccmpl_io[i]; + "Total: %x\n", tot_xmt); } - len += snprintf(buf + len, PAGE_SIZE - len, - "tot:xmit x%08x ccmpl x%08x cmpl x%08x rcv x%08x\n", - tot_xmt, tot_ccmpl, tot_cmpl, tot_rcv); return len; } @@ -2474,9 +2479,10 @@ lpfc_debugfs_cpucheck_write(struct file *file, const char __user *buf, struct lpfc_debug *debug = file->private_data; struct lpfc_vport *vport = (struct lpfc_vport *)debug->i_private; struct lpfc_hba *phba = vport->phba; + struct lpfc_sli4_hdw_queue *qp; char mybuf[64]; char *pbuf; - int i; + int i, j; if (nbytes > 64) nbytes = 64; @@ -2506,13 +2512,14 @@ lpfc_debugfs_cpucheck_write(struct file *file, const char __user *buf, return strlen(pbuf); } else if ((strncmp(pbuf, "zero", sizeof("zero") - 1) == 0)) { - for (i = 0; i < phba->sli4_hba.num_present_cpu; i++) { - if (i >= LPFC_CHECK_CPU_CNT) - break; - phba->cpucheck_rcv_io[i] = 0; - phba->cpucheck_xmt_io[i] = 0; - phba->cpucheck_cmpl_io[i] = 0; - phba->cpucheck_ccmpl_io[i] = 0; + for (i = 0; i < phba->cfg_hdw_queue; i++) { + qp = &phba->sli4_hba.hdwq[i]; + + for (j = 0; j < LPFC_CHECK_CPU_CNT; j++) { + qp->cpucheck_rcv_io[j] = 0; + qp->cpucheck_xmt_io[j] = 0; + qp->cpucheck_cmpl_io[j] = 0; + } } return strlen(pbuf); } @@ -5358,9 +5365,9 @@ lpfc_debugfs_initialize(struct lpfc_vport *vport) /* Setup hbqinfo */ snprintf(name, sizeof(name), "hbqinfo"); phba->debug_hbqinfo = - debugfs_create_file(name, S_IFREG|S_IRUGO|S_IWUSR, - phba->hba_debugfs_root, - phba, &lpfc_debugfs_op_hbqinfo); + debugfs_create_file(name, S_IFREG | 0644, + phba->hba_debugfs_root, + phba, &lpfc_debugfs_op_hbqinfo); /* Setup hdwqinfo */ snprintf(name, sizeof(name), "hdwqinfo"); @@ -5370,7 +5377,7 @@ lpfc_debugfs_initialize(struct lpfc_vport *vport) phba, &lpfc_debugfs_op_hdwqinfo); if (!phba->debug_hdwqinfo) { lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT, - "0411 Cant create debugfs hdwqinfo\n"); + "0511 Cant create debugfs hdwqinfo\n"); goto debug_failed; } diff --git a/drivers/scsi/lpfc/lpfc_nvme.c b/drivers/scsi/lpfc/lpfc_nvme.c index 0ecc73a6634f..fe0190b48abd 100644 --- a/drivers/scsi/lpfc/lpfc_nvme.c +++ b/drivers/scsi/lpfc/lpfc_nvme.c @@ -965,7 +965,7 @@ lpfc_nvme_io_cmd_wqe_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn, struct lpfc_nvme_fcpreq_priv *freqpriv; struct lpfc_nvme_lport *lport; struct lpfc_nvme_ctrl_stat *cstat; - uint32_t code, status, idx; + uint32_t code, status, idx, cpu; uint16_t cid, sqhd, data; uint32_t *ptr; @@ -1136,13 +1136,17 @@ out_err: lpfc_nvme_ktime(phba, lpfc_ncmd); } if (phba->cpucheck_on & LPFC_CHECK_NVME_IO) { - if (lpfc_ncmd->cpu != smp_processor_id()) - lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_IOERR, - "6701 CPU Check cmpl: " - "cpu %d expect %d\n", - smp_processor_id(), lpfc_ncmd->cpu); - if (lpfc_ncmd->cpu < LPFC_CHECK_CPU_CNT) - phba->cpucheck_cmpl_io[lpfc_ncmd->cpu]++; + idx = lpfc_ncmd->cur_iocbq.hba_wqidx; + cpu = smp_processor_id(); + if (cpu < LPFC_CHECK_CPU_CNT) { + if (lpfc_ncmd->cpu != cpu) + lpfc_printf_vlog(vport, + KERN_INFO, LOG_NVME_IOERR, + "6701 CPU Check cmpl: " + "cpu %d expect %d\n", + cpu, lpfc_ncmd->cpu); + phba->sli4_hba.hdwq[idx].cpucheck_cmpl_io[cpu]++; + } } #endif @@ -1421,7 +1425,7 @@ lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport, { int ret = 0; int expedite = 0; - int idx; + int idx, cpu; struct lpfc_nvme_lport *lport; struct lpfc_nvme_ctrl_stat *cstat; struct lpfc_vport *vport; @@ -1620,21 +1624,18 @@ lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport, lpfc_ncmd->ts_cmd_wqput = ktime_get_ns(); if (phba->cpucheck_on & LPFC_CHECK_NVME_IO) { - lpfc_ncmd->cpu = smp_processor_id(); - if (lpfc_ncmd->cpu != lpfc_queue_info->index) { - /* Check for admin queue */ - if (lpfc_queue_info->qidx) { + cpu = smp_processor_id(); + if (cpu < LPFC_CHECK_CPU_CNT) { + lpfc_ncmd->cpu = cpu; + if (idx != cpu) lpfc_printf_vlog(vport, - KERN_ERR, LOG_NVME_IOERR, + KERN_INFO, LOG_NVME_IOERR, "6702 CPU Check cmd: " "cpu %d wq %d\n", lpfc_ncmd->cpu, lpfc_queue_info->index); - } - lpfc_ncmd->cpu = lpfc_queue_info->index; + phba->sli4_hba.hdwq[idx].cpucheck_xmt_io[cpu]++; } - if (lpfc_ncmd->cpu < LPFC_CHECK_CPU_CNT) - phba->cpucheck_xmt_io[lpfc_ncmd->cpu]++; } #endif return 0; diff --git a/drivers/scsi/lpfc/lpfc_nvmet.c b/drivers/scsi/lpfc/lpfc_nvmet.c index f2a30ee9702b..b5e287cacc2a 100644 --- a/drivers/scsi/lpfc/lpfc_nvmet.c +++ b/drivers/scsi/lpfc/lpfc_nvmet.c @@ -744,16 +744,6 @@ lpfc_nvmet_xmt_fcp_op_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, ktime_get_ns(); } } - if (phba->cpucheck_on & LPFC_CHECK_NVMET_IO) { - id = smp_processor_id(); - if (ctxp->cpu != id) - lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, - "6703 CPU Check cmpl: " - "cpu %d expect %d\n", - id, ctxp->cpu); - if (ctxp->cpu < LPFC_CHECK_CPU_CNT) - phba->cpucheck_cmpl_io[id]++; - } #endif rsp->done(rsp); #ifdef CONFIG_SCSI_LPFC_DEBUG_FS @@ -771,19 +761,22 @@ lpfc_nvmet_xmt_fcp_op_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, ctxp->ts_isr_data = cmdwqe->isr_timestamp; ctxp->ts_data_nvme = ktime_get_ns(); } - if (phba->cpucheck_on & LPFC_CHECK_NVMET_IO) { - id = smp_processor_id(); +#endif + rsp->done(rsp); + } +#ifdef CONFIG_SCSI_LPFC_DEBUG_FS + if (phba->cpucheck_on & LPFC_CHECK_NVMET_IO) { + id = smp_processor_id(); + if (id < LPFC_CHECK_CPU_CNT) { if (ctxp->cpu != id) - lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, + lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR, "6704 CPU Check cmdcmpl: " "cpu %d expect %d\n", id, ctxp->cpu); - if (ctxp->cpu < LPFC_CHECK_CPU_CNT) - phba->cpucheck_ccmpl_io[id]++; + phba->sli4_hba.hdwq[rsp->hwqid].cpucheck_cmpl_io[id]++; } -#endif - rsp->done(rsp); } +#endif } static int @@ -910,16 +903,15 @@ lpfc_nvmet_xmt_fcp_op(struct nvmet_fc_target_port *tgtport, } if (phba->cpucheck_on & LPFC_CHECK_NVMET_IO) { int id = smp_processor_id(); - ctxp->cpu = id; - if (id < LPFC_CHECK_CPU_CNT) - phba->cpucheck_xmt_io[id]++; - if (rsp->hwqid != id) { - lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, - "6705 CPU Check OP: " - "cpu %d expect %d\n", - id, rsp->hwqid); - ctxp->cpu = rsp->hwqid; + if (id < LPFC_CHECK_CPU_CNT) { + if (rsp->hwqid != id) + lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR, + "6705 CPU Check OP: " + "cpu %d expect %d\n", + id, rsp->hwqid); + phba->sli4_hba.hdwq[rsp->hwqid].cpucheck_xmt_io[id]++; } + ctxp->cpu = id; /* Setup cpu for cmpl check */ } #endif @@ -1897,9 +1889,6 @@ lpfc_nvmet_unsol_fcp_buffer(struct lpfc_hba *phba, uint32_t size, oxid, sid, rc, qno; unsigned long iflag; int current_cpu; -#ifdef CONFIG_SCSI_LPFC_DEBUG_FS - uint32_t id; -#endif if (!IS_ENABLED(CONFIG_NVME_TARGET_FC)) return; @@ -1940,9 +1929,14 @@ lpfc_nvmet_unsol_fcp_buffer(struct lpfc_hba *phba, #ifdef CONFIG_SCSI_LPFC_DEBUG_FS if (phba->cpucheck_on & LPFC_CHECK_NVMET_RCV) { - id = smp_processor_id(); - if (id < LPFC_CHECK_CPU_CNT) - phba->cpucheck_rcv_io[id]++; + if (current_cpu < LPFC_CHECK_CPU_CNT) { + if (idx != current_cpu) + lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR, + "6703 CPU Check rcv: " + "cpu %d expect %d\n", + current_cpu, idx); + phba->sli4_hba.hdwq[idx].cpucheck_rcv_io[current_cpu]++; + } } #endif diff --git a/drivers/scsi/lpfc/lpfc_sli4.h b/drivers/scsi/lpfc/lpfc_sli4.h index 4862249732dd..8e3e99d52f75 100644 --- a/drivers/scsi/lpfc/lpfc_sli4.h +++ b/drivers/scsi/lpfc/lpfc_sli4.h @@ -20,6 +20,10 @@ * included with this package. * *******************************************************************/ +#if defined(CONFIG_DEBUG_FS) && !defined(CONFIG_SCSI_LPFC_DEBUG_FS) +#define CONFIG_SCSI_LPFC_DEBUG_FS +#endif + #define LPFC_ACTIVE_MBOX_WAIT_CNT 100 #define LPFC_XRI_EXCH_BUSY_WAIT_TMO 10000 #define LPFC_XRI_EXCH_BUSY_WAIT_T1 10 @@ -555,6 +559,13 @@ struct lpfc_sli4_hdw_queue { uint32_t empty_io_bufs; uint32_t abts_scsi_io_bufs; uint32_t abts_nvme_io_bufs; + +#ifdef CONFIG_SCSI_LPFC_DEBUG_FS +#define LPFC_CHECK_CPU_CNT 128 + uint32_t cpucheck_rcv_io[LPFC_CHECK_CPU_CNT]; + uint32_t cpucheck_xmt_io[LPFC_CHECK_CPU_CNT]; + uint32_t cpucheck_cmpl_io[LPFC_CHECK_CPU_CNT]; +#endif }; struct lpfc_sli4_hba { -- cgit v1.2.3 From 4c47efc140fa926f00aa59c248458d95bd7b5eab Mon Sep 17 00:00:00 2001 From: James Smart Date: Mon, 28 Jan 2019 11:14:25 -0800 Subject: scsi: lpfc: Move SCSI and NVME Stats to hardware queue structures Many io statistics were being sampled and saved using adapter-based data structures. This was creating a lot of contention and cache thrashing in the I/O path. Move the statistics to the hardware queue data structures. Given the per-queue data structures, use of atomic types is lessened. Add new sysfs and debugfs stat routines to collate the per hardware queue values and report at an adapter level. Signed-off-by: Dick Kennedy Signed-off-by: James Smart Reviewed-by: Hannes Reinecke Signed-off-by: Martin K. Petersen --- drivers/scsi/lpfc/lpfc.h | 9 +-- drivers/scsi/lpfc/lpfc_attr.c | 68 ++++++++++++++--- drivers/scsi/lpfc/lpfc_debugfs.c | 158 +++++++++++++++++++++++++++++++++++++-- drivers/scsi/lpfc/lpfc_debugfs.h | 3 + drivers/scsi/lpfc/lpfc_init.c | 40 ++++++---- drivers/scsi/lpfc/lpfc_nvme.c | 57 +++++--------- drivers/scsi/lpfc/lpfc_nvme.h | 11 +-- drivers/scsi/lpfc/lpfc_scsi.c | 47 ++++++++---- drivers/scsi/lpfc/lpfc_scsi.h | 3 + drivers/scsi/lpfc/lpfc_sli4.h | 11 +++ 10 files changed, 304 insertions(+), 103 deletions(-) (limited to 'drivers/scsi/lpfc/lpfc.h') diff --git a/drivers/scsi/lpfc/lpfc.h b/drivers/scsi/lpfc/lpfc.h index 310437b6b51a..9262c52e32d6 100644 --- a/drivers/scsi/lpfc/lpfc.h +++ b/drivers/scsi/lpfc/lpfc.h @@ -479,6 +479,7 @@ struct lpfc_vport { struct dentry *debug_disc_trc; struct dentry *debug_nodelist; struct dentry *debug_nvmestat; + struct dentry *debug_scsistat; struct dentry *debug_nvmektime; struct dentry *debug_cpucheck; struct dentry *vport_debugfs_root; @@ -946,14 +947,6 @@ struct lpfc_hba { struct timer_list eratt_poll; uint32_t eratt_poll_interval; - /* - * stat counters - */ - atomic_t fc4ScsiInputRequests; - atomic_t fc4ScsiOutputRequests; - atomic_t fc4ScsiControlRequests; - atomic_t fc4ScsiIoCmpls; - uint64_t bg_guard_err_cnt; uint64_t bg_apptag_err_cnt; uint64_t bg_reftag_err_cnt; diff --git a/drivers/scsi/lpfc/lpfc_attr.c b/drivers/scsi/lpfc/lpfc_attr.c index 1671d9371d3b..e10d930fcb6a 100644 --- a/drivers/scsi/lpfc/lpfc_attr.c +++ b/drivers/scsi/lpfc/lpfc_attr.c @@ -64,9 +64,6 @@ #define LPFC_MIN_MRQ_POST 512 #define LPFC_MAX_MRQ_POST 2048 -#define LPFC_MAX_NVME_INFO_TMP_LEN 100 -#define LPFC_NVME_INFO_MORE_STR "\nCould be more info...\n" - /* * Write key size should be multiple of 4. If write key is changed * make sure that library write key is also changed. @@ -155,7 +152,7 @@ lpfc_nvme_info_show(struct device *dev, struct device_attribute *attr, struct lpfc_nvme_rport *rport; struct lpfc_nodelist *ndlp; struct nvme_fc_remote_port *nrport; - struct lpfc_nvme_ctrl_stat *cstat; + struct lpfc_fc4_ctrl_stat *cstat; uint64_t data1, data2, data3; uint64_t totin, totout, tot; char *statep; @@ -457,12 +454,12 @@ lpfc_nvme_info_show(struct device *dev, struct device_attribute *attr, totin = 0; totout = 0; for (i = 0; i < phba->cfg_hdw_queue; i++) { - cstat = &lport->cstat[i]; - tot = atomic_read(&cstat->fc4NvmeIoCmpls); + cstat = &phba->sli4_hba.hdwq[i].nvme_cstat; + tot = cstat->io_cmpls; totin += tot; - data1 = atomic_read(&cstat->fc4NvmeInputRequests); - data2 = atomic_read(&cstat->fc4NvmeOutputRequests); - data3 = atomic_read(&cstat->fc4NvmeControlRequests); + data1 = cstat->input_requests; + data2 = cstat->output_requests; + data3 = cstat->control_requests; totout += (data1 + data2 + data3); } scnprintf(tmp, sizeof(tmp), @@ -508,6 +505,57 @@ buffer_done: return len; } +static ssize_t +lpfc_scsi_stat_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct Scsi_Host *shost = class_to_shost(dev); + struct lpfc_vport *vport = shost_priv(shost); + struct lpfc_hba *phba = vport->phba; + int len; + struct lpfc_fc4_ctrl_stat *cstat; + u64 data1, data2, data3; + u64 tot, totin, totout; + int i; + char tmp[LPFC_MAX_SCSI_INFO_TMP_LEN] = {0}; + + if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_FCP) || + (phba->sli_rev != LPFC_SLI_REV4)) + return 0; + + scnprintf(buf, PAGE_SIZE, "SCSI HDWQ Statistics\n"); + + totin = 0; + totout = 0; + for (i = 0; i < phba->cfg_hdw_queue; i++) { + cstat = &phba->sli4_hba.hdwq[i].scsi_cstat; + tot = cstat->io_cmpls; + totin += tot; + data1 = cstat->input_requests; + data2 = cstat->output_requests; + data3 = cstat->control_requests; + totout += (data1 + data2 + data3); + + scnprintf(tmp, sizeof(tmp), "HDWQ (%d): Rd %016llx Wr %016llx " + "IO %016llx ", i, data1, data2, data3); + if (strlcat(buf, tmp, PAGE_SIZE) >= PAGE_SIZE) + goto buffer_done; + + scnprintf(tmp, sizeof(tmp), "Cmpl %016llx OutIO %016llx\n", + tot, ((data1 + data2 + data3) - tot)); + if (strlcat(buf, tmp, PAGE_SIZE) >= PAGE_SIZE) + goto buffer_done; + } + scnprintf(tmp, sizeof(tmp), "Total FCP Cmpl %016llx Issue %016llx " + "OutIO %016llx\n", totin, totout, totout - totin); + strlcat(buf, tmp, PAGE_SIZE); + +buffer_done: + len = strnlen(buf, PAGE_SIZE); + + return len; +} + static ssize_t lpfc_bg_info_show(struct device *dev, struct device_attribute *attr, char *buf) @@ -2573,6 +2621,7 @@ lpfc_##attr##_store(struct device *dev, struct device_attribute *attr, \ static DEVICE_ATTR(nvme_info, 0444, lpfc_nvme_info_show, NULL); +static DEVICE_ATTR(scsi_stat, 0444, lpfc_scsi_stat_show, NULL); static DEVICE_ATTR(bg_info, S_IRUGO, lpfc_bg_info_show, NULL); static DEVICE_ATTR(bg_guard_err, S_IRUGO, lpfc_bg_guard_err_show, NULL); static DEVICE_ATTR(bg_apptag_err, S_IRUGO, lpfc_bg_apptag_err_show, NULL); @@ -5642,6 +5691,7 @@ LPFC_ATTR_RW(enable_dpp, 1, 0, 1, "Enable Direct Packet Push"); struct device_attribute *lpfc_hba_attrs[] = { &dev_attr_nvme_info, + &dev_attr_scsi_stat, &dev_attr_bg_info, &dev_attr_bg_guard_err, &dev_attr_bg_apptag_err, diff --git a/drivers/scsi/lpfc/lpfc_debugfs.c b/drivers/scsi/lpfc/lpfc_debugfs.c index d84fb8c088b1..218642d7c488 100644 --- a/drivers/scsi/lpfc/lpfc_debugfs.c +++ b/drivers/scsi/lpfc/lpfc_debugfs.c @@ -840,7 +840,7 @@ lpfc_debugfs_nvmestat_data(struct lpfc_vport *vport, char *buf, int size) struct lpfc_nvmet_tgtport *tgtp; struct lpfc_nvmet_rcv_ctx *ctxp, *next_ctxp; struct nvme_fc_local_port *localport; - struct lpfc_nvme_ctrl_stat *cstat; + struct lpfc_fc4_ctrl_stat *cstat; struct lpfc_nvme_lport *lport; uint64_t data1, data2, data3; uint64_t tot, totin, totout; @@ -979,7 +979,7 @@ lpfc_debugfs_nvmestat_data(struct lpfc_vport *vport, char *buf, int size) return len; len += snprintf(buf + len, size - len, - "\nNVME Lport Statistics\n"); + "\nNVME HDWQ Statistics\n"); len += snprintf(buf + len, size - len, "LS: Xmt %016x Cmpl %016x\n", @@ -993,12 +993,12 @@ lpfc_debugfs_nvmestat_data(struct lpfc_vport *vport, char *buf, int size) totin = 0; totout = 0; for (i = 0; i < phba->cfg_hdw_queue; i++) { - cstat = &lport->cstat[i]; - tot = atomic_read(&cstat->fc4NvmeIoCmpls); + cstat = &phba->sli4_hba.hdwq[i].nvme_cstat; + tot = cstat->io_cmpls; totin += tot; - data1 = atomic_read(&cstat->fc4NvmeInputRequests); - data2 = atomic_read(&cstat->fc4NvmeOutputRequests); - data3 = atomic_read(&cstat->fc4NvmeControlRequests); + data1 = cstat->input_requests; + data2 = cstat->output_requests; + data3 = cstat->control_requests; totout += (data1 + data2 + data3); /* Limit to 32, debugfs display buffer limitation */ @@ -1006,7 +1006,7 @@ lpfc_debugfs_nvmestat_data(struct lpfc_vport *vport, char *buf, int size) continue; len += snprintf(buf + len, PAGE_SIZE - len, - "FCP (%d): Rd %016llx Wr %016llx " + "HDWQ (%d): Rd %016llx Wr %016llx " "IO %016llx ", i, data1, data2, data3); len += snprintf(buf + len, PAGE_SIZE - len, @@ -1046,6 +1046,66 @@ lpfc_debugfs_nvmestat_data(struct lpfc_vport *vport, char *buf, int size) return len; } +/** + * lpfc_debugfs_scsistat_data - Dump target node list to a buffer + * @vport: The vport to gather target node info from. + * @buf: The buffer to dump log into. + * @size: The maximum amount of data to process. + * + * Description: + * This routine dumps the SCSI statistics associated with @vport + * + * Return Value: + * This routine returns the amount of bytes that were dumped into @buf and will + * not exceed @size. + **/ +static int +lpfc_debugfs_scsistat_data(struct lpfc_vport *vport, char *buf, int size) +{ + int len; + struct lpfc_hba *phba = vport->phba; + struct lpfc_fc4_ctrl_stat *cstat; + u64 data1, data2, data3; + u64 tot, totin, totout; + int i; + char tmp[LPFC_MAX_SCSI_INFO_TMP_LEN] = {0}; + + if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_FCP) || + (phba->sli_rev != LPFC_SLI_REV4)) + return 0; + + scnprintf(buf, size, "SCSI HDWQ Statistics\n"); + + totin = 0; + totout = 0; + for (i = 0; i < phba->cfg_hdw_queue; i++) { + cstat = &phba->sli4_hba.hdwq[i].scsi_cstat; + tot = cstat->io_cmpls; + totin += tot; + data1 = cstat->input_requests; + data2 = cstat->output_requests; + data3 = cstat->control_requests; + totout += (data1 + data2 + data3); + + scnprintf(tmp, sizeof(tmp), "HDWQ (%d): Rd %016llx Wr %016llx " + "IO %016llx ", i, data1, data2, data3); + if (strlcat(buf, tmp, size) >= size) + goto buffer_done; + + scnprintf(tmp, sizeof(tmp), "Cmpl %016llx OutIO %016llx\n", + tot, ((data1 + data2 + data3) - tot)); + if (strlcat(buf, tmp, size) >= size) + goto buffer_done; + } + scnprintf(tmp, sizeof(tmp), "Total FCP Cmpl %016llx Issue %016llx " + "OutIO %016llx\n", totin, totout, totout - totin); + strlcat(buf, tmp, size); + +buffer_done: + len = strnlen(buf, size); + + return len; +} /** * lpfc_debugfs_nvmektime_data - Dump target node list to a buffer @@ -2211,6 +2271,64 @@ lpfc_debugfs_nvmestat_write(struct file *file, const char __user *buf, return nbytes; } +static int +lpfc_debugfs_scsistat_open(struct inode *inode, struct file *file) +{ + struct lpfc_vport *vport = inode->i_private; + struct lpfc_debug *debug; + int rc = -ENOMEM; + + debug = kmalloc(sizeof(*debug), GFP_KERNEL); + if (!debug) + goto out; + + /* Round to page boundary */ + debug->buffer = kzalloc(LPFC_SCSISTAT_SIZE, GFP_KERNEL); + if (!debug->buffer) { + kfree(debug); + goto out; + } + + debug->len = lpfc_debugfs_scsistat_data(vport, debug->buffer, + LPFC_SCSISTAT_SIZE); + + debug->i_private = inode->i_private; + file->private_data = debug; + + rc = 0; +out: + return rc; +} + +static ssize_t +lpfc_debugfs_scsistat_write(struct file *file, const char __user *buf, + size_t nbytes, loff_t *ppos) +{ + struct lpfc_debug *debug = file->private_data; + struct lpfc_vport *vport = (struct lpfc_vport *)debug->i_private; + struct lpfc_hba *phba = vport->phba; + char mybuf[6] = {0}; + int i; + + /* Protect copy from user */ + if (!access_ok(buf, nbytes)) + return -EFAULT; + + if (copy_from_user(mybuf, buf, (nbytes >= sizeof(mybuf)) ? + (sizeof(mybuf) - 1) : nbytes)) + return -EFAULT; + + if ((strncmp(&mybuf[0], "reset", strlen("reset")) == 0) || + (strncmp(&mybuf[0], "zero", strlen("zero")) == 0)) { + for (i = 0; i < phba->cfg_hdw_queue; i++) { + memset(&phba->sli4_hba.hdwq[i].scsi_cstat, 0, + sizeof(phba->sli4_hba.hdwq[i].scsi_cstat)); + } + } + + return nbytes; +} + static int lpfc_debugfs_nvmektime_open(struct inode *inode, struct file *file) { @@ -4972,6 +5090,16 @@ static const struct file_operations lpfc_debugfs_op_nvmestat = { .release = lpfc_debugfs_release, }; +#undef lpfc_debugfs_op_scsistat +static const struct file_operations lpfc_debugfs_op_scsistat = { + .owner = THIS_MODULE, + .open = lpfc_debugfs_scsistat_open, + .llseek = lpfc_debugfs_lseek, + .read = lpfc_debugfs_read, + .write = lpfc_debugfs_scsistat_write, + .release = lpfc_debugfs_release, +}; + #undef lpfc_debugfs_op_nvmektime static const struct file_operations lpfc_debugfs_op_nvmektime = { .owner = THIS_MODULE, @@ -5612,6 +5740,17 @@ nvmeio_off: vport->vport_debugfs_root, vport, &lpfc_debugfs_op_nvmestat); + snprintf(name, sizeof(name), "scsistat"); + vport->debug_scsistat = + debugfs_create_file(name, 0644, + vport->vport_debugfs_root, + vport, &lpfc_debugfs_op_scsistat); + if (!vport->debug_scsistat) { + lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT, + "0811 Cannot create debugfs scsistat\n"); + goto debug_failed; + } + snprintf(name, sizeof(name), "nvmektime"); vport->debug_nvmektime = debugfs_create_file(name, 0644, @@ -5750,6 +5889,9 @@ lpfc_debugfs_terminate(struct lpfc_vport *vport) debugfs_remove(vport->debug_nvmestat); /* nvmestat */ vport->debug_nvmestat = NULL; + debugfs_remove(vport->debug_scsistat); /* scsistat */ + vport->debug_scsistat = NULL; + debugfs_remove(vport->debug_nvmektime); /* nvmektime */ vport->debug_nvmektime = NULL; diff --git a/drivers/scsi/lpfc/lpfc_debugfs.h b/drivers/scsi/lpfc/lpfc_debugfs.h index c904fa74dfbc..2360ec8fb848 100644 --- a/drivers/scsi/lpfc/lpfc_debugfs.h +++ b/drivers/scsi/lpfc/lpfc_debugfs.h @@ -50,6 +50,9 @@ #define LPFC_CPUCHECK_SIZE 8192 #define LPFC_NVMEIO_TRC_SIZE 8192 +/* scsistat output buffer size */ +#define LPFC_SCSISTAT_SIZE 8192 + #define LPFC_DEBUG_OUT_LINE_SZ 80 /* diff --git a/drivers/scsi/lpfc/lpfc_init.c b/drivers/scsi/lpfc/lpfc_init.c index ed0318b9003e..a15c3aa569b5 100644 --- a/drivers/scsi/lpfc/lpfc_init.c +++ b/drivers/scsi/lpfc/lpfc_init.c @@ -1282,7 +1282,7 @@ lpfc_hb_timeout_handler(struct lpfc_hba *phba) struct lpfc_register reg_data; struct nvme_fc_local_port *localport; struct lpfc_nvme_lport *lport; - struct lpfc_nvme_ctrl_stat *cstat; + struct lpfc_fc4_ctrl_stat *cstat; void __iomem *eqdreg = phba->sli4_hba.u.if_type2.EQDregaddr; vports = lpfc_create_vport_work_array(phba); @@ -1324,16 +1324,13 @@ lpfc_hb_timeout_handler(struct lpfc_hba *phba) tot = 0; for (i = 0; i < phba->cfg_hdw_queue; i++) { - cstat = &lport->cstat[i]; - data1 = atomic_read( - &cstat->fc4NvmeInputRequests); - data2 = atomic_read( - &cstat->fc4NvmeOutputRequests); - data3 = atomic_read( - &cstat->fc4NvmeControlRequests); + cstat = + &phba->sli4_hba.hdwq[i].nvme_cstat; + data1 = cstat->input_requests; + data2 = cstat->output_requests; + data3 = cstat->control_requests; tot += (data1 + data2 + data3); - tot -= atomic_read( - &cstat->fc4NvmeIoCmpls); + tot -= cstat->io_cmpls; } } } @@ -7221,10 +7218,6 @@ lpfc_create_shost(struct lpfc_hba *phba) phba->fc_arbtov = FF_DEF_ARBTOV; atomic_set(&phba->sdev_cnt, 0); - atomic_set(&phba->fc4ScsiInputRequests, 0); - atomic_set(&phba->fc4ScsiOutputRequests, 0); - atomic_set(&phba->fc4ScsiControlRequests, 0); - atomic_set(&phba->fc4ScsiIoCmpls, 0); vport = lpfc_create_port(phba, phba->brd_no, &phba->pcidev->dev); if (!vport) return -ENODEV; @@ -8776,6 +8769,25 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba) phba->sli4_hba.nvmet_mrq_data[idx] = qdesc; } } + +#if defined(BUILD_NVME) + /* Clear NVME stats */ + if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { + for (idx = 0; idx < phba->cfg_hdw_queue; idx++) { + memset(&phba->sli4_hba.hdwq[idx].nvme_cstat, 0, + sizeof(phba->sli4_hba.hdwq[idx].nvme_cstat)); + } + } +#endif + + /* Clear SCSI stats */ + if (phba->cfg_enable_fc4_type & LPFC_ENABLE_FCP) { + for (idx = 0; idx < phba->cfg_hdw_queue; idx++) { + memset(&phba->sli4_hba.hdwq[idx].scsi_cstat, 0, + sizeof(phba->sli4_hba.hdwq[idx].scsi_cstat)); + } + } + return 0; out_error: diff --git a/drivers/scsi/lpfc/lpfc_nvme.c b/drivers/scsi/lpfc/lpfc_nvme.c index fe0190b48abd..c13638a3c0e7 100644 --- a/drivers/scsi/lpfc/lpfc_nvme.c +++ b/drivers/scsi/lpfc/lpfc_nvme.c @@ -964,7 +964,6 @@ lpfc_nvme_io_cmd_wqe_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn, struct lpfc_nodelist *ndlp; struct lpfc_nvme_fcpreq_priv *freqpriv; struct lpfc_nvme_lport *lport; - struct lpfc_nvme_ctrl_stat *cstat; uint32_t code, status, idx, cpu; uint16_t cid, sqhd, data; uint32_t *ptr; @@ -991,17 +990,15 @@ lpfc_nvme_io_cmd_wqe_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn, nCmd = lpfc_ncmd->nvmeCmd; status = bf_get(lpfc_wcqe_c_status, wcqe); + idx = lpfc_ncmd->cur_iocbq.hba_wqidx; + phba->sli4_hba.hdwq[idx].nvme_cstat.io_cmpls++; + if (vport->localport) { lport = (struct lpfc_nvme_lport *)vport->localport->private; - if (lport) { - idx = lpfc_ncmd->cur_iocbq.hba_wqidx; - cstat = &lport->cstat[idx]; - atomic_inc(&cstat->fc4NvmeIoCmpls); - if (status) { - if (bf_get(lpfc_wcqe_c_xb, wcqe)) - atomic_inc(&lport->cmpl_fcp_xb); - atomic_inc(&lport->cmpl_fcp_err); - } + if (lport && status) { + if (bf_get(lpfc_wcqe_c_xb, wcqe)) + atomic_inc(&lport->cmpl_fcp_xb); + atomic_inc(&lport->cmpl_fcp_err); } } @@ -1186,7 +1183,7 @@ static int lpfc_nvme_prep_io_cmd(struct lpfc_vport *vport, struct lpfc_nvme_buf *lpfc_ncmd, struct lpfc_nodelist *pnode, - struct lpfc_nvme_ctrl_stat *cstat) + struct lpfc_fc4_ctrl_stat *cstat) { struct lpfc_hba *phba = vport->phba; struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd; @@ -1224,7 +1221,7 @@ lpfc_nvme_prep_io_cmd(struct lpfc_vport *vport, } else { wqe->fcp_iwrite.initial_xfer_len = 0; } - atomic_inc(&cstat->fc4NvmeOutputRequests); + cstat->output_requests++; } else { /* From the iread template, initialize words 7 - 11 */ memcpy(&wqe->words[7], @@ -1237,13 +1234,13 @@ lpfc_nvme_prep_io_cmd(struct lpfc_vport *vport, /* Word 5 */ wqe->fcp_iread.rsrvd5 = 0; - atomic_inc(&cstat->fc4NvmeInputRequests); + cstat->input_requests++; } } else { /* From the icmnd template, initialize words 4 - 11 */ memcpy(&wqe->words[4], &lpfc_icmnd_cmd_template.words[4], sizeof(uint32_t) * 8); - atomic_inc(&cstat->fc4NvmeControlRequests); + cstat->control_requests++; } /* * Finish initializing those WQE fields that are independent @@ -1427,7 +1424,7 @@ lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport, int expedite = 0; int idx, cpu; struct lpfc_nvme_lport *lport; - struct lpfc_nvme_ctrl_stat *cstat; + struct lpfc_fc4_ctrl_stat *cstat; struct lpfc_vport *vport; struct lpfc_hba *phba; struct lpfc_nodelist *ndlp; @@ -1590,7 +1587,7 @@ lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport, */ idx = lpfc_queue_info->index; lpfc_ncmd->cur_iocbq.hba_wqidx = idx; - cstat = &lport->cstat[idx]; + cstat = &phba->sli4_hba.hdwq[idx].nvme_cstat; lpfc_nvme_prep_io_cmd(vport, lpfc_ncmd, ndlp, cstat); ret = lpfc_nvme_prep_io_dma(vport, lpfc_ncmd); @@ -1643,11 +1640,11 @@ lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport, out_free_nvme_buf: if (lpfc_ncmd->nvmeCmd->sg_cnt) { if (lpfc_ncmd->nvmeCmd->io_dir == NVMEFC_FCP_WRITE) - atomic_dec(&cstat->fc4NvmeOutputRequests); + cstat->output_requests--; else - atomic_dec(&cstat->fc4NvmeInputRequests); + cstat->input_requests--; } else - atomic_dec(&cstat->fc4NvmeControlRequests); + cstat->control_requests--; lpfc_release_nvme_buf(phba, lpfc_ncmd); out_fail: return ret; @@ -2079,8 +2076,6 @@ lpfc_nvme_create_localport(struct lpfc_vport *vport) struct nvme_fc_port_info nfcp_info; struct nvme_fc_local_port *localport; struct lpfc_nvme_lport *lport; - struct lpfc_nvme_ctrl_stat *cstat; - int i; /* Initialize this localport instance. The vport wwn usage ensures * that NPIV is accounted for. @@ -2097,11 +2092,6 @@ lpfc_nvme_create_localport(struct lpfc_vport *vport) lpfc_nvme_template.max_sgl_segments = phba->cfg_nvme_seg_cnt + 1; lpfc_nvme_template.max_hw_queues = phba->cfg_hdw_queue; - cstat = kmalloc((sizeof(struct lpfc_nvme_ctrl_stat) * - phba->cfg_hdw_queue), GFP_KERNEL); - if (!cstat) - return -ENOMEM; - /* localport is allocated from the stack, but the registration * call allocates heap memory as well as the private area. */ @@ -2124,7 +2114,6 @@ lpfc_nvme_create_localport(struct lpfc_vport *vport) lport = (struct lpfc_nvme_lport *)localport->private; vport->localport = localport; lport->vport = vport; - lport->cstat = cstat; vport->nvmei_support = 1; atomic_set(&lport->xmt_fcp_noxri, 0); @@ -2141,16 +2130,7 @@ lpfc_nvme_create_localport(struct lpfc_vport *vport) atomic_set(&lport->cmpl_ls_err, 0); atomic_set(&lport->fc4NvmeLsRequests, 0); atomic_set(&lport->fc4NvmeLsCmpls, 0); - - for (i = 0; i < phba->cfg_hdw_queue; i++) { - cstat = &lport->cstat[i]; - atomic_set(&cstat->fc4NvmeInputRequests, 0); - atomic_set(&cstat->fc4NvmeOutputRequests, 0); - atomic_set(&cstat->fc4NvmeControlRequests, 0); - atomic_set(&cstat->fc4NvmeIoCmpls, 0); - } - } else - kfree(cstat); + } return ret; } @@ -2212,7 +2192,6 @@ lpfc_nvme_destroy_localport(struct lpfc_vport *vport) #if (IS_ENABLED(CONFIG_NVME_FC)) struct nvme_fc_local_port *localport; struct lpfc_nvme_lport *lport; - struct lpfc_nvme_ctrl_stat *cstat; int ret; if (vport->nvmei_support == 0) @@ -2221,7 +2200,6 @@ lpfc_nvme_destroy_localport(struct lpfc_vport *vport) localport = vport->localport; vport->localport = NULL; lport = (struct lpfc_nvme_lport *)localport->private; - cstat = lport->cstat; lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME, "6011 Destroying NVME localport %p\n", @@ -2237,7 +2215,6 @@ lpfc_nvme_destroy_localport(struct lpfc_vport *vport) * indefinitely or succeeds */ lpfc_nvme_lport_unreg_wait(vport, lport); - kfree(cstat); /* Regardless of the unregister upcall response, clear * nvmei_support. All rports are unregistered and the diff --git a/drivers/scsi/lpfc/lpfc_nvme.h b/drivers/scsi/lpfc/lpfc_nvme.h index 4a020b9c8fbf..974fbec7dd04 100644 --- a/drivers/scsi/lpfc/lpfc_nvme.h +++ b/drivers/scsi/lpfc/lpfc_nvme.h @@ -30,6 +30,9 @@ #define LPFC_NVME_FB_SHIFT 9 #define LPFC_NVME_MAX_FB (1 << 20) /* 1M */ +#define LPFC_MAX_NVME_INFO_TMP_LEN 100 +#define LPFC_NVME_INFO_MORE_STR "\nCould be more info...\n" + #define lpfc_ndlp_get_nrport(ndlp) \ ((!ndlp->nrport || (ndlp->upcall_flags & NLP_WAIT_FOR_UNREG)) \ ? NULL : ndlp->nrport) @@ -40,19 +43,11 @@ struct lpfc_nvme_qhandle { uint32_t cpu_id; /* current cpu id at time of create */ }; -struct lpfc_nvme_ctrl_stat { - atomic_t fc4NvmeInputRequests; - atomic_t fc4NvmeOutputRequests; - atomic_t fc4NvmeControlRequests; - atomic_t fc4NvmeIoCmpls; -}; - /* Declare nvme-based local and remote port definitions. */ struct lpfc_nvme_lport { struct lpfc_vport *vport; struct completion lport_unreg_done; /* Add stats counters here */ - struct lpfc_nvme_ctrl_stat *cstat; atomic_t fc4NvmeLsRequests; atomic_t fc4NvmeLsCmpls; atomic_t xmt_fcp_noxri; diff --git a/drivers/scsi/lpfc/lpfc_scsi.c b/drivers/scsi/lpfc/lpfc_scsi.c index 261d0ebf0093..08f284ca5f1e 100644 --- a/drivers/scsi/lpfc/lpfc_scsi.c +++ b/drivers/scsi/lpfc/lpfc_scsi.c @@ -3722,14 +3722,18 @@ lpfc_scsi_cmd_iocb_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pIocbIn, unsigned long flags; struct lpfc_fast_path_event *fast_path_evt; struct Scsi_Host *shost; + int idx; uint32_t logit = LOG_FCP; - atomic_inc(&phba->fc4ScsiIoCmpls); - /* Sanity check on return of outstanding command */ cmd = lpfc_cmd->pCmd; if (!cmd) return; + + idx = lpfc_cmd->cur_iocbq.hba_wqidx; + if (phba->sli4_hba.hdwq) + phba->sli4_hba.hdwq[idx].scsi_cstat.io_cmpls++; + shost = cmd->device->host; lpfc_cmd->result = (pIocbOut->iocb.un.ulpWord[4] & IOERR_PARAM_MASK); @@ -3986,7 +3990,9 @@ lpfc_scsi_prep_cmnd(struct lpfc_vport *vport, struct lpfc_scsi_buf *lpfc_cmd, struct fcp_cmnd *fcp_cmnd = lpfc_cmd->fcp_cmnd; IOCB_t *iocb_cmd = &lpfc_cmd->cur_iocbq.iocb; struct lpfc_iocbq *piocbq = &(lpfc_cmd->cur_iocbq); + struct lpfc_sli4_hdw_queue *hdwq = NULL; int datadir = scsi_cmnd->sc_data_direction; + int idx; uint8_t *ptr; bool sli4; uint32_t fcpdl; @@ -4012,6 +4018,9 @@ lpfc_scsi_prep_cmnd(struct lpfc_vport *vport, struct lpfc_scsi_buf *lpfc_cmd, sli4 = (phba->sli_rev == LPFC_SLI_REV4); piocbq->iocb.un.fcpi.fcpi_XRdy = 0; + idx = lpfc_cmd->hdwq; + if (phba->sli4_hba.hdwq) + hdwq = &phba->sli4_hba.hdwq[idx]; /* * There are three possibilities here - use scatter-gather segment, use @@ -4033,19 +4042,22 @@ lpfc_scsi_prep_cmnd(struct lpfc_vport *vport, struct lpfc_scsi_buf *lpfc_cmd, vport->cfg_first_burst_size; } fcp_cmnd->fcpCntl3 = WRITE_DATA; - atomic_inc(&phba->fc4ScsiOutputRequests); + if (hdwq) + hdwq->scsi_cstat.output_requests++; } else { iocb_cmd->ulpCommand = CMD_FCP_IREAD64_CR; iocb_cmd->ulpPU = PARM_READ_CHECK; fcp_cmnd->fcpCntl3 = READ_DATA; - atomic_inc(&phba->fc4ScsiInputRequests); + if (hdwq) + hdwq->scsi_cstat.input_requests++; } } else { iocb_cmd->ulpCommand = CMD_FCP_ICMND64_CR; iocb_cmd->un.fcpi.fcpi_parm = 0; iocb_cmd->ulpPU = 0; fcp_cmnd->fcpCntl3 = 0; - atomic_inc(&phba->fc4ScsiControlRequests); + if (hdwq) + hdwq->scsi_cstat.control_requests++; } if (phba->sli_rev == 3 && !(phba->sli3_options & LPFC_SLI3_BG_ENABLED)) @@ -4397,7 +4409,7 @@ lpfc_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *cmnd) struct lpfc_nodelist *ndlp; struct lpfc_scsi_buf *lpfc_cmd; struct fc_rport *rport = starget_to_rport(scsi_target(cmnd->device)); - int err; + int err, idx; rdata = lpfc_rport_data_from_scsi_device(cmnd->device); @@ -4532,16 +4544,6 @@ lpfc_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *cmnd) (uint32_t) (cmnd->request->timeout / 1000)); - switch (lpfc_cmd->fcp_cmnd->fcpCntl3) { - case WRITE_DATA: - atomic_dec(&phba->fc4ScsiOutputRequests); - break; - case READ_DATA: - atomic_dec(&phba->fc4ScsiInputRequests); - break; - default: - atomic_dec(&phba->fc4ScsiControlRequests); - } goto out_host_busy_free_buf; } if (phba->cfg_poll & ENABLE_FCP_RING_POLLING) { @@ -4555,7 +4557,20 @@ lpfc_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *cmnd) return 0; out_host_busy_free_buf: + idx = lpfc_cmd->hdwq; lpfc_scsi_unprep_dma_buf(phba, lpfc_cmd); + if (phba->sli4_hba.hdwq) { + switch (lpfc_cmd->fcp_cmnd->fcpCntl3) { + case WRITE_DATA: + phba->sli4_hba.hdwq[idx].scsi_cstat.output_requests--; + break; + case READ_DATA: + phba->sli4_hba.hdwq[idx].scsi_cstat.input_requests--; + break; + default: + phba->sli4_hba.hdwq[idx].scsi_cstat.control_requests--; + } + } lpfc_release_scsi_buf(phba, lpfc_cmd); out_host_busy: return SCSI_MLQUEUE_HOST_BUSY; diff --git a/drivers/scsi/lpfc/lpfc_scsi.h b/drivers/scsi/lpfc/lpfc_scsi.h index ebb5d5860cdb..68e6c4ad3936 100644 --- a/drivers/scsi/lpfc/lpfc_scsi.h +++ b/drivers/scsi/lpfc/lpfc_scsi.h @@ -196,5 +196,8 @@ struct lpfc_scsi_buf { #define TXRDY_PAYLOAD_LEN 12 +/* For sysfs/debugfs tmp string max len */ +#define LPFC_MAX_SCSI_INFO_TMP_LEN 79 + int lpfc_sli4_scmd_to_wqidx_distr(struct lpfc_hba *phba, struct lpfc_scsi_buf *lpfc_cmd); diff --git a/drivers/scsi/lpfc/lpfc_sli4.h b/drivers/scsi/lpfc/lpfc_sli4.h index 8e3e99d52f75..a33fb6715329 100644 --- a/drivers/scsi/lpfc/lpfc_sli4.h +++ b/drivers/scsi/lpfc/lpfc_sli4.h @@ -533,6 +533,13 @@ struct lpfc_vector_map_info { }; #define LPFC_VECTOR_MAP_EMPTY 0xffff +struct lpfc_fc4_ctrl_stat { + u32 input_requests; + u32 output_requests; + u32 control_requests; + u32 io_cmpls; +}; + /* SLI4 HBA data structure entries */ struct lpfc_sli4_hdw_queue { /* Pointers to the constructed SLI4 queues */ @@ -560,6 +567,10 @@ struct lpfc_sli4_hdw_queue { uint32_t abts_scsi_io_bufs; uint32_t abts_nvme_io_bufs; + /* FC-4 Stats counters */ + struct lpfc_fc4_ctrl_stat nvme_cstat; + struct lpfc_fc4_ctrl_stat scsi_cstat; + #ifdef CONFIG_SCSI_LPFC_DEBUG_FS #define LPFC_CHECK_CPU_CNT 128 uint32_t cpucheck_rcv_io[LPFC_CHECK_CPU_CNT]; -- cgit v1.2.3 From ace44e48b1fc36055a8ff05a778110e726387032 Mon Sep 17 00:00:00 2001 From: James Smart Date: Mon, 28 Jan 2019 11:14:27 -0800 Subject: scsi: lpfc: Synchronize hardware queues with SCSI MQ interface Now that the lower half has much better per-cpu parallelization using the hardware queues, the SCSI MQ support needs to be tied into it. The involves the following mods: - Use the hardware queue info from the midlayer to help select the hardware queue to utilize. This required change to the get_scsi-buf_xxx routines. - Remove lpfc_sli4_scmd_to_wqidx_distr() routine. No longer needed. - Includes fix for SLI-3 that does not have multi queue parallelization. Signed-off-by: Dick Kennedy Signed-off-by: James Smart Reviewed-by: Hannes Reinecke Signed-off-by: Martin K. Petersen --- drivers/scsi/lpfc/lpfc.h | 3 +- drivers/scsi/lpfc/lpfc_init.c | 8 +++-- drivers/scsi/lpfc/lpfc_scsi.c | 72 ++++++++++++------------------------------- drivers/scsi/lpfc/lpfc_scsi.h | 2 -- 4 files changed, 27 insertions(+), 58 deletions(-) (limited to 'drivers/scsi/lpfc/lpfc.h') diff --git a/drivers/scsi/lpfc/lpfc.h b/drivers/scsi/lpfc/lpfc.h index 9262c52e32d6..755bf49c272c 100644 --- a/drivers/scsi/lpfc/lpfc.h +++ b/drivers/scsi/lpfc/lpfc.h @@ -619,7 +619,8 @@ struct lpfc_ras_fwlog { struct lpfc_hba { /* SCSI interface function jump table entries */ struct lpfc_scsi_buf * (*lpfc_get_scsi_buf) - (struct lpfc_hba *, struct lpfc_nodelist *); + (struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, + struct scsi_cmnd *cmnd); int (*lpfc_scsi_prep_dma_buf) (struct lpfc_hba *, struct lpfc_scsi_buf *); void (*lpfc_scsi_unprep_dma_buf) diff --git a/drivers/scsi/lpfc/lpfc_init.c b/drivers/scsi/lpfc/lpfc_init.c index 36d9c32c9c87..88b1c3ca26dc 100644 --- a/drivers/scsi/lpfc/lpfc_init.c +++ b/drivers/scsi/lpfc/lpfc_init.c @@ -4063,12 +4063,16 @@ lpfc_create_port(struct lpfc_hba *phba, int instance, struct device *dev) shost->max_lun = vport->cfg_max_luns; shost->this_id = -1; shost->max_cmd_len = 16; - shost->nr_hw_queues = phba->cfg_hdw_queue; if (phba->sli_rev == LPFC_SLI_REV4) { + shost->nr_hw_queues = phba->cfg_hdw_queue; shost->dma_boundary = phba->sli4_hba.pc_sli4_params.sge_supp_len-1; shost->sg_tablesize = phba->cfg_scsi_seg_cnt; - } + } else + /* SLI-3 has a limited number of hardware queues (3), + * thus there is only one for FCP processing. + */ + shost->nr_hw_queues = 1; /* * Set initial can_queue value since 0 is no longer supported and diff --git a/drivers/scsi/lpfc/lpfc_scsi.c b/drivers/scsi/lpfc/lpfc_scsi.c index 55c58bbfee08..79a3765bdd9b 100644 --- a/drivers/scsi/lpfc/lpfc_scsi.c +++ b/drivers/scsi/lpfc/lpfc_scsi.c @@ -636,7 +636,8 @@ lpfc_sli4_fcp_xri_aborted(struct lpfc_hba *phba, * Pointer to lpfc_scsi_buf - Success **/ static struct lpfc_scsi_buf* -lpfc_get_scsi_buf_s3(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp) +lpfc_get_scsi_buf_s3(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, + struct scsi_cmnd *cmnd) { struct lpfc_scsi_buf * lpfc_cmd = NULL; struct list_head *scsi_buf_list_get = &phba->lpfc_scsi_buf_list_get; @@ -674,7 +675,8 @@ lpfc_get_scsi_buf_s3(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp) * Pointer to lpfc_scsi_buf - Success **/ static struct lpfc_scsi_buf* -lpfc_get_scsi_buf_s4(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp) +lpfc_get_scsi_buf_s4(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, + struct scsi_cmnd *cmnd) { struct lpfc_scsi_buf *lpfc_cmd, *lpfc_cmd_next; struct lpfc_sli4_hdw_queue *qp; @@ -685,12 +687,18 @@ lpfc_get_scsi_buf_s4(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp) dma_addr_t pdma_phys_fcp_cmd; uint32_t sgl_size, cpu, idx; int found = 0; + int tag; cpu = smp_processor_id(); - if (cpu < phba->cfg_hdw_queue) - idx = cpu; - else - idx = cpu % phba->cfg_hdw_queue; + if (cmnd) { + tag = blk_mq_unique_tag(cmnd->request); + idx = blk_mq_unique_tag_to_hwq(tag); + } else { + if (cpu < phba->cfg_hdw_queue) + idx = cpu; + else + idx = cpu % phba->cfg_hdw_queue; + } qp = &phba->sli4_hba.hdwq[idx]; spin_lock_irqsave(&qp->io_buf_list_get_lock, iflag); @@ -815,9 +823,10 @@ lpfc_get_scsi_buf_s4(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp) * Pointer to lpfc_scsi_buf - Success **/ static struct lpfc_scsi_buf* -lpfc_get_scsi_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp) +lpfc_get_scsi_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, + struct scsi_cmnd *cmnd) { - return phba->lpfc_get_scsi_buf(phba, ndlp); + return phba->lpfc_get_scsi_buf(phba, ndlp, cmnd); } /** @@ -3657,49 +3666,6 @@ lpfc_handle_fcp_err(struct lpfc_vport *vport, struct lpfc_scsi_buf *lpfc_cmd, lpfc_send_scsi_error_event(vport->phba, vport, lpfc_cmd, rsp_iocb); } -/** - * lpfc_sli4_scmd_to_wqidx_distr - scsi command to SLI4 WQ index distribution - * @phba: Pointer to HBA context object. - * - * This routine performs a roundrobin SCSI command to SLI4 FCP WQ index - * distribution. This is called by __lpfc_sli_issue_iocb_s4() with the hbalock - * held. - * If scsi-mq is enabled, get the default block layer mapping of software queues - * to hardware queues. This information is saved in request tag. - * - * Return: index into SLI4 fast-path FCP queue index. - **/ -int lpfc_sli4_scmd_to_wqidx_distr(struct lpfc_hba *phba, - struct lpfc_scsi_buf *lpfc_cmd) -{ - struct scsi_cmnd *cmnd = lpfc_cmd->pCmd; - struct lpfc_vector_map_info *cpup; - int chann, cpu; - uint32_t tag; - uint16_t hwq; - - if (cmnd) { - tag = blk_mq_unique_tag(cmnd->request); - hwq = blk_mq_unique_tag_to_hwq(tag); - - return hwq; - } - - if (phba->cfg_fcp_io_sched == LPFC_FCP_SCHED_BY_CPU && - phba->cfg_hdw_queue > 1) { - cpu = lpfc_cmd->cpu; - if (cpu < phba->sli4_hba.num_present_cpu) { - cpup = phba->sli4_hba.cpu_map; - cpup += cpu; - return cpup->channel_id; - } - } - chann = atomic_add_return(1, &phba->fcp_qidx); - chann = chann % phba->cfg_hdw_queue; - return chann; -} - - /** * lpfc_scsi_cmd_iocb_cmpl - Scsi cmnd IOCB completion routine * @phba: The Hba for which this call is being executed. @@ -4474,7 +4440,7 @@ lpfc_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *cmnd) } } - lpfc_cmd = lpfc_get_scsi_buf(phba, ndlp); + lpfc_cmd = lpfc_get_scsi_buf(phba, ndlp, cmnd); if (lpfc_cmd == NULL) { lpfc_rampdown_queue_depth(phba); @@ -4913,7 +4879,7 @@ lpfc_send_taskmgmt(struct lpfc_vport *vport, struct scsi_cmnd *cmnd, return FAILED; pnode = rdata->pnode; - lpfc_cmd = lpfc_get_scsi_buf(phba, pnode); + lpfc_cmd = lpfc_get_scsi_buf(phba, pnode, NULL); if (lpfc_cmd == NULL) return FAILED; lpfc_cmd->timeout = phba->cfg_task_mgmt_tmo; diff --git a/drivers/scsi/lpfc/lpfc_scsi.h b/drivers/scsi/lpfc/lpfc_scsi.h index 58cb6fd575c3..696898087244 100644 --- a/drivers/scsi/lpfc/lpfc_scsi.h +++ b/drivers/scsi/lpfc/lpfc_scsi.h @@ -200,5 +200,3 @@ struct lpfc_scsi_buf { /* For sysfs/debugfs tmp string max len */ #define LPFC_MAX_SCSI_INFO_TMP_LEN 79 -int lpfc_sli4_scmd_to_wqidx_distr(struct lpfc_hba *phba, - struct lpfc_scsi_buf *lpfc_cmd); -- cgit v1.2.3 From c490850a094794e7515737a6939146966c826577 Mon Sep 17 00:00:00 2001 From: James Smart Date: Mon, 28 Jan 2019 11:14:28 -0800 Subject: scsi: lpfc: Adapt partitioned XRI lists to efficient sharing The XRI get/put lists were partitioned per hardware queue. However, the adapter rarely had sufficient resources to give a large number of resources per queue. As such, it became common for a cpu to encounter a lack of XRI resource and request the upper io stack to retry after returning a BUSY condition. This occurred even though other cpus were idle and not using their resources. Create as efficient a scheme as possible to move resources to the cpus that need them. Each cpu maintains a small private pool which it allocates from for io. There is a watermark that the cpu attempts to keep in the private pool. The private pool, when empty, pulls from a global pool from the cpu. When the cpu's global pool is empty it will pull from other cpu's global pool. As there many cpu global pools (1 per cpu or hardware queue count) and as each cpu selects what cpu to pull from at different rates and at different times, it creates a radomizing effect that minimizes the number of cpu's that will contend with each other when the steal XRI's from another cpu's global pool. On io completion, a cpu will push the XRI back on to its private pool. A watermark level is maintained for the private pool such that when it is exceeded it will move XRI's to the CPU global pool so that other cpu's may allocate them. On NVME, as heartbeat commands are critical to get placed on the wire, a single expedite pool is maintained. When a heartbeat is to be sent, it will allocate an XRI from the expedite pool rather than the normal cpu private/global pools. On any io completion, if a reduction in the expedite pools is seen, it will be replenished before the XRI is placed on the cpu private pool. Statistics are added to aid understanding the XRI levels on each cpu and their behaviors. Signed-off-by: Dick Kennedy Signed-off-by: James Smart Reviewed-by: Hannes Reinecke Signed-off-by: Martin K. Petersen --- drivers/scsi/lpfc/lpfc.h | 26 +- drivers/scsi/lpfc/lpfc_attr.c | 9 + drivers/scsi/lpfc/lpfc_crtn.h | 16 + drivers/scsi/lpfc/lpfc_debugfs.c | 262 ++++++++++++++ drivers/scsi/lpfc/lpfc_debugfs.h | 3 + drivers/scsi/lpfc/lpfc_init.c | 326 ++++++++++++++++-- drivers/scsi/lpfc/lpfc_nvme.c | 91 ++--- drivers/scsi/lpfc/lpfc_nvme.h | 45 +-- drivers/scsi/lpfc/lpfc_scsi.c | 162 ++++----- drivers/scsi/lpfc/lpfc_scsi.h | 53 --- drivers/scsi/lpfc/lpfc_sli.c | 720 +++++++++++++++++++++++++++++++++++++-- drivers/scsi/lpfc/lpfc_sli.h | 85 +++++ drivers/scsi/lpfc/lpfc_sli4.h | 56 +++ 13 files changed, 1527 insertions(+), 327 deletions(-) (limited to 'drivers/scsi/lpfc/lpfc.h') diff --git a/drivers/scsi/lpfc/lpfc.h b/drivers/scsi/lpfc/lpfc.h index 755bf49c272c..0f8964fdfecf 100644 --- a/drivers/scsi/lpfc/lpfc.h +++ b/drivers/scsi/lpfc/lpfc.h @@ -235,8 +235,6 @@ typedef struct lpfc_vpd { } sli3Feat; } lpfc_vpd_t; -struct lpfc_scsi_buf; - /* * lpfc stat counters @@ -597,6 +595,13 @@ struct lpfc_mbox_ext_buf_ctx { struct list_head ext_dmabuf_list; }; +struct lpfc_epd_pool { + /* Expedite pool */ + struct list_head list; + u32 count; + spinlock_t lock; /* lock for expedite pool */ +}; + struct lpfc_ras_fwlog { uint8_t *fwlog_buff; uint32_t fw_buffcount; /* Buffer size posted to FW */ @@ -618,19 +623,19 @@ struct lpfc_ras_fwlog { struct lpfc_hba { /* SCSI interface function jump table entries */ - struct lpfc_scsi_buf * (*lpfc_get_scsi_buf) + struct lpfc_io_buf * (*lpfc_get_scsi_buf) (struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, struct scsi_cmnd *cmnd); int (*lpfc_scsi_prep_dma_buf) - (struct lpfc_hba *, struct lpfc_scsi_buf *); + (struct lpfc_hba *, struct lpfc_io_buf *); void (*lpfc_scsi_unprep_dma_buf) - (struct lpfc_hba *, struct lpfc_scsi_buf *); + (struct lpfc_hba *, struct lpfc_io_buf *); void (*lpfc_release_scsi_buf) - (struct lpfc_hba *, struct lpfc_scsi_buf *); + (struct lpfc_hba *, struct lpfc_io_buf *); void (*lpfc_rampdown_queue_depth) (struct lpfc_hba *); void (*lpfc_scsi_prep_cmnd) - (struct lpfc_vport *, struct lpfc_scsi_buf *, + (struct lpfc_vport *, struct lpfc_io_buf *, struct lpfc_nodelist *); /* IOCB interface function jump table entries */ @@ -673,9 +678,12 @@ struct lpfc_hba { (struct lpfc_hba *); int (*lpfc_bg_scsi_prep_dma_buf) - (struct lpfc_hba *, struct lpfc_scsi_buf *); + (struct lpfc_hba *, struct lpfc_io_buf *); /* Add new entries here */ + /* expedite pool */ + struct lpfc_epd_pool epd_pool; + /* SLI4 specific HBA data structure */ struct lpfc_sli4_hba sli4_hba; @@ -789,6 +797,7 @@ struct lpfc_hba { /* HBA Config Parameters */ uint32_t cfg_ack0; + uint32_t cfg_xri_rebalancing; uint32_t cfg_enable_npiv; uint32_t cfg_enable_rrq; uint32_t cfg_topology; @@ -1014,6 +1023,7 @@ struct lpfc_hba { #ifdef CONFIG_SCSI_LPFC_DEBUG_FS struct dentry *hba_debugfs_root; atomic_t debugfs_vport_count; + struct dentry *debug_multixri_pools; struct dentry *debug_hbqinfo; struct dentry *debug_dumpHostSlim; struct dentry *debug_dumpHBASlim; diff --git a/drivers/scsi/lpfc/lpfc_attr.c b/drivers/scsi/lpfc/lpfc_attr.c index e10d930fcb6a..47aa2af885a4 100644 --- a/drivers/scsi/lpfc/lpfc_attr.c +++ b/drivers/scsi/lpfc/lpfc_attr.c @@ -5266,6 +5266,12 @@ static DEVICE_ATTR_RW(lpfc_max_scsicmpl_time); */ LPFC_ATTR_R(ack0, 0, 0, 1, "Enable ACK0 support"); +/* +# lpfc_xri_rebalancing: enable or disable XRI rebalancing feature +# range is [0,1]. Default value is 1. +*/ +LPFC_ATTR_R(xri_rebalancing, 1, 0, 1, "Enable/Disable XRI rebalancing"); + /* * lpfc_io_sched: Determine scheduling algrithmn for issuing FCP cmds * range is [0,1]. Default value is 0. @@ -5723,6 +5729,7 @@ struct device_attribute *lpfc_hba_attrs[] = { &dev_attr_lpfc_use_adisc, &dev_attr_lpfc_first_burst_size, &dev_attr_lpfc_ack0, + &dev_attr_lpfc_xri_rebalancing, &dev_attr_lpfc_topology, &dev_attr_lpfc_scan_down, &dev_attr_lpfc_link_speed, @@ -6788,6 +6795,7 @@ lpfc_get_cfgparam(struct lpfc_hba *phba) lpfc_multi_ring_rctl_init(phba, lpfc_multi_ring_rctl); lpfc_multi_ring_type_init(phba, lpfc_multi_ring_type); lpfc_ack0_init(phba, lpfc_ack0); + lpfc_xri_rebalancing_init(phba, lpfc_xri_rebalancing); lpfc_topology_init(phba, lpfc_topology); lpfc_link_speed_init(phba, lpfc_link_speed); lpfc_poll_tmo_init(phba, lpfc_poll_tmo); @@ -6846,6 +6854,7 @@ lpfc_get_cfgparam(struct lpfc_hba *phba) phba->nvmet_support = 0; phba->cfg_enable_fc4_type = LPFC_ENABLE_FCP; phba->cfg_enable_bbcr = 0; + phba->cfg_xri_rebalancing = 0; } else { /* We MUST have FCP support */ if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_FCP)) diff --git a/drivers/scsi/lpfc/lpfc_crtn.h b/drivers/scsi/lpfc/lpfc_crtn.h index 1bd1362f39a0..726cd6a7c452 100644 --- a/drivers/scsi/lpfc/lpfc_crtn.h +++ b/drivers/scsi/lpfc/lpfc_crtn.h @@ -521,6 +521,7 @@ int lpfc_sli4_io_sgl_update(struct lpfc_hba *phba); int lpfc_sli4_post_io_sgl_list(struct lpfc_hba *phba, struct list_head *blist, int xricnt); int lpfc_new_io_buf(struct lpfc_hba *phba, int num_to_alloc); +void lpfc_io_free(struct lpfc_hba *phba); void lpfc_free_sgl_list(struct lpfc_hba *, struct list_head *); uint32_t lpfc_sli_port_speed_get(struct lpfc_hba *); int lpfc_sli4_request_firmware_update(struct lpfc_hba *, uint8_t); @@ -573,6 +574,21 @@ void lpfc_nvme_mod_param_dep(struct lpfc_hba *phba); void lpfc_nvme_abort_fcreq_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb, struct lpfc_wcqe_complete *abts_cmpl); +void lpfc_create_multixri_pools(struct lpfc_hba *phba); +void lpfc_create_destroy_pools(struct lpfc_hba *phba); +void lpfc_move_xri_pvt_to_pbl(struct lpfc_hba *phba, u32 hwqid); +void lpfc_move_xri_pbl_to_pvt(struct lpfc_hba *phba, u32 hwqid, u32 cnt); +void lpfc_adjust_high_watermark(struct lpfc_hba *phba, u32 hwqid); +void lpfc_keep_pvt_pool_above_lowwm(struct lpfc_hba *phba, u32 hwqid); +void lpfc_adjust_pvt_pool_count(struct lpfc_hba *phba, u32 hwqid); +#ifdef LPFC_MXP_STAT +void lpfc_snapshot_mxp(struct lpfc_hba *, u32); +#endif +struct lpfc_io_buf *lpfc_get_io_buf(struct lpfc_hba *phba, + struct lpfc_nodelist *ndlp, u32 hwqid, + int); +void lpfc_release_io_buf(struct lpfc_hba *phba, struct lpfc_io_buf *ncmd, + struct lpfc_sli4_hdw_queue *qp); void lpfc_nvme_cmd_template(void); void lpfc_nvmet_cmd_template(void); extern int lpfc_enable_nvmet_cnt; diff --git a/drivers/scsi/lpfc/lpfc_debugfs.c b/drivers/scsi/lpfc/lpfc_debugfs.c index 218642d7c488..2a2c46766eb6 100644 --- a/drivers/scsi/lpfc/lpfc_debugfs.c +++ b/drivers/scsi/lpfc/lpfc_debugfs.c @@ -378,6 +378,126 @@ skipit: return len; } +/** + * lpfc_debugfs_multixripools_data - Display multi-XRI pools information + * @phba: The HBA to gather host buffer info from. + * @buf: The buffer to dump log into. + * @size: The maximum amount of data to process. + * + * Description: + * This routine displays current multi-XRI pools information including XRI + * count in public, private and txcmplq. It also displays current high and + * low watermark. + * + * Return Value: + * This routine returns the amount of bytes that were dumped into @buf and will + * not exceed @size. + **/ +static int +lpfc_debugfs_multixripools_data(struct lpfc_hba *phba, char *buf, int size) +{ + u32 i; + u32 hwq_count; + struct lpfc_sli4_hdw_queue *qp; + struct lpfc_multixri_pool *multixri_pool; + struct lpfc_pvt_pool *pvt_pool; + struct lpfc_pbl_pool *pbl_pool; + u32 txcmplq_cnt; + char tmp[LPFC_DEBUG_OUT_LINE_SZ] = {0}; + + /* + * Pbl: Current number of free XRIs in public pool + * Pvt: Current number of free XRIs in private pool + * Busy: Current number of outstanding XRIs + * HWM: Current high watermark + * pvt_empty: Incremented by 1 when IO submission fails (no xri) + * pbl_empty: Incremented by 1 when all pbl_pool are empty during + * IO submission + */ + scnprintf(tmp, sizeof(tmp), + "HWQ: Pbl Pvt Busy HWM | pvt_empty pbl_empty "); + if (strlcat(buf, tmp, size) >= size) + return strnlen(buf, size); + +#ifdef LPFC_MXP_STAT + /* + * MAXH: Max high watermark seen so far + * above_lmt: Incremented by 1 if xri_owned > xri_limit during + * IO submission + * below_lmt: Incremented by 1 if xri_owned <= xri_limit during + * IO submission + * locPbl_hit: Incremented by 1 if successfully get a batch of XRI from + * local pbl_pool + * othPbl_hit: Incremented by 1 if successfully get a batch of XRI from + * other pbl_pool + */ + scnprintf(tmp, sizeof(tmp), + "MAXH above_lmt below_lmt locPbl_hit othPbl_hit"); + if (strlcat(buf, tmp, size) >= size) + return strnlen(buf, size); + + /* + * sPbl: snapshot of Pbl 15 sec after stat gets cleared + * sPvt: snapshot of Pvt 15 sec after stat gets cleared + * sBusy: snapshot of Busy 15 sec after stat gets cleared + */ + scnprintf(tmp, sizeof(tmp), + " | sPbl sPvt sBusy"); + if (strlcat(buf, tmp, size) >= size) + return strnlen(buf, size); +#endif + + scnprintf(tmp, sizeof(tmp), "\n"); + if (strlcat(buf, tmp, size) >= size) + return strnlen(buf, size); + + hwq_count = phba->cfg_hdw_queue; + for (i = 0; i < hwq_count; i++) { + qp = &phba->sli4_hba.hdwq[i]; + multixri_pool = qp->p_multixri_pool; + if (!multixri_pool) + continue; + pbl_pool = &multixri_pool->pbl_pool; + pvt_pool = &multixri_pool->pvt_pool; + txcmplq_cnt = qp->fcp_wq->pring->txcmplq_cnt; + if (qp->nvme_wq) + txcmplq_cnt += qp->nvme_wq->pring->txcmplq_cnt; + + scnprintf(tmp, sizeof(tmp), + "%03d: %4d %4d %4d %4d | %10d %10d ", + i, pbl_pool->count, pvt_pool->count, + txcmplq_cnt, pvt_pool->high_watermark, + qp->empty_io_bufs, multixri_pool->pbl_empty_count); + if (strlcat(buf, tmp, size) >= size) + break; + +#ifdef LPFC_MXP_STAT + scnprintf(tmp, sizeof(tmp), + "%4d %10d %10d %10d %10d", + multixri_pool->stat_max_hwm, + multixri_pool->above_limit_count, + multixri_pool->below_limit_count, + multixri_pool->local_pbl_hit_count, + multixri_pool->other_pbl_hit_count); + if (strlcat(buf, tmp, size) >= size) + break; + + scnprintf(tmp, sizeof(tmp), + " | %4d %4d %5d", + multixri_pool->stat_pbl_count, + multixri_pool->stat_pvt_count, + multixri_pool->stat_busy_count); + if (strlcat(buf, tmp, size) >= size) + break; +#endif + + scnprintf(tmp, sizeof(tmp), "\n"); + if (strlcat(buf, tmp, size) >= size) + break; + } + return strnlen(buf, size); +} + static int lpfc_debugfs_last_hdwq; /** @@ -1751,6 +1871,53 @@ out: return rc; } +/** + * lpfc_debugfs_multixripools_open - Open the multixripool debugfs buffer + * @inode: The inode pointer that contains a hba pointer. + * @file: The file pointer to attach the log output. + * + * Description: + * This routine is the entry point for the debugfs open file operation. It gets + * the hba from the i_private field in @inode, allocates the necessary buffer + * for the log, fills the buffer from the in-memory log for this hba, and then + * returns a pointer to that log in the private_data field in @file. + * + * Returns: + * This function returns zero if successful. On error it will return a negative + * error value. + **/ +static int +lpfc_debugfs_multixripools_open(struct inode *inode, struct file *file) +{ + struct lpfc_hba *phba = inode->i_private; + struct lpfc_debug *debug; + int rc = -ENOMEM; + + debug = kmalloc(sizeof(*debug), GFP_KERNEL); + if (!debug) + goto out; + + /* Round to page boundary */ + debug->buffer = kzalloc(LPFC_DUMP_MULTIXRIPOOL_SIZE, GFP_KERNEL); + if (!debug->buffer) { + kfree(debug); + goto out; + } + + if (phba->cfg_xri_rebalancing) + debug->len = lpfc_debugfs_multixripools_data( + phba, debug->buffer, LPFC_DUMP_MULTIXRIPOOL_SIZE); + else + debug->len = 0; + + debug->i_private = inode->i_private; + file->private_data = debug; + + rc = 0; +out: + return rc; +} + /** * lpfc_debugfs_hdwqinfo_open - Open the hdwqinfo debugfs buffer * @inode: The inode pointer that contains a vport pointer. @@ -2182,6 +2349,75 @@ lpfc_debugfs_dumpDataDif_release(struct inode *inode, struct file *file) return 0; } +/** + * lpfc_debugfs_multixripools_write - Clear multi-XRI pools statistics + * @file: The file pointer to read from. + * @buf: The buffer to copy the user data from. + * @nbytes: The number of bytes to get. + * @ppos: The position in the file to start reading from. + * + * Description: + * This routine clears multi-XRI pools statistics when buf contains "clear". + * + * Return Value: + * It returns the @nbytges passing in from debugfs user space when successful. + * In case of error conditions, it returns proper error code back to the user + * space. + **/ +static ssize_t +lpfc_debugfs_multixripools_write(struct file *file, const char __user *buf, + size_t nbytes, loff_t *ppos) +{ + struct lpfc_debug *debug = file->private_data; + struct lpfc_hba *phba = (struct lpfc_hba *)debug->i_private; + char mybuf[64]; + char *pbuf; + u32 i; + u32 hwq_count; + struct lpfc_sli4_hdw_queue *qp; + struct lpfc_multixri_pool *multixri_pool; + + if (nbytes > 64) + nbytes = 64; + + /* Protect copy from user */ + if (!access_ok(buf, nbytes)) + return -EFAULT; + + memset(mybuf, 0, sizeof(mybuf)); + + if (copy_from_user(mybuf, buf, nbytes)) + return -EFAULT; + pbuf = &mybuf[0]; + + if ((strncmp(pbuf, "clear", strlen("clear"))) == 0) { + hwq_count = phba->cfg_hdw_queue; + for (i = 0; i < hwq_count; i++) { + qp = &phba->sli4_hba.hdwq[i]; + multixri_pool = qp->p_multixri_pool; + if (!multixri_pool) + continue; + + qp->empty_io_bufs = 0; + multixri_pool->pbl_empty_count = 0; +#ifdef LPFC_MXP_STAT + multixri_pool->above_limit_count = 0; + multixri_pool->below_limit_count = 0; + multixri_pool->stat_max_hwm = 0; + multixri_pool->local_pbl_hit_count = 0; + multixri_pool->other_pbl_hit_count = 0; + + multixri_pool->stat_pbl_count = 0; + multixri_pool->stat_pvt_count = 0; + multixri_pool->stat_busy_count = 0; + multixri_pool->stat_snapshot_taken = 0; +#endif + } + return strlen(pbuf); + } + + return -EINVAL; +} static int lpfc_debugfs_nvmestat_open(struct inode *inode, struct file *file) @@ -5044,6 +5280,16 @@ static const struct file_operations lpfc_debugfs_op_nodelist = { .release = lpfc_debugfs_release, }; +#undef lpfc_debugfs_op_multixripools +static const struct file_operations lpfc_debugfs_op_multixripools = { + .owner = THIS_MODULE, + .open = lpfc_debugfs_multixripools_open, + .llseek = lpfc_debugfs_lseek, + .read = lpfc_debugfs_read, + .write = lpfc_debugfs_multixripools_write, + .release = lpfc_debugfs_release, +}; + #undef lpfc_debugfs_op_hbqinfo static const struct file_operations lpfc_debugfs_op_hbqinfo = { .owner = THIS_MODULE, @@ -5490,6 +5736,19 @@ lpfc_debugfs_initialize(struct lpfc_vport *vport) atomic_inc(&lpfc_debugfs_hba_count); atomic_set(&phba->debugfs_vport_count, 0); + /* Multi-XRI pools */ + snprintf(name, sizeof(name), "multixripools"); + phba->debug_multixri_pools = + debugfs_create_file(name, S_IFREG | 0644, + phba->hba_debugfs_root, + phba, + &lpfc_debugfs_op_multixripools); + if (!phba->debug_multixri_pools) { + lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT, + "0527 Cannot create debugfs multixripools\n"); + goto debug_failed; + } + /* Setup hbqinfo */ snprintf(name, sizeof(name), "hbqinfo"); phba->debug_hbqinfo = @@ -5906,6 +6165,9 @@ lpfc_debugfs_terminate(struct lpfc_vport *vport) if (atomic_read(&phba->debugfs_vport_count) == 0) { + debugfs_remove(phba->debug_multixri_pools); /* multixripools*/ + phba->debug_multixri_pools = NULL; + debugfs_remove(phba->debug_hbqinfo); /* hbqinfo */ phba->debug_hbqinfo = NULL; diff --git a/drivers/scsi/lpfc/lpfc_debugfs.h b/drivers/scsi/lpfc/lpfc_debugfs.h index 2360ec8fb848..cf256a6dca42 100644 --- a/drivers/scsi/lpfc/lpfc_debugfs.h +++ b/drivers/scsi/lpfc/lpfc_debugfs.h @@ -287,6 +287,9 @@ struct lpfc_idiag { #endif +/* multixripool output buffer size */ +#define LPFC_DUMP_MULTIXRIPOOL_SIZE 8192 + /* hdwqinfo output buffer size */ #define LPFC_HDWQINFO_SIZE 8192 diff --git a/drivers/scsi/lpfc/lpfc_init.c b/drivers/scsi/lpfc/lpfc_init.c index 88b1c3ca26dc..10e3ad5419f0 100644 --- a/drivers/scsi/lpfc/lpfc_init.c +++ b/drivers/scsi/lpfc/lpfc_init.c @@ -1037,7 +1037,7 @@ lpfc_hba_down_post_s3(struct lpfc_hba *phba) static int lpfc_hba_down_post_s4(struct lpfc_hba *phba) { - struct lpfc_scsi_buf *psb, *psb_next; + struct lpfc_io_buf *psb, *psb_next; struct lpfc_nvmet_rcv_ctx *ctxp, *ctxp_next; struct lpfc_sli4_hdw_queue *qp; LIST_HEAD(aborts); @@ -1249,6 +1249,33 @@ lpfc_hb_mbox_cmpl(struct lpfc_hba * phba, LPFC_MBOXQ_t * pmboxq) return; } +/** + * lpfc_hb_mxp_handler - Multi-XRI pools handler to adjust XRI distribution + * @phba: pointer to lpfc hba data structure. + * + * For each heartbeat, this routine does some heuristic methods to adjust + * XRI distribution. The goal is to fully utilize free XRIs. + **/ +static void lpfc_hb_mxp_handler(struct lpfc_hba *phba) +{ + u32 i; + u32 hwq_count; + + hwq_count = phba->cfg_hdw_queue; + for (i = 0; i < hwq_count; i++) { + /* Adjust XRIs in private pool */ + lpfc_adjust_pvt_pool_count(phba, i); + + /* Adjust high watermark */ + lpfc_adjust_high_watermark(phba, i); + +#ifdef LPFC_MXP_STAT + /* Snapshot pbl, pvt and busy count */ + lpfc_snapshot_mxp(phba, i); +#endif + } +} + /** * lpfc_hb_timeout_handler - The HBA-timer timeout handler * @phba: pointer to lpfc hba data structure. @@ -1285,6 +1312,11 @@ lpfc_hb_timeout_handler(struct lpfc_hba *phba) struct lpfc_fc4_ctrl_stat *cstat; void __iomem *eqdreg = phba->sli4_hba.u.if_type2.EQDregaddr; + if (phba->cfg_xri_rebalancing) { + /* Multi-XRI pools handler */ + lpfc_hb_mxp_handler(phba); + } + vports = lpfc_create_vport_work_array(phba); if (vports != NULL) for (i = 0; i <= phba->max_vports && vports[i] != NULL; i++) { @@ -3078,6 +3110,242 @@ lpfc_sli4_node_prep(struct lpfc_hba *phba) lpfc_destroy_vport_work_array(phba, vports); } +/** + * lpfc_create_expedite_pool - create expedite pool + * @phba: pointer to lpfc hba data structure. + * + * This routine moves a batch of XRIs from lpfc_io_buf_list_put of HWQ 0 + * to expedite pool. Mark them as expedite. + **/ +void lpfc_create_expedite_pool(struct lpfc_hba *phba) +{ + struct lpfc_sli4_hdw_queue *qp; + struct lpfc_io_buf *lpfc_ncmd; + struct lpfc_io_buf *lpfc_ncmd_next; + struct lpfc_epd_pool *epd_pool; + unsigned long iflag; + + epd_pool = &phba->epd_pool; + qp = &phba->sli4_hba.hdwq[0]; + + spin_lock_init(&epd_pool->lock); + spin_lock_irqsave(&qp->io_buf_list_put_lock, iflag); + spin_lock(&epd_pool->lock); + INIT_LIST_HEAD(&epd_pool->list); + list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next, + &qp->lpfc_io_buf_list_put, list) { + list_move_tail(&lpfc_ncmd->list, &epd_pool->list); + lpfc_ncmd->expedite = true; + qp->put_io_bufs--; + epd_pool->count++; + if (epd_pool->count >= XRI_BATCH) + break; + } + spin_unlock(&epd_pool->lock); + spin_unlock_irqrestore(&qp->io_buf_list_put_lock, iflag); +} + +/** + * lpfc_destroy_expedite_pool - destroy expedite pool + * @phba: pointer to lpfc hba data structure. + * + * This routine returns XRIs from expedite pool to lpfc_io_buf_list_put + * of HWQ 0. Clear the mark. + **/ +void lpfc_destroy_expedite_pool(struct lpfc_hba *phba) +{ + struct lpfc_sli4_hdw_queue *qp; + struct lpfc_io_buf *lpfc_ncmd; + struct lpfc_io_buf *lpfc_ncmd_next; + struct lpfc_epd_pool *epd_pool; + unsigned long iflag; + + epd_pool = &phba->epd_pool; + qp = &phba->sli4_hba.hdwq[0]; + + spin_lock_irqsave(&qp->io_buf_list_put_lock, iflag); + spin_lock(&epd_pool->lock); + list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next, + &epd_pool->list, list) { + list_move_tail(&lpfc_ncmd->list, + &qp->lpfc_io_buf_list_put); + lpfc_ncmd->flags = false; + qp->put_io_bufs++; + epd_pool->count--; + } + spin_unlock(&epd_pool->lock); + spin_unlock_irqrestore(&qp->io_buf_list_put_lock, iflag); +} + +/** + * lpfc_create_multixri_pools - create multi-XRI pools + * @phba: pointer to lpfc hba data structure. + * + * This routine initialize public, private per HWQ. Then, move XRIs from + * lpfc_io_buf_list_put to public pool. High and low watermark are also + * Initialized. + **/ +void lpfc_create_multixri_pools(struct lpfc_hba *phba) +{ + u32 i, j; + u32 hwq_count; + u32 count_per_hwq; + struct lpfc_io_buf *lpfc_ncmd; + struct lpfc_io_buf *lpfc_ncmd_next; + unsigned long iflag; + struct lpfc_sli4_hdw_queue *qp; + struct lpfc_multixri_pool *multixri_pool; + struct lpfc_pbl_pool *pbl_pool; + struct lpfc_pvt_pool *pvt_pool; + + lpfc_printf_log(phba, KERN_INFO, LOG_INIT, + "1234 num_hdw_queue=%d num_present_cpu=%d common_xri_cnt=%d\n", + phba->cfg_hdw_queue, phba->sli4_hba.num_present_cpu, + phba->sli4_hba.io_xri_cnt); + + if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) + lpfc_create_expedite_pool(phba); + + hwq_count = phba->cfg_hdw_queue; + count_per_hwq = phba->sli4_hba.io_xri_cnt / hwq_count; + + for (i = 0; i < hwq_count; i++) { + multixri_pool = kzalloc(sizeof(*multixri_pool), GFP_KERNEL); + + if (!multixri_pool) { + lpfc_printf_log(phba, KERN_INFO, LOG_INIT, + "1238 Failed to allocate memory for " + "multixri_pool\n"); + + if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) + lpfc_destroy_expedite_pool(phba); + + j = 0; + while (j < i) { + qp = &phba->sli4_hba.hdwq[j]; + kfree(qp->p_multixri_pool); + j++; + } + phba->cfg_xri_rebalancing = 0; + return; + } + + qp = &phba->sli4_hba.hdwq[i]; + qp->p_multixri_pool = multixri_pool; + + multixri_pool->xri_limit = count_per_hwq; + multixri_pool->rrb_next_hwqid = i; + + /* Deal with public free xri pool */ + pbl_pool = &multixri_pool->pbl_pool; + spin_lock_init(&pbl_pool->lock); + spin_lock_irqsave(&qp->io_buf_list_put_lock, iflag); + spin_lock(&pbl_pool->lock); + INIT_LIST_HEAD(&pbl_pool->list); + list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next, + &qp->lpfc_io_buf_list_put, list) { + list_move_tail(&lpfc_ncmd->list, &pbl_pool->list); + qp->put_io_bufs--; + pbl_pool->count++; + } + lpfc_printf_log(phba, KERN_INFO, LOG_INIT, + "1235 Moved %d buffers from PUT list over to pbl_pool[%d]\n", + pbl_pool->count, i); + spin_unlock(&pbl_pool->lock); + spin_unlock_irqrestore(&qp->io_buf_list_put_lock, iflag); + + /* Deal with private free xri pool */ + pvt_pool = &multixri_pool->pvt_pool; + pvt_pool->high_watermark = multixri_pool->xri_limit / 2; + pvt_pool->low_watermark = XRI_BATCH; + spin_lock_init(&pvt_pool->lock); + spin_lock_irqsave(&pvt_pool->lock, iflag); + INIT_LIST_HEAD(&pvt_pool->list); + pvt_pool->count = 0; + spin_unlock_irqrestore(&pvt_pool->lock, iflag); + } +} + +/** + * lpfc_destroy_multixri_pools - destroy multi-XRI pools + * @phba: pointer to lpfc hba data structure. + * + * This routine returns XRIs from public/private to lpfc_io_buf_list_put. + **/ +void lpfc_destroy_multixri_pools(struct lpfc_hba *phba) +{ + u32 i; + u32 hwq_count; + struct lpfc_io_buf *lpfc_ncmd; + struct lpfc_io_buf *lpfc_ncmd_next; + unsigned long iflag; + struct lpfc_sli4_hdw_queue *qp; + struct lpfc_multixri_pool *multixri_pool; + struct lpfc_pbl_pool *pbl_pool; + struct lpfc_pvt_pool *pvt_pool; + + if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) + lpfc_destroy_expedite_pool(phba); + + hwq_count = phba->cfg_hdw_queue; + + for (i = 0; i < hwq_count; i++) { + qp = &phba->sli4_hba.hdwq[i]; + multixri_pool = qp->p_multixri_pool; + if (!multixri_pool) + continue; + + qp->p_multixri_pool = NULL; + + spin_lock_irqsave(&qp->io_buf_list_put_lock, iflag); + + /* Deal with public free xri pool */ + pbl_pool = &multixri_pool->pbl_pool; + spin_lock(&pbl_pool->lock); + + lpfc_printf_log(phba, KERN_INFO, LOG_INIT, + "1236 Moving %d buffers from pbl_pool[%d] TO PUT list\n", + pbl_pool->count, i); + + list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next, + &pbl_pool->list, list) { + list_move_tail(&lpfc_ncmd->list, + &qp->lpfc_io_buf_list_put); + qp->put_io_bufs++; + pbl_pool->count--; + } + + INIT_LIST_HEAD(&pbl_pool->list); + pbl_pool->count = 0; + + spin_unlock(&pbl_pool->lock); + + /* Deal with private free xri pool */ + pvt_pool = &multixri_pool->pvt_pool; + spin_lock(&pvt_pool->lock); + + lpfc_printf_log(phba, KERN_INFO, LOG_INIT, + "1237 Moving %d buffers from pvt_pool[%d] TO PUT list\n", + pvt_pool->count, i); + + list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next, + &pvt_pool->list, list) { + list_move_tail(&lpfc_ncmd->list, + &qp->lpfc_io_buf_list_put); + qp->put_io_bufs++; + pvt_pool->count--; + } + + INIT_LIST_HEAD(&pvt_pool->list); + pvt_pool->count = 0; + + spin_unlock(&pvt_pool->lock); + spin_unlock_irqrestore(&qp->io_buf_list_put_lock, iflag); + + kfree(multixri_pool); + } +} + /** * lpfc_online - Initialize and bring a HBA online * @phba: pointer to lpfc hba data structure. @@ -3160,6 +3428,9 @@ lpfc_online(struct lpfc_hba *phba) } lpfc_destroy_vport_work_array(phba, vports); + if (phba->cfg_xri_rebalancing) + lpfc_create_multixri_pools(phba); + lpfc_unblock_mgmt_io(phba); return 0; } @@ -3318,6 +3589,9 @@ lpfc_offline(struct lpfc_hba *phba) spin_unlock_irq(shost->host_lock); } lpfc_destroy_vport_work_array(phba, vports); + + if (phba->cfg_xri_rebalancing) + lpfc_destroy_multixri_pools(phba); } /** @@ -3331,7 +3605,7 @@ lpfc_offline(struct lpfc_hba *phba) static void lpfc_scsi_free(struct lpfc_hba *phba) { - struct lpfc_scsi_buf *sb, *sb_next; + struct lpfc_io_buf *sb, *sb_next; if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_FCP)) return; @@ -3372,10 +3646,10 @@ lpfc_scsi_free(struct lpfc_hba *phba) * list back to kernel. It is called from lpfc_pci_remove_one to free * the internal resources before the device is removed from the system. **/ -static void +void lpfc_io_free(struct lpfc_hba *phba) { - struct lpfc_nvme_buf *lpfc_ncmd, *lpfc_ncmd_next; + struct lpfc_io_buf *lpfc_ncmd, *lpfc_ncmd_next; struct lpfc_sli4_hdw_queue *qp; int idx; @@ -3660,8 +3934,8 @@ lpfc_io_buf_flush(struct lpfc_hba *phba, struct list_head *cbuf) { LIST_HEAD(blist); struct lpfc_sli4_hdw_queue *qp; - struct lpfc_scsi_buf *lpfc_cmd; - struct lpfc_scsi_buf *iobufp, *prev_iobufp; + struct lpfc_io_buf *lpfc_cmd; + struct lpfc_io_buf *iobufp, *prev_iobufp; int idx, cnt, xri, inserted; cnt = 0; @@ -3689,7 +3963,7 @@ lpfc_io_buf_flush(struct lpfc_hba *phba, struct list_head *cbuf) * to post to the firmware. */ for (idx = 0; idx < cnt; idx++) { - list_remove_head(&blist, lpfc_cmd, struct lpfc_scsi_buf, list); + list_remove_head(&blist, lpfc_cmd, struct lpfc_io_buf, list); if (!lpfc_cmd) return cnt; if (idx == 0) { @@ -3721,7 +3995,7 @@ int lpfc_io_buf_replenish(struct lpfc_hba *phba, struct list_head *cbuf) { struct lpfc_sli4_hdw_queue *qp; - struct lpfc_scsi_buf *lpfc_cmd; + struct lpfc_io_buf *lpfc_cmd; int idx, cnt; qp = phba->sli4_hba.hdwq; @@ -3729,7 +4003,7 @@ lpfc_io_buf_replenish(struct lpfc_hba *phba, struct list_head *cbuf) while (!list_empty(cbuf)) { for (idx = 0; idx < phba->cfg_hdw_queue; idx++) { list_remove_head(cbuf, lpfc_cmd, - struct lpfc_scsi_buf, list); + struct lpfc_io_buf, list); if (!lpfc_cmd) return cnt; cnt++; @@ -3764,7 +4038,7 @@ lpfc_io_buf_replenish(struct lpfc_hba *phba, struct list_head *cbuf) int lpfc_sli4_io_sgl_update(struct lpfc_hba *phba) { - struct lpfc_nvme_buf *lpfc_ncmd = NULL, *lpfc_ncmd_next = NULL; + struct lpfc_io_buf *lpfc_ncmd = NULL, *lpfc_ncmd_next = NULL; uint16_t i, lxri, els_xri_cnt; uint16_t io_xri_cnt, io_xri_max; LIST_HEAD(io_sgl_list); @@ -3794,7 +4068,7 @@ lpfc_sli4_io_sgl_update(struct lpfc_hba *phba) /* release the extra allocated nvme buffers */ for (i = 0; i < io_xri_cnt; i++) { list_remove_head(&io_sgl_list, lpfc_ncmd, - struct lpfc_nvme_buf, list); + struct lpfc_io_buf, list); if (lpfc_ncmd) { dma_pool_free(phba->lpfc_sg_dma_buf_pool, lpfc_ncmd->data, @@ -3847,7 +4121,7 @@ out_free_mem: int lpfc_new_io_buf(struct lpfc_hba *phba, int num_to_alloc) { - struct lpfc_nvme_buf *lpfc_ncmd; + struct lpfc_io_buf *lpfc_ncmd; struct lpfc_iocbq *pwqeq; uint16_t iotag, lxri = 0; int bcnt, num_posted; @@ -3856,12 +4130,11 @@ lpfc_new_io_buf(struct lpfc_hba *phba, int num_to_alloc) LIST_HEAD(nvme_nblist); /* Sanity check to ensure our sizing is right for both SCSI and NVME */ - if ((sizeof(struct lpfc_scsi_buf) > LPFC_COMMON_IO_BUF_SZ) || - (sizeof(struct lpfc_nvme_buf) > LPFC_COMMON_IO_BUF_SZ)) { + if (sizeof(struct lpfc_io_buf) > LPFC_COMMON_IO_BUF_SZ) { lpfc_printf_log(phba, KERN_ERR, LOG_FCP, - "6426 Common buffer size mismatch: %ld %ld\n", - sizeof(struct lpfc_scsi_buf), - sizeof(struct lpfc_nvme_buf)); + "6426 Common buffer size %ld exceeds %d\n", + sizeof(struct lpfc_io_buf), + LPFC_COMMON_IO_BUF_SZ); return 0; } @@ -6434,6 +6707,8 @@ lpfc_sli4_driver_resource_setup(struct lpfc_hba *phba) " NVME_TARGET_FC infrastructure" " is not in kernel\n"); #endif + /* Not supported for NVMET */ + phba->cfg_xri_rebalancing = 0; break; } } @@ -11756,7 +12031,7 @@ lpfc_pci_probe_one_s4(struct pci_dev *pdev, const struct pci_device_id *pid) struct lpfc_hba *phba; struct lpfc_vport *vport = NULL; struct Scsi_Host *shost = NULL; - int error, len; + int error; uint32_t cfg_mode, intr_mode; /* Allocate memory for HBA structure */ @@ -11879,17 +12154,6 @@ lpfc_pci_probe_one_s4(struct pci_dev *pdev, const struct pci_device_id *pid) error); } } - /* Don't post more new bufs if repost already recovered - * the nvme sgls. - */ - if (phba->sli4_hba.io_xri_cnt == 0) { - len = lpfc_new_io_buf( - phba, phba->sli4_hba.io_xri_max); - if (len == 0) { - error = -ENOMEM; - goto out_free_sysfs_attr; - } - } } /* check for firmware upgrade or downgrade */ @@ -11972,6 +12236,10 @@ lpfc_pci_remove_one_s4(struct pci_dev *pdev) lpfc_nvmet_destroy_targetport(phba); lpfc_nvme_destroy_localport(vport); + /* De-allocate multi-XRI pools */ + if (phba->cfg_xri_rebalancing) + lpfc_destroy_multixri_pools(phba); + /* * Bring down the SLI Layer. This step disables all interrupts, * clears the rings, discards all mailbox commands, and resets diff --git a/drivers/scsi/lpfc/lpfc_nvme.c b/drivers/scsi/lpfc/lpfc_nvme.c index f1f697cd7e97..0c6c91d39e2f 100644 --- a/drivers/scsi/lpfc/lpfc_nvme.c +++ b/drivers/scsi/lpfc/lpfc_nvme.c @@ -56,12 +56,12 @@ /* NVME initiator-based functions */ -static struct lpfc_nvme_buf * +static struct lpfc_io_buf * lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, int idx, int expedite); static void -lpfc_release_nvme_buf(struct lpfc_hba *, struct lpfc_nvme_buf *); +lpfc_release_nvme_buf(struct lpfc_hba *, struct lpfc_io_buf *); static struct nvme_fc_port_template lpfc_nvme_template; @@ -760,7 +760,7 @@ lpfc_nvme_ls_abort(struct nvme_fc_local_port *pnvme_lport, /* Fix up the existing sgls for NVME IO. */ static inline void lpfc_nvme_adj_fcp_sgls(struct lpfc_vport *vport, - struct lpfc_nvme_buf *lpfc_ncmd, + struct lpfc_io_buf *lpfc_ncmd, struct nvmefc_fcp_req *nCmd) { struct lpfc_hba *phba = vport->phba; @@ -857,7 +857,7 @@ lpfc_nvme_adj_fcp_sgls(struct lpfc_vport *vport, #ifdef CONFIG_SCSI_LPFC_DEBUG_FS static void lpfc_nvme_ktime(struct lpfc_hba *phba, - struct lpfc_nvme_buf *lpfc_ncmd) + struct lpfc_io_buf *lpfc_ncmd) { uint64_t seg1, seg2, seg3, seg4; uint64_t segsum; @@ -955,8 +955,8 @@ static void lpfc_nvme_io_cmd_wqe_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn, struct lpfc_wcqe_complete *wcqe) { - struct lpfc_nvme_buf *lpfc_ncmd = - (struct lpfc_nvme_buf *)pwqeIn->context1; + struct lpfc_io_buf *lpfc_ncmd = + (struct lpfc_io_buf *)pwqeIn->context1; struct lpfc_vport *vport = pwqeIn->vport; struct nvmefc_fcp_req *nCmd; struct nvme_fc_ersp_iu *ep; @@ -1181,7 +1181,7 @@ out_err: **/ static int lpfc_nvme_prep_io_cmd(struct lpfc_vport *vport, - struct lpfc_nvme_buf *lpfc_ncmd, + struct lpfc_io_buf *lpfc_ncmd, struct lpfc_nodelist *pnode, struct lpfc_fc4_ctrl_stat *cstat) { @@ -1287,7 +1287,7 @@ lpfc_nvme_prep_io_cmd(struct lpfc_vport *vport, **/ static int lpfc_nvme_prep_io_dma(struct lpfc_vport *vport, - struct lpfc_nvme_buf *lpfc_ncmd) + struct lpfc_io_buf *lpfc_ncmd) { struct lpfc_hba *phba = vport->phba; struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd; @@ -1428,7 +1428,7 @@ lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport, struct lpfc_vport *vport; struct lpfc_hba *phba; struct lpfc_nodelist *ndlp; - struct lpfc_nvme_buf *lpfc_ncmd; + struct lpfc_io_buf *lpfc_ncmd; struct lpfc_nvme_rport *rport; struct lpfc_nvme_qhandle *lpfc_queue_info; struct lpfc_nvme_fcpreq_priv *freqpriv; @@ -1616,6 +1616,9 @@ lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport, goto out_free_nvme_buf; } + if (phba->cfg_xri_rebalancing) + lpfc_keep_pvt_pool_above_lowwm(phba, lpfc_ncmd->hdwq_no); + #ifdef CONFIG_SCSI_LPFC_DEBUG_FS if (lpfc_ncmd->ts_cmd_start) lpfc_ncmd->ts_cmd_wqput = ktime_get_ns(); @@ -1704,7 +1707,7 @@ lpfc_nvme_fcp_abort(struct nvme_fc_local_port *pnvme_lport, struct lpfc_nvme_lport *lport; struct lpfc_vport *vport; struct lpfc_hba *phba; - struct lpfc_nvme_buf *lpfc_nbuf; + struct lpfc_io_buf *lpfc_nbuf; struct lpfc_iocbq *abts_buf; struct lpfc_iocbq *nvmereq_wqe; struct lpfc_nvme_fcpreq_priv *freqpriv; @@ -1911,22 +1914,6 @@ static struct nvme_fc_port_template lpfc_nvme_template = { .fcprqst_priv_sz = sizeof(struct lpfc_nvme_fcpreq_priv), }; -static inline struct lpfc_nvme_buf * -lpfc_nvme_buf(struct lpfc_hba *phba, int idx) -{ - struct lpfc_sli4_hdw_queue *qp; - struct lpfc_nvme_buf *lpfc_ncmd, *lpfc_ncmd_next; - - qp = &phba->sli4_hba.hdwq[idx]; - list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next, - &qp->lpfc_io_buf_list_get, list) { - list_del_init(&lpfc_ncmd->list); - qp->get_io_bufs--; - return lpfc_ncmd; - } - return NULL; -} - /** * lpfc_get_nvme_buf - Get a nvme buffer from io_buf_list of the HBA * @phba: The HBA for which this call is being executed. @@ -1938,34 +1925,17 @@ lpfc_nvme_buf(struct lpfc_hba *phba, int idx) * NULL - Error * Pointer to lpfc_nvme_buf - Success **/ -static struct lpfc_nvme_buf * +static struct lpfc_io_buf * lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, int idx, int expedite) { - struct lpfc_nvme_buf *lpfc_ncmd = NULL; + struct lpfc_io_buf *lpfc_ncmd; struct lpfc_sli4_hdw_queue *qp; struct sli4_sge *sgl; struct lpfc_iocbq *pwqeq; union lpfc_wqe128 *wqe; - unsigned long iflag = 0; - qp = &phba->sli4_hba.hdwq[idx]; - spin_lock_irqsave(&qp->io_buf_list_get_lock, iflag); - if (qp->get_io_bufs > LPFC_NVME_EXPEDITE_XRICNT || expedite) - lpfc_ncmd = lpfc_nvme_buf(phba, idx); - if (!lpfc_ncmd) { - spin_lock(&qp->io_buf_list_put_lock); - list_splice(&qp->lpfc_io_buf_list_put, - &qp->lpfc_io_buf_list_get); - qp->get_io_bufs += qp->put_io_bufs; - INIT_LIST_HEAD(&qp->lpfc_io_buf_list_put); - qp->put_io_bufs = 0; - spin_unlock(&qp->io_buf_list_put_lock); - if (qp->get_io_bufs > LPFC_NVME_EXPEDITE_XRICNT || - expedite) - lpfc_ncmd = lpfc_nvme_buf(phba, idx); - } - spin_unlock_irqrestore(&qp->io_buf_list_get_lock, iflag); + lpfc_ncmd = lpfc_get_io_buf(phba, NULL, idx, expedite); if (lpfc_ncmd) { pwqeq = &(lpfc_ncmd->cur_iocbq); @@ -1978,8 +1948,6 @@ lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, pwqeq->wqe_cmpl = lpfc_nvme_io_cmd_wqe_cmpl; lpfc_ncmd->start_time = jiffies; lpfc_ncmd->flags = 0; - lpfc_ncmd->hdwq = qp; - lpfc_ncmd->hdwq_no = idx; /* Rsp SGE will be filled in when we rcv an IO * from the NVME Layer to be sent. @@ -1996,11 +1964,13 @@ lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, if (lpfc_ndlp_check_qdepth(phba, ndlp)) { atomic_inc(&ndlp->cmd_pending); - lpfc_ncmd->flags |= LPFC_BUMP_QDEPTH; + lpfc_ncmd->flags |= LPFC_SBUF_BUMP_QDEPTH; } - } else + } else { + qp = &phba->sli4_hba.hdwq[idx]; qp->empty_io_bufs++; + } return lpfc_ncmd; } @@ -2016,16 +1986,16 @@ lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, * aborted. **/ static void -lpfc_release_nvme_buf(struct lpfc_hba *phba, struct lpfc_nvme_buf *lpfc_ncmd) +lpfc_release_nvme_buf(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_ncmd) { struct lpfc_sli4_hdw_queue *qp; unsigned long iflag = 0; - if ((lpfc_ncmd->flags & LPFC_BUMP_QDEPTH) && lpfc_ncmd->ndlp) + if ((lpfc_ncmd->flags & LPFC_SBUF_BUMP_QDEPTH) && lpfc_ncmd->ndlp) atomic_dec(&lpfc_ncmd->ndlp->cmd_pending); lpfc_ncmd->ndlp = NULL; - lpfc_ncmd->flags &= ~LPFC_BUMP_QDEPTH; + lpfc_ncmd->flags &= ~LPFC_SBUF_BUMP_QDEPTH; qp = lpfc_ncmd->hdwq; if (lpfc_ncmd->flags & LPFC_SBUF_XBUSY) { @@ -2040,17 +2010,8 @@ lpfc_release_nvme_buf(struct lpfc_hba *phba, struct lpfc_nvme_buf *lpfc_ncmd) &qp->lpfc_abts_nvme_buf_list); qp->abts_nvme_io_bufs++; spin_unlock_irqrestore(&qp->abts_nvme_buf_list_lock, iflag); - } else { - /* MUST zero fields if buffer is reused by another protocol */ - lpfc_ncmd->nvmeCmd = NULL; - lpfc_ncmd->cur_iocbq.wqe_cmpl = NULL; - - spin_lock_irqsave(&qp->io_buf_list_put_lock, iflag); - list_add_tail(&lpfc_ncmd->list, - &qp->lpfc_io_buf_list_put); - qp->put_io_bufs++; - spin_unlock_irqrestore(&qp->io_buf_list_put_lock, iflag); - } + } else + lpfc_release_io_buf(phba, (struct lpfc_io_buf *)lpfc_ncmd, qp); } /** @@ -2510,7 +2471,7 @@ lpfc_sli4_nvme_xri_aborted(struct lpfc_hba *phba, struct sli4_wcqe_xri_aborted *axri, int idx) { uint16_t xri = bf_get(lpfc_wcqe_xa_xri, axri); - struct lpfc_nvme_buf *lpfc_ncmd, *next_lpfc_ncmd; + struct lpfc_io_buf *lpfc_ncmd, *next_lpfc_ncmd; struct nvmefc_fcp_req *nvme_cmd = NULL; struct lpfc_nodelist *ndlp; struct lpfc_sli4_hdw_queue *qp; diff --git a/drivers/scsi/lpfc/lpfc_nvme.h b/drivers/scsi/lpfc/lpfc_nvme.h index 13f900dd4eeb..07d5fa16dd39 100644 --- a/drivers/scsi/lpfc/lpfc_nvme.h +++ b/drivers/scsi/lpfc/lpfc_nvme.h @@ -71,49 +71,6 @@ struct lpfc_nvme_rport { struct completion rport_unreg_done; }; -struct lpfc_nvme_buf { - /* Common fields */ - struct list_head list; - void *data; - dma_addr_t dma_handle; - dma_addr_t dma_phys_sgl; - struct sli4_sge *dma_sgl; - struct lpfc_iocbq cur_iocbq; - struct lpfc_sli4_hdw_queue *hdwq; - uint16_t hdwq_no; - uint16_t cpu; - - /* NVME specific fields */ - struct nvmefc_fcp_req *nvmeCmd; - struct lpfc_nodelist *ndlp; - - uint32_t timeout; - - uint16_t flags; /* TBD convert exch_busy to flags */ -#define LPFC_SBUF_XBUSY 0x1 /* SLI4 hba reported XB on WCQE cmpl */ -#define LPFC_BUMP_QDEPTH 0x2 /* bumped queue depth counter */ - uint16_t exch_busy; /* SLI4 hba reported XB on complete WCQE */ - uint16_t status; /* From IOCB Word 7- ulpStatus */ - uint32_t result; /* From IOCB Word 4. */ - - uint32_t seg_cnt; /* Number of scatter-gather segments returned by - * dma_map_sg. The driver needs this for calls - * to dma_unmap_sg. - */ - wait_queue_head_t *waitq; - unsigned long start_time; - - uint16_t qidx; - -#ifdef CONFIG_SCSI_LPFC_DEBUG_FS - uint64_t ts_cmd_start; - uint64_t ts_last_cmd; - uint64_t ts_cmd_wqput; - uint64_t ts_isr_cmpl; - uint64_t ts_data_nvme; -#endif -}; - struct lpfc_nvme_fcpreq_priv { - struct lpfc_nvme_buf *nvme_buf; + struct lpfc_io_buf *nvme_buf; }; diff --git a/drivers/scsi/lpfc/lpfc_scsi.c b/drivers/scsi/lpfc/lpfc_scsi.c index 79a3765bdd9b..c824ed3be4f9 100644 --- a/drivers/scsi/lpfc/lpfc_scsi.c +++ b/drivers/scsi/lpfc/lpfc_scsi.c @@ -83,9 +83,9 @@ lpfc_rport_data_from_scsi_device(struct scsi_device *sdev) } static void -lpfc_release_scsi_buf_s4(struct lpfc_hba *phba, struct lpfc_scsi_buf *psb); +lpfc_release_scsi_buf_s4(struct lpfc_hba *phba, struct lpfc_io_buf *psb); static void -lpfc_release_scsi_buf_s3(struct lpfc_hba *phba, struct lpfc_scsi_buf *psb); +lpfc_release_scsi_buf_s3(struct lpfc_hba *phba, struct lpfc_io_buf *psb); static int lpfc_prot_group_type(struct lpfc_hba *phba, struct scsi_cmnd *sc); @@ -180,7 +180,7 @@ lpfc_cmd_guard_csum(struct scsi_cmnd *sc) **/ static void lpfc_sli4_set_rsp_sgl_last(struct lpfc_hba *phba, - struct lpfc_scsi_buf *lpfc_cmd) + struct lpfc_io_buf *lpfc_cmd) { struct sli4_sge *sgl = (struct sli4_sge *)lpfc_cmd->dma_sgl; if (sgl) { @@ -200,7 +200,7 @@ lpfc_sli4_set_rsp_sgl_last(struct lpfc_hba *phba, * function updates the statistical data for the command completion. **/ static void -lpfc_update_stats(struct lpfc_hba *phba, struct lpfc_scsi_buf *lpfc_cmd) +lpfc_update_stats(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_cmd) { struct lpfc_rport_data *rdata; struct lpfc_nodelist *pnode; @@ -389,7 +389,7 @@ static int lpfc_new_scsi_buf_s3(struct lpfc_vport *vport, int num_to_alloc) { struct lpfc_hba *phba = vport->phba; - struct lpfc_scsi_buf *psb; + struct lpfc_io_buf *psb; struct ulp_bde64 *bpl; IOCB_t *iocb; dma_addr_t pdma_phys_fcp_cmd; @@ -408,7 +408,7 @@ lpfc_new_scsi_buf_s3(struct lpfc_vport *vport, int num_to_alloc) (int)sizeof(struct fcp_rsp), bpl_size); for (bcnt = 0; bcnt < num_to_alloc; bcnt++) { - psb = kzalloc(sizeof(struct lpfc_scsi_buf), GFP_KERNEL); + psb = kzalloc(sizeof(struct lpfc_io_buf), GFP_KERNEL); if (!psb) break; @@ -442,7 +442,7 @@ lpfc_new_scsi_buf_s3(struct lpfc_vport *vport, int num_to_alloc) sizeof(struct fcp_rsp); /* Initialize local short-hand pointers. */ - bpl = psb->dma_sgl; + bpl = (struct ulp_bde64 *)psb->dma_sgl; pdma_phys_fcp_cmd = psb->dma_handle; pdma_phys_fcp_rsp = psb->dma_handle + sizeof(struct fcp_cmnd); pdma_phys_sgl = psb->dma_handle + sizeof(struct fcp_cmnd) + @@ -524,7 +524,7 @@ void lpfc_sli4_vport_delete_fcp_xri_aborted(struct lpfc_vport *vport) { struct lpfc_hba *phba = vport->phba; - struct lpfc_scsi_buf *psb, *next_psb; + struct lpfc_io_buf *psb, *next_psb; struct lpfc_sli4_hdw_queue *qp; unsigned long iflag = 0; int idx; @@ -562,7 +562,7 @@ lpfc_sli4_fcp_xri_aborted(struct lpfc_hba *phba, { uint16_t xri = bf_get(lpfc_wcqe_xa_xri, axri); uint16_t rxid = bf_get(lpfc_wcqe_xa_remote_xid, axri); - struct lpfc_scsi_buf *psb, *next_psb; + struct lpfc_io_buf *psb, *next_psb; struct lpfc_sli4_hdw_queue *qp; unsigned long iflag = 0; struct lpfc_iocbq *iocbq; @@ -613,7 +613,7 @@ lpfc_sli4_fcp_xri_aborted(struct lpfc_hba *phba, continue; if (iocbq->sli4_xritag != xri) continue; - psb = container_of(iocbq, struct lpfc_scsi_buf, cur_iocbq); + psb = container_of(iocbq, struct lpfc_io_buf, cur_iocbq); psb->exch_busy = 0; spin_unlock_irqrestore(&phba->hbalock, iflag); if (!list_empty(&pring->txq)) @@ -635,16 +635,16 @@ lpfc_sli4_fcp_xri_aborted(struct lpfc_hba *phba, * NULL - Error * Pointer to lpfc_scsi_buf - Success **/ -static struct lpfc_scsi_buf* +static struct lpfc_io_buf * lpfc_get_scsi_buf_s3(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, struct scsi_cmnd *cmnd) { - struct lpfc_scsi_buf * lpfc_cmd = NULL; + struct lpfc_io_buf *lpfc_cmd = NULL; struct list_head *scsi_buf_list_get = &phba->lpfc_scsi_buf_list_get; unsigned long iflag = 0; spin_lock_irqsave(&phba->scsi_buf_list_get_lock, iflag); - list_remove_head(scsi_buf_list_get, lpfc_cmd, struct lpfc_scsi_buf, + list_remove_head(scsi_buf_list_get, lpfc_cmd, struct lpfc_io_buf, list); if (!lpfc_cmd) { spin_lock(&phba->scsi_buf_list_put_lock); @@ -652,7 +652,7 @@ lpfc_get_scsi_buf_s3(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, &phba->lpfc_scsi_buf_list_get); INIT_LIST_HEAD(&phba->lpfc_scsi_buf_list_put); list_remove_head(scsi_buf_list_get, lpfc_cmd, - struct lpfc_scsi_buf, list); + struct lpfc_io_buf, list); spin_unlock(&phba->scsi_buf_list_put_lock); } spin_unlock_irqrestore(&phba->scsi_buf_list_get_lock, iflag); @@ -674,19 +674,17 @@ lpfc_get_scsi_buf_s3(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, * NULL - Error * Pointer to lpfc_scsi_buf - Success **/ -static struct lpfc_scsi_buf* +static struct lpfc_io_buf * lpfc_get_scsi_buf_s4(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, struct scsi_cmnd *cmnd) { - struct lpfc_scsi_buf *lpfc_cmd, *lpfc_cmd_next; + struct lpfc_io_buf *lpfc_cmd; struct lpfc_sli4_hdw_queue *qp; - unsigned long iflag = 0; struct sli4_sge *sgl; IOCB_t *iocb; dma_addr_t pdma_phys_fcp_rsp; dma_addr_t pdma_phys_fcp_cmd; uint32_t sgl_size, cpu, idx; - int found = 0; int tag; cpu = smp_processor_id(); @@ -700,40 +698,10 @@ lpfc_get_scsi_buf_s4(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, idx = cpu % phba->cfg_hdw_queue; } - qp = &phba->sli4_hba.hdwq[idx]; - spin_lock_irqsave(&qp->io_buf_list_get_lock, iflag); - list_for_each_entry_safe(lpfc_cmd, lpfc_cmd_next, - &qp->lpfc_io_buf_list_get, list) { - if (lpfc_test_rrq_active(phba, ndlp, - lpfc_cmd->cur_iocbq.sli4_lxritag)) - continue; - list_del_init(&lpfc_cmd->list); - qp->get_io_bufs--; - found = 1; - break; - } - if (!found) { - spin_lock(&qp->io_buf_list_put_lock); - list_splice(&qp->lpfc_io_buf_list_put, - &qp->lpfc_io_buf_list_get); - qp->get_io_bufs += qp->put_io_bufs; - INIT_LIST_HEAD(&qp->lpfc_io_buf_list_put); - qp->put_io_bufs = 0; - spin_unlock(&qp->io_buf_list_put_lock); - list_for_each_entry_safe(lpfc_cmd, lpfc_cmd_next, - &qp->lpfc_io_buf_list_get, - list) { - if (lpfc_test_rrq_active( - phba, ndlp, lpfc_cmd->cur_iocbq.sli4_lxritag)) - continue; - list_del_init(&lpfc_cmd->list); - qp->get_io_bufs--; - found = 1; - break; - } - } - spin_unlock_irqrestore(&qp->io_buf_list_get_lock, iflag); - if (!found) { + lpfc_cmd = lpfc_get_io_buf(phba, ndlp, idx, + !phba->cfg_xri_rebalancing); + if (!lpfc_cmd) { + qp = &phba->sli4_hba.hdwq[idx]; qp->empty_io_bufs++; return NULL; } @@ -756,9 +724,6 @@ lpfc_get_scsi_buf_s4(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, #ifdef CONFIG_SCSI_LPFC_DEBUG_FS lpfc_cmd->prot_data_type = 0; #endif - lpfc_cmd->hdwq = qp; - lpfc_cmd->hdwq_no = idx; - lpfc_cmd->fcp_cmnd = (lpfc_cmd->data + sgl_size); lpfc_cmd->fcp_rsp = (struct fcp_rsp *)((uint8_t *)lpfc_cmd->fcp_cmnd + sizeof(struct fcp_cmnd)); @@ -789,7 +754,7 @@ lpfc_get_scsi_buf_s4(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, /* * Since the IOCB for the FCP I/O is built into this - * lpfc_scsi_buf, initialize it with all known data now. + * lpfc_io_buf, initialize it with all known data now. */ iocb = &lpfc_cmd->cur_iocbq.iocb; iocb->un.fcpi64.bdl.ulpIoTag32 = 0; @@ -822,7 +787,7 @@ lpfc_get_scsi_buf_s4(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, * NULL - Error * Pointer to lpfc_scsi_buf - Success **/ -static struct lpfc_scsi_buf* +static struct lpfc_io_buf* lpfc_get_scsi_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, struct scsi_cmnd *cmnd) { @@ -838,7 +803,7 @@ lpfc_get_scsi_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, * lpfc_scsi_buf_list list. **/ static void -lpfc_release_scsi_buf_s3(struct lpfc_hba *phba, struct lpfc_scsi_buf *psb) +lpfc_release_scsi_buf_s3(struct lpfc_hba *phba, struct lpfc_io_buf *psb) { unsigned long iflag = 0; @@ -863,7 +828,7 @@ lpfc_release_scsi_buf_s3(struct lpfc_hba *phba, struct lpfc_scsi_buf *psb) * aborted. **/ static void -lpfc_release_scsi_buf_s4(struct lpfc_hba *phba, struct lpfc_scsi_buf *psb) +lpfc_release_scsi_buf_s4(struct lpfc_hba *phba, struct lpfc_io_buf *psb) { struct lpfc_sli4_hdw_queue *qp; unsigned long iflag = 0; @@ -879,14 +844,7 @@ lpfc_release_scsi_buf_s4(struct lpfc_hba *phba, struct lpfc_scsi_buf *psb) qp->abts_scsi_io_bufs++; spin_unlock_irqrestore(&qp->abts_scsi_buf_list_lock, iflag); } else { - /* MUST zero fields if buffer is reused by another protocol */ - psb->pCmd = NULL; - psb->cur_iocbq.iocb_cmpl = NULL; - - spin_lock_irqsave(&qp->io_buf_list_put_lock, iflag); - list_add_tail(&psb->list, &qp->lpfc_io_buf_list_put); - qp->put_io_bufs++; - spin_unlock_irqrestore(&qp->io_buf_list_put_lock, iflag); + lpfc_release_io_buf(phba, (struct lpfc_io_buf *)psb, qp); } } @@ -899,7 +857,7 @@ lpfc_release_scsi_buf_s4(struct lpfc_hba *phba, struct lpfc_scsi_buf *psb) * lpfc_scsi_buf_list list. **/ static void -lpfc_release_scsi_buf(struct lpfc_hba *phba, struct lpfc_scsi_buf *psb) +lpfc_release_scsi_buf(struct lpfc_hba *phba, struct lpfc_io_buf *psb) { if ((psb->flags & LPFC_SBUF_BUMP_QDEPTH) && psb->ndlp) atomic_dec(&psb->ndlp->cmd_pending); @@ -923,12 +881,12 @@ lpfc_release_scsi_buf(struct lpfc_hba *phba, struct lpfc_scsi_buf *psb) * 0 - Success **/ static int -lpfc_scsi_prep_dma_buf_s3(struct lpfc_hba *phba, struct lpfc_scsi_buf *lpfc_cmd) +lpfc_scsi_prep_dma_buf_s3(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_cmd) { struct scsi_cmnd *scsi_cmnd = lpfc_cmd->pCmd; struct scatterlist *sgel = NULL; struct fcp_cmnd *fcp_cmnd = lpfc_cmd->fcp_cmnd; - struct ulp_bde64 *bpl = lpfc_cmd->dma_sgl; + struct ulp_bde64 *bpl = (struct ulp_bde64 *)lpfc_cmd->dma_sgl; struct lpfc_iocbq *iocbq = &lpfc_cmd->cur_iocbq; IOCB_t *iocb_cmd = &lpfc_cmd->cur_iocbq.iocb; struct ulp_bde64 *data_bde = iocb_cmd->unsli3.fcp_ext.dbde; @@ -1075,7 +1033,7 @@ lpfc_bg_err_inject(struct lpfc_hba *phba, struct scsi_cmnd *sc, uint32_t *reftag, uint16_t *apptag, uint32_t new_guard) { struct scatterlist *sgpe; /* s/g prot entry */ - struct lpfc_scsi_buf *lpfc_cmd = NULL; + struct lpfc_io_buf *lpfc_cmd = NULL; struct scsi_dif_tuple *src = NULL; struct lpfc_nodelist *ndlp; struct lpfc_rport_data *rdata; @@ -1134,7 +1092,7 @@ lpfc_bg_err_inject(struct lpfc_hba *phba, struct scsi_cmnd *sc, if (sgpe) { src = (struct scsi_dif_tuple *)sg_virt(sgpe); src += blockoff; - lpfc_cmd = (struct lpfc_scsi_buf *)sc->host_scribble; + lpfc_cmd = (struct lpfc_io_buf *)sc->host_scribble; } /* Should we change the Reference Tag */ @@ -2439,7 +2397,7 @@ lpfc_prot_group_type(struct lpfc_hba *phba, struct scsi_cmnd *sc) **/ static int lpfc_bg_scsi_adjust_dl(struct lpfc_hba *phba, - struct lpfc_scsi_buf *lpfc_cmd) + struct lpfc_io_buf *lpfc_cmd) { struct scsi_cmnd *sc = lpfc_cmd->pCmd; int fcpdl; @@ -2479,11 +2437,11 @@ lpfc_bg_scsi_adjust_dl(struct lpfc_hba *phba, **/ static int lpfc_bg_scsi_prep_dma_buf_s3(struct lpfc_hba *phba, - struct lpfc_scsi_buf *lpfc_cmd) + struct lpfc_io_buf *lpfc_cmd) { struct scsi_cmnd *scsi_cmnd = lpfc_cmd->pCmd; struct fcp_cmnd *fcp_cmnd = lpfc_cmd->fcp_cmnd; - struct ulp_bde64 *bpl = lpfc_cmd->dma_sgl; + struct ulp_bde64 *bpl = (struct ulp_bde64 *)lpfc_cmd->dma_sgl; IOCB_t *iocb_cmd = &lpfc_cmd->cur_iocbq.iocb; uint32_t num_bde = 0; int datasegcnt, protsegcnt, datadir = scsi_cmnd->sc_data_direction; @@ -2659,7 +2617,7 @@ lpfc_bg_csum(uint8_t *data, int count) * what type of T10-DIF error occurred. */ static void -lpfc_calc_bg_err(struct lpfc_hba *phba, struct lpfc_scsi_buf *lpfc_cmd) +lpfc_calc_bg_err(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_cmd) { struct scatterlist *sgpe; /* s/g prot entry */ struct scatterlist *sgde; /* s/g data entry */ @@ -2844,8 +2802,8 @@ out: * -1 - Internal error (bad profile, ...etc) */ static int -lpfc_parse_bg_err(struct lpfc_hba *phba, struct lpfc_scsi_buf *lpfc_cmd, - struct lpfc_iocbq *pIocbOut) +lpfc_parse_bg_err(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_cmd, + struct lpfc_iocbq *pIocbOut) { struct scsi_cmnd *cmd = lpfc_cmd->pCmd; struct sli3_bg_fields *bgf = &pIocbOut->iocb.unsli3.sli3_bg; @@ -3011,7 +2969,7 @@ out: * 0 - Success **/ static int -lpfc_scsi_prep_dma_buf_s4(struct lpfc_hba *phba, struct lpfc_scsi_buf *lpfc_cmd) +lpfc_scsi_prep_dma_buf_s4(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_cmd) { struct scsi_cmnd *scsi_cmnd = lpfc_cmd->pCmd; struct scatterlist *sgel = NULL; @@ -3143,6 +3101,7 @@ lpfc_scsi_prep_dma_buf_s4(struct lpfc_hba *phba, struct lpfc_scsi_buf *lpfc_cmd) lpfc_cmd->cur_iocbq.priority = ((struct lpfc_device_data *) scsi_cmnd->device->hostdata)->priority; } + return 0; } @@ -3157,7 +3116,7 @@ lpfc_scsi_prep_dma_buf_s4(struct lpfc_hba *phba, struct lpfc_scsi_buf *lpfc_cmd) **/ static int lpfc_bg_scsi_prep_dma_buf_s4(struct lpfc_hba *phba, - struct lpfc_scsi_buf *lpfc_cmd) + struct lpfc_io_buf *lpfc_cmd) { struct scsi_cmnd *scsi_cmnd = lpfc_cmd->pCmd; struct fcp_cmnd *fcp_cmnd = lpfc_cmd->fcp_cmnd; @@ -3333,7 +3292,7 @@ err: * 0 - Success **/ static inline int -lpfc_scsi_prep_dma_buf(struct lpfc_hba *phba, struct lpfc_scsi_buf *lpfc_cmd) +lpfc_scsi_prep_dma_buf(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_cmd) { return phba->lpfc_scsi_prep_dma_buf(phba, lpfc_cmd); } @@ -3352,7 +3311,7 @@ lpfc_scsi_prep_dma_buf(struct lpfc_hba *phba, struct lpfc_scsi_buf *lpfc_cmd) * 0 - Success **/ static inline int -lpfc_bg_scsi_prep_dma_buf(struct lpfc_hba *phba, struct lpfc_scsi_buf *lpfc_cmd) +lpfc_bg_scsi_prep_dma_buf(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_cmd) { return phba->lpfc_bg_scsi_prep_dma_buf(phba, lpfc_cmd); } @@ -3369,7 +3328,7 @@ lpfc_bg_scsi_prep_dma_buf(struct lpfc_hba *phba, struct lpfc_scsi_buf *lpfc_cmd) **/ static void lpfc_send_scsi_error_event(struct lpfc_hba *phba, struct lpfc_vport *vport, - struct lpfc_scsi_buf *lpfc_cmd, struct lpfc_iocbq *rsp_iocb) { + struct lpfc_io_buf *lpfc_cmd, struct lpfc_iocbq *rsp_iocb) { struct scsi_cmnd *cmnd = lpfc_cmd->pCmd; struct fcp_rsp *fcprsp = lpfc_cmd->fcp_rsp; uint32_t resp_info = fcprsp->rspStatus2; @@ -3461,7 +3420,7 @@ lpfc_send_scsi_error_event(struct lpfc_hba *phba, struct lpfc_vport *vport, * field of @lpfc_cmd for device with SLI-3 interface spec. **/ static void -lpfc_scsi_unprep_dma_buf(struct lpfc_hba *phba, struct lpfc_scsi_buf *psb) +lpfc_scsi_unprep_dma_buf(struct lpfc_hba *phba, struct lpfc_io_buf *psb) { /* * There are only two special cases to consider. (1) the scsi command @@ -3480,7 +3439,7 @@ lpfc_scsi_unprep_dma_buf(struct lpfc_hba *phba, struct lpfc_scsi_buf *psb) /** * lpfc_handler_fcp_err - FCP response handler * @vport: The virtual port for which this call is being executed. - * @lpfc_cmd: Pointer to lpfc_scsi_buf data structure. + * @lpfc_cmd: Pointer to lpfc_io_buf data structure. * @rsp_iocb: The response IOCB which contains FCP error. * * This routine is called to process response IOCB with status field @@ -3488,7 +3447,7 @@ lpfc_scsi_unprep_dma_buf(struct lpfc_hba *phba, struct lpfc_scsi_buf *psb) * based upon SCSI and FCP error. **/ static void -lpfc_handle_fcp_err(struct lpfc_vport *vport, struct lpfc_scsi_buf *lpfc_cmd, +lpfc_handle_fcp_err(struct lpfc_vport *vport, struct lpfc_io_buf *lpfc_cmd, struct lpfc_iocbq *rsp_iocb) { struct lpfc_hba *phba = vport->phba; @@ -3680,8 +3639,8 @@ static void lpfc_scsi_cmd_iocb_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pIocbIn, struct lpfc_iocbq *pIocbOut) { - struct lpfc_scsi_buf *lpfc_cmd = - (struct lpfc_scsi_buf *) pIocbIn->context1; + struct lpfc_io_buf *lpfc_cmd = + (struct lpfc_io_buf *) pIocbIn->context1; struct lpfc_vport *vport = pIocbIn->vport; struct lpfc_rport_data *rdata = lpfc_cmd->rdata; struct lpfc_nodelist *pnode = rdata->pnode; @@ -3949,7 +3908,7 @@ lpfc_fcpcmd_to_iocb(uint8_t *data, struct fcp_cmnd *fcp_cmnd) * to transfer for device with SLI3 interface spec. **/ static void -lpfc_scsi_prep_cmnd(struct lpfc_vport *vport, struct lpfc_scsi_buf *lpfc_cmd, +lpfc_scsi_prep_cmnd(struct lpfc_vport *vport, struct lpfc_io_buf *lpfc_cmd, struct lpfc_nodelist *pnode) { struct lpfc_hba *phba = vport->phba; @@ -4052,7 +4011,7 @@ lpfc_scsi_prep_cmnd(struct lpfc_vport *vport, struct lpfc_scsi_buf *lpfc_cmd, /** * lpfc_scsi_prep_task_mgmt_cmd - Convert SLI3 scsi TM cmd to FCP info unit * @vport: The virtual port for which this call is being executed. - * @lpfc_cmd: Pointer to lpfc_scsi_buf data structure. + * @lpfc_cmd: Pointer to lpfc_io_buf data structure. * @lun: Logical unit number. * @task_mgmt_cmd: SCSI task management command. * @@ -4065,7 +4024,7 @@ lpfc_scsi_prep_cmnd(struct lpfc_vport *vport, struct lpfc_scsi_buf *lpfc_cmd, **/ static int lpfc_scsi_prep_task_mgmt_cmd(struct lpfc_vport *vport, - struct lpfc_scsi_buf *lpfc_cmd, + struct lpfc_io_buf *lpfc_cmd, uint64_t lun, uint8_t task_mgmt_cmd) { @@ -4174,8 +4133,8 @@ lpfc_tskmgmt_def_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocbq, struct lpfc_iocbq *rspiocbq) { - struct lpfc_scsi_buf *lpfc_cmd = - (struct lpfc_scsi_buf *) cmdiocbq->context1; + struct lpfc_io_buf *lpfc_cmd = + (struct lpfc_io_buf *) cmdiocbq->context1; if (lpfc_cmd) lpfc_release_scsi_buf(phba, lpfc_cmd); return; @@ -4374,7 +4333,7 @@ lpfc_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *cmnd) struct lpfc_hba *phba = vport->phba; struct lpfc_rport_data *rdata; struct lpfc_nodelist *ndlp; - struct lpfc_scsi_buf *lpfc_cmd; + struct lpfc_io_buf *lpfc_cmd; struct fc_rport *rport = starget_to_rport(scsi_target(cmnd->device)); int err, idx; @@ -4521,6 +4480,9 @@ lpfc_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *cmnd) lpfc_poll_rearm_timer(phba); } + if (phba->cfg_xri_rebalancing) + lpfc_keep_pvt_pool_above_lowwm(phba, lpfc_cmd->hdwq_no); + return 0; out_host_busy_free_buf: @@ -4569,7 +4531,7 @@ lpfc_abort_handler(struct scsi_cmnd *cmnd) struct lpfc_hba *phba = vport->phba; struct lpfc_iocbq *iocb; struct lpfc_iocbq *abtsiocb; - struct lpfc_scsi_buf *lpfc_cmd; + struct lpfc_io_buf *lpfc_cmd; IOCB_t *cmd, *icmd; int ret = SUCCESS, status = 0; struct lpfc_sli_ring *pring_s4 = NULL; @@ -4591,7 +4553,7 @@ lpfc_abort_handler(struct scsi_cmnd *cmnd) return FAILED; } - lpfc_cmd = (struct lpfc_scsi_buf *)cmnd->host_scribble; + lpfc_cmd = (struct lpfc_io_buf *)cmnd->host_scribble; if (!lpfc_cmd || !lpfc_cmd->pCmd) { spin_unlock_irqrestore(&phba->hbalock, flags); lpfc_printf_vlog(vport, KERN_WARNING, LOG_FCP, @@ -4621,7 +4583,7 @@ lpfc_abort_handler(struct scsi_cmnd *cmnd) return FAILED; } /* - * If pCmd field of the corresponding lpfc_scsi_buf structure + * If pCmd field of the corresponding lpfc_io_buf structure * points to a different SCSI command, then the driver has * already completed this command, but the midlayer did not * see the completion before the eh fired. Just return SUCCESS. @@ -4782,7 +4744,7 @@ lpfc_taskmgmt_name(uint8_t task_mgmt_cmd) /** * lpfc_check_fcp_rsp - check the returned fcp_rsp to see if task failed * @vport: The virtual port for which this call is being executed. - * @lpfc_cmd: Pointer to lpfc_scsi_buf data structure. + * @lpfc_cmd: Pointer to lpfc_io_buf data structure. * * This routine checks the FCP RSP INFO to see if the tsk mgmt command succeded * @@ -4791,7 +4753,7 @@ lpfc_taskmgmt_name(uint8_t task_mgmt_cmd) * 0x2002 - Success **/ static int -lpfc_check_fcp_rsp(struct lpfc_vport *vport, struct lpfc_scsi_buf *lpfc_cmd) +lpfc_check_fcp_rsp(struct lpfc_vport *vport, struct lpfc_io_buf *lpfc_cmd) { struct fcp_rsp *fcprsp = lpfc_cmd->fcp_rsp; uint32_t rsp_info; @@ -4866,7 +4828,7 @@ lpfc_send_taskmgmt(struct lpfc_vport *vport, struct scsi_cmnd *cmnd, uint8_t task_mgmt_cmd) { struct lpfc_hba *phba = vport->phba; - struct lpfc_scsi_buf *lpfc_cmd; + struct lpfc_io_buf *lpfc_cmd; struct lpfc_iocbq *iocbq; struct lpfc_iocbq *iocbqrsp; struct lpfc_rport_data *rdata; diff --git a/drivers/scsi/lpfc/lpfc_scsi.h b/drivers/scsi/lpfc/lpfc_scsi.h index 696898087244..9ee0ea4a1ef2 100644 --- a/drivers/scsi/lpfc/lpfc_scsi.h +++ b/drivers/scsi/lpfc/lpfc_scsi.h @@ -130,59 +130,6 @@ struct lpfc_scsicmd_bkt { uint32_t cmd_count; }; -struct lpfc_scsi_buf { - /* Common fields */ - struct list_head list; - void *data; - dma_addr_t dma_handle; - dma_addr_t dma_phys_sgl; - struct ulp_bde64 *dma_sgl; - struct lpfc_iocbq cur_iocbq; - struct lpfc_sli4_hdw_queue *hdwq; - uint16_t hdwq_no; - uint16_t cpu; - - /* SCSI specific fields */ - struct scsi_cmnd *pCmd; - struct lpfc_rport_data *rdata; - struct lpfc_nodelist *ndlp; - - uint32_t timeout; - - uint16_t flags; /* TBD convert exch_busy to flags */ -#define LPFC_SBUF_XBUSY 0x1 /* SLI4 hba reported XB on WCQE cmpl */ -#define LPFC_SBUF_BUMP_QDEPTH 0x2 /* bumped queue depth counter */ - uint16_t exch_busy; /* SLI4 hba reported XB on complete WCQE */ - uint16_t status; /* From IOCB Word 7- ulpStatus */ - uint32_t result; /* From IOCB Word 4. */ - - uint32_t seg_cnt; /* Number of scatter-gather segments returned by - * dma_map_sg. The driver needs this for calls - * to dma_unmap_sg. */ - uint32_t prot_seg_cnt; /* seg_cnt's counterpart for protection data */ - - /* - * data and dma_handle are the kernel virtual and bus address of the - * dma-able buffer containing the fcp_cmd, fcp_rsp and a scatter - * gather bde list that supports the sg_tablesize value. - */ - struct fcp_cmnd *fcp_cmnd; - struct fcp_rsp *fcp_rsp; - - wait_queue_head_t *waitq; - unsigned long start_time; - -#ifdef CONFIG_SCSI_LPFC_DEBUG_FS - /* Used to restore any changes to protection data for error injection */ - void *prot_data_segment; - uint32_t prot_data; - uint32_t prot_data_type; -#define LPFC_INJERR_REFTAG 1 -#define LPFC_INJERR_APPTAG 2 -#define LPFC_INJERR_GUARD 3 -#endif -}; - #define LPFC_SCSI_DMA_EXT_SIZE 264 #define LPFC_BPL_SIZE 1024 #define MDAC_DIRECT_CMD 0x22 diff --git a/drivers/scsi/lpfc/lpfc_sli.c b/drivers/scsi/lpfc/lpfc_sli.c index 90f039a41480..4443d0d43053 100644 --- a/drivers/scsi/lpfc/lpfc_sli.c +++ b/drivers/scsi/lpfc/lpfc_sli.c @@ -1133,14 +1133,14 @@ __lpfc_sli_get_els_sglq(struct lpfc_hba *phba, struct lpfc_iocbq *piocbq) struct list_head *lpfc_els_sgl_list = &phba->sli4_hba.lpfc_els_sgl_list; struct lpfc_sglq *sglq = NULL; struct lpfc_sglq *start_sglq = NULL; - struct lpfc_scsi_buf *lpfc_cmd; + struct lpfc_io_buf *lpfc_cmd; struct lpfc_nodelist *ndlp; int found = 0; lockdep_assert_held(&phba->hbalock); if (piocbq->iocb_flag & LPFC_IO_FCP) { - lpfc_cmd = (struct lpfc_scsi_buf *) piocbq->context1; + lpfc_cmd = (struct lpfc_io_buf *) piocbq->context1; ndlp = lpfc_cmd->rdata->pnode; } else if ((piocbq->iocb.ulpCommand == CMD_GEN_REQUEST64_CR) && !(piocbq->iocb_flag & LPFC_IO_LIBDFC)) { @@ -1596,6 +1596,7 @@ lpfc_sli_ringtxcmpl_put(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, list_add_tail(&piocb->list, &pring->txcmplq); piocb->iocb_flag |= LPFC_IO_ON_TXCMPLQ; + pring->txcmplq_cnt++; if ((unlikely(pring->ringno == LPFC_ELS_RING)) && (piocb->iocb.ulpCommand != CMD_ABORT_XRI_CN) && @@ -3008,6 +3009,7 @@ lpfc_sli_iocbq_lookup(struct lpfc_hba *phba, /* remove from txcmpl queue list */ list_del_init(&cmd_iocb->list); cmd_iocb->iocb_flag &= ~LPFC_IO_ON_TXCMPLQ; + pring->txcmplq_cnt--; return cmd_iocb; } } @@ -3045,6 +3047,7 @@ lpfc_sli_iocbq_lookup_by_tag(struct lpfc_hba *phba, /* remove from txcmpl queue list */ list_del_init(&cmd_iocb->list); cmd_iocb->iocb_flag &= ~LPFC_IO_ON_TXCMPLQ; + pring->txcmplq_cnt--; return cmd_iocb; } } @@ -7170,7 +7173,7 @@ lpfc_post_rq_buffer(struct lpfc_hba *phba, struct lpfc_queue *hrq, int lpfc_sli4_hba_setup(struct lpfc_hba *phba) { - int rc, i, cnt; + int rc, i, cnt, len; LPFC_MBOXQ_t *mboxq; struct lpfc_mqe *mqe; uint8_t *vpd; @@ -7648,6 +7651,25 @@ lpfc_sli4_hba_setup(struct lpfc_hba *phba) lpfc_sli_read_link_ste(phba); } + /* Don't post more new bufs if repost already recovered + * the nvme sgls. + */ + if (phba->nvmet_support == 0) { + if (phba->sli4_hba.io_xri_cnt == 0) { + len = lpfc_new_io_buf( + phba, phba->sli4_hba.io_xri_max); + if (len == 0) { + rc = -ENOMEM; + goto out_unset_queue; + } + + if (phba->cfg_xri_rebalancing) + lpfc_create_multixri_pools(phba); + } + } else { + phba->cfg_xri_rebalancing = 0; + } + /* Arm the CQs and then EQs on device */ lpfc_sli4_arm_cqeq_intr(phba); @@ -7727,18 +7749,21 @@ lpfc_sli4_hba_setup(struct lpfc_hba *phba) lpfc_printf_log(phba, KERN_ERR, LOG_INIT | LOG_SLI, "3104 Adapter failed to issue " "DOWN_LINK mbox cmd, rc:x%x\n", rc); - goto out_unset_queue; + goto out_io_buff_free; } } else if (phba->cfg_suppress_link_up == LPFC_INITIALIZE_LINK) { /* don't perform init_link on SLI4 FC port loopback test */ if (!(phba->link_flag & LS_LOOPBACK_MODE)) { rc = phba->lpfc_hba_init_link(phba, MBX_NOWAIT); if (rc) - goto out_unset_queue; + goto out_io_buff_free; } } mempool_free(mboxq, phba->mbox_mem_pool); return rc; +out_io_buff_free: + /* Free allocated IO Buffers */ + lpfc_io_free(phba); out_unset_queue: /* Unset all the queues set up in this routine when error out */ lpfc_sli4_queue_unset(phba); @@ -9472,7 +9497,7 @@ lpfc_sli4_iocb2wqe(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq, bf_set(wqe_pbde, &wqe->fcp_iwrite.wqe_com, 0); if (phba->fcp_embed_io) { - struct lpfc_scsi_buf *lpfc_cmd; + struct lpfc_io_buf *lpfc_cmd; struct sli4_sge *sgl; struct fcp_cmnd *fcp_cmnd; uint32_t *ptr; @@ -9536,7 +9561,7 @@ lpfc_sli4_iocb2wqe(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq, bf_set(wqe_pbde, &wqe->fcp_iread.wqe_com, 0); if (phba->fcp_embed_io) { - struct lpfc_scsi_buf *lpfc_cmd; + struct lpfc_io_buf *lpfc_cmd; struct sli4_sge *sgl; struct fcp_cmnd *fcp_cmnd; uint32_t *ptr; @@ -9593,7 +9618,7 @@ lpfc_sli4_iocb2wqe(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq, /* Note, word 10 is already initialized to 0 */ if (phba->fcp_embed_io) { - struct lpfc_scsi_buf *lpfc_cmd; + struct lpfc_io_buf *lpfc_cmd; struct sli4_sge *sgl; struct fcp_cmnd *fcp_cmnd; uint32_t *ptr; @@ -9994,7 +10019,7 @@ lpfc_sli_api_table_setup(struct lpfc_hba *phba, uint8_t dev_grp) struct lpfc_sli_ring * lpfc_sli4_calc_ring(struct lpfc_hba *phba, struct lpfc_iocbq *piocb) { - struct lpfc_scsi_buf *lpfc_cmd; + struct lpfc_io_buf *lpfc_cmd; if (piocb->iocb_flag & (LPFC_IO_FCP | LPFC_USE_FCPWQIDX)) { if (unlikely(!phba->sli4_hba.hdwq)) @@ -10004,7 +10029,7 @@ lpfc_sli4_calc_ring(struct lpfc_hba *phba, struct lpfc_iocbq *piocb) * be setup based on what work queue we used. */ if (!(piocb->iocb_flag & LPFC_USE_FCPWQIDX)) { - lpfc_cmd = (struct lpfc_scsi_buf *)piocb->context1; + lpfc_cmd = (struct lpfc_io_buf *)piocb->context1; piocb->hba_wqidx = lpfc_cmd->hdwq_no; } return phba->sli4_hba.hdwq[piocb->hba_wqidx].fcp_wq->pring; @@ -10494,6 +10519,7 @@ lpfc_sli4_queue_init(struct lpfc_hba *phba) pring = phba->sli4_hba.hdwq[i].fcp_wq->pring; pring->flag = 0; pring->ringno = LPFC_FCP_RING; + pring->txcmplq_cnt = 0; INIT_LIST_HEAD(&pring->txq); INIT_LIST_HEAD(&pring->txcmplq); INIT_LIST_HEAD(&pring->iocb_continueq); @@ -10502,6 +10528,7 @@ lpfc_sli4_queue_init(struct lpfc_hba *phba) pring = phba->sli4_hba.els_wq->pring; pring->flag = 0; pring->ringno = LPFC_ELS_RING; + pring->txcmplq_cnt = 0; INIT_LIST_HEAD(&pring->txq); INIT_LIST_HEAD(&pring->txcmplq); INIT_LIST_HEAD(&pring->iocb_continueq); @@ -10510,8 +10537,9 @@ lpfc_sli4_queue_init(struct lpfc_hba *phba) if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { for (i = 0; i < phba->cfg_hdw_queue; i++) { pring = phba->sli4_hba.hdwq[i].nvme_wq->pring; - pring->flag = 0; + pring->flag = 0; pring->ringno = LPFC_FCP_RING; + pring->txcmplq_cnt = 0; INIT_LIST_HEAD(&pring->txq); INIT_LIST_HEAD(&pring->txcmplq); INIT_LIST_HEAD(&pring->iocb_continueq); @@ -10520,6 +10548,7 @@ lpfc_sli4_queue_init(struct lpfc_hba *phba) pring = phba->sli4_hba.nvmels_wq->pring; pring->flag = 0; pring->ringno = LPFC_ELS_RING; + pring->txcmplq_cnt = 0; INIT_LIST_HEAD(&pring->txq); INIT_LIST_HEAD(&pring->txcmplq); INIT_LIST_HEAD(&pring->iocb_continueq); @@ -11433,7 +11462,7 @@ lpfc_sli_validate_fcp_iocb(struct lpfc_iocbq *iocbq, struct lpfc_vport *vport, uint16_t tgt_id, uint64_t lun_id, lpfc_ctx_cmd ctx_cmd) { - struct lpfc_scsi_buf *lpfc_cmd; + struct lpfc_io_buf *lpfc_cmd; int rc = 1; if (iocbq->vport != vport) @@ -11443,7 +11472,7 @@ lpfc_sli_validate_fcp_iocb(struct lpfc_iocbq *iocbq, struct lpfc_vport *vport, !(iocbq->iocb_flag & LPFC_IO_ON_TXCMPLQ)) return rc; - lpfc_cmd = container_of(iocbq, struct lpfc_scsi_buf, cur_iocbq); + lpfc_cmd = container_of(iocbq, struct lpfc_io_buf, cur_iocbq); if (lpfc_cmd->pCmd == NULL) return rc; @@ -11670,7 +11699,7 @@ lpfc_sli_abort_taskmgmt(struct lpfc_vport *vport, struct lpfc_sli_ring *pring, uint16_t tgt_id, uint64_t lun_id, lpfc_ctx_cmd cmd) { struct lpfc_hba *phba = vport->phba; - struct lpfc_scsi_buf *lpfc_cmd; + struct lpfc_io_buf *lpfc_cmd; struct lpfc_iocbq *abtsiocbq; struct lpfc_nodelist *ndlp; struct lpfc_iocbq *iocbq; @@ -11726,7 +11755,7 @@ lpfc_sli_abort_taskmgmt(struct lpfc_vport *vport, struct lpfc_sli_ring *pring, if (iocbq->iocb_flag & LPFC_IO_FOF) abtsiocbq->iocb_flag |= LPFC_IO_FOF; - lpfc_cmd = container_of(iocbq, struct lpfc_scsi_buf, cur_iocbq); + lpfc_cmd = container_of(iocbq, struct lpfc_io_buf, cur_iocbq); ndlp = lpfc_cmd->rdata->pnode; if (lpfc_is_link_up(phba) && @@ -11792,7 +11821,7 @@ lpfc_sli_wake_iocb_wait(struct lpfc_hba *phba, { wait_queue_head_t *pdone_q; unsigned long iflags; - struct lpfc_scsi_buf *lpfc_cmd; + struct lpfc_io_buf *lpfc_cmd; spin_lock_irqsave(&phba->hbalock, iflags); if (cmdiocbq->iocb_flag & LPFC_IO_WAKE_TMO) { @@ -11821,7 +11850,7 @@ lpfc_sli_wake_iocb_wait(struct lpfc_hba *phba, /* Set the exchange busy flag for task management commands */ if ((cmdiocbq->iocb_flag & LPFC_IO_FCP) && !(cmdiocbq->iocb_flag & LPFC_IO_LIBDFC)) { - lpfc_cmd = container_of(cmdiocbq, struct lpfc_scsi_buf, + lpfc_cmd = container_of(cmdiocbq, struct lpfc_io_buf, cur_iocbq); lpfc_cmd->exch_busy = rspiocbq->iocb_flag & LPFC_EXCHANGE_BUSY; } @@ -16627,7 +16656,7 @@ static int lpfc_sli4_post_io_sgl_block(struct lpfc_hba *phba, struct list_head *nblist, int count) { - struct lpfc_nvme_buf *lpfc_ncmd; + struct lpfc_io_buf *lpfc_ncmd; struct lpfc_mbx_post_uembed_sgl_page1 *sgl; struct sgl_page_pairs *sgl_pg_pairs; void *viraddr; @@ -16743,7 +16772,7 @@ int lpfc_sli4_post_io_sgl_list(struct lpfc_hba *phba, struct list_head *post_nblist, int sb_count) { - struct lpfc_nvme_buf *lpfc_ncmd, *lpfc_ncmd_next; + struct lpfc_io_buf *lpfc_ncmd, *lpfc_ncmd_next; int status, sgl_size; int post_cnt = 0, block_cnt = 0, num_posting = 0, num_posted = 0; dma_addr_t pdma_phys_sgl1; @@ -16801,11 +16830,13 @@ lpfc_sli4_post_io_sgl_list(struct lpfc_hba *phba, phba, lpfc_ncmd->dma_phys_sgl, pdma_phys_sgl1, cur_xritag); if (status) { - /* failure, put on abort nvme list */ - lpfc_ncmd->flags |= LPFC_SBUF_XBUSY; + /* Post error. Buffer unavailable. */ + lpfc_ncmd->flags |= + LPFC_SBUF_NOT_POSTED; } else { - /* success, put on NVME buffer list */ - lpfc_ncmd->flags &= ~LPFC_SBUF_XBUSY; + /* Post success. Bffer available. */ + lpfc_ncmd->flags &= + ~LPFC_SBUF_NOT_POSTED; lpfc_ncmd->status = IOSTAT_SUCCESS; num_posted++; } @@ -16832,13 +16863,13 @@ lpfc_sli4_post_io_sgl_list(struct lpfc_hba *phba, /* put posted NVME buffer-sgl posted on NVME buffer sgl list */ while (!list_empty(&blck_nblist)) { list_remove_head(&blck_nblist, lpfc_ncmd, - struct lpfc_nvme_buf, list); + struct lpfc_io_buf, list); if (status) { - /* failure, put on abort nvme list */ - lpfc_ncmd->flags |= LPFC_SBUF_XBUSY; + /* Post error. Mark buffer unavailable. */ + lpfc_ncmd->flags |= LPFC_SBUF_NOT_POSTED; } else { - /* success, put on NVME buffer list */ - lpfc_ncmd->flags &= ~LPFC_SBUF_XBUSY; + /* Post success, Mark buffer available. */ + lpfc_ncmd->flags &= ~LPFC_SBUF_NOT_POSTED; lpfc_ncmd->status = IOSTAT_SUCCESS; num_posted++; } @@ -19705,3 +19736,636 @@ lpfc_sli4_issue_wqe(struct lpfc_hba *phba, struct lpfc_sli4_hdw_queue *qp, } return WQE_ERROR; } + +#ifdef LPFC_MXP_STAT +/** + * lpfc_snapshot_mxp - Snapshot pbl, pvt and busy count + * @phba: pointer to lpfc hba data structure. + * @hwqid: belong to which HWQ. + * + * The purpose of this routine is to take a snapshot of pbl, pvt and busy count + * 15 seconds after a test case is running. + * + * The user should call lpfc_debugfs_multixripools_write before running a test + * case to clear stat_snapshot_taken. Then the user starts a test case. During + * test case is running, stat_snapshot_taken is incremented by 1 every time when + * this routine is called from heartbeat timer. When stat_snapshot_taken is + * equal to LPFC_MXP_SNAPSHOT_TAKEN, a snapshot is taken. + **/ +void lpfc_snapshot_mxp(struct lpfc_hba *phba, u32 hwqid) +{ + struct lpfc_sli4_hdw_queue *qp; + struct lpfc_multixri_pool *multixri_pool; + struct lpfc_pvt_pool *pvt_pool; + struct lpfc_pbl_pool *pbl_pool; + u32 txcmplq_cnt; + + qp = &phba->sli4_hba.hdwq[hwqid]; + multixri_pool = qp->p_multixri_pool; + if (!multixri_pool) + return; + + if (multixri_pool->stat_snapshot_taken == LPFC_MXP_SNAPSHOT_TAKEN) { + pvt_pool = &qp->p_multixri_pool->pvt_pool; + pbl_pool = &qp->p_multixri_pool->pbl_pool; + txcmplq_cnt = qp->fcp_wq->pring->txcmplq_cnt; + if (qp->nvme_wq) + txcmplq_cnt += qp->nvme_wq->pring->txcmplq_cnt; + + multixri_pool->stat_pbl_count = pbl_pool->count; + multixri_pool->stat_pvt_count = pvt_pool->count; + multixri_pool->stat_busy_count = txcmplq_cnt; + } + + multixri_pool->stat_snapshot_taken++; +} +#endif + +/** + * lpfc_adjust_pvt_pool_count - Adjust private pool count + * @phba: pointer to lpfc hba data structure. + * @hwqid: belong to which HWQ. + * + * This routine moves some XRIs from private to public pool when private pool + * is not busy. + **/ +void lpfc_adjust_pvt_pool_count(struct lpfc_hba *phba, u32 hwqid) +{ + struct lpfc_multixri_pool *multixri_pool; + u32 io_req_count; + u32 prev_io_req_count; + + multixri_pool = phba->sli4_hba.hdwq[hwqid].p_multixri_pool; + if (!multixri_pool) + return; + io_req_count = multixri_pool->io_req_count; + prev_io_req_count = multixri_pool->prev_io_req_count; + + if (prev_io_req_count != io_req_count) { + /* Private pool is busy */ + multixri_pool->prev_io_req_count = io_req_count; + } else { + /* Private pool is not busy. + * Move XRIs from private to public pool. + */ + lpfc_move_xri_pvt_to_pbl(phba, hwqid); + } +} + +/** + * lpfc_adjust_high_watermark - Adjust high watermark + * @phba: pointer to lpfc hba data structure. + * @hwqid: belong to which HWQ. + * + * This routine sets high watermark as number of outstanding XRIs, + * but make sure the new value is between xri_limit/2 and xri_limit. + **/ +void lpfc_adjust_high_watermark(struct lpfc_hba *phba, u32 hwqid) +{ + u32 new_watermark; + u32 watermark_max; + u32 watermark_min; + u32 xri_limit; + u32 txcmplq_cnt; + u32 abts_io_bufs; + struct lpfc_multixri_pool *multixri_pool; + struct lpfc_sli4_hdw_queue *qp; + + qp = &phba->sli4_hba.hdwq[hwqid]; + multixri_pool = qp->p_multixri_pool; + if (!multixri_pool) + return; + xri_limit = multixri_pool->xri_limit; + + watermark_max = xri_limit; + watermark_min = xri_limit / 2; + + txcmplq_cnt = qp->fcp_wq->pring->txcmplq_cnt; + abts_io_bufs = qp->abts_scsi_io_bufs; + if (qp->nvme_wq) { + txcmplq_cnt += qp->nvme_wq->pring->txcmplq_cnt; + abts_io_bufs += qp->abts_nvme_io_bufs; + } + + new_watermark = txcmplq_cnt + abts_io_bufs; + new_watermark = min(watermark_max, new_watermark); + new_watermark = max(watermark_min, new_watermark); + multixri_pool->pvt_pool.high_watermark = new_watermark; + +#ifdef LPFC_MXP_STAT + multixri_pool->stat_max_hwm = max(multixri_pool->stat_max_hwm, + new_watermark); +#endif +} + +/** + * lpfc_move_xri_pvt_to_pbl - Move some XRIs from private to public pool + * @phba: pointer to lpfc hba data structure. + * @hwqid: belong to which HWQ. + * + * This routine is called from hearbeat timer when pvt_pool is idle. + * All free XRIs are moved from private to public pool on hwqid with 2 steps. + * The first step moves (all - low_watermark) amount of XRIs. + * The second step moves the rest of XRIs. + **/ +void lpfc_move_xri_pvt_to_pbl(struct lpfc_hba *phba, u32 hwqid) +{ + struct lpfc_pbl_pool *pbl_pool; + struct lpfc_pvt_pool *pvt_pool; + struct lpfc_io_buf *lpfc_ncmd; + struct lpfc_io_buf *lpfc_ncmd_next; + unsigned long iflag; + struct list_head tmp_list; + u32 tmp_count; + + pbl_pool = &phba->sli4_hba.hdwq[hwqid].p_multixri_pool->pbl_pool; + pvt_pool = &phba->sli4_hba.hdwq[hwqid].p_multixri_pool->pvt_pool; + tmp_count = 0; + + spin_lock_irqsave(&pbl_pool->lock, iflag); + spin_lock(&pvt_pool->lock); + + if (pvt_pool->count > pvt_pool->low_watermark) { + /* Step 1: move (all - low_watermark) from pvt_pool + * to pbl_pool + */ + + /* Move low watermark of bufs from pvt_pool to tmp_list */ + INIT_LIST_HEAD(&tmp_list); + list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next, + &pvt_pool->list, list) { + list_move_tail(&lpfc_ncmd->list, &tmp_list); + tmp_count++; + if (tmp_count >= pvt_pool->low_watermark) + break; + } + + /* Move all bufs from pvt_pool to pbl_pool */ + list_splice_init(&pvt_pool->list, &pbl_pool->list); + + /* Move all bufs from tmp_list to pvt_pool */ + list_splice(&tmp_list, &pvt_pool->list); + + pbl_pool->count += (pvt_pool->count - tmp_count); + pvt_pool->count = tmp_count; + } else { + /* Step 2: move the rest from pvt_pool to pbl_pool */ + list_splice_init(&pvt_pool->list, &pbl_pool->list); + pbl_pool->count += pvt_pool->count; + pvt_pool->count = 0; + } + + spin_unlock(&pvt_pool->lock); + spin_unlock_irqrestore(&pbl_pool->lock, iflag); +} + +/** + * _lpfc_move_xri_pbl_to_pvt - Move some XRIs from public to private pool + * @phba: pointer to lpfc hba data structure + * @pbl_pool: specified public free XRI pool + * @pvt_pool: specified private free XRI pool + * @count: number of XRIs to move + * + * This routine tries to move some free common bufs from the specified pbl_pool + * to the specified pvt_pool. It might move less than count XRIs if there's not + * enough in public pool. + * + * Return: + * true - if XRIs are successfully moved from the specified pbl_pool to the + * specified pvt_pool + * false - if the specified pbl_pool is empty or locked by someone else + **/ +static bool +_lpfc_move_xri_pbl_to_pvt(struct lpfc_hba *phba, struct lpfc_pbl_pool *pbl_pool, + struct lpfc_pvt_pool *pvt_pool, u32 count) +{ + struct lpfc_io_buf *lpfc_ncmd; + struct lpfc_io_buf *lpfc_ncmd_next; + unsigned long iflag; + int ret; + + ret = spin_trylock_irqsave(&pbl_pool->lock, iflag); + if (ret) { + if (pbl_pool->count) { + /* Move a batch of XRIs from public to private pool */ + spin_lock(&pvt_pool->lock); + list_for_each_entry_safe(lpfc_ncmd, + lpfc_ncmd_next, + &pbl_pool->list, + list) { + list_move_tail(&lpfc_ncmd->list, + &pvt_pool->list); + pvt_pool->count++; + pbl_pool->count--; + count--; + if (count == 0) + break; + } + + spin_unlock(&pvt_pool->lock); + spin_unlock_irqrestore(&pbl_pool->lock, iflag); + return true; + } + spin_unlock_irqrestore(&pbl_pool->lock, iflag); + } + + return false; +} + +/** + * lpfc_move_xri_pbl_to_pvt - Move some XRIs from public to private pool + * @phba: pointer to lpfc hba data structure. + * @hwqid: belong to which HWQ. + * @count: number of XRIs to move + * + * This routine tries to find some free common bufs in one of public pools with + * Round Robin method. The search always starts from local hwqid, then the next + * HWQ which was found last time (rrb_next_hwqid). Once a public pool is found, + * a batch of free common bufs are moved to private pool on hwqid. + * It might move less than count XRIs if there's not enough in public pool. + **/ +void lpfc_move_xri_pbl_to_pvt(struct lpfc_hba *phba, u32 hwqid, u32 count) +{ + struct lpfc_multixri_pool *multixri_pool; + struct lpfc_multixri_pool *next_multixri_pool; + struct lpfc_pvt_pool *pvt_pool; + struct lpfc_pbl_pool *pbl_pool; + u32 next_hwqid; + u32 hwq_count; + int ret; + + multixri_pool = phba->sli4_hba.hdwq[hwqid].p_multixri_pool; + pvt_pool = &multixri_pool->pvt_pool; + pbl_pool = &multixri_pool->pbl_pool; + + /* Check if local pbl_pool is available */ + ret = _lpfc_move_xri_pbl_to_pvt(phba, pbl_pool, pvt_pool, count); + if (ret) { +#ifdef LPFC_MXP_STAT + multixri_pool->local_pbl_hit_count++; +#endif + return; + } + + hwq_count = phba->cfg_hdw_queue; + + /* Get the next hwqid which was found last time */ + next_hwqid = multixri_pool->rrb_next_hwqid; + + do { + /* Go to next hwq */ + next_hwqid = (next_hwqid + 1) % hwq_count; + + next_multixri_pool = + phba->sli4_hba.hdwq[next_hwqid].p_multixri_pool; + pbl_pool = &next_multixri_pool->pbl_pool; + + /* Check if the public free xri pool is available */ + ret = _lpfc_move_xri_pbl_to_pvt( + phba, pbl_pool, pvt_pool, count); + + /* Exit while-loop if success or all hwqid are checked */ + } while (!ret && next_hwqid != multixri_pool->rrb_next_hwqid); + + /* Starting point for the next time */ + multixri_pool->rrb_next_hwqid = next_hwqid; + + if (!ret) { + /* stats: all public pools are empty*/ + multixri_pool->pbl_empty_count++; + } + +#ifdef LPFC_MXP_STAT + if (ret) { + if (next_hwqid == hwqid) + multixri_pool->local_pbl_hit_count++; + else + multixri_pool->other_pbl_hit_count++; + } +#endif +} + +/** + * lpfc_keep_pvt_pool_above_lowwm - Keep pvt_pool above low watermark + * @phba: pointer to lpfc hba data structure. + * @qp: belong to which HWQ. + * + * This routine get a batch of XRIs from pbl_pool if pvt_pool is less than + * low watermark. + **/ +void lpfc_keep_pvt_pool_above_lowwm(struct lpfc_hba *phba, u32 hwqid) +{ + struct lpfc_multixri_pool *multixri_pool; + struct lpfc_pvt_pool *pvt_pool; + + multixri_pool = phba->sli4_hba.hdwq[hwqid].p_multixri_pool; + pvt_pool = &multixri_pool->pvt_pool; + + if (pvt_pool->count < pvt_pool->low_watermark) + lpfc_move_xri_pbl_to_pvt(phba, hwqid, XRI_BATCH); +} + +/** + * lpfc_release_io_buf - Return one IO buf back to free pool + * @phba: pointer to lpfc hba data structure. + * @lpfc_ncmd: IO buf to be returned. + * @qp: belong to which HWQ. + * + * This routine returns one IO buf back to free pool. If this is an urgent IO, + * the IO buf is returned to expedite pool. If cfg_xri_rebalancing==1, + * the IO buf is returned to pbl_pool or pvt_pool based on watermark and + * xri_limit. If cfg_xri_rebalancing==0, the IO buf is returned to + * lpfc_io_buf_list_put. + **/ +void lpfc_release_io_buf(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_ncmd, + struct lpfc_sli4_hdw_queue *qp) +{ + unsigned long iflag; + struct lpfc_pbl_pool *pbl_pool; + struct lpfc_pvt_pool *pvt_pool; + struct lpfc_epd_pool *epd_pool; + u32 txcmplq_cnt; + u32 xri_owned; + u32 xri_limit; + u32 abts_io_bufs; + + /* MUST zero fields if buffer is reused by another protocol */ + lpfc_ncmd->nvmeCmd = NULL; + lpfc_ncmd->cur_iocbq.wqe_cmpl = NULL; + lpfc_ncmd->cur_iocbq.iocb_cmpl = NULL; + + if (phba->cfg_xri_rebalancing) { + if (lpfc_ncmd->expedite) { + /* Return to expedite pool */ + epd_pool = &phba->epd_pool; + spin_lock_irqsave(&epd_pool->lock, iflag); + list_add_tail(&lpfc_ncmd->list, &epd_pool->list); + epd_pool->count++; + spin_unlock_irqrestore(&epd_pool->lock, iflag); + return; + } + + /* Avoid invalid access if an IO sneaks in and is being rejected + * just _after_ xri pools are destroyed in lpfc_offline. + * Nothing much can be done at this point. + */ + if (!qp->p_multixri_pool) + return; + + pbl_pool = &qp->p_multixri_pool->pbl_pool; + pvt_pool = &qp->p_multixri_pool->pvt_pool; + + txcmplq_cnt = qp->fcp_wq->pring->txcmplq_cnt; + abts_io_bufs = qp->abts_scsi_io_bufs; + if (qp->nvme_wq) { + txcmplq_cnt += qp->nvme_wq->pring->txcmplq_cnt; + abts_io_bufs += qp->abts_nvme_io_bufs; + } + + xri_owned = pvt_pool->count + txcmplq_cnt + abts_io_bufs; + xri_limit = qp->p_multixri_pool->xri_limit; + +#ifdef LPFC_MXP_STAT + if (xri_owned <= xri_limit) + qp->p_multixri_pool->below_limit_count++; + else + qp->p_multixri_pool->above_limit_count++; +#endif + + /* XRI goes to either public or private free xri pool + * based on watermark and xri_limit + */ + if ((pvt_pool->count < pvt_pool->low_watermark) || + (xri_owned < xri_limit && + pvt_pool->count < pvt_pool->high_watermark)) { + spin_lock_irqsave(&pvt_pool->lock, iflag); + list_add_tail(&lpfc_ncmd->list, + &pvt_pool->list); + pvt_pool->count++; + spin_unlock_irqrestore(&pvt_pool->lock, iflag); + } else { + spin_lock_irqsave(&pbl_pool->lock, iflag); + list_add_tail(&lpfc_ncmd->list, + &pbl_pool->list); + pbl_pool->count++; + spin_unlock_irqrestore(&pbl_pool->lock, iflag); + } + } else { + spin_lock_irqsave(&qp->io_buf_list_put_lock, iflag); + list_add_tail(&lpfc_ncmd->list, + &qp->lpfc_io_buf_list_put); + qp->put_io_bufs++; + spin_unlock_irqrestore(&qp->io_buf_list_put_lock, + iflag); + } +} + +/** + * lpfc_get_io_buf_from_private_pool - Get one free IO buf from private pool + * @phba: pointer to lpfc hba data structure. + * @pvt_pool: pointer to private pool data structure. + * @ndlp: pointer to lpfc nodelist data structure. + * + * This routine tries to get one free IO buf from private pool. + * + * Return: + * pointer to one free IO buf - if private pool is not empty + * NULL - if private pool is empty + **/ +static struct lpfc_io_buf * +lpfc_get_io_buf_from_private_pool(struct lpfc_hba *phba, + struct lpfc_pvt_pool *pvt_pool, + struct lpfc_nodelist *ndlp) +{ + struct lpfc_io_buf *lpfc_ncmd; + struct lpfc_io_buf *lpfc_ncmd_next; + unsigned long iflag; + + spin_lock_irqsave(&pvt_pool->lock, iflag); + list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next, + &pvt_pool->list, list) { + if (lpfc_test_rrq_active( + phba, ndlp, lpfc_ncmd->cur_iocbq.sli4_lxritag)) + continue; + list_del(&lpfc_ncmd->list); + pvt_pool->count--; + spin_unlock_irqrestore(&pvt_pool->lock, iflag); + return lpfc_ncmd; + } + spin_unlock_irqrestore(&pvt_pool->lock, iflag); + + return NULL; +} + +/** + * lpfc_get_io_buf_from_expedite_pool - Get one free IO buf from expedite pool + * @phba: pointer to lpfc hba data structure. + * + * This routine tries to get one free IO buf from expedite pool. + * + * Return: + * pointer to one free IO buf - if expedite pool is not empty + * NULL - if expedite pool is empty + **/ +static struct lpfc_io_buf * +lpfc_get_io_buf_from_expedite_pool(struct lpfc_hba *phba) +{ + struct lpfc_io_buf *lpfc_ncmd; + struct lpfc_io_buf *lpfc_ncmd_next; + unsigned long iflag; + struct lpfc_epd_pool *epd_pool; + + epd_pool = &phba->epd_pool; + lpfc_ncmd = NULL; + + spin_lock_irqsave(&epd_pool->lock, iflag); + if (epd_pool->count > 0) { + list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next, + &epd_pool->list, list) { + list_del(&lpfc_ncmd->list); + epd_pool->count--; + break; + } + } + spin_unlock_irqrestore(&epd_pool->lock, iflag); + + return lpfc_ncmd; +} + +/** + * lpfc_get_io_buf_from_multixri_pools - Get one free IO bufs + * @phba: pointer to lpfc hba data structure. + * @ndlp: pointer to lpfc nodelist data structure. + * @hwqid: belong to which HWQ + * @expedite: 1 means this request is urgent. + * + * This routine will do the following actions and then return a pointer to + * one free IO buf. + * + * 1. If private free xri count is empty, move some XRIs from public to + * private pool. + * 2. Get one XRI from private free xri pool. + * 3. If we fail to get one from pvt_pool and this is an expedite request, + * get one free xri from expedite pool. + * + * Note: ndlp is only used on SCSI side for RRQ testing. + * The caller should pass NULL for ndlp on NVME side. + * + * Return: + * pointer to one free IO buf - if private pool is not empty + * NULL - if private pool is empty + **/ +static struct lpfc_io_buf * +lpfc_get_io_buf_from_multixri_pools(struct lpfc_hba *phba, + struct lpfc_nodelist *ndlp, + int hwqid, int expedite) +{ + struct lpfc_sli4_hdw_queue *qp; + struct lpfc_multixri_pool *multixri_pool; + struct lpfc_pvt_pool *pvt_pool; + struct lpfc_io_buf *lpfc_ncmd; + + qp = &phba->sli4_hba.hdwq[hwqid]; + lpfc_ncmd = NULL; + multixri_pool = qp->p_multixri_pool; + pvt_pool = &multixri_pool->pvt_pool; + multixri_pool->io_req_count++; + + /* If pvt_pool is empty, move some XRIs from public to private pool */ + if (pvt_pool->count == 0) + lpfc_move_xri_pbl_to_pvt(phba, hwqid, XRI_BATCH); + + /* Get one XRI from private free xri pool */ + lpfc_ncmd = lpfc_get_io_buf_from_private_pool(phba, pvt_pool, ndlp); + + if (lpfc_ncmd) { + lpfc_ncmd->hdwq = qp; + lpfc_ncmd->hdwq_no = hwqid; + } else if (expedite) { + /* If we fail to get one from pvt_pool and this is an expedite + * request, get one free xri from expedite pool. + */ + lpfc_ncmd = lpfc_get_io_buf_from_expedite_pool(phba); + } + + return lpfc_ncmd; +} + +static inline struct lpfc_io_buf * +lpfc_io_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, int idx) +{ + struct lpfc_sli4_hdw_queue *qp; + struct lpfc_io_buf *lpfc_cmd, *lpfc_cmd_next; + + qp = &phba->sli4_hba.hdwq[idx]; + list_for_each_entry_safe(lpfc_cmd, lpfc_cmd_next, + &qp->lpfc_io_buf_list_get, list) { + if (lpfc_test_rrq_active(phba, ndlp, + lpfc_cmd->cur_iocbq.sli4_lxritag)) + continue; + + if (lpfc_cmd->flags & LPFC_SBUF_NOT_POSTED) + continue; + + list_del_init(&lpfc_cmd->list); + qp->get_io_bufs--; + lpfc_cmd->hdwq = qp; + lpfc_cmd->hdwq_no = idx; + return lpfc_cmd; + } + return NULL; +} + +/** + * lpfc_get_io_buf - Get one IO buffer from free pool + * @phba: The HBA for which this call is being executed. + * @ndlp: pointer to lpfc nodelist data structure. + * @hwqid: belong to which HWQ + * @expedite: 1 means this request is urgent. + * + * This routine gets one IO buffer from free pool. If cfg_xri_rebalancing==1, + * removes a IO buffer from multiXRI pools. If cfg_xri_rebalancing==0, removes + * a IO buffer from head of @hdwq io_buf_list and returns to caller. + * + * Note: ndlp is only used on SCSI side for RRQ testing. + * The caller should pass NULL for ndlp on NVME side. + * + * Return codes: + * NULL - Error + * Pointer to lpfc_io_buf - Success + **/ +struct lpfc_io_buf *lpfc_get_io_buf(struct lpfc_hba *phba, + struct lpfc_nodelist *ndlp, + u32 hwqid, int expedite) +{ + struct lpfc_sli4_hdw_queue *qp; + unsigned long iflag; + struct lpfc_io_buf *lpfc_cmd; + + qp = &phba->sli4_hba.hdwq[hwqid]; + lpfc_cmd = NULL; + + if (phba->cfg_xri_rebalancing) + lpfc_cmd = lpfc_get_io_buf_from_multixri_pools( + phba, ndlp, hwqid, expedite); + else { + spin_lock_irqsave(&qp->io_buf_list_get_lock, iflag); + if (qp->get_io_bufs > LPFC_NVME_EXPEDITE_XRICNT || expedite) + lpfc_cmd = lpfc_io_buf(phba, ndlp, hwqid); + if (!lpfc_cmd) { + spin_lock(&qp->io_buf_list_put_lock); + list_splice(&qp->lpfc_io_buf_list_put, + &qp->lpfc_io_buf_list_get); + qp->get_io_bufs += qp->put_io_bufs; + INIT_LIST_HEAD(&qp->lpfc_io_buf_list_put); + qp->put_io_bufs = 0; + spin_unlock(&qp->io_buf_list_put_lock); + if (qp->get_io_bufs > LPFC_NVME_EXPEDITE_XRICNT || + expedite) + lpfc_cmd = lpfc_io_buf(phba, ndlp, hwqid); + } + spin_unlock_irqrestore(&qp->io_buf_list_get_lock, iflag); + } + + return lpfc_cmd; +} diff --git a/drivers/scsi/lpfc/lpfc_sli.h b/drivers/scsi/lpfc/lpfc_sli.h index 3922e762ed31..658a40d3b1a2 100644 --- a/drivers/scsi/lpfc/lpfc_sli.h +++ b/drivers/scsi/lpfc/lpfc_sli.h @@ -20,6 +20,10 @@ * included with this package. * *******************************************************************/ +#if defined(CONFIG_DEBUG_FS) && !defined(CONFIG_SCSI_LPFC_DEBUG_FS) +#define CONFIG_SCSI_LPFC_DEBUG_FS +#endif + /* forward declaration for LPFC_IOCB_t's use */ struct lpfc_hba; struct lpfc_vport; @@ -352,3 +356,84 @@ struct lpfc_sli { #define LPFC_MBOX_SLI4_CONFIG_EXTENDED_TMO 300 /* Timeout for other flash-based outstanding mbox command (Seconds) */ #define LPFC_MBOX_TMO_FLASH_CMD 300 + +struct lpfc_io_buf { + /* Common fields */ + struct list_head list; + void *data; + dma_addr_t dma_handle; + dma_addr_t dma_phys_sgl; + struct sli4_sge *dma_sgl; + struct lpfc_iocbq cur_iocbq; + struct lpfc_sli4_hdw_queue *hdwq; + uint16_t hdwq_no; + uint16_t cpu; + + struct lpfc_nodelist *ndlp; + uint32_t timeout; + uint16_t flags; /* TBD convert exch_busy to flags */ +#define LPFC_SBUF_XBUSY 0x1 /* SLI4 hba reported XB on WCQE cmpl */ +#define LPFC_SBUF_BUMP_QDEPTH 0x2 /* bumped queue depth counter */ + /* External DIF device IO conversions */ +#define LPFC_SBUF_NORMAL_DIF 0x4 /* normal mode to insert/strip */ +#define LPFC_SBUF_PASS_DIF 0x8 /* insert/strip mode to passthru */ +#define LPFC_SBUF_NOT_POSTED 0x10 /* SGL failed post to FW. */ + uint16_t exch_busy; /* SLI4 hba reported XB on complete WCQE */ + uint16_t status; /* From IOCB Word 7- ulpStatus */ + uint32_t result; /* From IOCB Word 4. */ + + uint32_t seg_cnt; /* Number of scatter-gather segments returned by + * dma_map_sg. The driver needs this for calls + * to dma_unmap_sg. + */ + unsigned long start_time; + bool expedite; /* this is an expedite io_buf */ + + union { + /* SCSI specific fields */ + struct { + struct scsi_cmnd *pCmd; + struct lpfc_rport_data *rdata; + uint32_t prot_seg_cnt; /* seg_cnt's counterpart for + * protection data + */ + + /* + * data and dma_handle are the kernel virtual and bus + * address of the dma-able buffer containing the + * fcp_cmd, fcp_rsp and a scatter gather bde list that + * supports the sg_tablesize value. + */ + struct fcp_cmnd *fcp_cmnd; + struct fcp_rsp *fcp_rsp; + + wait_queue_head_t *waitq; + +#ifdef CONFIG_SCSI_LPFC_DEBUG_FS + /* Used to restore any changes to protection data for + * error injection + */ + void *prot_data_segment; + uint32_t prot_data; + uint32_t prot_data_type; +#define LPFC_INJERR_REFTAG 1 +#define LPFC_INJERR_APPTAG 2 +#define LPFC_INJERR_GUARD 3 +#endif + }; + + /* NVME specific fields */ + struct { + struct nvmefc_fcp_req *nvmeCmd; + uint16_t qidx; + +#ifdef CONFIG_SCSI_LPFC_DEBUG_FS + uint64_t ts_cmd_start; + uint64_t ts_last_cmd; + uint64_t ts_cmd_wqput; + uint64_t ts_isr_cmpl; + uint64_t ts_data_nvme; +#endif + }; + }; +}; diff --git a/drivers/scsi/lpfc/lpfc_sli4.h b/drivers/scsi/lpfc/lpfc_sli4.h index a33fb6715329..6893e8547431 100644 --- a/drivers/scsi/lpfc/lpfc_sli4.h +++ b/drivers/scsi/lpfc/lpfc_sli4.h @@ -533,6 +533,59 @@ struct lpfc_vector_map_info { }; #define LPFC_VECTOR_MAP_EMPTY 0xffff +/* Multi-XRI pool */ +#define XRI_BATCH 8 + +struct lpfc_pbl_pool { + struct list_head list; + u32 count; + spinlock_t lock; /* lock for pbl_pool*/ +}; + +struct lpfc_pvt_pool { + u32 low_watermark; + u32 high_watermark; + + struct list_head list; + u32 count; + spinlock_t lock; /* lock for pvt_pool */ +}; + +struct lpfc_multixri_pool { + u32 xri_limit; + + /* Starting point when searching a pbl_pool with round-robin method */ + u32 rrb_next_hwqid; + + /* Used by lpfc_adjust_pvt_pool_count. + * io_req_count is incremented by 1 during IO submission. The heartbeat + * handler uses these two variables to determine if pvt_pool is idle or + * busy. + */ + u32 prev_io_req_count; + u32 io_req_count; + + /* statistics */ + u32 pbl_empty_count; +#ifdef LPFC_MXP_STAT + u32 above_limit_count; + u32 below_limit_count; + u32 local_pbl_hit_count; + u32 other_pbl_hit_count; + u32 stat_max_hwm; + +#define LPFC_MXP_SNAPSHOT_TAKEN 3 /* snapshot is taken at 3rd heartbeats */ + u32 stat_pbl_count; + u32 stat_pvt_count; + u32 stat_busy_count; + u32 stat_snapshot_taken; +#endif + + /* TODO: Separate pvt_pool into get and put list */ + struct lpfc_pbl_pool pbl_pool; /* Public free XRI pool */ + struct lpfc_pvt_pool pvt_pool; /* Private free XRI pool */ +}; + struct lpfc_fc4_ctrl_stat { u32 input_requests; u32 output_requests; @@ -567,6 +620,9 @@ struct lpfc_sli4_hdw_queue { uint32_t abts_scsi_io_bufs; uint32_t abts_nvme_io_bufs; + /* Multi-XRI pool per HWQ */ + struct lpfc_multixri_pool *p_multixri_pool; + /* FC-4 Stats counters */ struct lpfc_fc4_ctrl_stat nvme_cstat; struct lpfc_fc4_ctrl_stat scsi_cstat; -- cgit v1.2.3 From 6a828b0f6192b4930894925d1c1d0dc1f1d99e6e Mon Sep 17 00:00:00 2001 From: James Smart Date: Mon, 28 Jan 2019 11:14:31 -0800 Subject: scsi: lpfc: Support non-uniform allocation of MSIX vectors to hardware queues So far MSIX vector allocation assumed it would be 1:1 with hardware queues. However, there are several reasons why fewer MSIX vectors may be allocated than hardware queues such as the platform being out of vectors or adapter limits being less than cpu count. This patch reworks the MSIX/EQ relationships with the per-cpu hardware queues so they can function independently. MSIX vectors will be equitably split been cpu sockets/cores and then the per-cpu hardware queues will be mapped to the vectors most efficient for them. Signed-off-by: Dick Kennedy Signed-off-by: James Smart Reviewed-by: Hannes Reinecke Signed-off-by: Martin K. Petersen --- drivers/scsi/lpfc/lpfc.h | 7 +- drivers/scsi/lpfc/lpfc_attr.c | 96 ++++---- drivers/scsi/lpfc/lpfc_crtn.h | 1 - drivers/scsi/lpfc/lpfc_debugfs.c | 303 ++++++++++++++++------- drivers/scsi/lpfc/lpfc_debugfs.h | 3 - drivers/scsi/lpfc/lpfc_hw4.h | 3 +- drivers/scsi/lpfc/lpfc_init.c | 503 ++++++++++++++++++++++++++++++++------- drivers/scsi/lpfc/lpfc_nvme.c | 18 +- drivers/scsi/lpfc/lpfc_scsi.c | 28 ++- drivers/scsi/lpfc/lpfc_sli.c | 148 +++++------- drivers/scsi/lpfc/lpfc_sli4.h | 64 ++++- 11 files changed, 831 insertions(+), 343 deletions(-) (limited to 'drivers/scsi/lpfc/lpfc.h') diff --git a/drivers/scsi/lpfc/lpfc.h b/drivers/scsi/lpfc/lpfc.h index 0f8964fdfecf..9fd2811ffa8b 100644 --- a/drivers/scsi/lpfc/lpfc.h +++ b/drivers/scsi/lpfc/lpfc.h @@ -84,8 +84,6 @@ struct lpfc_sli2_slim; #define LPFC_HB_MBOX_INTERVAL 5 /* Heart beat interval in seconds. */ #define LPFC_HB_MBOX_TIMEOUT 30 /* Heart beat timeout in seconds. */ -#define LPFC_LOOK_AHEAD_OFF 0 /* Look ahead logic is turned off */ - /* Error Attention event polling interval */ #define LPFC_ERATT_POLL_INTERVAL 5 /* EATT poll interval in seconds */ @@ -821,6 +819,7 @@ struct lpfc_hba { uint32_t cfg_fcp_imax; uint32_t cfg_fcp_cpu_map; uint32_t cfg_hdw_queue; + uint32_t cfg_irq_chann; uint32_t cfg_suppress_rsp; uint32_t cfg_nvme_oas; uint32_t cfg_nvme_embed_cmd; @@ -1042,6 +1041,9 @@ struct lpfc_hba { struct dentry *debug_nvmeio_trc; struct lpfc_debugfs_nvmeio_trc *nvmeio_trc; struct dentry *debug_hdwqinfo; +#ifdef LPFC_HDWQ_LOCK_STAT + struct dentry *debug_lockstat; +#endif atomic_t nvmeio_trc_cnt; uint32_t nvmeio_trc_size; uint32_t nvmeio_trc_output_idx; @@ -1161,6 +1163,7 @@ struct lpfc_hba { #define LPFC_CHECK_NVME_IO 1 #define LPFC_CHECK_NVMET_RCV 2 #define LPFC_CHECK_NVMET_IO 4 +#define LPFC_CHECK_SCSI_IO 8 uint16_t ktime_on; uint64_t ktime_data_samples; uint64_t ktime_status_samples; diff --git a/drivers/scsi/lpfc/lpfc_attr.c b/drivers/scsi/lpfc/lpfc_attr.c index 787812dd57a9..fc7f80d68638 100644 --- a/drivers/scsi/lpfc/lpfc_attr.c +++ b/drivers/scsi/lpfc/lpfc_attr.c @@ -4958,7 +4958,7 @@ lpfc_fcp_imax_store(struct device *dev, struct device_attribute *attr, phba->cfg_fcp_imax = (uint32_t)val; phba->initial_imax = phba->cfg_fcp_imax; - for (i = 0; i < phba->cfg_hdw_queue; i += LPFC_MAX_EQ_DELAY_EQID_CNT) + for (i = 0; i < phba->cfg_irq_chann; i += LPFC_MAX_EQ_DELAY_EQID_CNT) lpfc_modify_hba_eq_delay(phba, i, LPFC_MAX_EQ_DELAY_EQID_CNT, val); @@ -5059,13 +5059,6 @@ lpfc_fcp_cpu_map_show(struct device *dev, struct device_attribute *attr, phba->cfg_fcp_cpu_map, phba->sli4_hba.num_online_cpu); break; - case 2: - len += snprintf(buf + len, PAGE_SIZE-len, - "fcp_cpu_map: Driver centric mapping (%d): " - "%d online CPUs\n", - phba->cfg_fcp_cpu_map, - phba->sli4_hba.num_online_cpu); - break; } while (phba->sli4_hba.curr_disp_cpu < phba->sli4_hba.num_present_cpu) { @@ -5076,35 +5069,35 @@ lpfc_fcp_cpu_map_show(struct device *dev, struct device_attribute *attr, len += snprintf( buf + len, PAGE_SIZE - len, "CPU %02d hdwq None " - "physid %d coreid %d\n", + "physid %d coreid %d ht %d\n", phba->sli4_hba.curr_disp_cpu, cpup->phys_id, - cpup->core_id); + cpup->core_id, cpup->hyper); else len += snprintf( buf + len, PAGE_SIZE - len, - "CPU %02d hdwq %04d " - "physid %d coreid %d\n", + "CPU %02d EQ %04d hdwq %04d " + "physid %d coreid %d ht %d\n", phba->sli4_hba.curr_disp_cpu, - cpup->hdwq, cpup->phys_id, - cpup->core_id); + cpup->eq, cpup->hdwq, cpup->phys_id, + cpup->core_id, cpup->hyper); } else { if (cpup->hdwq == LPFC_VECTOR_MAP_EMPTY) len += snprintf( buf + len, PAGE_SIZE - len, "CPU %02d hdwq None " - "physid %d coreid %d IRQ %d\n", + "physid %d coreid %d ht %d IRQ %d\n", phba->sli4_hba.curr_disp_cpu, cpup->phys_id, - cpup->core_id, cpup->irq); + cpup->core_id, cpup->hyper, cpup->irq); else len += snprintf( buf + len, PAGE_SIZE - len, - "CPU %02d hdwq %04d " - "physid %d coreid %d IRQ %d\n", + "CPU %02d EQ %04d hdwq %04d " + "physid %d coreid %d ht %d IRQ %d\n", phba->sli4_hba.curr_disp_cpu, - cpup->hdwq, cpup->phys_id, - cpup->core_id, cpup->irq); + cpup->eq, cpup->hdwq, cpup->phys_id, + cpup->core_id, cpup->hyper, cpup->irq); } phba->sli4_hba.curr_disp_cpu++; @@ -5146,14 +5139,13 @@ lpfc_fcp_cpu_map_store(struct device *dev, struct device_attribute *attr, # lpfc_fcp_cpu_map: Defines how to map CPUs to IRQ vectors # for the HBA. # -# Value range is [0 to 2]. Default value is LPFC_DRIVER_CPU_MAP (2). +# Value range is [0 to 1]. Default value is LPFC_HBA_CPU_MAP (1). # 0 - Do not affinitze IRQ vectors # 1 - Affintize HBA vectors with respect to each HBA # (start with CPU0 for each HBA) -# 2 - Affintize HBA vectors with respect to the entire driver -# (round robin thru all CPUs across all HBAs) +# This also defines how Hardware Queues are mapped to specific CPUs. */ -static int lpfc_fcp_cpu_map = LPFC_DRIVER_CPU_MAP; +static int lpfc_fcp_cpu_map = LPFC_HBA_CPU_MAP; module_param(lpfc_fcp_cpu_map, int, S_IRUGO|S_IWUSR); MODULE_PARM_DESC(lpfc_fcp_cpu_map, "Defines how to map CPUs to IRQ vectors per HBA"); @@ -5187,7 +5179,7 @@ lpfc_fcp_cpu_map_init(struct lpfc_hba *phba, int val) lpfc_printf_log(phba, KERN_ERR, LOG_INIT, "3326 lpfc_fcp_cpu_map: %d out of range, using " "default\n", val); - phba->cfg_fcp_cpu_map = LPFC_DRIVER_CPU_MAP; + phba->cfg_fcp_cpu_map = LPFC_HBA_CPU_MAP; return 0; } @@ -5308,7 +5300,7 @@ LPFC_ATTR_R(xri_rebalancing, 1, 0, 1, "Enable/Disable XRI rebalancing"); * CPU. Otherwise, the default 0 (Round Robin) scheduling of FCP/NVME I/Os * through WQs will be used. */ -LPFC_ATTR_RW(fcp_io_sched, LPFC_FCP_SCHED_BY_HDWQ, +LPFC_ATTR_RW(fcp_io_sched, LPFC_FCP_SCHED_BY_CPU, LPFC_FCP_SCHED_BY_HDWQ, LPFC_FCP_SCHED_BY_CPU, "Determine scheduling algorithm for " @@ -5474,24 +5466,40 @@ LPFC_ATTR_RW(nvme_embed_cmd, 1, 0, 2, "Embed NVME Command in WQE"); /* - * lpfc_hdw_queue: Set the number of IO channels the driver + * lpfc_hdw_queue: Set the number of Hardware Queues the driver * will advertise it supports to the NVME and SCSI layers. This also - * will map to the number of EQ/CQ/WQs the driver will create. + * will map to the number of CQ/WQ pairs the driver will create. * * The NVME Layer will try to create this many, plus 1 administrative * hardware queue. The administrative queue will always map to WQ 0 - * A hardware IO queue maps (qidx) to a specific driver WQ. + * A hardware IO queue maps (qidx) to a specific driver CQ/WQ. * * 0 = Configure the number of hdw queues to the number of active CPUs. - * 1,64 = Manually specify how many hdw queues to use. + * 1,128 = Manually specify how many hdw queues to use. * - * Value range is [0,64]. Default value is 0. + * Value range is [0,128]. Default value is 0. */ LPFC_ATTR_R(hdw_queue, LPFC_HBA_HDWQ_DEF, LPFC_HBA_HDWQ_MIN, LPFC_HBA_HDWQ_MAX, "Set the number of I/O Hardware Queues"); +/* + * lpfc_irq_chann: Set the number of IRQ vectors that are available + * for Hardware Queues to utilize. This also will map to the number + * of EQ / MSI-X vectors the driver will create. This should never be + * more than the number of Hardware Queues + * + * 0 = Configure number of IRQ Channels to the number of active CPUs. + * 1,128 = Manually specify how many IRQ Channels to use. + * + * Value range is [0,128]. Default value is 0. + */ +LPFC_ATTR_R(irq_chann, + LPFC_HBA_HDWQ_DEF, + LPFC_HBA_HDWQ_MIN, LPFC_HBA_HDWQ_MAX, + "Set the number of I/O IRQ Channels"); + /* # lpfc_enable_hba_reset: Allow or prevent HBA resets to the hardware. # 0 = HBA resets disabled @@ -5532,16 +5540,6 @@ LPFC_ATTR_RW(XLanePriority, 0, 0x0, 0x7f, "CS_CTL for Express Lane Feature."); */ LPFC_ATTR_R(enable_bg, 0, 0, 1, "Enable BlockGuard Support"); -/* -# lpfc_fcp_look_ahead: Look ahead for completions in FCP start routine -# 0 = disabled (default) -# 1 = enabled -# Value range is [0,1]. Default value is 0. -# -# This feature in under investigation and may be supported in the future. -*/ -unsigned int lpfc_fcp_look_ahead = LPFC_LOOK_AHEAD_OFF; - /* # lpfc_prot_mask: i # - Bit mask of host protection capabilities used to register with the @@ -5788,6 +5786,7 @@ struct device_attribute *lpfc_hba_attrs[] = { &dev_attr_lpfc_fcp_imax, &dev_attr_lpfc_fcp_cpu_map, &dev_attr_lpfc_hdw_queue, + &dev_attr_lpfc_irq_chann, &dev_attr_lpfc_suppress_rsp, &dev_attr_lpfc_nvmet_mrq, &dev_attr_lpfc_nvmet_mrq_post, @@ -6867,6 +6866,7 @@ lpfc_get_cfgparam(struct lpfc_hba *phba) lpfc_nvme_enable_fb_init(phba, lpfc_nvme_enable_fb); lpfc_nvmet_fb_size_init(phba, lpfc_nvmet_fb_size); lpfc_hdw_queue_init(phba, lpfc_hdw_queue); + lpfc_irq_chann_init(phba, lpfc_irq_chann); lpfc_enable_bbcr_init(phba, lpfc_enable_bbcr); lpfc_enable_dpp_init(phba, lpfc_enable_dpp); @@ -6891,6 +6891,10 @@ lpfc_get_cfgparam(struct lpfc_hba *phba) /* A value of 0 means use the number of CPUs found in the system */ if (phba->cfg_hdw_queue == 0) phba->cfg_hdw_queue = phba->sli4_hba.num_present_cpu; + if (phba->cfg_irq_chann == 0) + phba->cfg_irq_chann = phba->sli4_hba.num_present_cpu; + if (phba->cfg_irq_chann > phba->cfg_hdw_queue) + phba->cfg_irq_chann = phba->cfg_hdw_queue; phba->cfg_soft_wwnn = 0L; phba->cfg_soft_wwpn = 0L; @@ -6933,6 +6937,10 @@ lpfc_nvme_mod_param_dep(struct lpfc_hba *phba) { if (phba->cfg_hdw_queue > phba->sli4_hba.num_present_cpu) phba->cfg_hdw_queue = phba->sli4_hba.num_present_cpu; + if (phba->cfg_irq_chann > phba->sli4_hba.num_present_cpu) + phba->cfg_irq_chann = phba->sli4_hba.num_present_cpu; + if (phba->cfg_irq_chann > phba->cfg_hdw_queue) + phba->cfg_irq_chann = phba->cfg_hdw_queue; if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME && phba->nvmet_support) { @@ -6953,11 +6961,11 @@ lpfc_nvme_mod_param_dep(struct lpfc_hba *phba) } if (!phba->cfg_nvmet_mrq) - phba->cfg_nvmet_mrq = phba->cfg_hdw_queue; + phba->cfg_nvmet_mrq = phba->cfg_irq_chann; /* Adjust lpfc_nvmet_mrq to avoid running out of WQE slots */ - if (phba->cfg_nvmet_mrq > phba->cfg_hdw_queue) { - phba->cfg_nvmet_mrq = phba->cfg_hdw_queue; + if (phba->cfg_nvmet_mrq > phba->cfg_irq_chann) { + phba->cfg_nvmet_mrq = phba->cfg_irq_chann; lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC, "6018 Adjust lpfc_nvmet_mrq to %d\n", phba->cfg_nvmet_mrq); diff --git a/drivers/scsi/lpfc/lpfc_crtn.h b/drivers/scsi/lpfc/lpfc_crtn.h index 726cd6a7c452..982401c31c12 100644 --- a/drivers/scsi/lpfc/lpfc_crtn.h +++ b/drivers/scsi/lpfc/lpfc_crtn.h @@ -440,7 +440,6 @@ extern spinlock_t _dump_buf_lock; extern int _dump_buf_done; extern spinlock_t pgcnt_lock; extern unsigned int pgcnt; -extern unsigned int lpfc_fcp_look_ahead; /* Interface exported by fabric iocb scheduler */ void lpfc_fabric_abort_nport(struct lpfc_nodelist *); diff --git a/drivers/scsi/lpfc/lpfc_debugfs.c b/drivers/scsi/lpfc/lpfc_debugfs.c index 2a2c46766eb6..72076b2cd4ff 100644 --- a/drivers/scsi/lpfc/lpfc_debugfs.c +++ b/drivers/scsi/lpfc/lpfc_debugfs.c @@ -378,6 +378,67 @@ skipit: return len; } +static int lpfc_debugfs_last_xripool; + +/** + * lpfc_debugfs_common_xri_data - Dump Hardware Queue info to a buffer + * @phba: The HBA to gather host buffer info from. + * @buf: The buffer to dump log into. + * @size: The maximum amount of data to process. + * + * Description: + * This routine dumps the Hardware Queue info from the @phba to @buf up to + * @size number of bytes. A header that describes the current hdwq state will be + * dumped to @buf first and then info on each hdwq entry will be dumped to @buf + * until @size bytes have been dumped or all the hdwq info has been dumped. + * + * Notes: + * This routine will rotate through each configured Hardware Queue each + * time called. + * + * Return Value: + * This routine returns the amount of bytes that were dumped into @buf and will + * not exceed @size. + **/ +static int +lpfc_debugfs_commonxripools_data(struct lpfc_hba *phba, char *buf, int size) +{ + struct lpfc_sli4_hdw_queue *qp; + int len = 0; + int i, out; + unsigned long iflag; + + for (i = 0; i < phba->cfg_hdw_queue; i++) { + if (len > (LPFC_DUMP_MULTIXRIPOOL_SIZE - 80)) + break; + qp = &phba->sli4_hba.hdwq[lpfc_debugfs_last_xripool]; + + len += snprintf(buf + len, size - len, "HdwQ %d Info ", i); + spin_lock_irqsave(&qp->abts_scsi_buf_list_lock, iflag); + spin_lock(&qp->abts_nvme_buf_list_lock); + spin_lock(&qp->io_buf_list_get_lock); + spin_lock(&qp->io_buf_list_put_lock); + out = qp->total_io_bufs - (qp->get_io_bufs + qp->put_io_bufs + + qp->abts_scsi_io_bufs + qp->abts_nvme_io_bufs); + len += snprintf(buf + len, size - len, + "tot:%d get:%d put:%d mt:%d " + "ABTS scsi:%d nvme:%d Out:%d\n", + qp->total_io_bufs, qp->get_io_bufs, qp->put_io_bufs, + qp->empty_io_bufs, qp->abts_scsi_io_bufs, + qp->abts_nvme_io_bufs, out); + spin_unlock(&qp->io_buf_list_put_lock); + spin_unlock(&qp->io_buf_list_get_lock); + spin_unlock(&qp->abts_nvme_buf_list_lock); + spin_unlock_irqrestore(&qp->abts_scsi_buf_list_lock, iflag); + + lpfc_debugfs_last_xripool++; + if (lpfc_debugfs_last_xripool >= phba->cfg_hdw_queue) + lpfc_debugfs_last_xripool = 0; + } + + return len; +} + /** * lpfc_debugfs_multixripools_data - Display multi-XRI pools information * @phba: The HBA to gather host buffer info from. @@ -405,6 +466,17 @@ lpfc_debugfs_multixripools_data(struct lpfc_hba *phba, char *buf, int size) u32 txcmplq_cnt; char tmp[LPFC_DEBUG_OUT_LINE_SZ] = {0}; + if (phba->sli_rev != LPFC_SLI_REV4) + return 0; + + if (!phba->sli4_hba.hdwq) + return 0; + + if (!phba->cfg_xri_rebalancing) { + i = lpfc_debugfs_commonxripools_data(phba, buf, size); + return i; + } + /* * Pbl: Current number of free XRIs in public pool * Pvt: Current number of free XRIs in private pool @@ -498,10 +570,12 @@ lpfc_debugfs_multixripools_data(struct lpfc_hba *phba, char *buf, int size) return strnlen(buf, size); } -static int lpfc_debugfs_last_hdwq; + +#ifdef LPFC_HDWQ_LOCK_STAT +static int lpfc_debugfs_last_lock; /** - * lpfc_debugfs_hdwqinfo_data - Dump Hardware Queue info to a buffer + * lpfc_debugfs_lockstat_data - Dump Hardware Queue info to a buffer * @phba: The HBA to gather host buffer info from. * @buf: The buffer to dump log into. * @size: The maximum amount of data to process. @@ -521,12 +595,11 @@ static int lpfc_debugfs_last_hdwq; * not exceed @size. **/ static int -lpfc_debugfs_hdwqinfo_data(struct lpfc_hba *phba, char *buf, int size) +lpfc_debugfs_lockstat_data(struct lpfc_hba *phba, char *buf, int size) { struct lpfc_sli4_hdw_queue *qp; int len = 0; - int i, out; - unsigned long iflag; + int i; if (phba->sli_rev != LPFC_SLI_REV4) return 0; @@ -535,35 +608,40 @@ lpfc_debugfs_hdwqinfo_data(struct lpfc_hba *phba, char *buf, int size) return 0; for (i = 0; i < phba->cfg_hdw_queue; i++) { - if (len > (LPFC_HDWQINFO_SIZE - 80)) + if (len > (LPFC_HDWQINFO_SIZE - 100)) break; - qp = &phba->sli4_hba.hdwq[lpfc_debugfs_last_hdwq]; + qp = &phba->sli4_hba.hdwq[lpfc_debugfs_last_lock]; - len += snprintf(buf + len, size - len, "HdwQ %d Info ", i); - spin_lock_irqsave(&qp->abts_scsi_buf_list_lock, iflag); - spin_lock(&qp->abts_nvme_buf_list_lock); - spin_lock(&qp->io_buf_list_get_lock); - spin_lock(&qp->io_buf_list_put_lock); - out = qp->total_io_bufs - (qp->get_io_bufs + qp->put_io_bufs + - qp->abts_scsi_io_bufs + qp->abts_nvme_io_bufs); - len += snprintf(buf + len, size - len, - "tot:%d get:%d put:%d mt:%d " - "ABTS scsi:%d nvme:%d Out:%d\n", - qp->total_io_bufs, qp->get_io_bufs, qp->put_io_bufs, - qp->empty_io_bufs, qp->abts_scsi_io_bufs, - qp->abts_nvme_io_bufs, out); - spin_unlock(&qp->io_buf_list_put_lock); - spin_unlock(&qp->io_buf_list_get_lock); - spin_unlock(&qp->abts_nvme_buf_list_lock); - spin_unlock_irqrestore(&qp->abts_scsi_buf_list_lock, iflag); + len += snprintf(buf + len, size - len, "HdwQ %03d Lock ", i); + if (phba->cfg_xri_rebalancing) { + len += snprintf(buf + len, size - len, + "get_pvt:%d mv_pvt:%d " + "mv2pub:%d mv2pvt:%d " + "put_pvt:%d put_pub:%d wq:%d\n", + qp->lock_conflict.alloc_pvt_pool, + qp->lock_conflict.mv_from_pvt_pool, + qp->lock_conflict.mv_to_pub_pool, + qp->lock_conflict.mv_to_pvt_pool, + qp->lock_conflict.free_pvt_pool, + qp->lock_conflict.free_pub_pool, + qp->lock_conflict.wq_access); + } else { + len += snprintf(buf + len, size - len, + "get:%d put:%d free:%d wq:%d\n", + qp->lock_conflict.alloc_xri_get, + qp->lock_conflict.alloc_xri_put, + qp->lock_conflict.free_xri, + qp->lock_conflict.wq_access); + } - lpfc_debugfs_last_hdwq++; - if (lpfc_debugfs_last_hdwq >= phba->cfg_hdw_queue) - lpfc_debugfs_last_hdwq = 0; + lpfc_debugfs_last_lock++; + if (lpfc_debugfs_last_lock >= phba->cfg_hdw_queue) + lpfc_debugfs_last_lock = 0; } return len; } +#endif static int lpfc_debugfs_last_hba_slim_off; @@ -964,7 +1042,7 @@ lpfc_debugfs_nvmestat_data(struct lpfc_vport *vport, char *buf, int size) struct lpfc_nvme_lport *lport; uint64_t data1, data2, data3; uint64_t tot, totin, totout; - int cnt, i, maxch; + int cnt, i; int len = 0; if (phba->nvmet_support) { @@ -1106,10 +1184,6 @@ lpfc_debugfs_nvmestat_data(struct lpfc_vport *vport, char *buf, int size) atomic_read(&lport->fc4NvmeLsRequests), atomic_read(&lport->fc4NvmeLsCmpls)); - if (phba->cfg_hdw_queue < LPFC_HBA_HDWQ_MAX) - maxch = phba->cfg_hdw_queue; - else - maxch = LPFC_HBA_HDWQ_MAX; totin = 0; totout = 0; for (i = 0; i < phba->cfg_hdw_queue; i++) { @@ -1547,7 +1621,7 @@ lpfc_debugfs_cpucheck_data(struct lpfc_vport *vport, char *buf, int size) { struct lpfc_hba *phba = vport->phba; struct lpfc_sli4_hdw_queue *qp; - int i, j; + int i, j, max_cnt; int len = 0; uint32_t tot_xmt; uint32_t tot_rcv; @@ -1565,6 +1639,7 @@ lpfc_debugfs_cpucheck_data(struct lpfc_vport *vport, char *buf, int size) } else { len += snprintf(buf + len, PAGE_SIZE - len, "\n"); } + max_cnt = size - LPFC_DEBUG_OUT_LINE_SZ; for (i = 0; i < phba->cfg_hdw_queue; i++) { qp = &phba->sli4_hba.hdwq[i]; @@ -1606,6 +1681,11 @@ lpfc_debugfs_cpucheck_data(struct lpfc_vport *vport, char *buf, int size) } len += snprintf(buf + len, PAGE_SIZE - len, "Total: %x\n", tot_xmt); + if (len >= max_cnt) { + len += snprintf(buf + len, PAGE_SIZE - len, + "Truncated ...\n"); + return len; + } } return len; } @@ -1904,11 +1984,8 @@ lpfc_debugfs_multixripools_open(struct inode *inode, struct file *file) goto out; } - if (phba->cfg_xri_rebalancing) - debug->len = lpfc_debugfs_multixripools_data( - phba, debug->buffer, LPFC_DUMP_MULTIXRIPOOL_SIZE); - else - debug->len = 0; + debug->len = lpfc_debugfs_multixripools_data( + phba, debug->buffer, LPFC_DUMP_MULTIXRIPOOL_SIZE); debug->i_private = inode->i_private; file->private_data = debug; @@ -1918,8 +1995,9 @@ out: return rc; } +#ifdef LPFC_HDWQ_LOCK_STAT /** - * lpfc_debugfs_hdwqinfo_open - Open the hdwqinfo debugfs buffer + * lpfc_debugfs_lockstat_open - Open the lockstat debugfs buffer * @inode: The inode pointer that contains a vport pointer. * @file: The file pointer to attach the log output. * @@ -1934,7 +2012,7 @@ out: * error value. **/ static int -lpfc_debugfs_hdwqinfo_open(struct inode *inode, struct file *file) +lpfc_debugfs_lockstat_open(struct inode *inode, struct file *file) { struct lpfc_hba *phba = inode->i_private; struct lpfc_debug *debug; @@ -1951,7 +2029,7 @@ lpfc_debugfs_hdwqinfo_open(struct inode *inode, struct file *file) goto out; } - debug->len = lpfc_debugfs_hdwqinfo_data(phba, debug->buffer, + debug->len = lpfc_debugfs_lockstat_data(phba, debug->buffer, LPFC_HBQINFO_SIZE); file->private_data = debug; @@ -1960,6 +2038,48 @@ out: return rc; } +static ssize_t +lpfc_debugfs_lockstat_write(struct file *file, const char __user *buf, + size_t nbytes, loff_t *ppos) +{ + struct lpfc_debug *debug = file->private_data; + struct lpfc_hba *phba = (struct lpfc_hba *)debug->i_private; + struct lpfc_sli4_hdw_queue *qp; + char mybuf[64]; + char *pbuf; + int i; + + /* Protect copy from user */ + if (!access_ok(buf, nbytes)) + return -EFAULT; + + memset(mybuf, 0, sizeof(mybuf)); + + if (copy_from_user(mybuf, buf, nbytes)) + return -EFAULT; + pbuf = &mybuf[0]; + + if ((strncmp(pbuf, "reset", strlen("reset")) == 0) || + (strncmp(pbuf, "zero", strlen("zero")) == 0)) { + for (i = 0; i < phba->cfg_hdw_queue; i++) { + qp = &phba->sli4_hba.hdwq[i]; + qp->lock_conflict.alloc_xri_get = 0; + qp->lock_conflict.alloc_xri_put = 0; + qp->lock_conflict.free_xri = 0; + qp->lock_conflict.wq_access = 0; + qp->lock_conflict.alloc_pvt_pool = 0; + qp->lock_conflict.mv_from_pvt_pool = 0; + qp->lock_conflict.mv_to_pub_pool = 0; + qp->lock_conflict.mv_to_pvt_pool = 0; + qp->lock_conflict.free_pvt_pool = 0; + qp->lock_conflict.free_pub_pool = 0; + qp->lock_conflict.wq_access = 0; + } + } + return nbytes; +} +#endif + /** * lpfc_debugfs_dumpHBASlim_open - Open the Dump HBA SLIM debugfs buffer * @inode: The inode pointer that contains a vport pointer. @@ -2816,7 +2936,7 @@ lpfc_debugfs_cpucheck_open(struct inode *inode, struct file *file) } debug->len = lpfc_debugfs_cpucheck_data(vport, debug->buffer, - LPFC_NVMEKTIME_SIZE); + LPFC_CPUCHECK_SIZE); debug->i_private = inode->i_private; file->private_data = debug; @@ -2848,11 +2968,21 @@ lpfc_debugfs_cpucheck_write(struct file *file, const char __user *buf, pbuf = &mybuf[0]; if ((strncmp(pbuf, "on", sizeof("on") - 1) == 0)) { + if (phba->nvmet_support) + phba->cpucheck_on |= LPFC_CHECK_NVMET_IO; + else + phba->cpucheck_on |= (LPFC_CHECK_NVME_IO | + LPFC_CHECK_SCSI_IO); + return strlen(pbuf); + } else if ((strncmp(pbuf, "nvme_on", sizeof("nvme_on") - 1) == 0)) { if (phba->nvmet_support) phba->cpucheck_on |= LPFC_CHECK_NVMET_IO; else phba->cpucheck_on |= LPFC_CHECK_NVME_IO; return strlen(pbuf); + } else if ((strncmp(pbuf, "scsi_on", sizeof("scsi_on") - 1) == 0)) { + phba->cpucheck_on |= LPFC_CHECK_SCSI_IO; + return strlen(pbuf); } else if ((strncmp(pbuf, "rcv", sizeof("rcv") - 1) == 0)) { if (phba->nvmet_support) @@ -3732,46 +3862,38 @@ lpfc_idiag_cqs_for_eq(struct lpfc_hba *phba, char *pbuffer, int *len, int max_cnt, int eqidx, int eq_id) { struct lpfc_queue *qp; - int qidx, rc; + int rc; - for (qidx = 0; qidx < phba->cfg_hdw_queue; qidx++) { - qp = phba->sli4_hba.hdwq[qidx].fcp_cq; - if (qp->assoc_qid != eq_id) - continue; + qp = phba->sli4_hba.hdwq[eqidx].fcp_cq; - *len = __lpfc_idiag_print_cq(qp, "FCP", pbuffer, *len); + *len = __lpfc_idiag_print_cq(qp, "FCP", pbuffer, *len); - /* Reset max counter */ - qp->CQ_max_cqe = 0; + /* Reset max counter */ + qp->CQ_max_cqe = 0; - if (*len >= max_cnt) - return 1; + if (*len >= max_cnt) + return 1; - rc = lpfc_idiag_wqs_for_cq(phba, "FCP", pbuffer, len, - max_cnt, qp->queue_id); - if (rc) - return 1; - } + rc = lpfc_idiag_wqs_for_cq(phba, "FCP", pbuffer, len, + max_cnt, qp->queue_id); + if (rc) + return 1; if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { - for (qidx = 0; qidx < phba->cfg_hdw_queue; qidx++) { - qp = phba->sli4_hba.hdwq[qidx].nvme_cq; - if (qp->assoc_qid != eq_id) - continue; + qp = phba->sli4_hba.hdwq[eqidx].nvme_cq; - *len = __lpfc_idiag_print_cq(qp, "NVME", pbuffer, *len); + *len = __lpfc_idiag_print_cq(qp, "NVME", pbuffer, *len); - /* Reset max counter */ - qp->CQ_max_cqe = 0; + /* Reset max counter */ + qp->CQ_max_cqe = 0; - if (*len >= max_cnt) - return 1; + if (*len >= max_cnt) + return 1; - rc = lpfc_idiag_wqs_for_cq(phba, "NVME", pbuffer, len, - max_cnt, qp->queue_id); - if (rc) - return 1; - } + rc = lpfc_idiag_wqs_for_cq(phba, "NVME", pbuffer, len, + max_cnt, qp->queue_id); + if (rc) + return 1; } if ((eqidx < phba->cfg_nvmet_mrq) && phba->nvmet_support) { @@ -3812,9 +3934,10 @@ __lpfc_idiag_print_eq(struct lpfc_queue *qp, char *eqtype, (unsigned long long)qp->q_cnt_4, qp->q_mode); len += snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len, "EQID[%02d], QE-CNT[%04d], QE-SZ[%04d], " - "HST-IDX[%04d], PRT-IDX[%04d], PST[%03d]", + "HST-IDX[%04d], PRT-IDX[%04d], PST[%03d] AFFIN[%03d]", qp->queue_id, qp->entry_count, qp->entry_size, - qp->host_index, qp->hba_index, qp->entry_repost); + qp->host_index, qp->hba_index, qp->entry_repost, + qp->chann); len += snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len, "\n"); return len; @@ -3869,7 +3992,7 @@ lpfc_idiag_queinfo_read(struct file *file, char __user *buf, size_t nbytes, phba->lpfc_idiag_last_eq = 0; len += snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len, - "EQ %d out of %d HBA EQs\n", + "HDWQ %d out of %d HBA HDWQs\n", x, phba->cfg_hdw_queue); /* Fast-path EQ */ @@ -5299,14 +5422,17 @@ static const struct file_operations lpfc_debugfs_op_hbqinfo = { .release = lpfc_debugfs_release, }; -#undef lpfc_debugfs_op_hdwqinfo -static const struct file_operations lpfc_debugfs_op_hdwqinfo = { +#ifdef LPFC_HDWQ_LOCK_STAT +#undef lpfc_debugfs_op_lockstat +static const struct file_operations lpfc_debugfs_op_lockstat = { .owner = THIS_MODULE, - .open = lpfc_debugfs_hdwqinfo_open, + .open = lpfc_debugfs_lockstat_open, .llseek = lpfc_debugfs_lseek, .read = lpfc_debugfs_read, + .write = lpfc_debugfs_lockstat_write, .release = lpfc_debugfs_release, }; +#endif #undef lpfc_debugfs_op_dumpHBASlim static const struct file_operations lpfc_debugfs_op_dumpHBASlim = { @@ -5756,17 +5882,19 @@ lpfc_debugfs_initialize(struct lpfc_vport *vport) phba->hba_debugfs_root, phba, &lpfc_debugfs_op_hbqinfo); - /* Setup hdwqinfo */ - snprintf(name, sizeof(name), "hdwqinfo"); - phba->debug_hdwqinfo = +#ifdef LPFC_HDWQ_LOCK_STAT + /* Setup lockstat */ + snprintf(name, sizeof(name), "lockstat"); + phba->debug_lockstat = debugfs_create_file(name, S_IFREG | 0644, phba->hba_debugfs_root, - phba, &lpfc_debugfs_op_hdwqinfo); - if (!phba->debug_hdwqinfo) { + phba, &lpfc_debugfs_op_lockstat); + if (!phba->debug_lockstat) { lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT, - "0511 Cant create debugfs hdwqinfo\n"); + "0913 Cant create debugfs lockstat\n"); goto debug_failed; } +#endif /* Setup dumpHBASlim */ if (phba->sli_rev < LPFC_SLI_REV4) { @@ -6006,7 +6134,7 @@ nvmeio_off: vport, &lpfc_debugfs_op_scsistat); if (!vport->debug_scsistat) { lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT, - "0811 Cannot create debugfs scsistat\n"); + "0914 Cannot create debugfs scsistat\n"); goto debug_failed; } @@ -6171,9 +6299,10 @@ lpfc_debugfs_terminate(struct lpfc_vport *vport) debugfs_remove(phba->debug_hbqinfo); /* hbqinfo */ phba->debug_hbqinfo = NULL; - debugfs_remove(phba->debug_hdwqinfo); /* hdwqinfo */ - phba->debug_hdwqinfo = NULL; - +#ifdef LPFC_HDWQ_LOCK_STAT + debugfs_remove(phba->debug_lockstat); /* lockstat */ + phba->debug_lockstat = NULL; +#endif debugfs_remove(phba->debug_dumpHBASlim); /* HBASlim */ phba->debug_dumpHBASlim = NULL; diff --git a/drivers/scsi/lpfc/lpfc_debugfs.h b/drivers/scsi/lpfc/lpfc_debugfs.h index cf256a6dca42..1fbee6496f85 100644 --- a/drivers/scsi/lpfc/lpfc_debugfs.h +++ b/drivers/scsi/lpfc/lpfc_debugfs.h @@ -290,9 +290,6 @@ struct lpfc_idiag { /* multixripool output buffer size */ #define LPFC_DUMP_MULTIXRIPOOL_SIZE 8192 -/* hdwqinfo output buffer size */ -#define LPFC_HDWQINFO_SIZE 8192 - enum { DUMP_FCP, DUMP_NVME, diff --git a/drivers/scsi/lpfc/lpfc_hw4.h b/drivers/scsi/lpfc/lpfc_hw4.h index cd39845c909f..665852291a4f 100644 --- a/drivers/scsi/lpfc/lpfc_hw4.h +++ b/drivers/scsi/lpfc/lpfc_hw4.h @@ -211,9 +211,8 @@ struct lpfc_sli_intf { #define LPFC_DEF_IMAX 150000 #define LPFC_MIN_CPU_MAP 0 -#define LPFC_MAX_CPU_MAP 2 +#define LPFC_MAX_CPU_MAP 1 #define LPFC_HBA_CPU_MAP 1 -#define LPFC_DRIVER_CPU_MAP 2 /* Default */ /* PORT_CAPABILITIES constants. */ #define LPFC_MAX_SUPPORTED_PAGES 8 diff --git a/drivers/scsi/lpfc/lpfc_init.c b/drivers/scsi/lpfc/lpfc_init.c index d9db29817f6b..145c08f112a3 100644 --- a/drivers/scsi/lpfc/lpfc_init.c +++ b/drivers/scsi/lpfc/lpfc_init.c @@ -37,6 +37,7 @@ #include #include #include +#include #include #include @@ -92,6 +93,8 @@ static void lpfc_sli4_cq_event_release_all(struct lpfc_hba *); static void lpfc_sli4_disable_intr(struct lpfc_hba *); static uint32_t lpfc_sli4_enable_intr(struct lpfc_hba *, uint32_t); static void lpfc_sli4_oas_verify(struct lpfc_hba *phba); +static uint16_t lpfc_find_eq_handle(struct lpfc_hba *, uint16_t); +static uint16_t lpfc_find_cpu_handle(struct lpfc_hba *, uint16_t, int); static struct scsi_transport_template *lpfc_transport_template = NULL; static struct scsi_transport_template *lpfc_vport_transport_template = NULL; @@ -1367,13 +1370,13 @@ lpfc_hb_timeout_handler(struct lpfc_hba *phba) } /* Interrupts per sec per EQ */ - val = phba->cfg_fcp_imax / phba->cfg_hdw_queue; + val = phba->cfg_fcp_imax / phba->cfg_irq_chann; tick_cqe = val / CONFIG_HZ; /* Per tick per EQ */ /* Assume 1 CQE/ISR, calc max CQEs allowed for time duration */ max_cqe = time_elapsed * tick_cqe; - for (i = 0; i < phba->cfg_hdw_queue; i++) { + for (i = 0; i < phba->cfg_irq_chann; i++) { /* Fast-path EQ */ qp = phba->sli4_hba.hdwq[i].hba_eq; if (!qp) @@ -1397,7 +1400,7 @@ lpfc_hb_timeout_handler(struct lpfc_hba *phba) if (val) { /* First, interrupts per sec per EQ */ val = phba->cfg_fcp_imax / - phba->cfg_hdw_queue; + phba->cfg_irq_chann; /* us delay between each interrupt */ val = LPFC_SEC_TO_USEC / val; @@ -4335,8 +4338,13 @@ lpfc_create_port(struct lpfc_hba *phba, int instance, struct device *dev) shost->max_lun = vport->cfg_max_luns; shost->this_id = -1; shost->max_cmd_len = 16; + if (phba->sli_rev == LPFC_SLI_REV4) { - shost->nr_hw_queues = phba->cfg_hdw_queue; + if (phba->cfg_fcp_io_sched == LPFC_FCP_SCHED_BY_HDWQ) + shost->nr_hw_queues = phba->cfg_hdw_queue; + else + shost->nr_hw_queues = phba->sli4_hba.num_present_cpu; + shost->dma_boundary = phba->sli4_hba.pc_sli4_params.sge_supp_len-1; shost->sg_tablesize = phba->cfg_scsi_seg_cnt; @@ -6819,7 +6827,7 @@ lpfc_sli4_driver_resource_setup(struct lpfc_hba *phba) goto out_remove_rpi_hdrs; } - phba->sli4_hba.hba_eq_hdl = kcalloc(phba->cfg_hdw_queue, + phba->sli4_hba.hba_eq_hdl = kcalloc(phba->cfg_irq_chann, sizeof(struct lpfc_hba_eq_hdl), GFP_KERNEL); if (!phba->sli4_hba.hba_eq_hdl) { @@ -8257,7 +8265,7 @@ lpfc_sli4_read_config(struct lpfc_hba *phba) struct lpfc_rsrc_desc_fcfcoe *desc; char *pdesc_0; uint16_t forced_link_speed; - uint32_t if_type; + uint32_t if_type, qmin; int length, i, rc = 0, rc2; pmb = (LPFC_MBOXQ_t *) mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL); @@ -8362,40 +8370,44 @@ lpfc_sli4_read_config(struct lpfc_hba *phba) phba->sli4_hba.max_cfg_param.max_rq); /* - * Calculate NVME queue resources based on how - * many WQ/CQs are available. + * Calculate queue resources based on how + * many WQ/CQ/EQs are available. */ - if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { - length = phba->sli4_hba.max_cfg_param.max_wq; - if (phba->sli4_hba.max_cfg_param.max_cq < - phba->sli4_hba.max_cfg_param.max_wq) - length = phba->sli4_hba.max_cfg_param.max_cq; + qmin = phba->sli4_hba.max_cfg_param.max_wq; + if (phba->sli4_hba.max_cfg_param.max_cq < qmin) + qmin = phba->sli4_hba.max_cfg_param.max_cq; + if (phba->sli4_hba.max_cfg_param.max_eq < qmin) + qmin = phba->sli4_hba.max_cfg_param.max_eq; + /* + * Whats left after this can go toward NVME / FCP. + * The minus 4 accounts for ELS, NVME LS, MBOX + * plus one extra. When configured for + * NVMET, FCP io channel WQs are not created. + */ + qmin -= 4; - /* - * Whats left after this can go toward NVME. - * The minus 6 accounts for ELS, NVME LS, MBOX - * plus a couple extra. When configured for - * NVMET, FCP io channel WQs are not created. - */ - length -= 6; - - /* Take off FCP queues */ - if (!phba->nvmet_support) - length -= phba->cfg_hdw_queue; - - /* Check to see if there is enough for NVME */ - if (phba->cfg_hdw_queue > length) { - lpfc_printf_log( - phba, KERN_ERR, LOG_SLI, - "2005 Reducing NVME IO channel to %d: " - "WQ %d CQ %d CommonIO %d\n", - length, + /* If NVME is configured, double the number of CQ/WQs needed */ + if ((phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) && + !phba->nvmet_support) + qmin /= 2; + + /* Check to see if there is enough for NVME */ + if ((phba->cfg_irq_chann > qmin) || + (phba->cfg_hdw_queue > qmin)) { + lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + "2005 Reducing Queues: " + "WQ %d CQ %d EQ %d: min %d: " + "IRQ %d HDWQ %d\n", phba->sli4_hba.max_cfg_param.max_wq, phba->sli4_hba.max_cfg_param.max_cq, + phba->sli4_hba.max_cfg_param.max_eq, + qmin, phba->cfg_irq_chann, phba->cfg_hdw_queue); - phba->cfg_hdw_queue = length; - } + if (phba->cfg_irq_chann > qmin) + phba->cfg_irq_chann = qmin; + if (phba->cfg_hdw_queue > qmin) + phba->cfg_hdw_queue = qmin; } } @@ -8612,25 +8624,17 @@ lpfc_sli4_queue_verify(struct lpfc_hba *phba) * device parameters */ - if (phba->cfg_hdw_queue > phba->sli4_hba.max_cfg_param.max_eq) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, - "2575 Reducing IO channels to match number of " - "available EQs: from %d to %d\n", - phba->cfg_hdw_queue, - phba->sli4_hba.max_cfg_param.max_eq); - phba->cfg_hdw_queue = phba->sli4_hba.max_cfg_param.max_eq; - } - if (phba->nvmet_support) { - if (phba->cfg_hdw_queue < phba->cfg_nvmet_mrq) - phba->cfg_nvmet_mrq = phba->cfg_hdw_queue; + if (phba->cfg_irq_chann < phba->cfg_nvmet_mrq) + phba->cfg_nvmet_mrq = phba->cfg_irq_chann; } if (phba->cfg_nvmet_mrq > LPFC_NVMET_MRQ_MAX) phba->cfg_nvmet_mrq = LPFC_NVMET_MRQ_MAX; lpfc_printf_log(phba, KERN_ERR, LOG_INIT, - "2574 IO channels: hdwQ %d MRQ: %d\n", - phba->cfg_hdw_queue, phba->cfg_nvmet_mrq); + "2574 IO channels: hdwQ %d IRQ %d MRQ: %d\n", + phba->cfg_hdw_queue, phba->cfg_irq_chann, + phba->cfg_nvmet_mrq); /* Get EQ depth from module parameter, fake the default for now */ phba->sli4_hba.eq_esize = LPFC_EQE_SIZE_4B; @@ -8658,6 +8662,7 @@ lpfc_alloc_nvme_wq_cq(struct lpfc_hba *phba, int wqidx) } qdesc->qe_valid = 1; qdesc->hdwq = wqidx; + qdesc->chann = lpfc_find_cpu_handle(phba, wqidx, LPFC_FIND_BY_HDWQ); phba->sli4_hba.hdwq[wqidx].nvme_cq = qdesc; qdesc = lpfc_sli4_queue_alloc(phba, LPFC_EXPANDED_PAGE_SIZE, @@ -8669,6 +8674,7 @@ lpfc_alloc_nvme_wq_cq(struct lpfc_hba *phba, int wqidx) return 1; } qdesc->hdwq = wqidx; + qdesc->chann = wqidx; phba->sli4_hba.hdwq[wqidx].nvme_wq = qdesc; list_add_tail(&qdesc->wq_list, &phba->sli4_hba.lpfc_wq_list); return 0; @@ -8698,6 +8704,7 @@ lpfc_alloc_fcp_wq_cq(struct lpfc_hba *phba, int wqidx) } qdesc->qe_valid = 1; qdesc->hdwq = wqidx; + qdesc->chann = lpfc_find_cpu_handle(phba, wqidx, LPFC_FIND_BY_HDWQ); phba->sli4_hba.hdwq[wqidx].fcp_cq = qdesc; /* Create Fast Path FCP WQs */ @@ -8720,6 +8727,7 @@ lpfc_alloc_fcp_wq_cq(struct lpfc_hba *phba, int wqidx) return 1; } qdesc->hdwq = wqidx; + qdesc->chann = wqidx; phba->sli4_hba.hdwq[wqidx].fcp_wq = qdesc; list_add_tail(&qdesc->wq_list, &phba->sli4_hba.lpfc_wq_list); return 0; @@ -8743,7 +8751,7 @@ int lpfc_sli4_queue_create(struct lpfc_hba *phba) { struct lpfc_queue *qdesc; - int idx; + int idx, eqidx; struct lpfc_sli4_hdw_queue *qp; /* @@ -8829,7 +8837,18 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba) /* Create HBA Event Queues (EQs) */ for (idx = 0; idx < phba->cfg_hdw_queue; idx++) { - /* Create EQs */ + /* + * If there are more Hardware Queues than available + * CQs, multiple Hardware Queues may share a common EQ. + */ + if (idx >= phba->cfg_irq_chann) { + /* Share an existing EQ */ + eqidx = lpfc_find_eq_handle(phba, idx); + phba->sli4_hba.hdwq[idx].hba_eq = + phba->sli4_hba.hdwq[eqidx].hba_eq; + continue; + } + /* Create an EQ */ qdesc = lpfc_sli4_queue_alloc(phba, LPFC_DEFAULT_PAGE_SIZE, phba->sli4_hba.eq_esize, phba->sli4_hba.eq_ecount); @@ -8840,20 +8859,27 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba) } qdesc->qe_valid = 1; qdesc->hdwq = idx; + + /* Save the CPU this EQ is affinitised to */ + eqidx = lpfc_find_eq_handle(phba, idx); + qdesc->chann = lpfc_find_cpu_handle(phba, eqidx, + LPFC_FIND_BY_EQ); phba->sli4_hba.hdwq[idx].hba_eq = qdesc; } /* Allocate SCSI SLI4 CQ/WQs */ - for (idx = 0; idx < phba->cfg_hdw_queue; idx++) + for (idx = 0; idx < phba->cfg_hdw_queue; idx++) { if (lpfc_alloc_fcp_wq_cq(phba, idx)) goto out_error; + } /* Allocate NVME SLI4 CQ/WQs */ if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { - for (idx = 0; idx < phba->cfg_hdw_queue; idx++) + for (idx = 0; idx < phba->cfg_hdw_queue; idx++) { if (lpfc_alloc_nvme_wq_cq(phba, idx)) goto out_error; + } if (phba->nvmet_support) { for (idx = 0; idx < phba->cfg_nvmet_mrq; idx++) { @@ -8871,6 +8897,7 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba) } qdesc->qe_valid = 1; qdesc->hdwq = idx; + qdesc->chann = idx; phba->sli4_hba.nvmet_cqset[idx] = qdesc; } } @@ -8902,6 +8929,7 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba) goto out_error; } qdesc->qe_valid = 1; + qdesc->chann = 0; phba->sli4_hba.els_cq = qdesc; @@ -8919,6 +8947,7 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba) "0505 Failed allocate slow-path MQ\n"); goto out_error; } + qdesc->chann = 0; phba->sli4_hba.mbx_wq = qdesc; /* @@ -8934,6 +8963,7 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba) "0504 Failed allocate slow-path ELS WQ\n"); goto out_error; } + qdesc->chann = 0; phba->sli4_hba.els_wq = qdesc; list_add_tail(&qdesc->wq_list, &phba->sli4_hba.lpfc_wq_list); @@ -8947,6 +8977,7 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba) "6079 Failed allocate NVME LS CQ\n"); goto out_error; } + qdesc->chann = 0; qdesc->qe_valid = 1; phba->sli4_hba.nvmels_cq = qdesc; @@ -8959,6 +8990,7 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba) "6080 Failed allocate NVME LS WQ\n"); goto out_error; } + qdesc->chann = 0; phba->sli4_hba.nvmels_wq = qdesc; list_add_tail(&qdesc->wq_list, &phba->sli4_hba.lpfc_wq_list); } @@ -9085,17 +9117,21 @@ lpfc_sli4_release_queues(struct lpfc_queue ***qs, int max) } static inline void -lpfc_sli4_release_hdwq(struct lpfc_sli4_hdw_queue *hdwq, int max) +lpfc_sli4_release_hdwq(struct lpfc_hba *phba) { + struct lpfc_sli4_hdw_queue *hdwq; uint32_t idx; - for (idx = 0; idx < max; idx++) { - lpfc_sli4_queue_free(hdwq[idx].hba_eq); + hdwq = phba->sli4_hba.hdwq; + for (idx = 0; idx < phba->cfg_hdw_queue; idx++) { + if (idx < phba->cfg_irq_chann) + lpfc_sli4_queue_free(hdwq[idx].hba_eq); + hdwq[idx].hba_eq = NULL; + lpfc_sli4_queue_free(hdwq[idx].fcp_cq); lpfc_sli4_queue_free(hdwq[idx].nvme_cq); lpfc_sli4_queue_free(hdwq[idx].fcp_wq); lpfc_sli4_queue_free(hdwq[idx].nvme_wq); - hdwq[idx].hba_eq = NULL; hdwq[idx].fcp_cq = NULL; hdwq[idx].nvme_cq = NULL; hdwq[idx].fcp_wq = NULL; @@ -9120,8 +9156,7 @@ lpfc_sli4_queue_destroy(struct lpfc_hba *phba) { /* Release HBA eqs */ if (phba->sli4_hba.hdwq) - lpfc_sli4_release_hdwq(phba->sli4_hba.hdwq, - phba->cfg_hdw_queue); + lpfc_sli4_release_hdwq(phba); if (phba->nvmet_support) { lpfc_sli4_release_queues(&phba->sli4_hba.nvmet_cqset, @@ -9202,7 +9237,6 @@ lpfc_create_wq_cq(struct lpfc_hba *phba, struct lpfc_queue *eq, qidx, (uint32_t)rc); return rc; } - cq->chann = qidx; if (qtype != LPFC_MBOX) { /* Setup cq_map for fast lookup */ @@ -9222,7 +9256,6 @@ lpfc_create_wq_cq(struct lpfc_hba *phba, struct lpfc_queue *eq, /* no need to tear down cq - caller will do so */ return rc; } - wq->chann = qidx; /* Bind this CQ/WQ to the NVME ring */ pring = wq->pring; @@ -9251,6 +9284,38 @@ lpfc_create_wq_cq(struct lpfc_hba *phba, struct lpfc_queue *eq, return 0; } +/** + * lpfc_setup_cq_lookup - Setup the CQ lookup table + * @phba: pointer to lpfc hba data structure. + * + * This routine will populate the cq_lookup table by all + * available CQ queue_id's. + **/ +void +lpfc_setup_cq_lookup(struct lpfc_hba *phba) +{ + struct lpfc_queue *eq, *childq; + struct lpfc_sli4_hdw_queue *qp; + int qidx; + + qp = phba->sli4_hba.hdwq; + memset(phba->sli4_hba.cq_lookup, 0, + (sizeof(struct lpfc_queue *) * (phba->sli4_hba.cq_max + 1))); + for (qidx = 0; qidx < phba->cfg_irq_chann; qidx++) { + eq = qp[qidx].hba_eq; + if (!eq) + continue; + list_for_each_entry(childq, &eq->child_list, list) { + if (childq->queue_id > phba->sli4_hba.cq_max) + continue; + if ((childq->subtype == LPFC_FCP) || + (childq->subtype == LPFC_NVME)) + phba->sli4_hba.cq_lookup[childq->queue_id] = + childq; + } + } +} + /** * lpfc_sli4_queue_setup - Set up all the SLI4 queues * @phba: pointer to lpfc hba data structure. @@ -9331,7 +9396,7 @@ lpfc_sli4_queue_setup(struct lpfc_hba *phba) rc = -ENOMEM; goto out_error; } - for (qidx = 0; qidx < phba->cfg_hdw_queue; qidx++) { + for (qidx = 0; qidx < phba->cfg_irq_chann; qidx++) { if (!qp[qidx].hba_eq) { lpfc_printf_log(phba, KERN_ERR, LOG_INIT, "0522 Fast-path EQ (%d) not " @@ -9578,11 +9643,23 @@ lpfc_sli4_queue_setup(struct lpfc_hba *phba) phba->sli4_hba.dat_rq->queue_id, phba->sli4_hba.els_cq->queue_id); - for (qidx = 0; qidx < phba->cfg_hdw_queue; + for (qidx = 0; qidx < phba->cfg_irq_chann; qidx += LPFC_MAX_EQ_DELAY_EQID_CNT) lpfc_modify_hba_eq_delay(phba, qidx, LPFC_MAX_EQ_DELAY_EQID_CNT, phba->cfg_fcp_imax); + if (phba->sli4_hba.cq_max) { + kfree(phba->sli4_hba.cq_lookup); + phba->sli4_hba.cq_lookup = kcalloc((phba->sli4_hba.cq_max + 1), + sizeof(struct lpfc_queue *), GFP_KERNEL); + if (!phba->sli4_hba.cq_lookup) { + lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + "0549 Failed setup of CQ Lookup table: " + "size 0x%x\n", phba->sli4_hba.cq_max); + goto out_destroy; + } + lpfc_setup_cq_lookup(phba); + } return 0; out_destroy: @@ -9664,9 +9741,14 @@ lpfc_sli4_queue_unset(struct lpfc_hba *phba) lpfc_wq_destroy(phba, qp->nvme_wq); lpfc_cq_destroy(phba, qp->fcp_cq); lpfc_cq_destroy(phba, qp->nvme_cq); - lpfc_eq_destroy(phba, qp->hba_eq); + if (qidx < phba->cfg_irq_chann) + lpfc_eq_destroy(phba, qp->hba_eq); } } + + kfree(phba->sli4_hba.cq_lookup); + phba->sli4_hba.cq_lookup = NULL; + phba->sli4_hba.cq_max = 0; } /** @@ -10445,23 +10527,199 @@ lpfc_sli_disable_intr(struct lpfc_hba *phba) phba->sli.slistat.sli_intr = 0; } +/** + * lpfc_find_cpu_handle - Find the CPU that corresponds to the specified EQ + * @phba: pointer to lpfc hba data structure. + * @id: EQ vector index or Hardware Queue index + * @match: LPFC_FIND_BY_EQ = match by EQ + * LPFC_FIND_BY_HDWQ = match by Hardware Queue + */ +static uint16_t +lpfc_find_cpu_handle(struct lpfc_hba *phba, uint16_t id, int match) +{ + struct lpfc_vector_map_info *cpup; + int cpu; + + /* Find the desired phys_id for the specified EQ */ + cpup = phba->sli4_hba.cpu_map; + for (cpu = 0; cpu < phba->sli4_hba.num_present_cpu; cpu++) { + if ((match == LPFC_FIND_BY_EQ) && + (cpup->irq != LPFC_VECTOR_MAP_EMPTY) && + (cpup->eq == id)) + return cpu; + if ((match == LPFC_FIND_BY_HDWQ) && (cpup->hdwq == id)) + return cpu; + cpup++; + } + return 0; +} + +/** + * lpfc_find_eq_handle - Find the EQ that corresponds to the specified + * Hardware Queue + * @phba: pointer to lpfc hba data structure. + * @hdwq: Hardware Queue index + */ +static uint16_t +lpfc_find_eq_handle(struct lpfc_hba *phba, uint16_t hdwq) +{ + struct lpfc_vector_map_info *cpup; + int cpu; + + /* Find the desired phys_id for the specified EQ */ + cpup = phba->sli4_hba.cpu_map; + for (cpu = 0; cpu < phba->sli4_hba.num_present_cpu; cpu++) { + if (cpup->hdwq == hdwq) + return cpup->eq; + cpup++; + } + return 0; +} + +/** + * lpfc_find_phys_id_eq - Find the next EQ that corresponds to the specified + * Physical Id. + * @phba: pointer to lpfc hba data structure. + * @eqidx: EQ index + * @phys_id: CPU package physical id + */ +static uint16_t +lpfc_find_phys_id_eq(struct lpfc_hba *phba, uint16_t eqidx, uint16_t phys_id) +{ + struct lpfc_vector_map_info *cpup; + int cpu, desired_phys_id; + + desired_phys_id = LPFC_VECTOR_MAP_EMPTY; + + /* Find the desired phys_id for the specified EQ */ + cpup = phba->sli4_hba.cpu_map; + for (cpu = 0; cpu < phba->sli4_hba.num_present_cpu; cpu++) { + if ((cpup->irq != LPFC_VECTOR_MAP_EMPTY) && + (cpup->eq == eqidx)) { + desired_phys_id = cpup->phys_id; + break; + } + cpup++; + } + if (phys_id == desired_phys_id) + return eqidx; + + /* Find a EQ thats on the specified phys_id */ + cpup = phba->sli4_hba.cpu_map; + for (cpu = 0; cpu < phba->sli4_hba.num_present_cpu; cpu++) { + if ((cpup->irq != LPFC_VECTOR_MAP_EMPTY) && + (cpup->phys_id == phys_id)) + return cpup->eq; + cpup++; + } + return 0; +} + +/** + * lpfc_find_cpu_map - Find next available CPU map entry that matches the + * phys_id and core_id. + * @phba: pointer to lpfc hba data structure. + * @phys_id: CPU package physical id + * @core_id: CPU core id + * @hdwqidx: Hardware Queue index + * @eqidx: EQ index + * @isr_avail: Should an IRQ be associated with this entry + */ +static struct lpfc_vector_map_info * +lpfc_find_cpu_map(struct lpfc_hba *phba, uint16_t phys_id, uint16_t core_id, + uint16_t hdwqidx, uint16_t eqidx, int isr_avail) +{ + struct lpfc_vector_map_info *cpup; + int cpu; + + cpup = phba->sli4_hba.cpu_map; + for (cpu = 0; cpu < phba->sli4_hba.num_present_cpu; cpu++) { + /* Does the cpup match the one we are looking for */ + if ((cpup->phys_id == phys_id) && + (cpup->core_id == core_id)) { + /* If it has been already assigned, then skip it */ + if (cpup->hdwq != LPFC_VECTOR_MAP_EMPTY) { + cpup++; + continue; + } + /* Ensure we are on the same phys_id as the first one */ + if (!isr_avail) + cpup->eq = lpfc_find_phys_id_eq(phba, eqidx, + phys_id); + else + cpup->eq = eqidx; + + cpup->hdwq = hdwqidx; + if (isr_avail) { + cpup->irq = + pci_irq_vector(phba->pcidev, eqidx); + + /* Now affinitize to the selected CPU */ + irq_set_affinity_hint(cpup->irq, + get_cpu_mask(cpu)); + irq_set_status_flags(cpup->irq, + IRQ_NO_BALANCING); + + lpfc_printf_log(phba, KERN_INFO, LOG_INIT, + "3330 Set Affinity: CPU %d " + "EQ %d irq %d (HDWQ %x)\n", + cpu, cpup->eq, + cpup->irq, cpup->hdwq); + } + return cpup; + } + cpup++; + } + return 0; +} + +#ifdef CONFIG_X86 +/** + * lpfc_find_hyper - Determine if the CPU map entry is hyper-threaded + * @phba: pointer to lpfc hba data structure. + * @cpu: CPU map index + * @phys_id: CPU package physical id + * @core_id: CPU core id + */ +static int +lpfc_find_hyper(struct lpfc_hba *phba, int cpu, + uint16_t phys_id, uint16_t core_id) +{ + struct lpfc_vector_map_info *cpup; + int idx; + + cpup = phba->sli4_hba.cpu_map; + for (idx = 0; idx < phba->sli4_hba.num_present_cpu; idx++) { + /* Does the cpup match the one we are looking for */ + if ((cpup->phys_id == phys_id) && + (cpup->core_id == core_id) && + (cpu != idx)) { + return 1; + } + cpup++; + } + return 0; +} +#endif + /** * lpfc_cpu_affinity_check - Check vector CPU affinity mappings * @phba: pointer to lpfc hba data structure. + * @vectors: number of msix vectors allocated. * * The routine will figure out the CPU affinity assignment for every - * MSI-X vector allocated for the HBA. The hba_eq_hdl will be updated - * with a pointer to the CPU mask that defines ALL the CPUs this vector - * can be associated with. If the vector can be unquely associated with - * a single CPU, that CPU will be recorded in hba_eq_hdl[index].cpu. + * MSI-X vector allocated for the HBA. * In addition, the CPU to IO channel mapping will be calculated * and the phba->sli4_hba.cpu_map array will reflect this. */ static void -lpfc_cpu_affinity_check(struct lpfc_hba *phba) +lpfc_cpu_affinity_check(struct lpfc_hba *phba, int vectors) { + int i, j, idx, phys_id; + int max_phys_id, min_phys_id; + int max_core_id, min_core_id; struct lpfc_vector_map_info *cpup; - int cpu, idx; + int cpu, eqidx, hdwqidx, isr_avail; #ifdef CONFIG_X86 struct cpuinfo_x86 *cpuinfo; #endif @@ -10471,6 +10729,12 @@ lpfc_cpu_affinity_check(struct lpfc_hba *phba) (sizeof(struct lpfc_vector_map_info) * phba->sli4_hba.num_present_cpu)); + max_phys_id = 0; + min_phys_id = 0xffff; + max_core_id = 0; + min_core_id = 0xffff; + phys_id = 0; + /* Update CPU map with physical id and core id of each CPU */ cpup = phba->sli4_hba.cpu_map; for (cpu = 0; cpu < phba->sli4_hba.num_present_cpu; cpu++) { @@ -10478,34 +10742,91 @@ lpfc_cpu_affinity_check(struct lpfc_hba *phba) cpuinfo = &cpu_data(cpu); cpup->phys_id = cpuinfo->phys_proc_id; cpup->core_id = cpuinfo->cpu_core_id; + cpup->hyper = lpfc_find_hyper(phba, cpu, + cpup->phys_id, cpup->core_id); #else /* No distinction between CPUs for other platforms */ cpup->phys_id = 0; - cpup->core_id = 0; + cpup->core_id = cpu; + cpup->hyper = 0; #endif + lpfc_printf_log(phba, KERN_INFO, LOG_INIT, "3328 CPU physid %d coreid %d\n", cpup->phys_id, cpup->core_id); + + if (cpup->phys_id > max_phys_id) + max_phys_id = cpup->phys_id; + if (cpup->phys_id < min_phys_id) + min_phys_id = cpup->phys_id; + + if (cpup->core_id > max_core_id) + max_core_id = cpup->core_id; + if (cpup->core_id < min_core_id) + min_core_id = cpup->core_id; + cpup++; } - for (idx = 0; idx < phba->cfg_hdw_queue; idx++) { - cpup = &phba->sli4_hba.cpu_map[idx]; - cpup->irq = pci_irq_vector(phba->pcidev, idx); + /* + * If the number of IRQ vectors == number of CPUs, + * mapping is pretty simple: 1 to 1. + * This is the desired path if NVME is enabled. + */ + if (vectors == phba->sli4_hba.num_present_cpu) { + cpup = phba->sli4_hba.cpu_map; + for (idx = 0; idx < vectors; idx++) { + cpup->eq = idx; + cpup->hdwq = idx; + cpup->irq = pci_irq_vector(phba->pcidev, idx); + + /* Now affinitize to the selected CPU */ + irq_set_affinity_hint( + pci_irq_vector(phba->pcidev, idx), + get_cpu_mask(idx)); + irq_set_status_flags(cpup->irq, IRQ_NO_BALANCING); - /* For now assume vector N maps to CPU N */ - irq_set_affinity_hint(cpup->irq, get_cpu_mask(idx)); - cpup->hdwq = idx; + lpfc_printf_log(phba, KERN_INFO, LOG_INIT, + "3336 Set Affinity: CPU %d " + "EQ %d irq %d\n", + idx, cpup->eq, + pci_irq_vector(phba->pcidev, idx)); + cpup++; + } + return; + } - lpfc_printf_log(phba, KERN_INFO, LOG_INIT, - "3336 Set Affinity: CPU %d " - "hdwq %d irq %d\n", - cpu, cpup->hdwq, cpup->irq); + idx = 0; + isr_avail = 1; + eqidx = 0; + hdwqidx = 0; + + /* Mapping is more complicated for this case. Hardware Queues are + * assigned in a "ping pong" fashion, ping pong-ing between the + * available phys_id's. + */ + while (idx < phba->sli4_hba.num_present_cpu) { + for (i = min_core_id; i <= max_core_id; i++) { + for (j = min_phys_id; j <= max_phys_id; j++) { + cpup = lpfc_find_cpu_map(phba, j, i, hdwqidx, + eqidx, isr_avail); + if (!cpup) + continue; + idx++; + hdwqidx++; + if (hdwqidx >= phba->cfg_hdw_queue) + hdwqidx = 0; + eqidx++; + if (eqidx >= phba->cfg_irq_chann) { + isr_avail = 0; + eqidx = 0; + } + } + } } return; } - /** * lpfc_sli4_enable_msix - Enable MSI-X interrupt mode to SLI-4 device * @phba: pointer to lpfc hba data structure. @@ -10524,7 +10845,7 @@ lpfc_sli4_enable_msix(struct lpfc_hba *phba) char *name; /* Set up MSI-X multi-message vectors */ - vectors = phba->cfg_hdw_queue; + vectors = phba->cfg_irq_chann; rc = pci_alloc_irq_vectors(phba->pcidev, (phba->nvmet_support) ? 1 : 2, @@ -10545,7 +10866,6 @@ lpfc_sli4_enable_msix(struct lpfc_hba *phba) phba->sli4_hba.hba_eq_hdl[index].idx = index; phba->sli4_hba.hba_eq_hdl[index].phba = phba; - atomic_set(&phba->sli4_hba.hba_eq_hdl[index].hba_eq_in_use, 1); rc = request_irq(pci_irq_vector(phba->pcidev, index), &lpfc_sli4_hba_intr_handler, 0, name, @@ -10558,17 +10878,16 @@ lpfc_sli4_enable_msix(struct lpfc_hba *phba) } } - if (vectors != phba->cfg_hdw_queue) { + if (vectors != phba->cfg_irq_chann) { lpfc_printf_log(phba, KERN_ERR, LOG_INIT, "3238 Reducing IO channels to match number of " "MSI-X vectors, requested %d got %d\n", - phba->cfg_hdw_queue, vectors); - if (phba->cfg_hdw_queue > vectors) - phba->cfg_hdw_queue = vectors; + phba->cfg_irq_chann, vectors); + if (phba->cfg_irq_chann > vectors) + phba->cfg_irq_chann = vectors; if (phba->cfg_nvmet_mrq > vectors) phba->cfg_nvmet_mrq = vectors; } - lpfc_cpu_affinity_check(phba); return rc; @@ -10623,7 +10942,7 @@ lpfc_sli4_enable_msi(struct lpfc_hba *phba) return rc; } - for (index = 0; index < phba->cfg_hdw_queue; index++) { + for (index = 0; index < phba->cfg_irq_chann; index++) { phba->sli4_hba.hba_eq_hdl[index].idx = index; phba->sli4_hba.hba_eq_hdl[index].phba = phba; } @@ -10688,11 +11007,10 @@ lpfc_sli4_enable_intr(struct lpfc_hba *phba, uint32_t cfg_mode) phba->intr_type = INTx; intr_mode = 0; - for (idx = 0; idx < phba->cfg_hdw_queue; idx++) { + for (idx = 0; idx < phba->cfg_irq_chann; idx++) { eqhdl = &phba->sli4_hba.hba_eq_hdl[idx]; eqhdl->idx = idx; eqhdl->phba = phba; - atomic_set(&eqhdl->hba_eq_in_use, 1); } } } @@ -10716,7 +11034,7 @@ lpfc_sli4_disable_intr(struct lpfc_hba *phba) int index; /* Free up MSI-X multi-message vectors */ - for (index = 0; index < phba->cfg_hdw_queue; index++) { + for (index = 0; index < phba->cfg_irq_chann; index++) { irq_set_affinity_hint( pci_irq_vector(phba->pcidev, index), NULL); @@ -12092,12 +12410,13 @@ lpfc_pci_probe_one_s4(struct pci_dev *pdev, const struct pci_device_id *pid) } /* Default to single EQ for non-MSI-X */ if (phba->intr_type != MSIX) { - phba->cfg_hdw_queue = 1; + phba->cfg_irq_chann = 1; if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { if (phba->nvmet_support) phba->cfg_nvmet_mrq = 1; } } + lpfc_cpu_affinity_check(phba, phba->cfg_irq_chann); /* Create SCSI host to the physical port */ error = lpfc_create_shost(phba); diff --git a/drivers/scsi/lpfc/lpfc_nvme.c b/drivers/scsi/lpfc/lpfc_nvme.c index c9aacd56a449..9480257c5143 100644 --- a/drivers/scsi/lpfc/lpfc_nvme.c +++ b/drivers/scsi/lpfc/lpfc_nvme.c @@ -239,7 +239,7 @@ lpfc_nvme_create_queue(struct nvme_fc_local_port *pnvme_lport, if (qidx) { str = "IO "; /* IO queue */ qhandle->index = ((qidx - 1) % - vport->phba->cfg_hdw_queue); + lpfc_nvme_template.max_hw_queues); } else { str = "ADM"; /* Admin queue */ qhandle->index = qidx; @@ -1546,14 +1546,12 @@ lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport, } } + /* Lookup Hardware Queue index based on fcp_io_sched module parameter */ if (phba->cfg_fcp_io_sched == LPFC_FCP_SCHED_BY_HDWQ) { idx = lpfc_queue_info->index; } else { cpu = smp_processor_id(); - if (cpu < phba->cfg_hdw_queue) - idx = cpu; - else - idx = cpu % phba->cfg_hdw_queue; + idx = phba->sli4_hba.cpu_map[cpu].hdwq; } lpfc_ncmd = lpfc_get_nvme_buf(phba, ndlp, idx, expedite); @@ -2060,7 +2058,13 @@ lpfc_nvme_create_localport(struct lpfc_vport *vport) * allocate + 3, one for cmd, one for rsp and one for this alignment */ lpfc_nvme_template.max_sgl_segments = phba->cfg_nvme_seg_cnt + 1; - lpfc_nvme_template.max_hw_queues = phba->cfg_hdw_queue; + + /* Advertise how many hw queues we support based on fcp_io_sched */ + if (phba->cfg_fcp_io_sched == LPFC_FCP_SCHED_BY_HDWQ) + lpfc_nvme_template.max_hw_queues = phba->cfg_hdw_queue; + else + lpfc_nvme_template.max_hw_queues = + phba->sli4_hba.num_present_cpu; /* localport is allocated from the stack, but the registration * call allocates heap memory as well as the private area. @@ -2554,6 +2558,8 @@ lpfc_nvme_wait_for_io_drain(struct lpfc_hba *phba) * WQEs have been removed from the txcmplqs. */ for (i = 0; i < phba->cfg_hdw_queue; i++) { + if (!phba->sli4_hba.hdwq[i].nvme_wq) + continue; pring = phba->sli4_hba.hdwq[i].nvme_wq->pring; if (!pring) diff --git a/drivers/scsi/lpfc/lpfc_scsi.c b/drivers/scsi/lpfc/lpfc_scsi.c index 7b22cc995d7f..a827520789f1 100644 --- a/drivers/scsi/lpfc/lpfc_scsi.c +++ b/drivers/scsi/lpfc/lpfc_scsi.c @@ -692,10 +692,7 @@ lpfc_get_scsi_buf_s4(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, tag = blk_mq_unique_tag(cmnd->request); idx = blk_mq_unique_tag_to_hwq(tag); } else { - if (cpu < phba->cfg_hdw_queue) - idx = cpu; - else - idx = cpu % phba->cfg_hdw_queue; + idx = phba->sli4_hba.cpu_map[cpu].hdwq; } lpfc_cmd = lpfc_get_io_buf(phba, ndlp, idx, @@ -3650,6 +3647,9 @@ lpfc_scsi_cmd_iocb_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pIocbIn, struct Scsi_Host *shost; int idx; uint32_t logit = LOG_FCP; +#ifdef CONFIG_SCSI_LPFC_DEBUG_FS + int cpu; +#endif /* Sanity check on return of outstanding command */ cmd = lpfc_cmd->pCmd; @@ -3660,6 +3660,13 @@ lpfc_scsi_cmd_iocb_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pIocbIn, if (phba->sli4_hba.hdwq) phba->sli4_hba.hdwq[idx].scsi_cstat.io_cmpls++; +#ifdef CONFIG_SCSI_LPFC_DEBUG_FS + if (phba->cpucheck_on & LPFC_CHECK_SCSI_IO) { + cpu = smp_processor_id(); + if (cpu < LPFC_CHECK_CPU_CNT) + phba->sli4_hba.hdwq[idx].cpucheck_cmpl_io[cpu]++; + } +#endif shost = cmd->device->host; lpfc_cmd->result = (pIocbOut->iocb.un.ulpWord[4] & IOERR_PARAM_MASK); @@ -4336,6 +4343,9 @@ lpfc_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *cmnd) struct lpfc_io_buf *lpfc_cmd; struct fc_rport *rport = starget_to_rport(scsi_target(cmnd->device)); int err, idx; +#ifdef CONFIG_SCSI_LPFC_DEBUG_FS + int cpu; +#endif rdata = lpfc_rport_data_from_scsi_device(cmnd->device); @@ -4450,6 +4460,16 @@ lpfc_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *cmnd) lpfc_scsi_prep_cmnd(vport, lpfc_cmd, ndlp); +#ifdef CONFIG_SCSI_LPFC_DEBUG_FS + if (phba->cpucheck_on & LPFC_CHECK_SCSI_IO) { + cpu = smp_processor_id(); + if (cpu < LPFC_CHECK_CPU_CNT) { + struct lpfc_sli4_hdw_queue *hdwq = + &phba->sli4_hba.hdwq[lpfc_cmd->hdwq_no]; + hdwq->cpucheck_xmt_io[cpu]++; + } + } +#endif err = lpfc_sli_issue_iocb(phba, LPFC_FCP_RING, &lpfc_cmd->cur_iocbq, SLI_IOCB_RET_IOCB); if (err) { diff --git a/drivers/scsi/lpfc/lpfc_sli.c b/drivers/scsi/lpfc/lpfc_sli.c index c0f0adccdea7..0cc81321643d 100644 --- a/drivers/scsi/lpfc/lpfc_sli.c +++ b/drivers/scsi/lpfc/lpfc_sli.c @@ -5586,7 +5586,7 @@ lpfc_sli4_arm_cqeq_intr(struct lpfc_hba *phba) LPFC_QUEUE_REARM); } - for (qidx = 0; qidx < phba->cfg_hdw_queue; qidx++) + for (qidx = 0; qidx < phba->cfg_irq_chann; qidx++) sli4_hba->sli4_eq_release(qp[qidx].hba_eq, LPFC_QUEUE_REARM); } @@ -7878,7 +7878,7 @@ lpfc_sli4_process_missed_mbox_completions(struct lpfc_hba *phba) /* Find the eq associated with the mcq */ if (sli4_hba->hdwq) - for (eqidx = 0; eqidx < phba->cfg_hdw_queue; eqidx++) + for (eqidx = 0; eqidx < phba->cfg_irq_chann; eqidx++) if (sli4_hba->hdwq[eqidx].hba_eq->queue_id == sli4_hba->mbx_cq->assoc_qid) { fpeq = sli4_hba->hdwq[eqidx].hba_eq; @@ -10058,12 +10058,9 @@ int lpfc_sli_issue_iocb(struct lpfc_hba *phba, uint32_t ring_number, struct lpfc_iocbq *piocb, uint32_t flag) { - struct lpfc_hba_eq_hdl *hba_eq_hdl; struct lpfc_sli_ring *pring; - struct lpfc_queue *fpeq; - struct lpfc_eqe *eqe; unsigned long iflags; - int rc, idx; + int rc; if (phba->sli_rev == LPFC_SLI_REV4) { pring = lpfc_sli4_calc_ring(phba, piocb); @@ -10073,34 +10070,6 @@ lpfc_sli_issue_iocb(struct lpfc_hba *phba, uint32_t ring_number, spin_lock_irqsave(&pring->ring_lock, iflags); rc = __lpfc_sli_issue_iocb(phba, ring_number, piocb, flag); spin_unlock_irqrestore(&pring->ring_lock, iflags); - - if (lpfc_fcp_look_ahead && (piocb->iocb_flag & LPFC_IO_FCP)) { - idx = piocb->hba_wqidx; - hba_eq_hdl = &phba->sli4_hba.hba_eq_hdl[idx]; - - if (atomic_dec_and_test(&hba_eq_hdl->hba_eq_in_use)) { - - /* Get associated EQ with this index */ - fpeq = phba->sli4_hba.hdwq[idx].hba_eq; - - /* Turn off interrupts from this EQ */ - phba->sli4_hba.sli4_eq_clr_intr(fpeq); - - /* - * Process all the events on FCP EQ - */ - while ((eqe = lpfc_sli4_eq_get(fpeq))) { - lpfc_sli4_hba_handle_eqe(phba, - eqe, idx); - fpeq->EQ_processed++; - } - - /* Always clear and re-arm the EQ */ - phba->sli4_hba.sli4_eq_release(fpeq, - LPFC_QUEUE_REARM); - } - atomic_inc(&hba_eq_hdl->hba_eq_in_use); - } } else { /* For now, SLI2/3 will still use hbalock */ spin_lock_irqsave(&phba->hbalock, iflags); @@ -13651,7 +13620,7 @@ lpfc_sli4_sp_handle_eqe(struct lpfc_hba *phba, struct lpfc_eqe *eqe, /* Save EQ associated with this CQ */ cq->assoc_qp = speq; - if (!queue_work(phba->wq, &cq->spwork)) + if (!queue_work_on(cq->chann, phba->wq, &cq->spwork)) lpfc_printf_log(phba, KERN_ERR, LOG_SLI, "0390 Cannot schedule soft IRQ " "for CQ eqcqid=%d, cqid=%d on CPU %d\n", @@ -14057,18 +14026,11 @@ lpfc_sli4_hba_handle_eqe(struct lpfc_hba *phba, struct lpfc_eqe *eqe, /* Get the reference to the corresponding CQ */ cqid = bf_get_le32(lpfc_eqe_resource_id, eqe); - /* First check for NVME/SCSI completion */ - if ((phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) && - (cqid == phba->sli4_hba.hdwq[qidx].nvme_cq_map)) { - /* Process NVME / NVMET command completion */ - cq = phba->sli4_hba.hdwq[qidx].nvme_cq; - goto process_cq; - } - - if (cqid == phba->sli4_hba.hdwq[qidx].fcp_cq_map) { - /* Process FCP command completion */ - cq = phba->sli4_hba.hdwq[qidx].fcp_cq; - goto process_cq; + /* Use the fast lookup method first */ + if (cqid <= phba->sli4_hba.cq_max) { + cq = phba->sli4_hba.cq_lookup[cqid]; + if (cq) + goto work_cq; } /* Next check for NVMET completion */ @@ -14103,9 +14065,7 @@ process_cq: return; } - /* Save EQ associated with this CQ */ - cq->assoc_qp = phba->sli4_hba.hdwq[qidx].hba_eq; - +work_cq: if (!queue_work_on(cq->chann, phba->wq, &cq->irqwork)) lpfc_printf_log(phba, KERN_ERR, LOG_SLI, "0363 Cannot schedule soft IRQ " @@ -14233,15 +14193,6 @@ lpfc_sli4_hba_intr_handler(int irq, void *dev_id) if (unlikely(!fpeq)) return IRQ_NONE; - if (lpfc_fcp_look_ahead) { - if (atomic_dec_and_test(&hba_eq_hdl->hba_eq_in_use)) - phba->sli4_hba.sli4_eq_clr_intr(fpeq); - else { - atomic_inc(&hba_eq_hdl->hba_eq_in_use); - return IRQ_NONE; - } - } - /* Check device state for handling interrupt */ if (unlikely(lpfc_intr_state_check(phba))) { /* Check again for link_state with lock held */ @@ -14250,8 +14201,6 @@ lpfc_sli4_hba_intr_handler(int irq, void *dev_id) /* Flush, clear interrupt, and rearm the EQ */ lpfc_sli4_eq_flush(phba, fpeq); spin_unlock_irqrestore(&phba->hbalock, iflag); - if (lpfc_fcp_look_ahead) - atomic_inc(&hba_eq_hdl->hba_eq_in_use); return IRQ_NONE; } @@ -14274,12 +14223,6 @@ lpfc_sli4_hba_intr_handler(int irq, void *dev_id) if (unlikely(ecount == 0)) { fpeq->EQ_no_entry++; - - if (lpfc_fcp_look_ahead) { - atomic_inc(&hba_eq_hdl->hba_eq_in_use); - return IRQ_NONE; - } - if (phba->intr_type == MSIX) /* MSI-X treated interrupt served as no EQ share INT */ lpfc_printf_log(phba, KERN_WARNING, LOG_SLI, @@ -14289,9 +14232,6 @@ lpfc_sli4_hba_intr_handler(int irq, void *dev_id) return IRQ_NONE; } - if (lpfc_fcp_look_ahead) - atomic_inc(&hba_eq_hdl->hba_eq_in_use); - return IRQ_HANDLED; } /* lpfc_sli4_fp_intr_handler */ @@ -14329,7 +14269,7 @@ lpfc_sli4_intr_handler(int irq, void *dev_id) /* * Invoke fast-path host attention interrupt handling as appropriate. */ - for (qidx = 0; qidx < phba->cfg_hdw_queue; qidx++) { + for (qidx = 0; qidx < phba->cfg_irq_chann; qidx++) { hba_irq_rc = lpfc_sli4_hba_intr_handler(irq, &phba->sli4_hba.hba_eq_hdl[qidx]); if (hba_irq_rc == IRQ_HANDLED) @@ -14516,7 +14456,7 @@ lpfc_modify_hba_eq_delay(struct lpfc_hba *phba, uint32_t startq, union lpfc_sli4_cfg_shdr *shdr; uint16_t dmult; - if (startq >= phba->cfg_hdw_queue) + if (startq >= phba->cfg_irq_chann) return 0; mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL); @@ -14530,7 +14470,7 @@ lpfc_modify_hba_eq_delay(struct lpfc_hba *phba, uint32_t startq, eq_delay = &mbox->u.mqe.un.eq_delay; /* Calculate delay multiper from maximum interrupt per second */ - result = imax / phba->cfg_hdw_queue; + result = imax / phba->cfg_irq_chann; if (result > LPFC_DMULT_CONST || result == 0) dmult = 0; else @@ -14539,7 +14479,7 @@ lpfc_modify_hba_eq_delay(struct lpfc_hba *phba, uint32_t startq, dmult = LPFC_DMULT_MAX; cnt = 0; - for (qidx = startq; qidx < phba->cfg_hdw_queue; qidx++) { + for (qidx = startq; qidx < phba->cfg_irq_chann; qidx++) { eq = phba->sli4_hba.hdwq[qidx].hba_eq; if (!eq) continue; @@ -14557,7 +14497,7 @@ lpfc_modify_hba_eq_delay(struct lpfc_hba *phba, uint32_t startq, val = phba->cfg_fcp_imax; if (val) { /* First, interrupts per sec per EQ */ - val = phba->cfg_fcp_imax / phba->cfg_hdw_queue; + val = phba->cfg_fcp_imax / phba->cfg_irq_chann; /* us delay between each interrupt */ val = LPFC_SEC_TO_USEC / val; @@ -14852,10 +14792,13 @@ lpfc_cq_create(struct lpfc_hba *phba, struct lpfc_queue *cq, cq->subtype = subtype; cq->queue_id = bf_get(lpfc_mbx_cq_create_q_id, &cq_create->u.response); cq->assoc_qid = eq->queue_id; + cq->assoc_qp = eq; cq->host_index = 0; cq->hba_index = 0; cq->entry_repost = LPFC_CQ_REPOST; + if (cq->queue_id > phba->sli4_hba.cq_max) + phba->sli4_hba.cq_max = cq->queue_id; out: mempool_free(mbox, phba->mbox_mem_pool); return status; @@ -15061,6 +15004,7 @@ lpfc_cq_create_set(struct lpfc_hba *phba, struct lpfc_queue **cqp, cq->type = type; cq->subtype = subtype; cq->assoc_qid = eq->queue_id; + cq->assoc_qp = eq; cq->host_index = 0; cq->hba_index = 0; cq->entry_repost = LPFC_CQ_REPOST; @@ -15101,6 +15045,8 @@ lpfc_cq_create_set(struct lpfc_hba *phba, struct lpfc_queue **cqp, for (idx = 0; idx < numcq; idx++) { cq = cqp[idx]; cq->queue_id = rc + idx; + if (cq->queue_id > phba->sli4_hba.cq_max) + phba->sli4_hba.cq_max = cq->queue_id; } out: @@ -19664,7 +19610,8 @@ lpfc_sli4_issue_wqe(struct lpfc_hba *phba, struct lpfc_sli4_hdw_queue *qp, /* NVME_LS and NVME_LS ABTS requests. */ if (pwqe->iocb_flag & LPFC_IO_NVME_LS) { pring = phba->sli4_hba.nvmels_wq->pring; - spin_lock_irqsave(&pring->ring_lock, iflags); + lpfc_qp_spin_lock_irqsave(&pring->ring_lock, iflags, + qp, wq_access); sglq = __lpfc_sli_get_els_sglq(phba, pwqe); if (!sglq) { spin_unlock_irqrestore(&pring->ring_lock, iflags); @@ -19697,7 +19644,8 @@ lpfc_sli4_issue_wqe(struct lpfc_hba *phba, struct lpfc_sli4_hdw_queue *qp, bf_set(wqe_cqid, &wqe->generic.wqe_com, qp->nvme_cq_map); - spin_lock_irqsave(&pring->ring_lock, iflags); + lpfc_qp_spin_lock_irqsave(&pring->ring_lock, iflags, + qp, wq_access); ret = lpfc_sli4_wq_put(wq, wqe); if (ret) { spin_unlock_irqrestore(&pring->ring_lock, iflags); @@ -19724,7 +19672,8 @@ lpfc_sli4_issue_wqe(struct lpfc_hba *phba, struct lpfc_sli4_hdw_queue *qp, pwqe->sli4_xritag); bf_set(wqe_cqid, &wqe->generic.wqe_com, qp->nvme_cq_map); - spin_lock_irqsave(&pring->ring_lock, iflags); + lpfc_qp_spin_lock_irqsave(&pring->ring_lock, iflags, + qp, wq_access); ret = lpfc_sli4_wq_put(wq, wqe); if (ret) { spin_unlock_irqrestore(&pring->ring_lock, iflags); @@ -19872,18 +19821,20 @@ void lpfc_move_xri_pvt_to_pbl(struct lpfc_hba *phba, u32 hwqid) { struct lpfc_pbl_pool *pbl_pool; struct lpfc_pvt_pool *pvt_pool; + struct lpfc_sli4_hdw_queue *qp; struct lpfc_io_buf *lpfc_ncmd; struct lpfc_io_buf *lpfc_ncmd_next; unsigned long iflag; struct list_head tmp_list; u32 tmp_count; - pbl_pool = &phba->sli4_hba.hdwq[hwqid].p_multixri_pool->pbl_pool; - pvt_pool = &phba->sli4_hba.hdwq[hwqid].p_multixri_pool->pvt_pool; + qp = &phba->sli4_hba.hdwq[hwqid]; + pbl_pool = &qp->p_multixri_pool->pbl_pool; + pvt_pool = &qp->p_multixri_pool->pvt_pool; tmp_count = 0; - spin_lock_irqsave(&pbl_pool->lock, iflag); - spin_lock(&pvt_pool->lock); + lpfc_qp_spin_lock_irqsave(&pbl_pool->lock, iflag, qp, mv_to_pub_pool); + lpfc_qp_spin_lock(&pvt_pool->lock, qp, mv_from_pvt_pool); if (pvt_pool->count > pvt_pool->low_watermark) { /* Step 1: move (all - low_watermark) from pvt_pool @@ -19936,7 +19887,8 @@ void lpfc_move_xri_pvt_to_pbl(struct lpfc_hba *phba, u32 hwqid) * false - if the specified pbl_pool is empty or locked by someone else **/ static bool -_lpfc_move_xri_pbl_to_pvt(struct lpfc_hba *phba, struct lpfc_pbl_pool *pbl_pool, +_lpfc_move_xri_pbl_to_pvt(struct lpfc_hba *phba, struct lpfc_sli4_hdw_queue *qp, + struct lpfc_pbl_pool *pbl_pool, struct lpfc_pvt_pool *pvt_pool, u32 count) { struct lpfc_io_buf *lpfc_ncmd; @@ -19948,7 +19900,7 @@ _lpfc_move_xri_pbl_to_pvt(struct lpfc_hba *phba, struct lpfc_pbl_pool *pbl_pool, if (ret) { if (pbl_pool->count) { /* Move a batch of XRIs from public to private pool */ - spin_lock(&pvt_pool->lock); + lpfc_qp_spin_lock(&pvt_pool->lock, qp, mv_to_pvt_pool); list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next, &pbl_pool->list, @@ -19990,16 +19942,18 @@ void lpfc_move_xri_pbl_to_pvt(struct lpfc_hba *phba, u32 hwqid, u32 count) struct lpfc_multixri_pool *next_multixri_pool; struct lpfc_pvt_pool *pvt_pool; struct lpfc_pbl_pool *pbl_pool; + struct lpfc_sli4_hdw_queue *qp; u32 next_hwqid; u32 hwq_count; int ret; - multixri_pool = phba->sli4_hba.hdwq[hwqid].p_multixri_pool; + qp = &phba->sli4_hba.hdwq[hwqid]; + multixri_pool = qp->p_multixri_pool; pvt_pool = &multixri_pool->pvt_pool; pbl_pool = &multixri_pool->pbl_pool; /* Check if local pbl_pool is available */ - ret = _lpfc_move_xri_pbl_to_pvt(phba, pbl_pool, pvt_pool, count); + ret = _lpfc_move_xri_pbl_to_pvt(phba, qp, pbl_pool, pvt_pool, count); if (ret) { #ifdef LPFC_MXP_STAT multixri_pool->local_pbl_hit_count++; @@ -20022,7 +19976,7 @@ void lpfc_move_xri_pbl_to_pvt(struct lpfc_hba *phba, u32 hwqid, u32 count) /* Check if the public free xri pool is available */ ret = _lpfc_move_xri_pbl_to_pvt( - phba, pbl_pool, pvt_pool, count); + phba, qp, pbl_pool, pvt_pool, count); /* Exit while-loop if success or all hwqid are checked */ } while (!ret && next_hwqid != multixri_pool->rrb_next_hwqid); @@ -20138,20 +20092,23 @@ void lpfc_release_io_buf(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_ncmd, if ((pvt_pool->count < pvt_pool->low_watermark) || (xri_owned < xri_limit && pvt_pool->count < pvt_pool->high_watermark)) { - spin_lock_irqsave(&pvt_pool->lock, iflag); + lpfc_qp_spin_lock_irqsave(&pvt_pool->lock, iflag, + qp, free_pvt_pool); list_add_tail(&lpfc_ncmd->list, &pvt_pool->list); pvt_pool->count++; spin_unlock_irqrestore(&pvt_pool->lock, iflag); } else { - spin_lock_irqsave(&pbl_pool->lock, iflag); + lpfc_qp_spin_lock_irqsave(&pbl_pool->lock, iflag, + qp, free_pub_pool); list_add_tail(&lpfc_ncmd->list, &pbl_pool->list); pbl_pool->count++; spin_unlock_irqrestore(&pbl_pool->lock, iflag); } } else { - spin_lock_irqsave(&qp->io_buf_list_put_lock, iflag); + lpfc_qp_spin_lock_irqsave(&qp->io_buf_list_put_lock, iflag, + qp, free_xri); list_add_tail(&lpfc_ncmd->list, &qp->lpfc_io_buf_list_put); qp->put_io_bufs++; @@ -20174,6 +20131,7 @@ void lpfc_release_io_buf(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_ncmd, **/ static struct lpfc_io_buf * lpfc_get_io_buf_from_private_pool(struct lpfc_hba *phba, + struct lpfc_sli4_hdw_queue *qp, struct lpfc_pvt_pool *pvt_pool, struct lpfc_nodelist *ndlp) { @@ -20181,7 +20139,7 @@ lpfc_get_io_buf_from_private_pool(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_ncmd_next; unsigned long iflag; - spin_lock_irqsave(&pvt_pool->lock, iflag); + lpfc_qp_spin_lock_irqsave(&pvt_pool->lock, iflag, qp, alloc_pvt_pool); list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next, &pvt_pool->list, list) { if (lpfc_test_rrq_active( @@ -20276,7 +20234,7 @@ lpfc_get_io_buf_from_multixri_pools(struct lpfc_hba *phba, lpfc_move_xri_pbl_to_pvt(phba, hwqid, XRI_BATCH); /* Get one XRI from private free xri pool */ - lpfc_ncmd = lpfc_get_io_buf_from_private_pool(phba, pvt_pool, ndlp); + lpfc_ncmd = lpfc_get_io_buf_from_private_pool(phba, qp, pvt_pool, ndlp); if (lpfc_ncmd) { lpfc_ncmd->hdwq = qp; @@ -20349,11 +20307,13 @@ struct lpfc_io_buf *lpfc_get_io_buf(struct lpfc_hba *phba, lpfc_cmd = lpfc_get_io_buf_from_multixri_pools( phba, ndlp, hwqid, expedite); else { - spin_lock_irqsave(&qp->io_buf_list_get_lock, iflag); + lpfc_qp_spin_lock_irqsave(&qp->io_buf_list_get_lock, iflag, + qp, alloc_xri_get); if (qp->get_io_bufs > LPFC_NVME_EXPEDITE_XRICNT || expedite) lpfc_cmd = lpfc_io_buf(phba, ndlp, hwqid); if (!lpfc_cmd) { - spin_lock(&qp->io_buf_list_put_lock); + lpfc_qp_spin_lock(&qp->io_buf_list_put_lock, + qp, alloc_xri_put); list_splice(&qp->lpfc_io_buf_list_put, &qp->lpfc_io_buf_list_get); qp->get_io_bufs += qp->put_io_bufs; diff --git a/drivers/scsi/lpfc/lpfc_sli4.h b/drivers/scsi/lpfc/lpfc_sli4.h index f5e58cd4c6ac..c381f2cb4909 100644 --- a/drivers/scsi/lpfc/lpfc_sli4.h +++ b/drivers/scsi/lpfc/lpfc_sli4.h @@ -41,7 +41,7 @@ /* Multi-queue arrangement for FCP EQ/CQ/WQ tuples */ #define LPFC_HBA_HDWQ_MIN 0 -#define LPFC_HBA_HDWQ_MAX 64 +#define LPFC_HBA_HDWQ_MAX 128 #define LPFC_HBA_HDWQ_DEF 0 /* Common buffer size to accomidate SCSI and NVME IO buffers */ @@ -166,16 +166,19 @@ struct lpfc_queue { uint32_t assoc_qid; /* Queue ID associated with, for CQ/WQ/MQ */ uint32_t host_index; /* The host's index for putting or getting */ uint32_t hba_index; /* The last known hba index for get or put */ + uint32_t q_mode; struct lpfc_sli_ring *pring; /* ptr to io ring associated with q */ struct lpfc_rqb *rqbp; /* ptr to RQ buffers */ - uint32_t q_mode; uint16_t page_count; /* Number of pages allocated for this queue */ uint16_t page_size; /* size of page allocated for this queue */ #define LPFC_EXPANDED_PAGE_SIZE 16384 #define LPFC_DEFAULT_PAGE_SIZE 4096 - uint16_t chann; /* IO channel this queue is associated with */ + uint16_t chann; /* Hardware Queue association WQ/CQ */ + /* CPU affinity for EQ */ +#define LPFC_FIND_BY_EQ 0 +#define LPFC_FIND_BY_HDWQ 1 uint8_t db_format; #define LPFC_DB_RING_FORMAT 0x01 #define LPFC_DB_LIST_FORMAT 0x02 @@ -431,11 +434,6 @@ struct lpfc_hba_eq_hdl { uint32_t idx; char handler_name[LPFC_SLI4_HANDLER_NAME_SZ]; struct lpfc_hba *phba; - atomic_t hba_eq_in_use; - struct cpumask *cpumask; - /* CPU affinitsed to or 0xffffffff if multiple */ - uint32_t cpu; -#define LPFC_MULTI_CPU_AFFINITY 0xffffffff }; /*BB Credit recovery value*/ @@ -529,7 +527,9 @@ struct lpfc_vector_map_info { uint16_t phys_id; uint16_t core_id; uint16_t irq; + uint16_t eq; uint16_t hdwq; + uint16_t hyper; }; #define LPFC_VECTOR_MAP_EMPTY 0xffff @@ -593,6 +593,21 @@ struct lpfc_fc4_ctrl_stat { u32 io_cmpls; }; +#ifdef LPFC_HDWQ_LOCK_STAT +struct lpfc_lock_stat { + uint32_t alloc_xri_get; + uint32_t alloc_xri_put; + uint32_t free_xri; + uint32_t wq_access; + uint32_t alloc_pvt_pool; + uint32_t mv_from_pvt_pool; + uint32_t mv_to_pub_pool; + uint32_t mv_to_pvt_pool; + uint32_t free_pub_pool; + uint32_t free_pvt_pool; +}; +#endif + /* SLI4 HBA data structure entries */ struct lpfc_sli4_hdw_queue { /* Pointers to the constructed SLI4 queues */ @@ -626,6 +641,9 @@ struct lpfc_sli4_hdw_queue { /* FC-4 Stats counters */ struct lpfc_fc4_ctrl_stat nvme_cstat; struct lpfc_fc4_ctrl_stat scsi_cstat; +#ifdef LPFC_HDWQ_LOCK_STAT + struct lpfc_lock_stat lock_conflict; +#endif #ifdef CONFIG_SCSI_LPFC_DEBUG_FS #define LPFC_CHECK_CPU_CNT 128 @@ -635,6 +653,34 @@ struct lpfc_sli4_hdw_queue { #endif }; +#ifdef LPFC_HDWQ_LOCK_STAT +/* compile time trylock stats */ +#define lpfc_qp_spin_lock_irqsave(lock, flag, qp, lstat) \ + { \ + int only_once = 1; \ + while (spin_trylock_irqsave(lock, flag) == 0) { \ + if (only_once) { \ + only_once = 0; \ + qp->lock_conflict.lstat++; \ + } \ + } \ + } +#define lpfc_qp_spin_lock(lock, qp, lstat) \ + { \ + int only_once = 1; \ + while (spin_trylock(lock) == 0) { \ + if (only_once) { \ + only_once = 0; \ + qp->lock_conflict.lstat++; \ + } \ + } \ + } +#else +#define lpfc_qp_spin_lock_irqsave(lock, flag, qp, lstat) \ + spin_lock_irqsave(lock, flag) +#define lpfc_qp_spin_lock(lock, qp, lstat) spin_lock(lock) +#endif + struct lpfc_sli4_hba { void __iomem *conf_regs_memmap_p; /* Kernel memory mapped address for * config space registers @@ -764,6 +810,8 @@ struct lpfc_sli4_hba { uint16_t nvmet_xri_cnt; uint16_t nvmet_io_wait_cnt; uint16_t nvmet_io_wait_total; + uint16_t cq_max; + struct lpfc_queue **cq_lookup; struct list_head lpfc_els_sgl_list; struct list_head lpfc_abts_els_sgl_list; spinlock_t abts_scsi_buf_list_lock; /* list of aborted SCSI IOs */ -- cgit v1.2.3 From 32517fc0975bf8dd3967e43c2a6350f038a3af28 Mon Sep 17 00:00:00 2001 From: James Smart Date: Mon, 28 Jan 2019 11:14:33 -0800 Subject: scsi: lpfc: Rework EQ/CQ processing to address interrupt coalescing When driving high iop counts, auto_imax coalescing kicks in and drives the performance to extremely small iops levels. There are two issues: 1) auto_imax is enabled by default. The auto algorithm, when iops gets high, divides the iops by the hdwq count and uses that value to calculate EQ_Delay. The EQ_Delay is set uniformly on all EQs whether they have load or not. The EQ_delay is only manipulated every 5s (a long time). Thus there were large 5s swings of no interrupt delay followed by large/maximum delay, before repeating. 2) When processing a CQ, the driver got mixed up on the rate of when to ring the doorbell to keep the chip appraised of the eqe or cqe consumption as well as how how long to sit in the thread and process queue entries. Currently, the driver capped its work at 64 entries (very small) and exited/rearmed the CQ. Thus, on heavy loads, additional overheads were taken to exit and re-enter the interrupt handler. Worse, if in the large/maximum coalescing windows,k it could be a while before getting back to servicing. The issues are corrected by the following: - A change in defaults. Auto_imax is turned OFF and fcp_imax is set to 0. Thus all interrupts are immediate. - Cleanup of field names and their meanings. Existing names were non-intuitive or used for duplicate things. - Added max_proc_limit field, to control the length of time the handlers would service completions. - Reworked EQ handling: Added common routine that walks eq, applying notify interval and max processing limits. Use queue_claimed to claim ownership of the queue while processing. Always rearm the queue whenever the common routine is called. Rework queue element processing, namely to eliminate hba_index vs host_index. Only one index is necessary. The queue entry can be marked invalid and the host_index updated immediately after eqe processing. After rework, xx_release routines are now DB write functions. Renamed the routines as such. Moved lpfc_sli4_eq_flush(), which does similar action, to same area. Replaced the 2 individual loops that walk an eq with a call to the common routine. Slightly revised lpfc_sli4_hba_handle_eqe() calling syntax. Added per-cpu counters to detect interrupt rates and scale interrupt coalescing values. - Reworked CQ handling: Added common routine that walks cq, applying notify interval and max processing limits. Use queue_claimed to claim ownership of the queue while processing. Always rearm the queue whenever the common routine is called. Rework queue element processing, namely to eliminate hba_index vs host_index. Only one index is necessary. The queue entry can be marked invalid and the host_index updated immediately after cqe processing. After rework, xx_release routines are now DB write functions. Renamed the routines as such. Replaced the 3 individual loops that walk a cq with a call to the common routine. Redefined lpfc_sli4_sp_handle_mcqe() to commong handler definition with queue reference. Add increment for mbox completion to handler. - Added a new module/sysfs attribute: lpfc_cq_max_proc_limit To allow dynamic changing of the CQ max_proc_limit value being used. Although this leaves an EQ as an immediate interrupt, that interrupt will only occur if a CQ bound to it is in an armed state and has cqe's to process. By staying in the cq processing routine longer, high loads will avoid generating more interrupts as they will only rearm as the processing thread exits. The immediately interrupt is also beneficial to idle or lower-processing CQ's as they get serviced immediately without being penalized by sharing an EQ with a more loaded CQ. Signed-off-by: Dick Kennedy Signed-off-by: James Smart Reviewed-by: Hannes Reinecke Signed-off-by: Martin K. Petersen --- drivers/scsi/lpfc/lpfc.h | 25 +- drivers/scsi/lpfc/lpfc_attr.c | 141 +++++++- drivers/scsi/lpfc/lpfc_debugfs.c | 22 +- drivers/scsi/lpfc/lpfc_hw4.h | 9 +- drivers/scsi/lpfc/lpfc_init.c | 205 +++++------ drivers/scsi/lpfc/lpfc_sli.c | 733 ++++++++++++++++++++++----------------- drivers/scsi/lpfc/lpfc_sli4.h | 70 +++- 7 files changed, 729 insertions(+), 476 deletions(-) (limited to 'drivers/scsi/lpfc/lpfc.h') diff --git a/drivers/scsi/lpfc/lpfc.h b/drivers/scsi/lpfc/lpfc.h index 9fd2811ffa8b..0bc498172add 100644 --- a/drivers/scsi/lpfc/lpfc.h +++ b/drivers/scsi/lpfc/lpfc.h @@ -686,6 +686,7 @@ struct lpfc_hba { struct lpfc_sli4_hba sli4_hba; struct workqueue_struct *wq; + struct delayed_work eq_delay_work; struct lpfc_sli sli; uint8_t pci_dev_grp; /* lpfc PCI dev group: 0x0, 0x1, 0x2,... */ @@ -789,7 +790,6 @@ struct lpfc_hba { uint8_t nvmet_support; /* driver supports NVMET */ #define LPFC_NVMET_MAX_PORTS 32 uint8_t mds_diags_support; - uint32_t initial_imax; uint8_t bbcredit_support; uint8_t enab_exp_wqcq_pages; @@ -817,6 +817,8 @@ struct lpfc_hba { uint32_t cfg_use_msi; uint32_t cfg_auto_imax; uint32_t cfg_fcp_imax; + uint32_t cfg_cq_poll_threshold; + uint32_t cfg_cq_max_proc_limit; uint32_t cfg_fcp_cpu_map; uint32_t cfg_hdw_queue; uint32_t cfg_irq_chann; @@ -1084,7 +1086,6 @@ struct lpfc_hba { uint8_t temp_sensor_support; /* Fields used for heart beat. */ - unsigned long last_eqdelay_time; unsigned long last_completion_time; unsigned long skipped_hb; struct timer_list hb_tmofunc; @@ -1287,3 +1288,23 @@ lpfc_phba_elsring(struct lpfc_hba *phba) } return &phba->sli.sli3_ring[LPFC_ELS_RING]; } + +/** + * lpfc_sli4_mod_hba_eq_delay - update EQ delay + * @phba: Pointer to HBA context object. + * @q: The Event Queue to update. + * @delay: The delay value (in us) to be written. + * + **/ +static inline void +lpfc_sli4_mod_hba_eq_delay(struct lpfc_hba *phba, struct lpfc_queue *eq, + u32 delay) +{ + struct lpfc_register reg_data; + + reg_data.word0 = 0; + bf_set(lpfc_sliport_eqdelay_id, ®_data, eq->queue_id); + bf_set(lpfc_sliport_eqdelay_delay, ®_data, delay); + writel(reg_data.word0, phba->sli4_hba.u.if_type2.EQDregaddr); + eq->q_mode = delay; +} diff --git a/drivers/scsi/lpfc/lpfc_attr.c b/drivers/scsi/lpfc/lpfc_attr.c index ed8caeefe3a2..2864cb53b1e8 100644 --- a/drivers/scsi/lpfc/lpfc_attr.c +++ b/drivers/scsi/lpfc/lpfc_attr.c @@ -4935,6 +4935,7 @@ lpfc_fcp_imax_store(struct device *dev, struct device_attribute *attr, struct Scsi_Host *shost = class_to_shost(dev); struct lpfc_vport *vport = (struct lpfc_vport *)shost->hostdata; struct lpfc_hba *phba = vport->phba; + struct lpfc_eq_intr_info *eqi; uint32_t usdelay; int val = 0, i; @@ -4956,8 +4957,18 @@ lpfc_fcp_imax_store(struct device *dev, struct device_attribute *attr, if (val && (val < LPFC_MIN_IMAX || val > LPFC_MAX_IMAX)) return -EINVAL; + phba->cfg_auto_imax = (val) ? 0 : 1; + if (phba->cfg_fcp_imax && !val) { + queue_delayed_work(phba->wq, &phba->eq_delay_work, + msecs_to_jiffies(LPFC_EQ_DELAY_MSECS)); + + for_each_present_cpu(i) { + eqi = per_cpu_ptr(phba->sli4_hba.eq_info, i); + eqi->icnt = 0; + } + } + phba->cfg_fcp_imax = (uint32_t)val; - phba->initial_imax = phba->cfg_fcp_imax; if (phba->cfg_fcp_imax) usdelay = LPFC_SEC_TO_USEC / phba->cfg_fcp_imax; @@ -5020,15 +5031,119 @@ lpfc_fcp_imax_init(struct lpfc_hba *phba, int val) static DEVICE_ATTR_RW(lpfc_fcp_imax); +/** + * lpfc_cq_max_proc_limit_store + * + * @dev: class device that is converted into a Scsi_host. + * @attr: device attribute, not used. + * @buf: string with the cq max processing limit of cqes + * @count: unused variable. + * + * Description: + * If val is in a valid range, then set value on each cq + * + * Returns: + * The length of the buf: if successful + * -ERANGE: if val is not in the valid range + * -EINVAL: if bad value format or intended mode is not supported. + **/ +static ssize_t +lpfc_cq_max_proc_limit_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct Scsi_Host *shost = class_to_shost(dev); + struct lpfc_vport *vport = (struct lpfc_vport *)shost->hostdata; + struct lpfc_hba *phba = vport->phba; + struct lpfc_queue *eq, *cq; + unsigned long val; + int i; + + /* cq_max_proc_limit is only valid for SLI4 */ + if (phba->sli_rev != LPFC_SLI_REV4) + return -EINVAL; + + /* Sanity check on user data */ + if (!isdigit(buf[0])) + return -EINVAL; + if (kstrtoul(buf, 0, &val)) + return -EINVAL; + + if (val < LPFC_CQ_MIN_PROC_LIMIT || val > LPFC_CQ_MAX_PROC_LIMIT) + return -ERANGE; + + phba->cfg_cq_max_proc_limit = (uint32_t)val; + + /* set the values on the cq's */ + for (i = 0; i < phba->cfg_irq_chann; i++) { + eq = phba->sli4_hba.hdwq[i].hba_eq; + if (!eq) + continue; + + list_for_each_entry(cq, &eq->child_list, list) + cq->max_proc_limit = min(phba->cfg_cq_max_proc_limit, + cq->entry_count); + } + + return strlen(buf); +} + /* - * lpfc_auto_imax: Controls Auto-interrupt coalescing values support. - * 0 No auto_imax support - * 1 auto imax on - * Auto imax will change the value of fcp_imax on a per EQ basis, using - * the EQ Delay Multiplier, depending on the activity for that EQ. - * Value range [0,1]. Default value is 1. + * lpfc_cq_max_proc_limit: The maximum number CQE entries processed in an + * itteration of CQ processing. */ -LPFC_ATTR_RW(auto_imax, 1, 0, 1, "Enable Auto imax"); +static int lpfc_cq_max_proc_limit = LPFC_CQ_DEF_MAX_PROC_LIMIT; +module_param(lpfc_cq_max_proc_limit, int, 0644); +MODULE_PARM_DESC(lpfc_cq_max_proc_limit, + "Set the maximum number CQEs processed in an iteration of " + "CQ processing"); +lpfc_param_show(cq_max_proc_limit) + +/* + * lpfc_cq_poll_threshold: Set the threshold of CQE completions in a + * single handler call which should request a polled completion rather + * than re-enabling interrupts. + */ +LPFC_ATTR_RW(cq_poll_threshold, LPFC_CQ_DEF_THRESHOLD_TO_POLL, + LPFC_CQ_MIN_THRESHOLD_TO_POLL, + LPFC_CQ_MAX_THRESHOLD_TO_POLL, + "CQE Processing Threshold to enable Polling"); + +/** + * lpfc_cq_max_proc_limit_init - Set the initial cq max_proc_limit + * @phba: lpfc_hba pointer. + * @val: entry limit + * + * Description: + * If val is in a valid range, then initialize the adapter's maximum + * value. + * + * Returns: + * Always returns 0 for success, even if value not always set to + * requested value. If value out of range or not supported, will fall + * back to default. + **/ +static int +lpfc_cq_max_proc_limit_init(struct lpfc_hba *phba, int val) +{ + phba->cfg_cq_max_proc_limit = LPFC_CQ_DEF_MAX_PROC_LIMIT; + + if (phba->sli_rev != LPFC_SLI_REV4) + return 0; + + if (val >= LPFC_CQ_MIN_PROC_LIMIT && val <= LPFC_CQ_MAX_PROC_LIMIT) { + phba->cfg_cq_max_proc_limit = val; + return 0; + } + + lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + "0371 "LPFC_DRIVER_NAME"_cq_max_proc_limit: " + "%d out of range, using default\n", + phba->cfg_cq_max_proc_limit); + + return 0; +} + +static DEVICE_ATTR_RW(lpfc_cq_max_proc_limit); /** * lpfc_state_show - Display current driver CPU affinity @@ -5788,8 +5903,9 @@ struct device_attribute *lpfc_hba_attrs[] = { &dev_attr_lpfc_use_msi, &dev_attr_lpfc_nvme_oas, &dev_attr_lpfc_nvme_embed_cmd, - &dev_attr_lpfc_auto_imax, &dev_attr_lpfc_fcp_imax, + &dev_attr_lpfc_cq_poll_threshold, + &dev_attr_lpfc_cq_max_proc_limit, &dev_attr_lpfc_fcp_cpu_map, &dev_attr_lpfc_hdw_queue, &dev_attr_lpfc_irq_chann, @@ -6834,8 +6950,9 @@ lpfc_get_cfgparam(struct lpfc_hba *phba) lpfc_use_msi_init(phba, lpfc_use_msi); lpfc_nvme_oas_init(phba, lpfc_nvme_oas); lpfc_nvme_embed_cmd_init(phba, lpfc_nvme_embed_cmd); - lpfc_auto_imax_init(phba, lpfc_auto_imax); lpfc_fcp_imax_init(phba, lpfc_fcp_imax); + lpfc_cq_poll_threshold_init(phba, lpfc_cq_poll_threshold); + lpfc_cq_max_proc_limit_init(phba, lpfc_cq_max_proc_limit); lpfc_fcp_cpu_map_init(phba, lpfc_fcp_cpu_map); lpfc_enable_hba_reset_init(phba, lpfc_enable_hba_reset); lpfc_enable_hba_heartbeat_init(phba, lpfc_enable_hba_heartbeat); @@ -6888,9 +7005,7 @@ lpfc_get_cfgparam(struct lpfc_hba *phba) phba->cfg_enable_fc4_type |= LPFC_ENABLE_FCP; } - if (phba->cfg_auto_imax && !phba->cfg_fcp_imax) - phba->cfg_auto_imax = 0; - phba->initial_imax = phba->cfg_fcp_imax; + phba->cfg_auto_imax = (phba->cfg_fcp_imax) ? 0 : 1; phba->cfg_enable_pbde = 0; diff --git a/drivers/scsi/lpfc/lpfc_debugfs.c b/drivers/scsi/lpfc/lpfc_debugfs.c index 72076b2cd4ff..16b461778e59 100644 --- a/drivers/scsi/lpfc/lpfc_debugfs.c +++ b/drivers/scsi/lpfc/lpfc_debugfs.c @@ -3766,10 +3766,10 @@ __lpfc_idiag_print_wq(struct lpfc_queue *qp, char *wqtype, (unsigned long long)qp->q_cnt_4); len += snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len, "\t\tWQID[%02d], QE-CNT[%04d], QE-SZ[%04d], " - "HST-IDX[%04d], PRT-IDX[%04d], PST[%03d]", + "HST-IDX[%04d], PRT-IDX[%04d], NTFI[%03d]", qp->queue_id, qp->entry_count, qp->entry_size, qp->host_index, - qp->hba_index, qp->entry_repost); + qp->hba_index, qp->notify_interval); len += snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len, "\n"); return len; @@ -3819,10 +3819,10 @@ __lpfc_idiag_print_cq(struct lpfc_queue *qp, char *cqtype, qp->q_cnt_3, (unsigned long long)qp->q_cnt_4); len += snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len, "\tCQID[%02d], QE-CNT[%04d], QE-SZ[%04d], " - "HST-IDX[%04d], PRT-IDX[%04d], PST[%03d]", + "HST-IDX[%04d], NTFI[%03d], PLMT[%03d]", qp->queue_id, qp->entry_count, qp->entry_size, qp->host_index, - qp->hba_index, qp->entry_repost); + qp->notify_interval, qp->max_proc_limit); len += snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len, "\n"); @@ -3845,15 +3845,15 @@ __lpfc_idiag_print_rqpair(struct lpfc_queue *qp, struct lpfc_queue *datqp, qp->q_cnt_3, (unsigned long long)qp->q_cnt_4); len += snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len, "\t\tHQID[%02d], QE-CNT[%04d], QE-SZ[%04d], " - "HST-IDX[%04d], PRT-IDX[%04d], PST[%03d]\n", + "HST-IDX[%04d], PRT-IDX[%04d], NTFI[%03d]\n", qp->queue_id, qp->entry_count, qp->entry_size, - qp->host_index, qp->hba_index, qp->entry_repost); + qp->host_index, qp->hba_index, qp->notify_interval); len += snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len, "\t\tDQID[%02d], QE-CNT[%04d], QE-SZ[%04d], " - "HST-IDX[%04d], PRT-IDX[%04d], PST[%03d]\n", + "HST-IDX[%04d], PRT-IDX[%04d], NTFI[%03d]\n", datqp->queue_id, datqp->entry_count, datqp->entry_size, datqp->host_index, - datqp->hba_index, datqp->entry_repost); + datqp->hba_index, datqp->notify_interval); return len; } @@ -3934,10 +3934,10 @@ __lpfc_idiag_print_eq(struct lpfc_queue *qp, char *eqtype, (unsigned long long)qp->q_cnt_4, qp->q_mode); len += snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len, "EQID[%02d], QE-CNT[%04d], QE-SZ[%04d], " - "HST-IDX[%04d], PRT-IDX[%04d], PST[%03d] AFFIN[%03d]", + "HST-IDX[%04d], NTFI[%03d], PLMT[%03d], AFFIN[%03d]", qp->queue_id, qp->entry_count, qp->entry_size, - qp->host_index, qp->hba_index, qp->entry_repost, - qp->chann); + qp->host_index, qp->notify_interval, + qp->max_proc_limit, qp->chann); len += snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len, "\n"); return len; diff --git a/drivers/scsi/lpfc/lpfc_hw4.h b/drivers/scsi/lpfc/lpfc_hw4.h index 665852291a4f..c9a056ef321a 100644 --- a/drivers/scsi/lpfc/lpfc_hw4.h +++ b/drivers/scsi/lpfc/lpfc_hw4.h @@ -208,7 +208,14 @@ struct lpfc_sli_intf { /* Configuration of Interrupts / sec for entire HBA port */ #define LPFC_MIN_IMAX 5000 #define LPFC_MAX_IMAX 5000000 -#define LPFC_DEF_IMAX 150000 +#define LPFC_DEF_IMAX 0 + +#define LPFC_IMAX_THRESHOLD 1000 +#define LPFC_MAX_AUTO_EQ_DELAY 120 +#define LPFC_EQ_DELAY_STEP 15 +#define LPFC_EQD_ISR_TRIGGER 20000 +/* 1s intervals */ +#define LPFC_EQ_DELAY_MSECS 1000 #define LPFC_MIN_CPU_MAP 0 #define LPFC_MAX_CPU_MAP 1 diff --git a/drivers/scsi/lpfc/lpfc_init.c b/drivers/scsi/lpfc/lpfc_init.c index 22aec1a9b689..3fa7240a8e31 100644 --- a/drivers/scsi/lpfc/lpfc_init.c +++ b/drivers/scsi/lpfc/lpfc_init.c @@ -1251,6 +1251,69 @@ lpfc_hb_mbox_cmpl(struct lpfc_hba * phba, LPFC_MBOXQ_t * pmboxq) return; } +static void +lpfc_hb_eq_delay_work(struct work_struct *work) +{ + struct lpfc_hba *phba = container_of(to_delayed_work(work), + struct lpfc_hba, eq_delay_work); + struct lpfc_eq_intr_info *eqi, *eqi_new; + struct lpfc_queue *eq, *eq_next; + unsigned char *eqcnt = NULL; + uint32_t usdelay; + int i; + + if (!phba->cfg_auto_imax || phba->pport->load_flag & FC_UNLOADING) + return; + + if (phba->link_state == LPFC_HBA_ERROR || + phba->pport->fc_flag & FC_OFFLINE_MODE) + goto requeue; + + eqcnt = kcalloc(num_possible_cpus(), sizeof(unsigned char), + GFP_KERNEL); + if (!eqcnt) + goto requeue; + + for (i = 0; i < phba->cfg_irq_chann; i++) { + eq = phba->sli4_hba.hdwq[i].hba_eq; + if (eq && eqcnt[eq->last_cpu] < 2) + eqcnt[eq->last_cpu]++; + continue; + } + + for_each_present_cpu(i) { + if (phba->cfg_irq_chann > 1 && eqcnt[i] < 2) + continue; + + eqi = per_cpu_ptr(phba->sli4_hba.eq_info, i); + + usdelay = (eqi->icnt / LPFC_IMAX_THRESHOLD) * + LPFC_EQ_DELAY_STEP; + if (usdelay > LPFC_MAX_AUTO_EQ_DELAY) + usdelay = LPFC_MAX_AUTO_EQ_DELAY; + + eqi->icnt = 0; + + list_for_each_entry_safe(eq, eq_next, &eqi->list, cpu_list) { + if (eq->last_cpu != i) { + eqi_new = per_cpu_ptr(phba->sli4_hba.eq_info, + eq->last_cpu); + list_move_tail(&eq->cpu_list, &eqi_new->list); + continue; + } + if (usdelay != eq->q_mode) + lpfc_modify_hba_eq_delay(phba, eq->hdwq, 1, + usdelay); + } + } + + kfree(eqcnt); + +requeue: + queue_delayed_work(phba->wq, &phba->eq_delay_work, + msecs_to_jiffies(LPFC_EQ_DELAY_MSECS)); +} + /** * lpfc_hb_mxp_handler - Multi-XRI pools handler to adjust XRI distribution * @phba: pointer to lpfc hba data structure. @@ -1303,16 +1366,6 @@ lpfc_hb_timeout_handler(struct lpfc_hba *phba) int retval, i; struct lpfc_sli *psli = &phba->sli; LIST_HEAD(completions); - struct lpfc_queue *qp; - unsigned long time_elapsed; - uint32_t tick_cqe, max_cqe, val; - uint64_t tot, data1, data2, data3; - struct lpfc_nvmet_tgtport *tgtp; - struct lpfc_register reg_data; - struct nvme_fc_local_port *localport; - struct lpfc_nvme_lport *lport; - struct lpfc_fc4_ctrl_stat *cstat; - void __iomem *eqdreg = phba->sli4_hba.u.if_type2.EQDregaddr; if (phba->cfg_xri_rebalancing) { /* Multi-XRI pools handler */ @@ -1332,104 +1385,6 @@ lpfc_hb_timeout_handler(struct lpfc_hba *phba) (phba->pport->fc_flag & FC_OFFLINE_MODE)) return; - if (phba->cfg_auto_imax) { - if (!phba->last_eqdelay_time) { - phba->last_eqdelay_time = jiffies; - goto skip_eqdelay; - } - time_elapsed = jiffies - phba->last_eqdelay_time; - phba->last_eqdelay_time = jiffies; - - tot = 0xffff; - /* Check outstanding IO count */ - if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { - if (phba->nvmet_support) { - tgtp = phba->targetport->private; - /* Calculate outstanding IOs */ - tot = atomic_read(&tgtp->rcv_fcp_cmd_drop); - tot += atomic_read(&tgtp->xmt_fcp_release); - tot = atomic_read(&tgtp->rcv_fcp_cmd_in) - tot; - } else { - localport = phba->pport->localport; - if (!localport || !localport->private) - goto skip_eqdelay; - lport = (struct lpfc_nvme_lport *) - localport->private; - tot = 0; - for (i = 0; - i < phba->cfg_hdw_queue; i++) { - cstat = - &phba->sli4_hba.hdwq[i].nvme_cstat; - data1 = cstat->input_requests; - data2 = cstat->output_requests; - data3 = cstat->control_requests; - tot += (data1 + data2 + data3); - tot -= cstat->io_cmpls; - } - } - } - - /* Interrupts per sec per EQ */ - val = phba->cfg_fcp_imax / phba->cfg_irq_chann; - tick_cqe = val / CONFIG_HZ; /* Per tick per EQ */ - - /* Assume 1 CQE/ISR, calc max CQEs allowed for time duration */ - max_cqe = time_elapsed * tick_cqe; - - for (i = 0; i < phba->cfg_irq_chann; i++) { - /* Fast-path EQ */ - qp = phba->sli4_hba.hdwq[i].hba_eq; - if (!qp) - continue; - - /* Use no EQ delay if we don't have many outstanding - * IOs, or if we are only processing 1 CQE/ISR or less. - * Otherwise, assume we can process up to lpfc_fcp_imax - * interrupts per HBA. - */ - if (tot < LPFC_NODELAY_MAX_IO || - qp->EQ_cqe_cnt <= max_cqe) - val = 0; - else - val = phba->cfg_fcp_imax; - - if (phba->sli.sli_flag & LPFC_SLI_USE_EQDR) { - /* Use EQ Delay Register method */ - - /* Convert for EQ Delay register */ - if (val) { - /* First, interrupts per sec per EQ */ - val = phba->cfg_fcp_imax / - phba->cfg_irq_chann; - - /* us delay between each interrupt */ - val = LPFC_SEC_TO_USEC / val; - } - if (val != qp->q_mode) { - reg_data.word0 = 0; - bf_set(lpfc_sliport_eqdelay_id, - ®_data, qp->queue_id); - bf_set(lpfc_sliport_eqdelay_delay, - ®_data, val); - writel(reg_data.word0, eqdreg); - } - } else { - /* Use mbox command method */ - if (val != qp->q_mode) - lpfc_modify_hba_eq_delay(phba, i, - 1, val); - } - - /* - * val is cfg_fcp_imax or 0 for mbox delay or us delay - * between interrupts for EQDR. - */ - qp->q_mode = val; - qp->EQ_cqe_cnt = 0; - } - } - -skip_eqdelay: spin_lock_irq(&phba->pport->work_port_lock); if (time_after(phba->last_completion_time + @@ -2986,6 +2941,7 @@ lpfc_stop_hba_timers(struct lpfc_hba *phba) { if (phba->pport) lpfc_stop_vport_timers(phba->pport); + cancel_delayed_work_sync(&phba->eq_delay_work); del_timer_sync(&phba->sli.mbox_tmo); del_timer_sync(&phba->fabric_block_timer); del_timer_sync(&phba->eratt_poll); @@ -6234,6 +6190,8 @@ lpfc_setup_driver_resource_phase1(struct lpfc_hba *phba) /* Heartbeat timer */ timer_setup(&phba->hb_tmofunc, lpfc_hb_timeout, 0); + INIT_DELAYED_WORK(&phba->eq_delay_work, lpfc_hb_eq_delay_work); + return 0; } @@ -6849,6 +6807,13 @@ lpfc_sli4_driver_resource_setup(struct lpfc_hba *phba) goto out_free_hba_eq_hdl; } + phba->sli4_hba.eq_info = alloc_percpu(struct lpfc_eq_intr_info); + if (!phba->sli4_hba.eq_info) { + lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + "3321 Failed allocation for per_cpu stats\n"); + rc = -ENOMEM; + goto out_free_hba_cpu_map; + } /* * Enable sr-iov virtual functions if supported and configured * through the module parameter. @@ -6868,6 +6833,8 @@ lpfc_sli4_driver_resource_setup(struct lpfc_hba *phba) return 0; +out_free_hba_cpu_map: + kfree(phba->sli4_hba.cpu_map); out_free_hba_eq_hdl: kfree(phba->sli4_hba.hba_eq_hdl); out_free_fcf_rr_bmask: @@ -6897,6 +6864,8 @@ lpfc_sli4_driver_resource_unset(struct lpfc_hba *phba) { struct lpfc_fcf_conn_entry *conn_entry, *next_conn_entry; + free_percpu(phba->sli4_hba.eq_info); + /* Free memory allocated for msi-x interrupt vector to CPU mapping */ kfree(phba->sli4_hba.cpu_map); phba->sli4_hba.num_present_cpu = 0; @@ -8753,6 +8722,7 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba) struct lpfc_queue *qdesc; int idx, eqidx; struct lpfc_sli4_hdw_queue *qp; + struct lpfc_eq_intr_info *eqi; /* * Create HBA Record arrays. @@ -8865,6 +8835,9 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba) qdesc->chann = lpfc_find_cpu_handle(phba, eqidx, LPFC_FIND_BY_EQ); phba->sli4_hba.hdwq[idx].hba_eq = qdesc; + qdesc->last_cpu = qdesc->chann; + eqi = per_cpu_ptr(phba->sli4_hba.eq_info, qdesc->last_cpu); + list_add(&qdesc->cpu_list, &eqi->list); } @@ -10246,13 +10219,13 @@ lpfc_sli4_pci_mem_setup(struct lpfc_hba *phba) case LPFC_SLI_INTF_IF_TYPE_0: case LPFC_SLI_INTF_IF_TYPE_2: phba->sli4_hba.sli4_eq_clr_intr = lpfc_sli4_eq_clr_intr; - phba->sli4_hba.sli4_eq_release = lpfc_sli4_eq_release; - phba->sli4_hba.sli4_cq_release = lpfc_sli4_cq_release; + phba->sli4_hba.sli4_write_eq_db = lpfc_sli4_write_eq_db; + phba->sli4_hba.sli4_write_cq_db = lpfc_sli4_write_cq_db; break; case LPFC_SLI_INTF_IF_TYPE_6: phba->sli4_hba.sli4_eq_clr_intr = lpfc_sli4_if6_eq_clr_intr; - phba->sli4_hba.sli4_eq_release = lpfc_sli4_if6_eq_release; - phba->sli4_hba.sli4_cq_release = lpfc_sli4_if6_cq_release; + phba->sli4_hba.sli4_write_eq_db = lpfc_sli4_if6_write_eq_db; + phba->sli4_hba.sli4_write_cq_db = lpfc_sli4_if6_write_cq_db; break; default: break; @@ -10773,6 +10746,14 @@ lpfc_cpu_affinity_check(struct lpfc_hba *phba, int vectors) cpup++; } + for_each_possible_cpu(i) { + struct lpfc_eq_intr_info *eqi = + per_cpu_ptr(phba->sli4_hba.eq_info, i); + + INIT_LIST_HEAD(&eqi->list); + eqi->icnt = 0; + } + /* * If the number of IRQ vectors == number of CPUs, * mapping is pretty simple: 1 to 1. diff --git a/drivers/scsi/lpfc/lpfc_sli.c b/drivers/scsi/lpfc/lpfc_sli.c index a3477cecb0c9..ec385d3413f2 100644 --- a/drivers/scsi/lpfc/lpfc_sli.c +++ b/drivers/scsi/lpfc/lpfc_sli.c @@ -78,12 +78,13 @@ static void lpfc_sli4_send_seq_to_ulp(struct lpfc_vport *, struct hbq_dmabuf *); static void lpfc_sli4_handle_mds_loopback(struct lpfc_vport *vport, struct hbq_dmabuf *dmabuf); -static int lpfc_sli4_fp_handle_cqe(struct lpfc_hba *, struct lpfc_queue *, - struct lpfc_cqe *); +static bool lpfc_sli4_fp_handle_cqe(struct lpfc_hba *phba, + struct lpfc_queue *cq, struct lpfc_cqe *cqe); static int lpfc_sli4_post_sgl_list(struct lpfc_hba *, struct list_head *, int); static void lpfc_sli4_hba_handle_eqe(struct lpfc_hba *phba, - struct lpfc_eqe *eqe, uint32_t qidx); + struct lpfc_queue *eq, + struct lpfc_eqe *eqe); static bool lpfc_sli4_mbox_completions_pending(struct lpfc_hba *phba); static bool lpfc_sli4_process_missed_mbox_completions(struct lpfc_hba *phba); static int lpfc_sli4_abort_nvme_io(struct lpfc_hba *phba, @@ -160,7 +161,7 @@ lpfc_sli4_wq_put(struct lpfc_queue *q, union lpfc_wqe128 *wqe) } q->WQ_posted++; /* set consumption flag every once in a while */ - if (!((q->host_index + 1) % q->entry_repost)) + if (!((q->host_index + 1) % q->notify_interval)) bf_set(wqe_wqec, &wqe->generic.wqe_com, 1); else bf_set(wqe_wqec, &wqe->generic.wqe_com, 0); @@ -325,29 +326,16 @@ lpfc_sli4_mq_release(struct lpfc_queue *q) static struct lpfc_eqe * lpfc_sli4_eq_get(struct lpfc_queue *q) { - struct lpfc_hba *phba; struct lpfc_eqe *eqe; - uint32_t idx; /* sanity check on queue memory */ if (unlikely(!q)) return NULL; - phba = q->phba; - eqe = q->qe[q->hba_index].eqe; + eqe = q->qe[q->host_index].eqe; /* If the next EQE is not valid then we are done */ if (bf_get_le32(lpfc_eqe_valid, eqe) != q->qe_valid) return NULL; - /* If the host has not yet processed the next entry then we are done */ - idx = ((q->hba_index + 1) % q->entry_count); - if (idx == q->host_index) - return NULL; - - q->hba_index = idx; - /* if the index wrapped around, toggle the valid bit */ - if (phba->sli4_hba.pc_sli4_params.eqav && !q->hba_index) - q->qe_valid = (q->qe_valid) ? 0 : 1; - /* * insert barrier for instruction interlock : data from the hardware @@ -397,44 +385,25 @@ lpfc_sli4_if6_eq_clr_intr(struct lpfc_queue *q) } /** - * lpfc_sli4_eq_release - Indicates the host has finished processing an EQ + * lpfc_sli4_write_eq_db - write EQ DB for eqe's consumed or arm state + * @phba: adapter with EQ * @q: The Event Queue that the host has completed processing for. + * @count: Number of elements that have been consumed * @arm: Indicates whether the host wants to arms this CQ. * - * This routine will mark all Event Queue Entries on @q, from the last - * known completed entry to the last entry that was processed, as completed - * by clearing the valid bit for each completion queue entry. Then it will - * notify the HBA, by ringing the doorbell, that the EQEs have been processed. - * The internal host index in the @q will be updated by this routine to indicate - * that the host has finished processing the entries. The @arm parameter - * indicates that the queue should be rearmed when ringing the doorbell. - * - * This function will return the number of EQEs that were popped. + * This routine will notify the HBA, by ringing the doorbell, that count + * number of EQEs have been processed. The @arm parameter indicates whether + * the queue should be rearmed when ringing the doorbell. **/ -uint32_t -lpfc_sli4_eq_release(struct lpfc_queue *q, bool arm) +void +lpfc_sli4_write_eq_db(struct lpfc_hba *phba, struct lpfc_queue *q, + uint32_t count, bool arm) { - uint32_t released = 0; - struct lpfc_hba *phba; - struct lpfc_eqe *temp_eqe; struct lpfc_register doorbell; /* sanity check on queue memory */ - if (unlikely(!q)) - return 0; - phba = q->phba; - - /* while there are valid entries */ - while (q->hba_index != q->host_index) { - if (!phba->sli4_hba.pc_sli4_params.eqav) { - temp_eqe = q->qe[q->host_index].eqe; - bf_set_le32(lpfc_eqe_valid, temp_eqe, 0); - } - released++; - q->host_index = ((q->host_index + 1) % q->entry_count); - } - if (unlikely(released == 0 && !arm)) - return 0; + if (unlikely(!q || (count == 0 && !arm))) + return; /* ring doorbell for number popped */ doorbell.word0 = 0; @@ -442,7 +411,7 @@ lpfc_sli4_eq_release(struct lpfc_queue *q, bool arm) bf_set(lpfc_eqcq_doorbell_arm, &doorbell, 1); bf_set(lpfc_eqcq_doorbell_eqci, &doorbell, 1); } - bf_set(lpfc_eqcq_doorbell_num_released, &doorbell, released); + bf_set(lpfc_eqcq_doorbell_num_released, &doorbell, count); bf_set(lpfc_eqcq_doorbell_qt, &doorbell, LPFC_QUEUE_TYPE_EVENT); bf_set(lpfc_eqcq_doorbell_eqid_hi, &doorbell, (q->queue_id >> LPFC_EQID_HI_FIELD_SHIFT)); @@ -451,60 +420,112 @@ lpfc_sli4_eq_release(struct lpfc_queue *q, bool arm) /* PCI read to flush PCI pipeline on re-arming for INTx mode */ if ((q->phba->intr_type == INTx) && (arm == LPFC_QUEUE_REARM)) readl(q->phba->sli4_hba.EQDBregaddr); - return released; } /** - * lpfc_sli4_if6_eq_release - Indicates the host has finished processing an EQ + * lpfc_sli4_if6_write_eq_db - write EQ DB for eqe's consumed or arm state + * @phba: adapter with EQ * @q: The Event Queue that the host has completed processing for. + * @count: Number of elements that have been consumed * @arm: Indicates whether the host wants to arms this CQ. * - * This routine will mark all Event Queue Entries on @q, from the last - * known completed entry to the last entry that was processed, as completed - * by clearing the valid bit for each completion queue entry. Then it will - * notify the HBA, by ringing the doorbell, that the EQEs have been processed. - * The internal host index in the @q will be updated by this routine to indicate - * that the host has finished processing the entries. The @arm parameter - * indicates that the queue should be rearmed when ringing the doorbell. - * - * This function will return the number of EQEs that were popped. + * This routine will notify the HBA, by ringing the doorbell, that count + * number of EQEs have been processed. The @arm parameter indicates whether + * the queue should be rearmed when ringing the doorbell. **/ -uint32_t -lpfc_sli4_if6_eq_release(struct lpfc_queue *q, bool arm) +void +lpfc_sli4_if6_write_eq_db(struct lpfc_hba *phba, struct lpfc_queue *q, + uint32_t count, bool arm) { - uint32_t released = 0; - struct lpfc_hba *phba; - struct lpfc_eqe *temp_eqe; struct lpfc_register doorbell; /* sanity check on queue memory */ - if (unlikely(!q)) - return 0; - phba = q->phba; - - /* while there are valid entries */ - while (q->hba_index != q->host_index) { - if (!phba->sli4_hba.pc_sli4_params.eqav) { - temp_eqe = q->qe[q->host_index].eqe; - bf_set_le32(lpfc_eqe_valid, temp_eqe, 0); - } - released++; - q->host_index = ((q->host_index + 1) % q->entry_count); - } - if (unlikely(released == 0 && !arm)) - return 0; + if (unlikely(!q || (count == 0 && !arm))) + return; /* ring doorbell for number popped */ doorbell.word0 = 0; if (arm) bf_set(lpfc_if6_eq_doorbell_arm, &doorbell, 1); - bf_set(lpfc_if6_eq_doorbell_num_released, &doorbell, released); + bf_set(lpfc_if6_eq_doorbell_num_released, &doorbell, count); bf_set(lpfc_if6_eq_doorbell_eqid, &doorbell, q->queue_id); writel(doorbell.word0, q->phba->sli4_hba.EQDBregaddr); /* PCI read to flush PCI pipeline on re-arming for INTx mode */ if ((q->phba->intr_type == INTx) && (arm == LPFC_QUEUE_REARM)) readl(q->phba->sli4_hba.EQDBregaddr); - return released; +} + +static void +__lpfc_sli4_consume_eqe(struct lpfc_hba *phba, struct lpfc_queue *eq, + struct lpfc_eqe *eqe) +{ + if (!phba->sli4_hba.pc_sli4_params.eqav) + bf_set_le32(lpfc_eqe_valid, eqe, 0); + + eq->host_index = ((eq->host_index + 1) % eq->entry_count); + + /* if the index wrapped around, toggle the valid bit */ + if (phba->sli4_hba.pc_sli4_params.eqav && !eq->host_index) + eq->qe_valid = (eq->qe_valid) ? 0 : 1; +} + +static void +lpfc_sli4_eq_flush(struct lpfc_hba *phba, struct lpfc_queue *eq) +{ + struct lpfc_eqe *eqe; + uint32_t count = 0; + + /* walk all the EQ entries and drop on the floor */ + eqe = lpfc_sli4_eq_get(eq); + while (eqe) { + __lpfc_sli4_consume_eqe(phba, eq, eqe); + count++; + eqe = lpfc_sli4_eq_get(eq); + } + + /* Clear and re-arm the EQ */ + phba->sli4_hba.sli4_write_eq_db(phba, eq, count, LPFC_QUEUE_REARM); +} + +static int +lpfc_sli4_process_eq(struct lpfc_hba *phba, struct lpfc_queue *eq) +{ + struct lpfc_eqe *eqe; + int count = 0, consumed = 0; + + if (cmpxchg(&eq->queue_claimed, 0, 1) != 0) + goto rearm_and_exit; + + eqe = lpfc_sli4_eq_get(eq); + while (eqe) { + lpfc_sli4_hba_handle_eqe(phba, eq, eqe); + __lpfc_sli4_consume_eqe(phba, eq, eqe); + + consumed++; + if (!(++count % eq->max_proc_limit)) + break; + + if (!(count % eq->notify_interval)) { + phba->sli4_hba.sli4_write_eq_db(phba, eq, consumed, + LPFC_QUEUE_NOARM); + consumed = 0; + } + + eqe = lpfc_sli4_eq_get(eq); + } + eq->EQ_processed += count; + + /* Track the max number of EQEs processed in 1 intr */ + if (count > eq->EQ_max_eqe) + eq->EQ_max_eqe = count; + + eq->queue_claimed = 0; + +rearm_and_exit: + /* Always clear and re-arm the EQ */ + phba->sli4_hba.sli4_write_eq_db(phba, eq, consumed, LPFC_QUEUE_REARM); + + return count; } /** @@ -519,28 +540,16 @@ lpfc_sli4_if6_eq_release(struct lpfc_queue *q, bool arm) static struct lpfc_cqe * lpfc_sli4_cq_get(struct lpfc_queue *q) { - struct lpfc_hba *phba; struct lpfc_cqe *cqe; - uint32_t idx; /* sanity check on queue memory */ if (unlikely(!q)) return NULL; - phba = q->phba; - cqe = q->qe[q->hba_index].cqe; + cqe = q->qe[q->host_index].cqe; /* If the next CQE is not valid then we are done */ if (bf_get_le32(lpfc_cqe_valid, cqe) != q->qe_valid) return NULL; - /* If the host has not yet processed the next entry then we are done */ - idx = ((q->hba_index + 1) % q->entry_count); - if (idx == q->host_index) - return NULL; - - q->hba_index = idx; - /* if the index wrapped around, toggle the valid bit */ - if (phba->sli4_hba.pc_sli4_params.cqav && !q->hba_index) - q->qe_valid = (q->qe_valid) ? 0 : 1; /* * insert barrier for instruction interlock : data from the hardware @@ -554,107 +563,81 @@ lpfc_sli4_cq_get(struct lpfc_queue *q) return cqe; } +static void +__lpfc_sli4_consume_cqe(struct lpfc_hba *phba, struct lpfc_queue *cq, + struct lpfc_cqe *cqe) +{ + if (!phba->sli4_hba.pc_sli4_params.cqav) + bf_set_le32(lpfc_cqe_valid, cqe, 0); + + cq->host_index = ((cq->host_index + 1) % cq->entry_count); + + /* if the index wrapped around, toggle the valid bit */ + if (phba->sli4_hba.pc_sli4_params.cqav && !cq->host_index) + cq->qe_valid = (cq->qe_valid) ? 0 : 1; +} + /** - * lpfc_sli4_cq_release - Indicates the host has finished processing a CQ + * lpfc_sli4_write_cq_db - write cq DB for entries consumed or arm state. + * @phba: the adapter with the CQ * @q: The Completion Queue that the host has completed processing for. + * @count: the number of elements that were consumed * @arm: Indicates whether the host wants to arms this CQ. * - * This routine will mark all Completion queue entries on @q, from the last - * known completed entry to the last entry that was processed, as completed - * by clearing the valid bit for each completion queue entry. Then it will - * notify the HBA, by ringing the doorbell, that the CQEs have been processed. - * The internal host index in the @q will be updated by this routine to indicate - * that the host has finished processing the entries. The @arm parameter - * indicates that the queue should be rearmed when ringing the doorbell. - * - * This function will return the number of CQEs that were released. + * This routine will notify the HBA, by ringing the doorbell, that the + * CQEs have been processed. The @arm parameter specifies whether the + * queue should be rearmed when ringing the doorbell. **/ -uint32_t -lpfc_sli4_cq_release(struct lpfc_queue *q, bool arm) +void +lpfc_sli4_write_cq_db(struct lpfc_hba *phba, struct lpfc_queue *q, + uint32_t count, bool arm) { - uint32_t released = 0; - struct lpfc_hba *phba; - struct lpfc_cqe *temp_qe; struct lpfc_register doorbell; /* sanity check on queue memory */ - if (unlikely(!q)) - return 0; - phba = q->phba; - - /* while there are valid entries */ - while (q->hba_index != q->host_index) { - if (!phba->sli4_hba.pc_sli4_params.cqav) { - temp_qe = q->qe[q->host_index].cqe; - bf_set_le32(lpfc_cqe_valid, temp_qe, 0); - } - released++; - q->host_index = ((q->host_index + 1) % q->entry_count); - } - if (unlikely(released == 0 && !arm)) - return 0; + if (unlikely(!q || (count == 0 && !arm))) + return; /* ring doorbell for number popped */ doorbell.word0 = 0; if (arm) bf_set(lpfc_eqcq_doorbell_arm, &doorbell, 1); - bf_set(lpfc_eqcq_doorbell_num_released, &doorbell, released); + bf_set(lpfc_eqcq_doorbell_num_released, &doorbell, count); bf_set(lpfc_eqcq_doorbell_qt, &doorbell, LPFC_QUEUE_TYPE_COMPLETION); bf_set(lpfc_eqcq_doorbell_cqid_hi, &doorbell, (q->queue_id >> LPFC_CQID_HI_FIELD_SHIFT)); bf_set(lpfc_eqcq_doorbell_cqid_lo, &doorbell, q->queue_id); writel(doorbell.word0, q->phba->sli4_hba.CQDBregaddr); - return released; } /** - * lpfc_sli4_if6_cq_release - Indicates the host has finished processing a CQ + * lpfc_sli4_if6_write_cq_db - write cq DB for entries consumed or arm state. + * @phba: the adapter with the CQ * @q: The Completion Queue that the host has completed processing for. + * @count: the number of elements that were consumed * @arm: Indicates whether the host wants to arms this CQ. * - * This routine will mark all Completion queue entries on @q, from the last - * known completed entry to the last entry that was processed, as completed - * by clearing the valid bit for each completion queue entry. Then it will - * notify the HBA, by ringing the doorbell, that the CQEs have been processed. - * The internal host index in the @q will be updated by this routine to indicate - * that the host has finished processing the entries. The @arm parameter - * indicates that the queue should be rearmed when ringing the doorbell. - * - * This function will return the number of CQEs that were released. + * This routine will notify the HBA, by ringing the doorbell, that the + * CQEs have been processed. The @arm parameter specifies whether the + * queue should be rearmed when ringing the doorbell. **/ -uint32_t -lpfc_sli4_if6_cq_release(struct lpfc_queue *q, bool arm) +void +lpfc_sli4_if6_write_cq_db(struct lpfc_hba *phba, struct lpfc_queue *q, + uint32_t count, bool arm) { - uint32_t released = 0; - struct lpfc_hba *phba; - struct lpfc_cqe *temp_qe; struct lpfc_register doorbell; /* sanity check on queue memory */ - if (unlikely(!q)) - return 0; - phba = q->phba; - - /* while there are valid entries */ - while (q->hba_index != q->host_index) { - if (!phba->sli4_hba.pc_sli4_params.cqav) { - temp_qe = q->qe[q->host_index].cqe; - bf_set_le32(lpfc_cqe_valid, temp_qe, 0); - } - released++; - q->host_index = ((q->host_index + 1) % q->entry_count); - } - if (unlikely(released == 0 && !arm)) - return 0; + if (unlikely(!q || (count == 0 && !arm))) + return; /* ring doorbell for number popped */ doorbell.word0 = 0; if (arm) bf_set(lpfc_if6_cq_doorbell_arm, &doorbell, 1); - bf_set(lpfc_if6_cq_doorbell_num_released, &doorbell, released); + bf_set(lpfc_if6_cq_doorbell_num_released, &doorbell, count); bf_set(lpfc_if6_cq_doorbell_cqid, &doorbell, q->queue_id); writel(doorbell.word0, q->phba->sli4_hba.CQDBregaddr); - return released; } /** @@ -703,15 +686,15 @@ lpfc_sli4_rq_put(struct lpfc_queue *hq, struct lpfc_queue *dq, hq->RQ_buf_posted++; /* Ring The Header Receive Queue Doorbell */ - if (!(hq->host_index % hq->entry_repost)) { + if (!(hq->host_index % hq->notify_interval)) { doorbell.word0 = 0; if (hq->db_format == LPFC_DB_RING_FORMAT) { bf_set(lpfc_rq_db_ring_fm_num_posted, &doorbell, - hq->entry_repost); + hq->notify_interval); bf_set(lpfc_rq_db_ring_fm_id, &doorbell, hq->queue_id); } else if (hq->db_format == LPFC_DB_LIST_FORMAT) { bf_set(lpfc_rq_db_list_fm_num_posted, &doorbell, - hq->entry_repost); + hq->notify_interval); bf_set(lpfc_rq_db_list_fm_index, &doorbell, hq->host_index); bf_set(lpfc_rq_db_list_fm_id, &doorbell, hq->queue_id); @@ -5571,30 +5554,30 @@ lpfc_sli4_arm_cqeq_intr(struct lpfc_hba *phba) struct lpfc_sli4_hba *sli4_hba = &phba->sli4_hba; struct lpfc_sli4_hdw_queue *qp; - sli4_hba->sli4_cq_release(sli4_hba->mbx_cq, LPFC_QUEUE_REARM); - sli4_hba->sli4_cq_release(sli4_hba->els_cq, LPFC_QUEUE_REARM); + sli4_hba->sli4_write_cq_db(phba, sli4_hba->mbx_cq, 0, LPFC_QUEUE_REARM); + sli4_hba->sli4_write_cq_db(phba, sli4_hba->els_cq, 0, LPFC_QUEUE_REARM); if (sli4_hba->nvmels_cq) - sli4_hba->sli4_cq_release(sli4_hba->nvmels_cq, - LPFC_QUEUE_REARM); + sli4_hba->sli4_write_cq_db(phba, sli4_hba->nvmels_cq, 0, + LPFC_QUEUE_REARM); qp = sli4_hba->hdwq; if (sli4_hba->hdwq) { for (qidx = 0; qidx < phba->cfg_hdw_queue; qidx++) { - sli4_hba->sli4_cq_release(qp[qidx].fcp_cq, - LPFC_QUEUE_REARM); - sli4_hba->sli4_cq_release(qp[qidx].nvme_cq, - LPFC_QUEUE_REARM); + sli4_hba->sli4_write_cq_db(phba, qp[qidx].fcp_cq, 0, + LPFC_QUEUE_REARM); + sli4_hba->sli4_write_cq_db(phba, qp[qidx].nvme_cq, 0, + LPFC_QUEUE_REARM); } for (qidx = 0; qidx < phba->cfg_irq_chann; qidx++) - sli4_hba->sli4_eq_release(qp[qidx].hba_eq, - LPFC_QUEUE_REARM); + sli4_hba->sli4_write_eq_db(phba, qp[qidx].hba_eq, + 0, LPFC_QUEUE_REARM); } if (phba->nvmet_support) { for (qidx = 0; qidx < phba->cfg_nvmet_mrq; qidx++) { - sli4_hba->sli4_cq_release( - sli4_hba->nvmet_cqset[qidx], + sli4_hba->sli4_write_cq_db(phba, + sli4_hba->nvmet_cqset[qidx], 0, LPFC_QUEUE_REARM); } } @@ -7698,6 +7681,11 @@ lpfc_sli4_hba_setup(struct lpfc_hba *phba) phba->hb_outstanding = 0; phba->last_completion_time = jiffies; + /* start eq_delay heartbeat */ + if (phba->cfg_auto_imax) + queue_delayed_work(phba->wq, &phba->eq_delay_work, + msecs_to_jiffies(LPFC_EQ_DELAY_MSECS)); + /* Start error attention (ERATT) polling timer */ mod_timer(&phba->eratt_poll, jiffies + msecs_to_jiffies(1000 * phba->eratt_poll_interval)); @@ -7869,7 +7857,6 @@ lpfc_sli4_process_missed_mbox_completions(struct lpfc_hba *phba) struct lpfc_sli4_hba *sli4_hba = &phba->sli4_hba; uint32_t eqidx; struct lpfc_queue *fpeq = NULL; - struct lpfc_eqe *eqe; bool mbox_pending; if (unlikely(!phba) || (phba->sli_rev != LPFC_SLI_REV4)) @@ -7903,14 +7890,11 @@ lpfc_sli4_process_missed_mbox_completions(struct lpfc_hba *phba) */ if (mbox_pending) - while ((eqe = lpfc_sli4_eq_get(fpeq))) { - lpfc_sli4_hba_handle_eqe(phba, eqe, eqidx); - fpeq->EQ_processed++; - } - - /* Always clear and re-arm the EQ */ - - sli4_hba->sli4_eq_release(fpeq, LPFC_QUEUE_REARM); + /* process and rearm the EQ */ + lpfc_sli4_process_eq(phba, fpeq); + else + /* Always clear and re-arm the EQ */ + sli4_hba->sli4_write_eq_db(phba, fpeq, 0, LPFC_QUEUE_REARM); return mbox_pending; @@ -13265,11 +13249,14 @@ out_no_mqe_complete: * Return: true if work posted to worker thread, otherwise false. **/ static bool -lpfc_sli4_sp_handle_mcqe(struct lpfc_hba *phba, struct lpfc_cqe *cqe) +lpfc_sli4_sp_handle_mcqe(struct lpfc_hba *phba, struct lpfc_queue *cq, + struct lpfc_cqe *cqe) { struct lpfc_mcqe mcqe; bool workposted; + cq->CQ_mbox++; + /* Copy the mailbox MCQE and convert endian order as needed */ lpfc_sli4_pcimem_bcopy(cqe, &mcqe, sizeof(struct lpfc_mcqe)); @@ -13528,7 +13515,7 @@ out: * lpfc_sli4_sp_handle_cqe - Process a slow path completion queue entry * @phba: Pointer to HBA context object. * @cq: Pointer to the completion queue. - * @wcqe: Pointer to a completion queue entry. + * @cqe: Pointer to a completion queue entry. * * This routine process a slow-path work-queue or receive queue completion queue * entry. @@ -13628,60 +13615,129 @@ lpfc_sli4_sp_handle_eqe(struct lpfc_hba *phba, struct lpfc_eqe *eqe, } /** - * lpfc_sli4_sp_process_cq - Process a slow-path event queue entry + * __lpfc_sli4_process_cq - Process elements of a CQ * @phba: Pointer to HBA context object. + * @cq: Pointer to CQ to be processed + * @handler: Routine to process each cqe + * @delay: Pointer to usdelay to set in case of rescheduling of the handler * - * This routine process a event queue entry from the slow-path event queue. - * It will check the MajorCode and MinorCode to determine this is for a - * completion event on a completion queue, if not, an error shall be logged - * and just return. Otherwise, it will get to the corresponding completion - * queue and process all the entries on that completion queue, rearm the - * completion queue, and then return. + * This routine processes completion queue entries in a CQ. While a valid + * queue element is found, the handler is called. During processing checks + * are made for periodic doorbell writes to let the hardware know of + * element consumption. * + * If the max limit on cqes to process is hit, or there are no more valid + * entries, the loop stops. If we processed a sufficient number of elements, + * meaning there is sufficient load, rather than rearming and generating + * another interrupt, a cq rescheduling delay will be set. A delay of 0 + * indicates no rescheduling. + * + * Returns True if work scheduled, False otherwise. **/ -static void -lpfc_sli4_sp_process_cq(struct work_struct *work) +static bool +__lpfc_sli4_process_cq(struct lpfc_hba *phba, struct lpfc_queue *cq, + bool (*handler)(struct lpfc_hba *, struct lpfc_queue *, + struct lpfc_cqe *), unsigned long *delay) { - struct lpfc_queue *cq = - container_of(work, struct lpfc_queue, spwork); - struct lpfc_hba *phba = cq->phba; struct lpfc_cqe *cqe; bool workposted = false; - int ccount = 0; + int count = 0, consumed = 0; + bool arm = true; + + /* default - no reschedule */ + *delay = 0; + + if (cmpxchg(&cq->queue_claimed, 0, 1) != 0) + goto rearm_and_exit; /* Process all the entries to the CQ */ + cqe = lpfc_sli4_cq_get(cq); + while (cqe) { +#if defined(CONFIG_SCSI_LPFC_DEBUG_FS) && defined(BUILD_NVME) + if (phba->ktime_on) + cq->isr_timestamp = ktime_get_ns(); + else + cq->isr_timestamp = 0; +#endif + workposted |= handler(phba, cq, cqe); + __lpfc_sli4_consume_cqe(phba, cq, cqe); + + consumed++; + if (!(++count % cq->max_proc_limit)) + break; + + if (!(count % cq->notify_interval)) { + phba->sli4_hba.sli4_write_cq_db(phba, cq, consumed, + LPFC_QUEUE_NOARM); + consumed = 0; + } + + cqe = lpfc_sli4_cq_get(cq); + } + if (count >= phba->cfg_cq_poll_threshold) { + *delay = 1; + arm = false; + } + + /* Track the max number of CQEs processed in 1 EQ */ + if (count > cq->CQ_max_cqe) + cq->CQ_max_cqe = count; + + cq->assoc_qp->EQ_cqe_cnt += count; + + /* Catch the no cq entry condition */ + if (unlikely(count == 0)) + lpfc_printf_log(phba, KERN_INFO, LOG_SLI, + "0369 No entry from completion queue " + "qid=%d\n", cq->queue_id); + + cq->queue_claimed = 0; + +rearm_and_exit: + phba->sli4_hba.sli4_write_cq_db(phba, cq, consumed, + arm ? LPFC_QUEUE_REARM : LPFC_QUEUE_NOARM); + + return workposted; +} + +/** + * lpfc_sli4_sp_process_cq - Process a slow-path event queue entry + * @cq: pointer to CQ to process + * + * This routine calls the cq processing routine with a handler specific + * to the type of queue bound to it. + * + * The CQ routine returns two values: the first is the calling status, + * which indicates whether work was queued to the background discovery + * thread. If true, the routine should wakeup the discovery thread; + * the second is the delay parameter. If non-zero, rather than rearming + * the CQ and yet another interrupt, the CQ handler should be queued so + * that it is processed in a subsequent polling action. The value of + * the delay indicates when to reschedule it. + **/ +static void +__lpfc_sli4_sp_process_cq(struct lpfc_queue *cq) +{ + struct lpfc_hba *phba = cq->phba; + unsigned long delay; + bool workposted = false; + + /* Process and rearm the CQ */ switch (cq->type) { case LPFC_MCQ: - while ((cqe = lpfc_sli4_cq_get(cq))) { - workposted |= lpfc_sli4_sp_handle_mcqe(phba, cqe); - if (!(++ccount % cq->entry_repost)) - break; - cq->CQ_mbox++; - } + workposted |= __lpfc_sli4_process_cq(phba, cq, + lpfc_sli4_sp_handle_mcqe, + &delay); break; case LPFC_WCQ: - while ((cqe = lpfc_sli4_cq_get(cq))) { - if (cq->subtype == LPFC_FCP || - cq->subtype == LPFC_NVME) { -#ifdef CONFIG_SCSI_LPFC_DEBUG_FS - if (phba->ktime_on) - cq->isr_timestamp = ktime_get_ns(); - else - cq->isr_timestamp = 0; -#endif - workposted |= lpfc_sli4_fp_handle_cqe(phba, cq, - cqe); - } else { - workposted |= lpfc_sli4_sp_handle_cqe(phba, cq, - cqe); - } - if (!(++ccount % cq->entry_repost)) - break; - } - - /* Track the max number of CQEs processed in 1 EQ */ - if (ccount > cq->CQ_max_cqe) - cq->CQ_max_cqe = ccount; + if (cq->subtype == LPFC_FCP || cq->subtype == LPFC_NVME) + workposted |= __lpfc_sli4_process_cq(phba, cq, + lpfc_sli4_fp_handle_cqe, + &delay); + else + workposted |= __lpfc_sli4_process_cq(phba, cq, + lpfc_sli4_sp_handle_cqe, + &delay); break; default: lpfc_printf_log(phba, KERN_ERR, LOG_SLI, @@ -13690,20 +13746,50 @@ lpfc_sli4_sp_process_cq(struct work_struct *work) return; } - /* Catch the no cq entry condition, log an error */ - if (unlikely(ccount == 0)) - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, - "0371 No entry from the CQ: identifier " - "(x%x), type (%d)\n", cq->queue_id, cq->type); - - /* In any case, flash and re-arm the RCQ */ - phba->sli4_hba.sli4_cq_release(cq, LPFC_QUEUE_REARM); + if (delay) { + if (!queue_delayed_work_on(cq->chann, phba->wq, + &cq->sched_spwork, delay)) + lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + "0394 Cannot schedule soft IRQ " + "for cqid=%d on CPU %d\n", + cq->queue_id, cq->chann); + } /* wake up worker thread if there are works to be done */ if (workposted) lpfc_worker_wake_up(phba); } +/** + * lpfc_sli4_sp_process_cq - slow-path work handler when started by + * interrupt + * @work: pointer to work element + * + * translates from the work handler and calls the slow-path handler. + **/ +static void +lpfc_sli4_sp_process_cq(struct work_struct *work) +{ + struct lpfc_queue *cq = container_of(work, struct lpfc_queue, spwork); + + __lpfc_sli4_sp_process_cq(cq); +} + +/** + * lpfc_sli4_dly_sp_process_cq - slow-path work handler when started by timer + * @work: pointer to work element + * + * translates from the work handler and calls the slow-path handler. + **/ +static void +lpfc_sli4_dly_sp_process_cq(struct work_struct *work) +{ + struct lpfc_queue *cq = container_of(to_delayed_work(work), + struct lpfc_queue, sched_spwork); + + __lpfc_sli4_sp_process_cq(cq); +} + /** * lpfc_sli4_fp_handle_fcp_wcqe - Process fast-path work queue completion entry * @phba: Pointer to HBA context object. @@ -13935,13 +14021,16 @@ out: /** * lpfc_sli4_fp_handle_cqe - Process fast-path work queue completion entry + * @phba: adapter with cq * @cq: Pointer to the completion queue. * @eqe: Pointer to fast-path completion queue entry. * * This routine process a fast-path work queue completion entry from fast-path * event queue for FCP command response completion. + * + * Return: true if work posted to worker thread, otherwise false. **/ -static int +static bool lpfc_sli4_fp_handle_cqe(struct lpfc_hba *phba, struct lpfc_queue *cq, struct lpfc_cqe *cqe) { @@ -14008,10 +14097,11 @@ lpfc_sli4_fp_handle_cqe(struct lpfc_hba *phba, struct lpfc_queue *cq, * completion queue, and then return. **/ static void -lpfc_sli4_hba_handle_eqe(struct lpfc_hba *phba, struct lpfc_eqe *eqe, - uint32_t qidx) +lpfc_sli4_hba_handle_eqe(struct lpfc_hba *phba, struct lpfc_queue *eq, + struct lpfc_eqe *eqe) { struct lpfc_queue *cq = NULL; + uint32_t qidx = eq->hdwq; uint16_t cqid, id; if (unlikely(bf_get_le32(lpfc_eqe_major_code, eqe) != 0)) { @@ -14074,72 +14164,74 @@ work_cq: } /** - * lpfc_sli4_hba_process_cq - Process a fast-path event queue entry - * @phba: Pointer to HBA context object. - * @eqe: Pointer to fast-path event queue entry. + * __lpfc_sli4_hba_process_cq - Process a fast-path event queue entry + * @cq: Pointer to CQ to be processed * - * This routine process a event queue entry from the fast-path event queue. - * It will check the MajorCode and MinorCode to determine this is for a - * completion event on a completion queue, if not, an error shall be logged - * and just return. Otherwise, it will get to the corresponding completion - * queue and process all the entries on the completion queue, rearm the - * completion queue, and then return. + * This routine calls the cq processing routine with the handler for + * fast path CQEs. + * + * The CQ routine returns two values: the first is the calling status, + * which indicates whether work was queued to the background discovery + * thread. If true, the routine should wakeup the discovery thread; + * the second is the delay parameter. If non-zero, rather than rearming + * the CQ and yet another interrupt, the CQ handler should be queued so + * that it is processed in a subsequent polling action. The value of + * the delay indicates when to reschedule it. **/ static void -lpfc_sli4_hba_process_cq(struct work_struct *work) +__lpfc_sli4_hba_process_cq(struct lpfc_queue *cq) { - struct lpfc_queue *cq = - container_of(work, struct lpfc_queue, irqwork); struct lpfc_hba *phba = cq->phba; - struct lpfc_cqe *cqe; + unsigned long delay; bool workposted = false; - int ccount = 0; - - /* Process all the entries to the CQ */ - while ((cqe = lpfc_sli4_cq_get(cq))) { -#ifdef CONFIG_SCSI_LPFC_DEBUG_FS - if (phba->ktime_on) - cq->isr_timestamp = ktime_get_ns(); - else - cq->isr_timestamp = 0; -#endif - workposted |= lpfc_sli4_fp_handle_cqe(phba, cq, cqe); - if (!(++ccount % cq->entry_repost)) - break; - } - - /* Track the max number of CQEs processed in 1 EQ */ - if (ccount > cq->CQ_max_cqe) - cq->CQ_max_cqe = ccount; - cq->assoc_qp->EQ_cqe_cnt += ccount; - /* Catch the no cq entry condition */ - if (unlikely(ccount == 0)) - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, - "0369 No entry from fast-path completion " - "queue fcpcqid=%d\n", cq->queue_id); + /* process and rearm the CQ */ + workposted |= __lpfc_sli4_process_cq(phba, cq, lpfc_sli4_fp_handle_cqe, + &delay); - /* In any case, flash and re-arm the CQ */ - phba->sli4_hba.sli4_cq_release(cq, LPFC_QUEUE_REARM); + if (delay) { + if (!queue_delayed_work_on(cq->chann, phba->wq, + &cq->sched_irqwork, delay)) + lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + "0367 Cannot schedule soft IRQ " + "for cqid=%d on CPU %d\n", + cq->queue_id, cq->chann); + } /* wake up worker thread if there are works to be done */ if (workposted) lpfc_worker_wake_up(phba); } +/** + * lpfc_sli4_hba_process_cq - fast-path work handler when started by + * interrupt + * @work: pointer to work element + * + * translates from the work handler and calls the fast-path handler. + **/ static void -lpfc_sli4_eq_flush(struct lpfc_hba *phba, struct lpfc_queue *eq) +lpfc_sli4_hba_process_cq(struct work_struct *work) { - struct lpfc_eqe *eqe; - - /* walk all the EQ entries and drop on the floor */ - while ((eqe = lpfc_sli4_eq_get(eq))) - ; + struct lpfc_queue *cq = container_of(work, struct lpfc_queue, irqwork); - /* Clear and re-arm the EQ */ - phba->sli4_hba.sli4_eq_release(eq, LPFC_QUEUE_REARM); + __lpfc_sli4_hba_process_cq(cq); } +/** + * lpfc_sli4_hba_process_cq - fast-path work handler when started by timer + * @work: pointer to work element + * + * translates from the work handler and calls the fast-path handler. + **/ +static void +lpfc_sli4_dly_hba_process_cq(struct work_struct *work) +{ + struct lpfc_queue *cq = container_of(to_delayed_work(work), + struct lpfc_queue, sched_irqwork); + + __lpfc_sli4_hba_process_cq(cq); +} /** * lpfc_sli4_hba_intr_handler - HBA interrupt handler to SLI-4 device @@ -14173,10 +14265,11 @@ lpfc_sli4_hba_intr_handler(int irq, void *dev_id) struct lpfc_hba *phba; struct lpfc_hba_eq_hdl *hba_eq_hdl; struct lpfc_queue *fpeq; - struct lpfc_eqe *eqe; unsigned long iflag; int ecount = 0; int hba_eqidx; + struct lpfc_eq_intr_info *eqi; + uint32_t icnt; /* Get the driver's phba structure from the dev_id */ hba_eq_hdl = (struct lpfc_hba_eq_hdl *)dev_id; @@ -14204,22 +14297,19 @@ lpfc_sli4_hba_intr_handler(int irq, void *dev_id) return IRQ_NONE; } - /* - * Process all the event on FCP fast-path EQ - */ - while ((eqe = lpfc_sli4_eq_get(fpeq))) { - lpfc_sli4_hba_handle_eqe(phba, eqe, hba_eqidx); - if (!(++ecount % fpeq->entry_repost)) - break; - fpeq->EQ_processed++; - } + eqi = phba->sli4_hba.eq_info; + icnt = this_cpu_inc_return(eqi->icnt); + fpeq->last_cpu = smp_processor_id(); - /* Track the max number of EQEs processed in 1 intr */ - if (ecount > fpeq->EQ_max_eqe) - fpeq->EQ_max_eqe = ecount; + if (icnt > LPFC_EQD_ISR_TRIGGER && + phba->cfg_irq_chann == 1 && + phba->cfg_auto_imax && + fpeq->q_mode != LPFC_MAX_AUTO_EQ_DELAY && + phba->sli.sli_flag & LPFC_SLI_USE_EQDR) + lpfc_sli4_mod_hba_eq_delay(phba, fpeq, LPFC_MAX_AUTO_EQ_DELAY); - /* Always clear and re-arm the fast-path EQ */ - phba->sli4_hba.sli4_eq_release(fpeq, LPFC_QUEUE_REARM); + /* process and rearm the EQ */ + ecount = lpfc_sli4_process_eq(phba, fpeq); if (unlikely(ecount == 0)) { fpeq->EQ_no_entry++; @@ -14307,6 +14397,9 @@ lpfc_sli4_queue_free(struct lpfc_queue *queue) kfree(queue->rqbp); } + if (!list_empty(&queue->cpu_list)) + list_del(&queue->cpu_list); + if (!list_empty(&queue->wq_list)) list_del(&queue->wq_list); @@ -14355,6 +14448,7 @@ lpfc_sli4_queue_alloc(struct lpfc_hba *phba, uint32_t page_size, INIT_LIST_HEAD(&queue->wqfull_list); INIT_LIST_HEAD(&queue->page_list); INIT_LIST_HEAD(&queue->child_list); + INIT_LIST_HEAD(&queue->cpu_list); /* Set queue parameters now. If the system cannot provide memory * resources, the free routine needs to know what was allocated. @@ -14387,8 +14481,10 @@ lpfc_sli4_queue_alloc(struct lpfc_hba *phba, uint32_t page_size, } INIT_WORK(&queue->irqwork, lpfc_sli4_hba_process_cq); INIT_WORK(&queue->spwork, lpfc_sli4_sp_process_cq); + INIT_DELAYED_WORK(&queue->sched_irqwork, lpfc_sli4_dly_hba_process_cq); + INIT_DELAYED_WORK(&queue->sched_spwork, lpfc_sli4_dly_sp_process_cq); - /* entry_repost will be set during q creation */ + /* notify_interval will be set during q creation */ return queue; out_fail: @@ -14457,7 +14553,6 @@ lpfc_modify_hba_eq_delay(struct lpfc_hba *phba, uint32_t startq, int cnt = 0, rc, length; uint32_t shdr_status, shdr_add_status; uint32_t dmult; - struct lpfc_register reg_data; int qidx; union lpfc_sli4_cfg_shdr *shdr; @@ -14478,16 +14573,7 @@ lpfc_modify_hba_eq_delay(struct lpfc_hba *phba, uint32_t startq, if (!eq) continue; - /* save value last set */ - eq->q_mode = usdelay; - - /* write register */ - reg_data.word0 = 0; - bf_set(lpfc_sliport_eqdelay_id, ®_data, - eq->queue_id); - bf_set(lpfc_sliport_eqdelay_delay, ®_data, usdelay); - writel(reg_data.word0, - phba->sli4_hba.u.if_type2.EQDregaddr); + lpfc_sli4_mod_hba_eq_delay(phba, eq, usdelay); if (++cnt >= numq) break; @@ -14673,8 +14759,8 @@ lpfc_eq_create(struct lpfc_hba *phba, struct lpfc_queue *eq, uint32_t imax) if (eq->queue_id == 0xFFFF) status = -ENXIO; eq->host_index = 0; - eq->hba_index = 0; - eq->entry_repost = LPFC_EQ_REPOST; + eq->notify_interval = LPFC_EQ_NOTIFY_INTRVL; + eq->max_proc_limit = LPFC_EQ_MAX_PROC_LIMIT; mempool_free(mbox, phba->mbox_mem_pool); return status; @@ -14814,8 +14900,8 @@ lpfc_cq_create(struct lpfc_hba *phba, struct lpfc_queue *cq, cq->assoc_qid = eq->queue_id; cq->assoc_qp = eq; cq->host_index = 0; - cq->hba_index = 0; - cq->entry_repost = LPFC_CQ_REPOST; + cq->notify_interval = LPFC_CQ_NOTIFY_INTRVL; + cq->max_proc_limit = min(phba->cfg_cq_max_proc_limit, cq->entry_count); if (cq->queue_id > phba->sli4_hba.cq_max) phba->sli4_hba.cq_max = cq->queue_id; @@ -15026,8 +15112,9 @@ lpfc_cq_create_set(struct lpfc_hba *phba, struct lpfc_queue **cqp, cq->assoc_qid = eq->queue_id; cq->assoc_qp = eq; cq->host_index = 0; - cq->hba_index = 0; - cq->entry_repost = LPFC_CQ_REPOST; + cq->notify_interval = LPFC_CQ_NOTIFY_INTRVL; + cq->max_proc_limit = min(phba->cfg_cq_max_proc_limit, + cq->entry_count); cq->chann = idx; rc = 0; @@ -15279,7 +15366,6 @@ lpfc_mq_create(struct lpfc_hba *phba, struct lpfc_queue *mq, mq->subtype = subtype; mq->host_index = 0; mq->hba_index = 0; - mq->entry_repost = LPFC_MQ_REPOST; /* link the mq onto the parent cq child list */ list_add_tail(&mq->list, &cq->child_list); @@ -15545,7 +15631,7 @@ lpfc_wq_create(struct lpfc_hba *phba, struct lpfc_queue *wq, wq->subtype = subtype; wq->host_index = 0; wq->hba_index = 0; - wq->entry_repost = LPFC_RELEASE_NOTIFICATION_INTERVAL; + wq->notify_interval = LPFC_WQ_NOTIFY_INTRVL; /* link the wq onto the parent cq child list */ list_add_tail(&wq->list, &cq->child_list); @@ -15739,7 +15825,7 @@ lpfc_rq_create(struct lpfc_hba *phba, struct lpfc_queue *hrq, hrq->subtype = subtype; hrq->host_index = 0; hrq->hba_index = 0; - hrq->entry_repost = LPFC_RQ_REPOST; + hrq->notify_interval = LPFC_RQ_NOTIFY_INTRVL; /* now create the data queue */ lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_FCOE, @@ -15832,7 +15918,7 @@ lpfc_rq_create(struct lpfc_hba *phba, struct lpfc_queue *hrq, drq->subtype = subtype; drq->host_index = 0; drq->hba_index = 0; - drq->entry_repost = LPFC_RQ_REPOST; + drq->notify_interval = LPFC_RQ_NOTIFY_INTRVL; /* link the header and data RQs onto the parent cq child list */ list_add_tail(&hrq->list, &cq->child_list); @@ -15990,7 +16076,7 @@ lpfc_mrq_create(struct lpfc_hba *phba, struct lpfc_queue **hrqp, hrq->subtype = subtype; hrq->host_index = 0; hrq->hba_index = 0; - hrq->entry_repost = LPFC_RQ_REPOST; + hrq->notify_interval = LPFC_RQ_NOTIFY_INTRVL; drq->db_format = LPFC_DB_RING_FORMAT; drq->db_regaddr = phba->sli4_hba.RQDBregaddr; @@ -15999,7 +16085,7 @@ lpfc_mrq_create(struct lpfc_hba *phba, struct lpfc_queue **hrqp, drq->subtype = subtype; drq->host_index = 0; drq->hba_index = 0; - drq->entry_repost = LPFC_RQ_REPOST; + drq->notify_interval = LPFC_RQ_NOTIFY_INTRVL; list_add_tail(&hrq->list, &cq->child_list); list_add_tail(&drq->list, &cq->child_list); @@ -16059,6 +16145,7 @@ lpfc_eq_destroy(struct lpfc_hba *phba, struct lpfc_queue *eq) /* sanity check on queue memory */ if (!eq) return -ENODEV; + mbox = mempool_alloc(eq->phba->mbox_mem_pool, GFP_KERNEL); if (!mbox) return -ENOMEM; diff --git a/drivers/scsi/lpfc/lpfc_sli4.h b/drivers/scsi/lpfc/lpfc_sli4.h index accccca3a027..20566c506e5f 100644 --- a/drivers/scsi/lpfc/lpfc_sli4.h +++ b/drivers/scsi/lpfc/lpfc_sli4.h @@ -154,14 +154,41 @@ struct lpfc_queue { struct list_head child_list; struct list_head page_list; struct list_head sgl_list; + struct list_head cpu_list; uint32_t entry_count; /* Number of entries to support on the queue */ uint32_t entry_size; /* Size of each queue entry. */ - uint32_t entry_repost; /* Count of entries before doorbell is rung */ -#define LPFC_EQ_REPOST 8 -#define LPFC_MQ_REPOST 8 -#define LPFC_CQ_REPOST 64 -#define LPFC_RQ_REPOST 64 -#define LPFC_RELEASE_NOTIFICATION_INTERVAL 32 /* For WQs */ + uint32_t notify_interval; /* Queue Notification Interval + * For chip->host queues (EQ, CQ, RQ): + * specifies the interval (number of + * entries) where the doorbell is rung to + * notify the chip of entry consumption. + * For host->chip queues (WQ): + * specifies the interval (number of + * entries) where consumption CQE is + * requested to indicate WQ entries + * consumed by the chip. + * Not used on an MQ. + */ +#define LPFC_EQ_NOTIFY_INTRVL 16 +#define LPFC_CQ_NOTIFY_INTRVL 16 +#define LPFC_WQ_NOTIFY_INTRVL 16 +#define LPFC_RQ_NOTIFY_INTRVL 16 + uint32_t max_proc_limit; /* Queue Processing Limit + * For chip->host queues (EQ, CQ): + * specifies the maximum number of + * entries to be consumed in one + * processing iteration sequence. Queue + * will be rearmed after each iteration. + * Not used on an MQ, RQ or WQ. + */ +#define LPFC_EQ_MAX_PROC_LIMIT 256 +#define LPFC_CQ_MIN_PROC_LIMIT 64 +#define LPFC_CQ_MAX_PROC_LIMIT LPFC_CQE_EXP_COUNT // 4096 +#define LPFC_CQ_DEF_MAX_PROC_LIMIT LPFC_CQE_DEF_COUNT // 1024 +#define LPFC_CQ_MIN_THRESHOLD_TO_POLL 64 +#define LPFC_CQ_MAX_THRESHOLD_TO_POLL LPFC_CQ_DEF_MAX_PROC_LIMIT +#define LPFC_CQ_DEF_THRESHOLD_TO_POLL LPFC_CQ_DEF_MAX_PROC_LIMIT + uint32_t queue_claimed; /* indicates queue is being processed */ uint32_t queue_id; /* Queue ID assigned by the hardware */ uint32_t assoc_qid; /* Queue ID associated with, for CQ/WQ/MQ */ uint32_t host_index; /* The host's index for putting or getting */ @@ -217,11 +244,14 @@ struct lpfc_queue { #define RQ_buf_posted q_cnt_3 #define RQ_rcv_buf q_cnt_4 - struct work_struct irqwork; - struct work_struct spwork; + struct work_struct irqwork; + struct work_struct spwork; + struct delayed_work sched_irqwork; + struct delayed_work sched_spwork; uint64_t isr_timestamp; uint16_t hdwq; + uint16_t last_cpu; /* most recent cpu */ uint8_t qe_valid; struct lpfc_queue *assoc_qp; union sli4_qe qe[1]; /* array to index entries (must be last) */ @@ -608,6 +638,11 @@ struct lpfc_lock_stat { }; #endif +struct lpfc_eq_intr_info { + struct list_head list; + uint32_t icnt; +}; + /* SLI4 HBA data structure entries */ struct lpfc_sli4_hdw_queue { /* Pointers to the constructed SLI4 queues */ @@ -749,8 +784,10 @@ struct lpfc_sli4_hba { struct lpfc_hba_eq_hdl *hba_eq_hdl; /* HBA per-WQ handle */ void (*sli4_eq_clr_intr)(struct lpfc_queue *q); - uint32_t (*sli4_eq_release)(struct lpfc_queue *q, bool arm); - uint32_t (*sli4_cq_release)(struct lpfc_queue *q, bool arm); + void (*sli4_write_eq_db)(struct lpfc_hba *phba, struct lpfc_queue *eq, + uint32_t count, bool arm); + void (*sli4_write_cq_db)(struct lpfc_hba *phba, struct lpfc_queue *cq, + uint32_t count, bool arm); /* Pointers to the constructed SLI4 queues */ struct lpfc_sli4_hdw_queue *hdwq; @@ -856,6 +893,7 @@ struct lpfc_sli4_hba { uint16_t num_online_cpu; uint16_t num_present_cpu; uint16_t curr_disp_cpu; + struct lpfc_eq_intr_info __percpu *eq_info; uint32_t conf_trunk; #define lpfc_conf_trunk_port0_WORD conf_trunk #define lpfc_conf_trunk_port0_SHIFT 0 @@ -1020,11 +1058,15 @@ int lpfc_sli4_get_els_iocb_cnt(struct lpfc_hba *); int lpfc_sli4_get_iocb_cnt(struct lpfc_hba *phba); int lpfc_sli4_init_vpi(struct lpfc_vport *); inline void lpfc_sli4_eq_clr_intr(struct lpfc_queue *); -uint32_t lpfc_sli4_cq_release(struct lpfc_queue *, bool); -uint32_t lpfc_sli4_eq_release(struct lpfc_queue *, bool); +void lpfc_sli4_write_cq_db(struct lpfc_hba *phba, struct lpfc_queue *q, + uint32_t count, bool arm); +void lpfc_sli4_write_eq_db(struct lpfc_hba *phba, struct lpfc_queue *q, + uint32_t count, bool arm); inline void lpfc_sli4_if6_eq_clr_intr(struct lpfc_queue *q); -uint32_t lpfc_sli4_if6_cq_release(struct lpfc_queue *q, bool arm); -uint32_t lpfc_sli4_if6_eq_release(struct lpfc_queue *q, bool arm); +void lpfc_sli4_if6_write_cq_db(struct lpfc_hba *phba, struct lpfc_queue *q, + uint32_t count, bool arm); +void lpfc_sli4_if6_write_eq_db(struct lpfc_hba *phba, struct lpfc_queue *q, + uint32_t count, bool arm); void lpfc_sli4_fcfi_unreg(struct lpfc_hba *, uint16_t); int lpfc_sli4_fcf_scan_read_fcf_rec(struct lpfc_hba *, uint16_t); int lpfc_sli4_fcf_rr_read_fcf_rec(struct lpfc_hba *, uint16_t); -- cgit v1.2.3 From f6e84790520ac7a14abd921db5a2a1e790e363f9 Mon Sep 17 00:00:00 2001 From: James Smart Date: Mon, 28 Jan 2019 11:14:38 -0800 Subject: scsi: lpfc: Fix default driver parameter collision for allowing NPIV support The conversion to enable SCSI and NVME fc4 support ran into an issue with NPIV support. With NVME, NPIV is not currently supported, but with SCSI it was. The driver reverted to its lowest setting meaning NPIV with SCSI was not allowed. Convert the NPIV checks and implementation so that SCSI can continue to allow NPIV support. Signed-off-by: Dick Kennedy Signed-off-by: James Smart Reviewed-by: Hannes Reinecke Signed-off-by: Martin K. Petersen --- drivers/scsi/lpfc/lpfc.h | 3 ++- drivers/scsi/lpfc/lpfc_attr.c | 4 ++-- drivers/scsi/lpfc/lpfc_ct.c | 16 ++++++++-------- drivers/scsi/lpfc/lpfc_debugfs.c | 4 ++-- drivers/scsi/lpfc/lpfc_els.c | 4 ++-- drivers/scsi/lpfc/lpfc_hbadisc.c | 36 ++++++++++++++++-------------------- drivers/scsi/lpfc/lpfc_init.c | 3 +++ drivers/scsi/lpfc/lpfc_nportdisc.c | 8 ++++---- drivers/scsi/lpfc/lpfc_scsi.c | 2 +- drivers/scsi/lpfc/lpfc_vport.c | 25 ++++++------------------- 10 files changed, 46 insertions(+), 59 deletions(-) (limited to 'drivers/scsi/lpfc/lpfc.h') diff --git a/drivers/scsi/lpfc/lpfc.h b/drivers/scsi/lpfc/lpfc.h index 0bc498172add..b710994a352e 100644 --- a/drivers/scsi/lpfc/lpfc.h +++ b/drivers/scsi/lpfc/lpfc.h @@ -462,6 +462,7 @@ struct lpfc_vport { uint32_t cfg_use_adisc; uint32_t cfg_discovery_threads; uint32_t cfg_log_verbose; + uint32_t cfg_enable_fc4_type; uint32_t cfg_max_luns; uint32_t cfg_enable_da_id; uint32_t cfg_max_scsicmpl_time; @@ -860,6 +861,7 @@ struct lpfc_hba { uint32_t cfg_prot_guard; uint32_t cfg_hostmem_hgp; uint32_t cfg_log_verbose; + uint32_t cfg_enable_fc4_type; uint32_t cfg_aer_support; uint32_t cfg_sriov_nr_virtfn; uint32_t cfg_request_firmware_upgrade; @@ -880,7 +882,6 @@ struct lpfc_hba { uint32_t cfg_ras_fwlog_level; uint32_t cfg_ras_fwlog_buffsize; uint32_t cfg_ras_fwlog_func; - uint32_t cfg_enable_fc4_type; uint32_t cfg_enable_bbcr; /* Enable BB Credit Recovery */ uint32_t cfg_enable_dpp; /* Enable Direct Packet Push */ #define LPFC_ENABLE_FCP 1 diff --git a/drivers/scsi/lpfc/lpfc_attr.c b/drivers/scsi/lpfc/lpfc_attr.c index 4006cb425f16..212bfae1966a 100644 --- a/drivers/scsi/lpfc/lpfc_attr.c +++ b/drivers/scsi/lpfc/lpfc_attr.c @@ -160,7 +160,7 @@ lpfc_nvme_info_show(struct device *dev, struct device_attribute *attr, int len = 0; char tmp[LPFC_MAX_NVME_INFO_TMP_LEN] = {0}; - if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME)) { + if (!(vport->cfg_enable_fc4_type & LPFC_ENABLE_NVME)) { len = scnprintf(buf, PAGE_SIZE, "NVME Disabled\n"); return len; } @@ -519,7 +519,7 @@ lpfc_scsi_stat_show(struct device *dev, struct device_attribute *attr, int i; char tmp[LPFC_MAX_SCSI_INFO_TMP_LEN] = {0}; - if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_FCP) || + if (!(vport->cfg_enable_fc4_type & LPFC_ENABLE_FCP) || (phba->sli_rev != LPFC_SLI_REV4)) return 0; diff --git a/drivers/scsi/lpfc/lpfc_ct.c b/drivers/scsi/lpfc/lpfc_ct.c index 552da8bf43e4..98faa3aae35c 100644 --- a/drivers/scsi/lpfc/lpfc_ct.c +++ b/drivers/scsi/lpfc/lpfc_ct.c @@ -1656,16 +1656,16 @@ lpfc_ns_cmd(struct lpfc_vport *vport, int cmdcode, CtReq->un.rft.PortId = cpu_to_be32(vport->fc_myDID); /* Register FC4 FCP type if enabled. */ - if ((phba->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) || - (phba->cfg_enable_fc4_type == LPFC_ENABLE_FCP)) + if (vport->cfg_enable_fc4_type == LPFC_ENABLE_BOTH || + vport->cfg_enable_fc4_type == LPFC_ENABLE_FCP) CtReq->un.rft.fcpReg = 1; /* Register NVME type if enabled. Defined LE and swapped. * rsvd[0] is used as word1 because of the hard-coded * word0 usage in the ct_request data structure. */ - if ((phba->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) || - (phba->cfg_enable_fc4_type == LPFC_ENABLE_NVME)) + if (vport->cfg_enable_fc4_type == LPFC_ENABLE_BOTH || + vport->cfg_enable_fc4_type == LPFC_ENABLE_NVME) CtReq->un.rft.rsvd[0] = cpu_to_be32(LPFC_FC4_TYPE_BITMASK); @@ -1732,8 +1732,8 @@ lpfc_ns_cmd(struct lpfc_vport *vport, int cmdcode, * caller can specify NVME (type x28) as well. But only * these that FC4 type is supported. */ - if (((phba->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) || - (phba->cfg_enable_fc4_type == LPFC_ENABLE_NVME)) && + if (((vport->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) || + (vport->cfg_enable_fc4_type == LPFC_ENABLE_NVME)) && (context == FC_TYPE_NVME)) { if ((vport == phba->pport) && phba->nvmet_support) { CtReq->un.rff.fbits = (FC4_FEATURE_TARGET | @@ -1744,8 +1744,8 @@ lpfc_ns_cmd(struct lpfc_vport *vport, int cmdcode, } CtReq->un.rff.type_code = context; - } else if (((phba->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) || - (phba->cfg_enable_fc4_type == LPFC_ENABLE_FCP)) && + } else if (((vport->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) || + (vport->cfg_enable_fc4_type == LPFC_ENABLE_FCP)) && (context == FC_TYPE_FCP)) CtReq->un.rff.type_code = context; diff --git a/drivers/scsi/lpfc/lpfc_debugfs.c b/drivers/scsi/lpfc/lpfc_debugfs.c index 16b461778e59..19d13a8789b9 100644 --- a/drivers/scsi/lpfc/lpfc_debugfs.c +++ b/drivers/scsi/lpfc/lpfc_debugfs.c @@ -1166,7 +1166,7 @@ lpfc_debugfs_nvmestat_data(struct lpfc_vport *vport, char *buf, int size) phba->sli4_hba.nvmet_io_wait_total, tot); } else { - if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME)) + if (!(vport->cfg_enable_fc4_type & LPFC_ENABLE_NVME)) return len; localport = vport->localport; @@ -1264,7 +1264,7 @@ lpfc_debugfs_scsistat_data(struct lpfc_vport *vport, char *buf, int size) int i; char tmp[LPFC_MAX_SCSI_INFO_TMP_LEN] = {0}; - if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_FCP) || + if (!(vport->cfg_enable_fc4_type & LPFC_ENABLE_FCP) || (phba->sli_rev != LPFC_SLI_REV4)) return 0; diff --git a/drivers/scsi/lpfc/lpfc_els.c b/drivers/scsi/lpfc/lpfc_els.c index b3a4789468c3..3df2cfd02b41 100644 --- a/drivers/scsi/lpfc/lpfc_els.c +++ b/drivers/scsi/lpfc/lpfc_els.c @@ -2827,8 +2827,8 @@ out: !(vport->fc_flag & FC_PT2PT_PLOGI)) { phba->pport->fc_myDID = 0; - if ((phba->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) || - (phba->cfg_enable_fc4_type == LPFC_ENABLE_NVME)) { + if ((vport->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) || + (vport->cfg_enable_fc4_type == LPFC_ENABLE_NVME)) { if (phba->nvmet_support) lpfc_nvmet_update_targetport(phba); else diff --git a/drivers/scsi/lpfc/lpfc_hbadisc.c b/drivers/scsi/lpfc/lpfc_hbadisc.c index 62689a06c188..47608d4b8e8c 100644 --- a/drivers/scsi/lpfc/lpfc_hbadisc.c +++ b/drivers/scsi/lpfc/lpfc_hbadisc.c @@ -857,10 +857,9 @@ lpfc_port_link_failure(struct lpfc_vport *vport) void lpfc_linkdown_port(struct lpfc_vport *vport) { - struct lpfc_hba *phba = vport->phba; struct Scsi_Host *shost = lpfc_shost_from_vport(vport); - if (phba->cfg_enable_fc4_type != LPFC_ENABLE_NVME) + if (vport->cfg_enable_fc4_type != LPFC_ENABLE_NVME) fc_host_post_event(shost, fc_get_event_number(), FCH_EVT_LINKDOWN, 0); @@ -923,8 +922,8 @@ lpfc_linkdown(struct lpfc_hba *phba) vports[i]->fc_myDID = 0; - if ((phba->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) || - (phba->cfg_enable_fc4_type == LPFC_ENABLE_NVME)) { + if ((vport->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) || + (vport->cfg_enable_fc4_type == LPFC_ENABLE_NVME)) { if (phba->nvmet_support) lpfc_nvmet_update_targetport(phba); else @@ -1010,7 +1009,7 @@ lpfc_linkup_port(struct lpfc_vport *vport) (vport != phba->pport)) return; - if (phba->cfg_enable_fc4_type != LPFC_ENABLE_NVME) + if (vport->cfg_enable_fc4_type != LPFC_ENABLE_NVME) fc_host_post_event(shost, fc_get_event_number(), FCH_EVT_LINKUP, 0); @@ -3658,8 +3657,8 @@ lpfc_mbx_cmpl_reg_vpi(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb) spin_unlock_irq(shost->host_lock); vport->fc_myDID = 0; - if ((phba->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) || - (phba->cfg_enable_fc4_type == LPFC_ENABLE_NVME)) { + if ((vport->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) || + (vport->cfg_enable_fc4_type == LPFC_ENABLE_NVME)) { if (phba->nvmet_support) lpfc_nvmet_update_targetport(phba); else @@ -3921,11 +3920,9 @@ lpfc_mbx_cmpl_fabric_reg_login(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb) int lpfc_issue_gidft(struct lpfc_vport *vport) { - struct lpfc_hba *phba = vport->phba; - /* Good status, issue CT Request to NameServer */ - if ((phba->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) || - (phba->cfg_enable_fc4_type == LPFC_ENABLE_FCP)) { + if ((vport->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) || + (vport->cfg_enable_fc4_type == LPFC_ENABLE_FCP)) { if (lpfc_ns_cmd(vport, SLI_CTNS_GID_FT, 0, SLI_CTPT_FCP)) { /* Cannot issue NameServer FCP Query, so finish up * discovery @@ -3940,8 +3937,8 @@ lpfc_issue_gidft(struct lpfc_vport *vport) vport->gidft_inp++; } - if ((phba->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) || - (phba->cfg_enable_fc4_type == LPFC_ENABLE_NVME)) { + if ((vport->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) || + (vport->cfg_enable_fc4_type == LPFC_ENABLE_NVME)) { if (lpfc_ns_cmd(vport, SLI_CTNS_GID_FT, 0, SLI_CTPT_NVME)) { /* Cannot issue NameServer NVME Query, so finish up * discovery @@ -4057,12 +4054,12 @@ out: lpfc_ns_cmd(vport, SLI_CTNS_RSPN_ID, 0, 0); lpfc_ns_cmd(vport, SLI_CTNS_RFT_ID, 0, 0); - if ((phba->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) || - (phba->cfg_enable_fc4_type == LPFC_ENABLE_FCP)) + if ((vport->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) || + (vport->cfg_enable_fc4_type == LPFC_ENABLE_FCP)) lpfc_ns_cmd(vport, SLI_CTNS_RFF_ID, 0, FC_TYPE_FCP); - if ((phba->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) || - (phba->cfg_enable_fc4_type == LPFC_ENABLE_NVME)) + if ((vport->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) || + (vport->cfg_enable_fc4_type == LPFC_ENABLE_NVME)) lpfc_ns_cmd(vport, SLI_CTNS_RFF_ID, 0, FC_TYPE_NVME); @@ -4098,7 +4095,7 @@ lpfc_register_remote_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp) struct fc_rport_identifiers rport_ids; struct lpfc_hba *phba = vport->phba; - if (phba->cfg_enable_fc4_type == LPFC_ENABLE_NVME) + if (vport->cfg_enable_fc4_type == LPFC_ENABLE_NVME) return; /* Remote port has reappeared. Re-register w/ FC transport */ @@ -4173,9 +4170,8 @@ lpfc_unregister_remote_port(struct lpfc_nodelist *ndlp) { struct fc_rport *rport = ndlp->rport; struct lpfc_vport *vport = ndlp->vport; - struct lpfc_hba *phba = vport->phba; - if (phba->cfg_enable_fc4_type == LPFC_ENABLE_NVME) + if (vport->cfg_enable_fc4_type == LPFC_ENABLE_NVME) return; lpfc_debugfs_disc_trc(vport, LPFC_DISC_TRC_RPORT, diff --git a/drivers/scsi/lpfc/lpfc_init.c b/drivers/scsi/lpfc/lpfc_init.c index 1188354ecd99..563b6009ffce 100644 --- a/drivers/scsi/lpfc/lpfc_init.c +++ b/drivers/scsi/lpfc/lpfc_init.c @@ -4290,6 +4290,9 @@ lpfc_create_port(struct lpfc_hba *phba, int instance, struct device *dev) vport->fc_rscn_flush = 0; lpfc_get_vport_cfgparam(vport); + /* Adjust value in vport */ + vport->cfg_enable_fc4_type = phba->cfg_enable_fc4_type; + shost->unique_id = instance; shost->max_id = LPFC_MAX_TARGET; shost->max_lun = vport->cfg_max_luns; diff --git a/drivers/scsi/lpfc/lpfc_nportdisc.c b/drivers/scsi/lpfc/lpfc_nportdisc.c index 96bc3789a166..acfce67dacde 100644 --- a/drivers/scsi/lpfc/lpfc_nportdisc.c +++ b/drivers/scsi/lpfc/lpfc_nportdisc.c @@ -825,7 +825,7 @@ lpfc_rcv_prli(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp, "rport rolechg: role:x%x did:x%x flg:x%x", roles, ndlp->nlp_DID, ndlp->nlp_flag); - if (phba->cfg_enable_fc4_type != LPFC_ENABLE_NVME) + if (vport->cfg_enable_fc4_type != LPFC_ENABLE_NVME) fc_remote_port_rolechg(rport, roles); } } @@ -1789,8 +1789,8 @@ lpfc_cmpl_reglogin_reglogin_issue(struct lpfc_vport *vport, * is configured try it. */ ndlp->nlp_fc4_type |= NLP_FC4_FCP; - if ((phba->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) || - (phba->cfg_enable_fc4_type == LPFC_ENABLE_NVME)) { + if ((vport->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) || + (vport->cfg_enable_fc4_type == LPFC_ENABLE_NVME)) { ndlp->nlp_fc4_type |= NLP_FC4_NVME; /* We need to update the localport also */ lpfc_nvme_update_localport(vport); @@ -1804,7 +1804,7 @@ lpfc_cmpl_reglogin_reglogin_issue(struct lpfc_vport *vport, * should just issue PRLI for FCP. Otherwise issue * GFT_ID to determine if remote port supports NVME. */ - if (phba->cfg_enable_fc4_type != LPFC_ENABLE_FCP) { + if (vport->cfg_enable_fc4_type != LPFC_ENABLE_FCP) { rc = lpfc_ns_cmd(vport, SLI_CTNS_GFT_ID, 0, ndlp->nlp_DID); return ndlp->nlp_state; diff --git a/drivers/scsi/lpfc/lpfc_scsi.c b/drivers/scsi/lpfc/lpfc_scsi.c index c0065452bc9a..af263b0e8bad 100644 --- a/drivers/scsi/lpfc/lpfc_scsi.c +++ b/drivers/scsi/lpfc/lpfc_scsi.c @@ -530,7 +530,7 @@ lpfc_sli4_vport_delete_fcp_xri_aborted(struct lpfc_vport *vport) unsigned long iflag = 0; int idx; - if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_FCP)) + if (!(vport->cfg_enable_fc4_type & LPFC_ENABLE_FCP)) return; spin_lock_irqsave(&phba->hbalock, iflag); diff --git a/drivers/scsi/lpfc/lpfc_vport.c b/drivers/scsi/lpfc/lpfc_vport.c index 102a011ff6d4..0d712eb04e2b 100644 --- a/drivers/scsi/lpfc/lpfc_vport.c +++ b/drivers/scsi/lpfc/lpfc_vport.c @@ -313,11 +313,11 @@ lpfc_vport_create(struct fc_vport *fc_vport, bool disable) goto error_out; } - /* NPIV is not supported if HBA has NVME enabled */ - if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { + /* NPIV is not supported if HBA has NVME Target enabled */ + if (phba->nvmet_support) { lpfc_printf_log(phba, KERN_ERR, LOG_VPORT, "3189 Create VPORT failed: " - "NPIV is not supported on NVME\n"); + "NPIV is not supported on NVME Target\n"); rc = VPORT_INVAL; goto error_out; } @@ -403,6 +403,9 @@ lpfc_vport_create(struct fc_vport *fc_vport, bool disable) /* Set the DFT_LUN_Q_DEPTH accordingly */ vport->cfg_lun_queue_depth = phba->pport->cfg_lun_queue_depth; + /* Only the physical port can support NVME for now */ + vport->cfg_enable_fc4_type = LPFC_ENABLE_FCP; + *(struct lpfc_vport **)fc_vport->dd_data = vport; vport->fc_vport = fc_vport; @@ -415,22 +418,6 @@ lpfc_vport_create(struct fc_vport *fc_vport, bool disable) vport->fdmi_port_mask = phba->pport->fdmi_port_mask; } - if ((phba->nvmet_support == 0) && - ((phba->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) || - (phba->cfg_enable_fc4_type == LPFC_ENABLE_NVME))) { - /* Create NVME binding with nvme_fc_transport. This - * ensures the vport is initialized. - */ - rc = lpfc_nvme_create_localport(vport); - if (rc) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, - "6003 %s status x%x\n", - "NVME registration failed, ", - rc); - goto error_out; - } - } - /* * In SLI4, the vpi must be activated before it can be used * by the port. -- cgit v1.2.3 From 472e146d1cf3410a898b49834500fa9e33ac41a2 Mon Sep 17 00:00:00 2001 From: James Smart Date: Mon, 28 Jan 2019 11:14:39 -0800 Subject: scsi: lpfc: Correct upcalling nvmet_fc transport during io done downcall When the transport calls into the lpfc target to release an IO job structure, which corresponds to an exchange, and if the driver was waiting for an exchange in order to post a previously received command to the transport, the driver immediately takes the IO job and reuses the context for the prior command and calls nvmet_fc_rcv_fcp_req() to tell the transport about a newly received command. Problem is, the execution of the IO job release may be in the context of the back end driver and its bio completion handlers, thus it may be in a irq context and protection code kicks in in the bio and request layers that are subsequently called. Rework lpfc so that instead of immediately upcalling, queue it to a deferred work thread and have the thread make the upcall. Took advantage of this change to remove duplicated code with the normal command receive path that preps the IO job and upcalls nvmet_fc. Created a common routine both paths use. Also corrected some errors that were found during review of the context freeing and reuse - basically unlocked operations and a somewhat disjoint set of calls to release associated job elements. Cleaned up this path and added locks for coherency. Signed-off-by: Dick Kennedy Signed-off-by: James Smart Reviewed-by: Hannes Reinecke Signed-off-by: Martin K. Petersen --- drivers/scsi/lpfc/lpfc.h | 1 + drivers/scsi/lpfc/lpfc_nvmet.c | 247 ++++++++++++++++++++++------------------- drivers/scsi/lpfc/lpfc_nvmet.h | 1 + 3 files changed, 137 insertions(+), 112 deletions(-) (limited to 'drivers/scsi/lpfc/lpfc.h') diff --git a/drivers/scsi/lpfc/lpfc.h b/drivers/scsi/lpfc/lpfc.h index b710994a352e..ea97d82f99f9 100644 --- a/drivers/scsi/lpfc/lpfc.h +++ b/drivers/scsi/lpfc/lpfc.h @@ -144,6 +144,7 @@ struct lpfc_nvmet_ctxbuf { struct lpfc_nvmet_rcv_ctx *context; struct lpfc_iocbq *iocbq; struct lpfc_sglq *sglq; + struct work_struct defer_work; }; struct lpfc_dma_pool { diff --git a/drivers/scsi/lpfc/lpfc_nvmet.c b/drivers/scsi/lpfc/lpfc_nvmet.c index 0b27e8c5ae32..0d10dfc74018 100644 --- a/drivers/scsi/lpfc/lpfc_nvmet.c +++ b/drivers/scsi/lpfc/lpfc_nvmet.c @@ -73,6 +73,9 @@ static int lpfc_nvmet_unsol_ls_issue_abort(struct lpfc_hba *, uint32_t, uint16_t); static void lpfc_nvmet_wqfull_flush(struct lpfc_hba *, struct lpfc_queue *, struct lpfc_nvmet_rcv_ctx *); +static void lpfc_nvmet_fcp_rqst_defer_work(struct work_struct *); + +static void lpfc_nvmet_process_rcv_fcp_req(struct lpfc_nvmet_ctxbuf *ctx_buf); static union lpfc_wqe128 lpfc_tsend_cmd_template; static union lpfc_wqe128 lpfc_treceive_cmd_template; @@ -220,21 +223,19 @@ lpfc_nvmet_cmd_template(void) void lpfc_nvmet_defer_release(struct lpfc_hba *phba, struct lpfc_nvmet_rcv_ctx *ctxp) { - unsigned long iflag; + lockdep_assert_held(&ctxp->ctxlock); lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, "6313 NVMET Defer ctx release xri x%x flg x%x\n", ctxp->oxid, ctxp->flag); - spin_lock_irqsave(&phba->sli4_hba.abts_nvmet_buf_list_lock, iflag); - if (ctxp->flag & LPFC_NVMET_CTX_RLS) { - spin_unlock_irqrestore(&phba->sli4_hba.abts_nvmet_buf_list_lock, - iflag); + if (ctxp->flag & LPFC_NVMET_CTX_RLS) return; - } + ctxp->flag |= LPFC_NVMET_CTX_RLS; + spin_lock(&phba->sli4_hba.abts_nvmet_buf_list_lock); list_add_tail(&ctxp->list, &phba->sli4_hba.lpfc_abts_nvmet_ctx_list); - spin_unlock_irqrestore(&phba->sli4_hba.abts_nvmet_buf_list_lock, iflag); + spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock); } /** @@ -325,7 +326,7 @@ lpfc_nvmet_ctxbuf_post(struct lpfc_hba *phba, struct lpfc_nvmet_ctxbuf *ctx_buf) struct rqb_dmabuf *nvmebuf; struct lpfc_nvmet_ctx_info *infop; uint32_t *payload; - uint32_t size, oxid, sid, rc; + uint32_t size, oxid, sid; int cpu; unsigned long iflag; @@ -341,6 +342,20 @@ lpfc_nvmet_ctxbuf_post(struct lpfc_hba *phba, struct lpfc_nvmet_ctxbuf *ctx_buf) "6411 NVMET free, already free IO x%x: %d %d\n", ctxp->oxid, ctxp->state, ctxp->entry_cnt); } + + if (ctxp->rqb_buffer) { + nvmebuf = ctxp->rqb_buffer; + spin_lock_irqsave(&ctxp->ctxlock, iflag); + ctxp->rqb_buffer = NULL; + if (ctxp->flag & LPFC_NVMET_CTX_REUSE_WQ) { + ctxp->flag &= ~LPFC_NVMET_CTX_REUSE_WQ; + spin_unlock_irqrestore(&ctxp->ctxlock, iflag); + nvmebuf->hrq->rqbp->rqb_free_buffer(phba, nvmebuf); + } else { + spin_unlock_irqrestore(&ctxp->ctxlock, iflag); + lpfc_rq_buf_free(phba, &nvmebuf->hbuf); /* repost */ + } + } ctxp->state = LPFC_NVMET_STE_FREE; spin_lock_irqsave(&phba->sli4_hba.nvmet_io_wait_lock, iflag); @@ -388,46 +403,30 @@ lpfc_nvmet_ctxbuf_post(struct lpfc_hba *phba, struct lpfc_nvmet_ctxbuf *ctx_buf) } #endif atomic_inc(&tgtp->rcv_fcp_cmd_in); - /* - * The calling sequence should be: - * nvmet_fc_rcv_fcp_req->lpfc_nvmet_xmt_fcp_op/cmp- req->done - * lpfc_nvmet_xmt_fcp_op_cmp should free the allocated ctxp. - * When we return from nvmet_fc_rcv_fcp_req, all relevant info - * the NVME command / FC header is stored. - * A buffer has already been reposted for this IO, so just free - * the nvmebuf. - */ - rc = nvmet_fc_rcv_fcp_req(phba->targetport, &ctxp->ctx.fcp_req, - payload, size); - /* Process FCP command */ - if (rc == 0) { - ctxp->rqb_buffer = NULL; - atomic_inc(&tgtp->rcv_fcp_cmd_out); - nvmebuf->hrq->rqbp->rqb_free_buffer(phba, nvmebuf); - return; - } + /* flag new work queued, replacement buffer has already + * been reposted + */ + spin_lock_irqsave(&ctxp->ctxlock, iflag); + ctxp->flag |= LPFC_NVMET_CTX_REUSE_WQ; + spin_unlock_irqrestore(&ctxp->ctxlock, iflag); - /* Processing of FCP command is deferred */ - if (rc == -EOVERFLOW) { - lpfc_nvmeio_data(phba, - "NVMET RCV BUSY: xri x%x sz %d " - "from %06x\n", - oxid, size, sid); - atomic_inc(&tgtp->rcv_fcp_cmd_out); - return; + if (!queue_work(phba->wq, &ctx_buf->defer_work)) { + atomic_inc(&tgtp->rcv_fcp_cmd_drop); + lpfc_printf_log(phba, KERN_ERR, LOG_NVME, + "6181 Unable to queue deferred work " + "for oxid x%x. " + "FCP Drop IO [x%x x%x x%x]\n", + ctxp->oxid, + atomic_read(&tgtp->rcv_fcp_cmd_in), + atomic_read(&tgtp->rcv_fcp_cmd_out), + atomic_read(&tgtp->xmt_fcp_release)); + + spin_lock_irqsave(&ctxp->ctxlock, iflag); + lpfc_nvmet_defer_release(phba, ctxp); + spin_unlock_irqrestore(&ctxp->ctxlock, iflag); + lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, sid, oxid); } - atomic_inc(&tgtp->rcv_fcp_cmd_drop); - lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, - "2582 FCP Drop IO x%x: err x%x: x%x x%x x%x\n", - ctxp->oxid, rc, - atomic_read(&tgtp->rcv_fcp_cmd_in), - atomic_read(&tgtp->rcv_fcp_cmd_out), - atomic_read(&tgtp->xmt_fcp_release)); - - lpfc_nvmet_defer_release(phba, ctxp); - lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, sid, oxid); - nvmebuf->hrq->rqbp->rqb_free_buffer(phba, nvmebuf); return; } spin_unlock_irqrestore(&phba->sli4_hba.nvmet_io_wait_lock, iflag); @@ -1113,6 +1112,8 @@ lpfc_nvmet_defer_rcv(struct nvmet_fc_target_port *tgtport, container_of(rsp, struct lpfc_nvmet_rcv_ctx, ctx.fcp_req); struct rqb_dmabuf *nvmebuf = ctxp->rqb_buffer; struct lpfc_hba *phba = ctxp->phba; + unsigned long iflag; + lpfc_nvmeio_data(phba, "NVMET DEFERRCV: xri x%x sz %d CPU %02x\n", ctxp->oxid, ctxp->size, smp_processor_id()); @@ -1131,6 +1132,9 @@ lpfc_nvmet_defer_rcv(struct nvmet_fc_target_port *tgtport, /* Free the nvmebuf since a new buffer already replaced it */ nvmebuf->hrq->rqbp->rqb_free_buffer(phba, nvmebuf); + spin_lock_irqsave(&ctxp->ctxlock, iflag); + ctxp->rqb_buffer = NULL; + spin_unlock_irqrestore(&ctxp->ctxlock, iflag); } static struct nvmet_fc_target_template lpfc_tgttemplate = { @@ -1323,6 +1327,7 @@ lpfc_nvmet_setup_io_context(struct lpfc_hba *phba) "6407 Ran out of NVMET XRIs\n"); return -ENOMEM; } + INIT_WORK(&ctx_buf->defer_work, lpfc_nvmet_fcp_rqst_defer_work); /* * Add ctx to MRQidx context list. Our initial assumption @@ -1824,6 +1829,86 @@ dropit: #endif } +static void +lpfc_nvmet_process_rcv_fcp_req(struct lpfc_nvmet_ctxbuf *ctx_buf) +{ +#if (IS_ENABLED(CONFIG_NVME_TARGET_FC)) + struct lpfc_nvmet_rcv_ctx *ctxp = ctx_buf->context; + struct lpfc_hba *phba = ctxp->phba; + struct rqb_dmabuf *nvmebuf = ctxp->rqb_buffer; + struct lpfc_nvmet_tgtport *tgtp; + uint32_t *payload; + uint32_t rc; + unsigned long iflags; + + if (!nvmebuf) { + lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, + "6159 process_rcv_fcp_req, nvmebuf is NULL, " + "oxid: x%x flg: x%x state: x%x\n", + ctxp->oxid, ctxp->flag, ctxp->state); + spin_lock_irqsave(&ctxp->ctxlock, iflags); + lpfc_nvmet_defer_release(phba, ctxp); + spin_unlock_irqrestore(&ctxp->ctxlock, iflags); + lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, ctxp->sid, + ctxp->oxid); + return; + } + + payload = (uint32_t *)(nvmebuf->dbuf.virt); + tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; + /* + * The calling sequence should be: + * nvmet_fc_rcv_fcp_req->lpfc_nvmet_xmt_fcp_op/cmp- req->done + * lpfc_nvmet_xmt_fcp_op_cmp should free the allocated ctxp. + * When we return from nvmet_fc_rcv_fcp_req, all relevant info + * the NVME command / FC header is stored. + * A buffer has already been reposted for this IO, so just free + * the nvmebuf. + */ + rc = nvmet_fc_rcv_fcp_req(phba->targetport, &ctxp->ctx.fcp_req, + payload, ctxp->size); + /* Process FCP command */ + if (rc == 0) { + atomic_inc(&tgtp->rcv_fcp_cmd_out); + return; + } + + /* Processing of FCP command is deferred */ + if (rc == -EOVERFLOW) { + lpfc_nvmeio_data(phba, "NVMET RCV BUSY: xri x%x sz %d " + "from %06x\n", + ctxp->oxid, ctxp->size, ctxp->sid); + atomic_inc(&tgtp->rcv_fcp_cmd_out); + atomic_inc(&tgtp->defer_fod); + return; + } + atomic_inc(&tgtp->rcv_fcp_cmd_drop); + lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, + "2582 FCP Drop IO x%x: err x%x: x%x x%x x%x\n", + ctxp->oxid, rc, + atomic_read(&tgtp->rcv_fcp_cmd_in), + atomic_read(&tgtp->rcv_fcp_cmd_out), + atomic_read(&tgtp->xmt_fcp_release)); + lpfc_nvmeio_data(phba, "NVMET FCP DROP: xri x%x sz %d from %06x\n", + ctxp->oxid, ctxp->size, ctxp->sid); + spin_lock_irqsave(&ctxp->ctxlock, iflags); + lpfc_nvmet_defer_release(phba, ctxp); + spin_unlock_irqrestore(&ctxp->ctxlock, iflags); + lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, ctxp->sid, ctxp->oxid); +#endif +} + +static void +lpfc_nvmet_fcp_rqst_defer_work(struct work_struct *work) +{ +#if (IS_ENABLED(CONFIG_NVME_TARGET_FC)) + struct lpfc_nvmet_ctxbuf *ctx_buf = + container_of(work, struct lpfc_nvmet_ctxbuf, defer_work); + + lpfc_nvmet_process_rcv_fcp_req(ctx_buf); +#endif +} + static struct lpfc_nvmet_ctxbuf * lpfc_nvmet_replenish_context(struct lpfc_hba *phba, struct lpfc_nvmet_ctx_info *current_infop) @@ -1906,7 +1991,7 @@ lpfc_nvmet_unsol_fcp_buffer(struct lpfc_hba *phba, struct lpfc_nvmet_ctxbuf *ctx_buf; struct lpfc_nvmet_ctx_info *current_infop; uint32_t *payload; - uint32_t size, oxid, sid, rc, qno; + uint32_t size, oxid, sid, qno; unsigned long iflag; int current_cpu; @@ -1917,11 +2002,9 @@ lpfc_nvmet_unsol_fcp_buffer(struct lpfc_hba *phba, if (!nvmebuf || !phba->targetport) { lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, "6157 NVMET FCP Drop IO\n"); - oxid = 0; - size = 0; - sid = 0; - ctxp = NULL; - goto dropit; + if (nvmebuf) + lpfc_rq_buf_free(phba, &nvmebuf->hbuf); + return; } /* @@ -2028,67 +2111,7 @@ lpfc_nvmet_unsol_fcp_buffer(struct lpfc_hba *phba, #endif atomic_inc(&tgtp->rcv_fcp_cmd_in); - /* - * The calling sequence should be: - * nvmet_fc_rcv_fcp_req -> lpfc_nvmet_xmt_fcp_op/cmp -> req->done - * lpfc_nvmet_xmt_fcp_op_cmp should free the allocated ctxp. - * When we return from nvmet_fc_rcv_fcp_req, all relevant info in - * the NVME command / FC header is stored, so we are free to repost - * the buffer. - */ - rc = nvmet_fc_rcv_fcp_req(phba->targetport, &ctxp->ctx.fcp_req, - payload, size); - - /* Process FCP command */ - if (rc == 0) { - ctxp->rqb_buffer = NULL; - atomic_inc(&tgtp->rcv_fcp_cmd_out); - lpfc_rq_buf_free(phba, &nvmebuf->hbuf); /* repost */ - return; - } - - /* Processing of FCP command is deferred */ - if (rc == -EOVERFLOW) { - /* - * Post a brand new DMA buffer to RQ and defer - * freeing rcv buffer till .defer_rcv callback - */ - qno = nvmebuf->idx; - lpfc_post_rq_buffer( - phba, phba->sli4_hba.nvmet_mrq_hdr[qno], - phba->sli4_hba.nvmet_mrq_data[qno], 1, qno); - - lpfc_nvmeio_data(phba, - "NVMET RCV BUSY: xri x%x sz %d from %06x\n", - oxid, size, sid); - atomic_inc(&tgtp->rcv_fcp_cmd_out); - atomic_inc(&tgtp->defer_fod); - return; - } - ctxp->rqb_buffer = nvmebuf; - - atomic_inc(&tgtp->rcv_fcp_cmd_drop); - lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, - "6159 FCP Drop IO x%x: err x%x: x%x x%x x%x\n", - ctxp->oxid, rc, - atomic_read(&tgtp->rcv_fcp_cmd_in), - atomic_read(&tgtp->rcv_fcp_cmd_out), - atomic_read(&tgtp->xmt_fcp_release)); -dropit: - lpfc_nvmeio_data(phba, "NVMET FCP DROP: xri x%x sz %d from %06x\n", - oxid, size, sid); - if (oxid) { - lpfc_nvmet_defer_release(phba, ctxp); - lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, sid, oxid); - lpfc_rq_buf_free(phba, &nvmebuf->hbuf); /* repost */ - return; - } - - if (ctx_buf) - lpfc_nvmet_ctxbuf_post(phba, ctx_buf); - - if (nvmebuf) - lpfc_rq_buf_free(phba, &nvmebuf->hbuf); /* repost */ + lpfc_nvmet_process_rcv_fcp_req(ctx_buf); } /** diff --git a/drivers/scsi/lpfc/lpfc_nvmet.h b/drivers/scsi/lpfc/lpfc_nvmet.h index 9b767c59de3d..b8c342a41d98 100644 --- a/drivers/scsi/lpfc/lpfc_nvmet.h +++ b/drivers/scsi/lpfc/lpfc_nvmet.h @@ -137,6 +137,7 @@ struct lpfc_nvmet_rcv_ctx { #define LPFC_NVMET_XBUSY 0x4 /* XB bit set on IO cmpl */ #define LPFC_NVMET_CTX_RLS 0x8 /* ctx free requested */ #define LPFC_NVMET_ABTS_RCV 0x10 /* ABTS received on exchange */ +#define LPFC_NVMET_CTX_REUSE_WQ 0x20 /* ctx reused via WQ */ #define LPFC_NVMET_DEFER_WQFULL 0x40 /* Waiting on a free WQE */ struct rqb_dmabuf *rqb_buffer; struct lpfc_nvmet_ctxbuf *ctxbuf; -- cgit v1.2.3 From 0d041215f0b4420bf193f3b6e13a1887ffc8320c Mon Sep 17 00:00:00 2001 From: James Smart Date: Mon, 28 Jan 2019 11:14:41 -0800 Subject: scsi: lpfc: Update 12.2.0.0 file copyrights to 2019 For files modified as part of 12.2.0.0 patches, update copyright to 2019 Signed-off-by: Dick Kennedy Signed-off-by: James Smart Reviewed-by: Hannes Reinecke Signed-off-by: Martin K. Petersen --- drivers/scsi/lpfc/lpfc.h | 2 +- drivers/scsi/lpfc/lpfc_attr.c | 2 +- drivers/scsi/lpfc/lpfc_crtn.h | 2 +- drivers/scsi/lpfc/lpfc_ct.c | 2 +- drivers/scsi/lpfc/lpfc_debugfs.c | 2 +- drivers/scsi/lpfc/lpfc_debugfs.h | 2 +- drivers/scsi/lpfc/lpfc_els.c | 2 +- drivers/scsi/lpfc/lpfc_hbadisc.c | 2 +- drivers/scsi/lpfc/lpfc_hw4.h | 2 +- drivers/scsi/lpfc/lpfc_init.c | 2 +- drivers/scsi/lpfc/lpfc_nportdisc.c | 2 +- drivers/scsi/lpfc/lpfc_nvme.c | 2 +- drivers/scsi/lpfc/lpfc_nvme.h | 2 +- drivers/scsi/lpfc/lpfc_nvmet.c | 2 +- drivers/scsi/lpfc/lpfc_nvmet.h | 2 +- drivers/scsi/lpfc/lpfc_scsi.c | 2 +- drivers/scsi/lpfc/lpfc_scsi.h | 2 +- drivers/scsi/lpfc/lpfc_sli.c | 2 +- drivers/scsi/lpfc/lpfc_sli.h | 2 +- drivers/scsi/lpfc/lpfc_sli4.h | 2 +- drivers/scsi/lpfc/lpfc_version.h | 2 +- drivers/scsi/lpfc/lpfc_vport.c | 2 +- 22 files changed, 22 insertions(+), 22 deletions(-) (limited to 'drivers/scsi/lpfc/lpfc.h') diff --git a/drivers/scsi/lpfc/lpfc.h b/drivers/scsi/lpfc/lpfc.h index ea97d82f99f9..41d849f283f6 100644 --- a/drivers/scsi/lpfc/lpfc.h +++ b/drivers/scsi/lpfc/lpfc.h @@ -1,7 +1,7 @@ /******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channel Host Bus Adapters. * - * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term * + * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * * Copyright (C) 2004-2016 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * diff --git a/drivers/scsi/lpfc/lpfc_attr.c b/drivers/scsi/lpfc/lpfc_attr.c index 212bfae1966a..ce3e541434dc 100644 --- a/drivers/scsi/lpfc/lpfc_attr.c +++ b/drivers/scsi/lpfc/lpfc_attr.c @@ -1,7 +1,7 @@ /******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channel Host Bus Adapters. * - * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term * + * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * * Copyright (C) 2004-2016 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * diff --git a/drivers/scsi/lpfc/lpfc_crtn.h b/drivers/scsi/lpfc/lpfc_crtn.h index 982401c31c12..e0b14d791b8c 100644 --- a/drivers/scsi/lpfc/lpfc_crtn.h +++ b/drivers/scsi/lpfc/lpfc_crtn.h @@ -1,7 +1,7 @@ /******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channel Host Bus Adapters. * - * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term * + * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * * Copyright (C) 2004-2016 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * diff --git a/drivers/scsi/lpfc/lpfc_ct.c b/drivers/scsi/lpfc/lpfc_ct.c index 98faa3aae35c..7290573110fe 100644 --- a/drivers/scsi/lpfc/lpfc_ct.c +++ b/drivers/scsi/lpfc/lpfc_ct.c @@ -1,7 +1,7 @@ /******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channel Host Bus Adapters. * - * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term * + * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * * Copyright (C) 2004-2016 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * diff --git a/drivers/scsi/lpfc/lpfc_debugfs.c b/drivers/scsi/lpfc/lpfc_debugfs.c index 19d13a8789b9..ee98ea1c68f9 100644 --- a/drivers/scsi/lpfc/lpfc_debugfs.c +++ b/drivers/scsi/lpfc/lpfc_debugfs.c @@ -1,7 +1,7 @@ /******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channel Host Bus Adapters. * - * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term * + * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * * Copyright (C) 2007-2015 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * diff --git a/drivers/scsi/lpfc/lpfc_debugfs.h b/drivers/scsi/lpfc/lpfc_debugfs.h index 1fbee6496f85..93ab7dfb8ee0 100644 --- a/drivers/scsi/lpfc/lpfc_debugfs.h +++ b/drivers/scsi/lpfc/lpfc_debugfs.h @@ -1,7 +1,7 @@ /******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channel Host Bus Adapters. * - * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term * + * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * * Copyright (C) 2007-2011 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * diff --git a/drivers/scsi/lpfc/lpfc_els.c b/drivers/scsi/lpfc/lpfc_els.c index 3df2cfd02b41..fc077cb87900 100644 --- a/drivers/scsi/lpfc/lpfc_els.c +++ b/drivers/scsi/lpfc/lpfc_els.c @@ -1,7 +1,7 @@ /******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channel Host Bus Adapters. * - * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term * + * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * * Copyright (C) 2004-2016 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * diff --git a/drivers/scsi/lpfc/lpfc_hbadisc.c b/drivers/scsi/lpfc/lpfc_hbadisc.c index 47608d4b8e8c..aa4961a2caf8 100644 --- a/drivers/scsi/lpfc/lpfc_hbadisc.c +++ b/drivers/scsi/lpfc/lpfc_hbadisc.c @@ -1,7 +1,7 @@ /******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channel Host Bus Adapters. * - * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term * + * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * * Copyright (C) 2004-2016 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * diff --git a/drivers/scsi/lpfc/lpfc_hw4.h b/drivers/scsi/lpfc/lpfc_hw4.h index c9a056ef321a..ff875b833192 100644 --- a/drivers/scsi/lpfc/lpfc_hw4.h +++ b/drivers/scsi/lpfc/lpfc_hw4.h @@ -1,7 +1,7 @@ /******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channel Host Bus Adapters. * - * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term * + * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * * Copyright (C) 2009-2016 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * diff --git a/drivers/scsi/lpfc/lpfc_init.c b/drivers/scsi/lpfc/lpfc_init.c index 563b6009ffce..d24af076afa8 100644 --- a/drivers/scsi/lpfc/lpfc_init.c +++ b/drivers/scsi/lpfc/lpfc_init.c @@ -1,7 +1,7 @@ /******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channel Host Bus Adapters. * - * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term * + * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * * Copyright (C) 2004-2016 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * diff --git a/drivers/scsi/lpfc/lpfc_nportdisc.c b/drivers/scsi/lpfc/lpfc_nportdisc.c index acfce67dacde..6172682a24ba 100644 --- a/drivers/scsi/lpfc/lpfc_nportdisc.c +++ b/drivers/scsi/lpfc/lpfc_nportdisc.c @@ -1,7 +1,7 @@ /******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channel Host Bus Adapters. * - * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term * + * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * * Copyright (C) 2004-2016 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * diff --git a/drivers/scsi/lpfc/lpfc_nvme.c b/drivers/scsi/lpfc/lpfc_nvme.c index 271ad42be7f4..e73895674f39 100644 --- a/drivers/scsi/lpfc/lpfc_nvme.c +++ b/drivers/scsi/lpfc/lpfc_nvme.c @@ -1,7 +1,7 @@ /******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channel Host Bus Adapters. * - * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term * + * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * * Copyright (C) 2004-2016 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * diff --git a/drivers/scsi/lpfc/lpfc_nvme.h b/drivers/scsi/lpfc/lpfc_nvme.h index 07d5fa16dd39..6b6d88cefdb8 100644 --- a/drivers/scsi/lpfc/lpfc_nvme.h +++ b/drivers/scsi/lpfc/lpfc_nvme.h @@ -1,7 +1,7 @@ /******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channel Host Bus Adapters. * - * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term * + * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * * Copyright (C) 2004-2016 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * diff --git a/drivers/scsi/lpfc/lpfc_nvmet.c b/drivers/scsi/lpfc/lpfc_nvmet.c index 4aadb3d5e718..caf641ee6210 100644 --- a/drivers/scsi/lpfc/lpfc_nvmet.c +++ b/drivers/scsi/lpfc/lpfc_nvmet.c @@ -1,7 +1,7 @@ /******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channsel Host Bus Adapters. * - * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term * + * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * * Copyright (C) 2004-2016 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * diff --git a/drivers/scsi/lpfc/lpfc_nvmet.h b/drivers/scsi/lpfc/lpfc_nvmet.h index b8c342a41d98..c3ca93d55856 100644 --- a/drivers/scsi/lpfc/lpfc_nvmet.h +++ b/drivers/scsi/lpfc/lpfc_nvmet.h @@ -1,7 +1,7 @@ /******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channel Host Bus Adapters. * - * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term * + * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * * Copyright (C) 2004-2016 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * diff --git a/drivers/scsi/lpfc/lpfc_scsi.c b/drivers/scsi/lpfc/lpfc_scsi.c index af263b0e8bad..c98f264f1d83 100644 --- a/drivers/scsi/lpfc/lpfc_scsi.c +++ b/drivers/scsi/lpfc/lpfc_scsi.c @@ -1,7 +1,7 @@ /******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channel Host Bus Adapters. * - * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term * + * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * * Copyright (C) 2004-2016 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * diff --git a/drivers/scsi/lpfc/lpfc_scsi.h b/drivers/scsi/lpfc/lpfc_scsi.h index 9ee0ea4a1ef2..f76667b7da7b 100644 --- a/drivers/scsi/lpfc/lpfc_scsi.h +++ b/drivers/scsi/lpfc/lpfc_scsi.h @@ -1,7 +1,7 @@ /******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channel Host Bus Adapters. * - * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term * + * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * * “Broadcom” refers to Broadcom Inc and/or its subsidiaries. * * Copyright (C) 2004-2016 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * diff --git a/drivers/scsi/lpfc/lpfc_sli.c b/drivers/scsi/lpfc/lpfc_sli.c index d548066a7e91..c4c319245a66 100644 --- a/drivers/scsi/lpfc/lpfc_sli.c +++ b/drivers/scsi/lpfc/lpfc_sli.c @@ -1,7 +1,7 @@ /******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channel Host Bus Adapters. * - * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term * + * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * * Copyright (C) 2004-2016 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * diff --git a/drivers/scsi/lpfc/lpfc_sli.h b/drivers/scsi/lpfc/lpfc_sli.h index d203a666497f..7a1a761efdd6 100644 --- a/drivers/scsi/lpfc/lpfc_sli.h +++ b/drivers/scsi/lpfc/lpfc_sli.h @@ -1,7 +1,7 @@ /******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channel Host Bus Adapters. * - * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term * + * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * * Copyright (C) 2004-2016 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * diff --git a/drivers/scsi/lpfc/lpfc_sli4.h b/drivers/scsi/lpfc/lpfc_sli4.h index 1e3d7f534eaa..40c85091c805 100644 --- a/drivers/scsi/lpfc/lpfc_sli4.h +++ b/drivers/scsi/lpfc/lpfc_sli4.h @@ -1,7 +1,7 @@ /******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channel Host Bus Adapters. * - * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term * + * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * * Copyright (C) 2009-2016 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * diff --git a/drivers/scsi/lpfc/lpfc_version.h b/drivers/scsi/lpfc/lpfc_version.h index 3f4398ffb567..a248a895c7d0 100644 --- a/drivers/scsi/lpfc/lpfc_version.h +++ b/drivers/scsi/lpfc/lpfc_version.h @@ -1,7 +1,7 @@ /******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channel Host Bus Adapters. * - * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term * + * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * * Copyright (C) 2004-2016 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * diff --git a/drivers/scsi/lpfc/lpfc_vport.c b/drivers/scsi/lpfc/lpfc_vport.c index 0d712eb04e2b..343bc71d4615 100644 --- a/drivers/scsi/lpfc/lpfc_vport.c +++ b/drivers/scsi/lpfc/lpfc_vport.c @@ -1,7 +1,7 @@ /******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channel Host Bus Adapters. * - * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term * + * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * * Copyright (C) 2004-2016 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * -- cgit v1.2.3