diff options
| author | Linus Torvalds <torvalds@linux-foundation.org> | 2018-02-06 20:09:45 +0100 |
|---|---|---|
| committer | Linus Torvalds <torvalds@linux-foundation.org> | 2018-02-06 20:09:45 +0100 |
| commit | 2246edfaf88dc368e8671b04afd54412625df60a (patch) | |
| tree | 0597235e022e707eae23ab4c23aa6d4bbd545803 /drivers/infiniband | |
| parent | Merge tag 'libnvdimm-for-4.16' of git://git.kernel.org/pub/scm/linux/kernel/g... (diff) | |
| parent | net/mlx5: increase async EQ to avoid EQ overrun (diff) | |
| download | linux-2246edfaf88dc368e8671b04afd54412625df60a.tar.xz linux-2246edfaf88dc368e8671b04afd54412625df60a.zip | |
Merge tag 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma
Pull more rdma updates from Doug Ledford:
"Items of note:
- two patches fix a regression in the 4.15 kernel. The 4.14 kernel
worked fine with NVMe over Fabrics and mlx5 adapters. That broke in
4.15. The fix is here.
- one of the patches (the endian notation patch from Lijun) looks
like a lot of lines of change, but it's mostly mechanical in
nature. It amounts to the biggest chunk of change in it (it's about
2/3rds of the overall pull request).
Summary:
- Clean up some function signatures in rxe for clarity
- Tidy the RDMA netlink header to remove unimplemented constants
- bnxt_re driver fixes, one is a regression this window.
- Minor hns driver fixes
- Various fixes from Dan Carpenter and his tool
- Fix IRQ cleanup race in HFI1
- HF1 performance optimizations and a fix to report counters in the right units
- Fix for an IPoIB startup sequence race with the external manager
- Oops fix for the new kabi path
- Endian cleanups for hns
- Fix for mlx5 related to the new automatic affinity support"
* tag 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma: (38 commits)
net/mlx5: increase async EQ to avoid EQ overrun
mlx5: fix mlx5_get_vector_affinity to start from completion vector 0
RDMA/hns: Fix the endian problem for hns
IB/uverbs: Use the standard kConfig format for experimental
IB: Update references to libibverbs
IB/hfi1: Add 16B rcvhdr trace support
IB/hfi1: Convert kzalloc_node and kcalloc to use kcalloc_node
IB/core: Avoid a potential OOPs for an unused optional parameter
IB/core: Map iWarp AH type to undefined in rdma_ah_find_type
IB/ipoib: Fix for potential no-carrier state
IB/hfi1: Show fault stats in both TX and RX directions
IB/hfi1: Remove blind constants from 16B update
IB/hfi1: Convert PortXmitWait/PortVLXmitWait counters to flit times
IB/hfi1: Do not override given pcie_pset value
IB/hfi1: Optimize process_receive_ib()
IB/hfi1: Remove unnecessary fecn and becn fields
IB/hfi1: Look up ibport using a pointer in receive path
IB/hfi1: Optimize packet type comparison using 9B and bypass code paths
IB/hfi1: Compute BTH only for RDMA_WRITE_LAST/SEND_LAST packet
IB/hfi1: Remove dependence on qp->s_hdrwords
...
Diffstat (limited to 'drivers/infiniband')
48 files changed, 882 insertions, 620 deletions
diff --git a/drivers/infiniband/Kconfig b/drivers/infiniband/Kconfig index 5cd700421695..8517d6ea91a6 100644 --- a/drivers/infiniband/Kconfig +++ b/drivers/infiniband/Kconfig @@ -20,7 +20,8 @@ config INFINIBAND_USER_MAD Userspace InfiniBand Management Datagram (MAD) support. This is the kernel side of the userspace MAD support, which allows userspace processes to send and receive MADs. You will also - need libibumad from <http://www.openfabrics.org/downloads/management/>. + need libibumad from rdma-core + <https://github.com/linux-rdma/rdma-core>. config INFINIBAND_USER_ACCESS tristate "InfiniBand userspace access (verbs and CM)" @@ -32,10 +33,10 @@ config INFINIBAND_USER_ACCESS to set up connections and directly access InfiniBand hardware for fast-path operations. You will also need libibverbs, libibcm and a hardware driver library from - <http://www.openfabrics.org/git/>. + rdma-core <https://github.com/linux-rdma/rdma-core>. config INFINIBAND_EXP_USER_ACCESS - bool "Allow experimental support for Infiniband ABI" + bool "Enable the full uverbs ioctl interface (EXPERIMENTAL)" depends on INFINIBAND_USER_ACCESS ---help--- IOCTL based ABI support for Infiniband. This allows userspace diff --git a/drivers/infiniband/core/nldev.c b/drivers/infiniband/core/nldev.c index fa8655e3b3ed..5326a684555f 100644 --- a/drivers/infiniband/core/nldev.c +++ b/drivers/infiniband/core/nldev.c @@ -499,8 +499,10 @@ static int nldev_res_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh, return -EINVAL; msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); - if (!msg) + if (!msg) { + ret = -ENOMEM; goto err; + } nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq, RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_RES_GET), diff --git a/drivers/infiniband/core/uverbs_std_types.c b/drivers/infiniband/core/uverbs_std_types.c index b571176babbe..cab0ac3556eb 100644 --- a/drivers/infiniband/core/uverbs_std_types.c +++ b/drivers/infiniband/core/uverbs_std_types.c @@ -316,7 +316,7 @@ static int uverbs_create_cq_handler(struct ib_device *ib_dev, cq->uobject = &obj->uobject; cq->comp_handler = ib_uverbs_comp_handler; cq->event_handler = ib_uverbs_cq_event_handler; - cq->cq_context = &ev_file->ev_queue; + cq->cq_context = ev_file ? &ev_file->ev_queue : NULL; obj->uobject.object = cq; obj->uobject.user_handle = user_handle; atomic_set(&cq->usecnt, 0); diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c index 9b8fa77b8831..ae9e9ff54826 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c @@ -1314,7 +1314,7 @@ int bnxt_re_destroy_srq(struct ib_srq *ib_srq) return rc; } - if (srq->umem && !IS_ERR(srq->umem)) + if (srq->umem) ib_umem_release(srq->umem); kfree(srq); atomic_dec(&rdev->srq_count); @@ -1430,11 +1430,8 @@ struct ib_srq *bnxt_re_create_srq(struct ib_pd *ib_pd, return &srq->ib_srq; fail: - if (udata && srq->umem && !IS_ERR(srq->umem)) { + if (srq->umem) ib_umem_release(srq->umem); - srq->umem = NULL; - } - kfree(srq); exit: return ERR_PTR(rc); diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.c b/drivers/infiniband/hw/bnxt_re/qplib_fp.c index 8b5f11ac0e42..1b0e94697fe3 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_fp.c +++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.c @@ -557,8 +557,10 @@ int bnxt_qplib_create_srq(struct bnxt_qplib_res *res, srq->swq = kcalloc(srq->hwq.max_elements, sizeof(*srq->swq), GFP_KERNEL); - if (!srq->swq) + if (!srq->swq) { + rc = -ENOMEM; goto fail; + } RCFW_CMD_PREP(req, CREATE_SRQ, cmd_flags); diff --git a/drivers/infiniband/hw/bnxt_re/qplib_res.c b/drivers/infiniband/hw/bnxt_re/qplib_res.c index ad37d54affcc..539a5d44e6db 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_res.c +++ b/drivers/infiniband/hw/bnxt_re/qplib_res.c @@ -705,12 +705,8 @@ static int bnxt_qplib_alloc_dpi_tbl(struct bnxt_qplib_res *res, dpit->max = dbr_len / PAGE_SIZE; dpit->app_tbl = kcalloc(dpit->max, sizeof(void *), GFP_KERNEL); - if (!dpit->app_tbl) { - pci_iounmap(res->pdev, dpit->dbr_bar_reg_iomem); - dev_err(&res->pdev->dev, - "QPLIB: DPI app tbl allocation failed"); - return -ENOMEM; - } + if (!dpit->app_tbl) + goto unmap_io; bytes = dpit->max >> 3; if (!bytes) @@ -718,18 +714,18 @@ static int bnxt_qplib_alloc_dpi_tbl(struct bnxt_qplib_res *res, dpit->tbl = kmalloc(bytes, GFP_KERNEL); if (!dpit->tbl) { - pci_iounmap(res->pdev, dpit->dbr_bar_reg_iomem); kfree(dpit->app_tbl); dpit->app_tbl = NULL; - dev_err(&res->pdev->dev, - "QPLIB: DPI tbl allocation failed for size = %d", - bytes); - return -ENOMEM; + goto unmap_io; } memset((u8 *)dpit->tbl, 0xFF, bytes); return 0; + +unmap_io: + pci_iounmap(res->pdev, dpit->dbr_bar_reg_iomem); + return -ENOMEM; } /* PKEYs */ diff --git a/drivers/infiniband/hw/hfi1/chip.c b/drivers/infiniband/hw/hfi1/chip.c index 6660f920f42e..e6a60fa59f2b 100644 --- a/drivers/infiniband/hw/hfi1/chip.c +++ b/drivers/infiniband/hw/hfi1/chip.c @@ -1083,6 +1083,7 @@ static int qos_rmt_entries(struct hfi1_devdata *dd, unsigned int *mp, static void clear_full_mgmt_pkey(struct hfi1_pportdata *ppd); static int wait_link_transfer_active(struct hfi1_devdata *dd, int wait_ms); static void clear_rsm_rule(struct hfi1_devdata *dd, u8 rule_index); +static void update_xmit_counters(struct hfi1_pportdata *ppd, u16 link_width); /* * Error interrupt table entry. This is used as input to the interrupt @@ -6905,6 +6906,32 @@ void handle_freeze(struct work_struct *work) /* no longer frozen */ } +/** + * update_xmit_counters - update PortXmitWait/PortVlXmitWait + * counters. + * @ppd: info of physical Hfi port + * @link_width: new link width after link up or downgrade + * + * Update the PortXmitWait and PortVlXmitWait counters after + * a link up or downgrade event to reflect a link width change. + */ +static void update_xmit_counters(struct hfi1_pportdata *ppd, u16 link_width) +{ + int i; + u16 tx_width; + u16 link_speed; + + tx_width = tx_link_width(link_width); + link_speed = get_link_speed(ppd->link_speed_active); + + /* + * There are C_VL_COUNT number of PortVLXmitWait counters. + * Adding 1 to C_VL_COUNT to include the PortXmitWait counter. + */ + for (i = 0; i < C_VL_COUNT + 1; i++) + get_xmit_wait_counters(ppd, tx_width, link_speed, i); +} + /* * Handle a link up interrupt from the 8051. * @@ -7526,18 +7553,29 @@ void handle_verify_cap(struct work_struct *work) set_link_state(ppd, HLS_GOING_UP); } -/* - * Apply the link width downgrade enabled policy against the current active - * link widths. +/** + * apply_link_downgrade_policy - Apply the link width downgrade enabled + * policy against the current active link widths. + * @ppd: info of physical Hfi port + * @refresh_widths: True indicates link downgrade event + * @return: True indicates a successful link downgrade. False indicates + * link downgrade event failed and the link will bounce back to + * default link width. * - * Called when the enabled policy changes or the active link widths change. + * Called when the enabled policy changes or the active link widths + * change. + * Refresh_widths indicates that a link downgrade occurred. The + * link_downgraded variable is set by refresh_widths and + * determines the success/failure of the policy application. */ -void apply_link_downgrade_policy(struct hfi1_pportdata *ppd, int refresh_widths) +bool apply_link_downgrade_policy(struct hfi1_pportdata *ppd, + bool refresh_widths) { int do_bounce = 0; int tries; u16 lwde; u16 tx, rx; + bool link_downgraded = refresh_widths; /* use the hls lock to avoid a race with actual link up */ tries = 0; @@ -7571,6 +7609,7 @@ retry: ppd->link_width_downgrade_rx_active == 0) { /* the 8051 reported a dead link as a downgrade */ dd_dev_err(ppd->dd, "Link downgrade is really a link down, ignoring\n"); + link_downgraded = false; } else if (lwde == 0) { /* downgrade is disabled */ @@ -7587,6 +7626,7 @@ retry: ppd->link_width_downgrade_tx_active, ppd->link_width_downgrade_rx_active); do_bounce = 1; + link_downgraded = false; } } else if ((lwde & ppd->link_width_downgrade_tx_active) == 0 || (lwde & ppd->link_width_downgrade_rx_active) == 0) { @@ -7598,6 +7638,7 @@ retry: lwde, ppd->link_width_downgrade_tx_active, ppd->link_width_downgrade_rx_active); do_bounce = 1; + link_downgraded = false; } done: @@ -7609,6 +7650,8 @@ done: set_link_state(ppd, HLS_DN_OFFLINE); start_link(ppd); } + + return link_downgraded; } /* @@ -7622,7 +7665,8 @@ void handle_link_downgrade(struct work_struct *work) link_downgrade_work); dd_dev_info(ppd->dd, "8051: Link width downgrade\n"); - apply_link_downgrade_policy(ppd, 1); + if (apply_link_downgrade_policy(ppd, true)) + update_xmit_counters(ppd, ppd->link_width_downgrade_tx_active); } static char *dcc_err_string(char *buf, int buf_len, u64 flags) @@ -8264,8 +8308,8 @@ static irqreturn_t sdma_interrupt(int irq, void *data) /* handle the interrupt(s) */ sdma_engine_interrupt(sde, status); } else { - dd_dev_err_ratelimited(dd, "SDMA engine %u interrupt, but no status bits set\n", - sde->this_idx); + dd_dev_info_ratelimited(dd, "SDMA engine %u interrupt, but no status bits set\n", + sde->this_idx); } return IRQ_HANDLED; } @@ -10597,6 +10641,14 @@ int set_link_state(struct hfi1_pportdata *ppd, u32 state) add_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK); handle_linkup_change(dd, 1); + + /* + * After link up, a new link width will have been set. + * Update the xmit counters with regards to the new + * link width. + */ + update_xmit_counters(ppd, ppd->link_width_active); + ppd->host_link_state = HLS_UP_INIT; update_statusp(ppd, IB_PORT_INIT); break; @@ -12960,7 +13012,14 @@ static void disable_intx(struct pci_dev *pdev) pci_intx(pdev, 0); } -static void clean_up_interrupts(struct hfi1_devdata *dd) +/** + * hfi1_clean_up_interrupts() - Free all IRQ resources + * @dd: valid device data data structure + * + * Free the MSI or INTx IRQs and assoicated PCI resources, + * if they have been allocated. + */ +void hfi1_clean_up_interrupts(struct hfi1_devdata *dd) { int i; @@ -13321,7 +13380,7 @@ static int set_up_interrupts(struct hfi1_devdata *dd) return 0; fail: - clean_up_interrupts(dd); + hfi1_clean_up_interrupts(dd); return ret; } @@ -14748,7 +14807,6 @@ void hfi1_start_cleanup(struct hfi1_devdata *dd) aspm_exit(dd); free_cntrs(dd); free_rcverr(dd); - clean_up_interrupts(dd); finish_chip_resources(dd); } @@ -15204,7 +15262,7 @@ bail_free_rcverr: bail_free_cntrs: free_cntrs(dd); bail_clear_intr: - clean_up_interrupts(dd); + hfi1_clean_up_interrupts(dd); bail_cleanup: hfi1_pcie_ddcleanup(dd); bail_free: diff --git a/drivers/infiniband/hw/hfi1/chip.h b/drivers/infiniband/hw/hfi1/chip.h index 21fca8ec5076..c0d70f255050 100644 --- a/drivers/infiniband/hw/hfi1/chip.h +++ b/drivers/infiniband/hw/hfi1/chip.h @@ -736,8 +736,8 @@ int read_8051_config(struct hfi1_devdata *, u8, u8, u32 *); int start_link(struct hfi1_pportdata *ppd); int bringup_serdes(struct hfi1_pportdata *ppd); void set_intr_state(struct hfi1_devdata *dd, u32 enable); -void apply_link_downgrade_policy(struct hfi1_pportdata *ppd, - int refresh_widths); +bool apply_link_downgrade_policy(struct hfi1_pportdata *ppd, + bool refresh_widths); void update_usrhead(struct hfi1_ctxtdata *rcd, u32 hd, u32 updegr, u32 egrhd, u32 intr_adjust, u32 npkts); int stop_drain_data_vls(struct hfi1_devdata *dd); diff --git a/drivers/infiniband/hw/hfi1/debugfs.c b/drivers/infiniband/hw/hfi1/debugfs.c index 2e6e0c516041..852173bf05d0 100644 --- a/drivers/infiniband/hw/hfi1/debugfs.c +++ b/drivers/infiniband/hw/hfi1/debugfs.c @@ -1,5 +1,5 @@ /* - * Copyright(c) 2015-2017 Intel Corporation. + * Copyright(c) 2015-2018 Intel Corporation. * * This file is provided under a dual BSD/GPLv2 license. When using or * redistributing this file, you may do so under either license. @@ -1201,6 +1201,13 @@ static int _fault_stats_seq_show(struct seq_file *s, void *v) } hfi1_rcd_put(rcd); } + for_each_possible_cpu(j) { + struct hfi1_opcode_stats_perctx *sp = + per_cpu_ptr(dd->tx_opstats, j); + + n_packets += sp->stats[i].n_packets; + n_bytes += sp->stats[i].n_bytes; + } if (!n_packets && !n_bytes) return SEQ_SKIP; if (!ibd->fault_opcode->n_rxfaults[i] && diff --git a/drivers/infiniband/hw/hfi1/driver.c b/drivers/infiniband/hw/hfi1/driver.c index 067b29f35f21..addc68e83606 100644 --- a/drivers/infiniband/hw/hfi1/driver.c +++ b/drivers/infiniband/hw/hfi1/driver.c @@ -256,7 +256,12 @@ static void rcv_hdrerr(struct hfi1_ctxtdata *rcd, struct hfi1_pportdata *ppd, u32 mlid_base; struct hfi1_ibport *ibp = rcd_to_iport(rcd); struct hfi1_devdata *dd = ppd->dd; - struct rvt_dev_info *rdi = &dd->verbs_dev.rdi; + struct hfi1_ibdev *verbs_dev = &dd->verbs_dev; + struct rvt_dev_info *rdi = &verbs_dev->rdi; + + if ((packet->rhf & RHF_DC_ERR) && + hfi1_dbg_fault_suppress_err(verbs_dev)) + return; if (packet->rhf & (RHF_VCRC_ERR | RHF_ICRC_ERR)) return; @@ -634,9 +639,10 @@ next: } } -static void process_rcv_qp_work(struct hfi1_ctxtdata *rcd) +static void process_rcv_qp_work(struct hfi1_packet *packet) { struct rvt_qp *qp, *nqp; + struct hfi1_ctxtdata *rcd = packet->rcd; /* * Iterate over all QPs waiting to respond. @@ -646,7 +652,8 @@ static void process_rcv_qp_work(struct hfi1_ctxtdata *rcd) list_del_init(&qp->rspwait); if (qp->r_flags & RVT_R_RSP_NAK) { qp->r_flags &= ~RVT_R_RSP_NAK; - hfi1_send_rc_ack(rcd, qp, 0); + packet->qp = qp; + hfi1_send_rc_ack(packet, 0); } if (qp->r_flags & RVT_R_RSP_SEND) { unsigned long flags; @@ -667,7 +674,7 @@ static noinline int max_packet_exceeded(struct hfi1_packet *packet, int thread) if (thread) { if ((packet->numpkt & (MAX_PKT_RECV_THREAD - 1)) == 0) /* allow defered processing */ - process_rcv_qp_work(packet->rcd); + process_rcv_qp_work(packet); cond_resched(); return RCV_PKT_OK; } else { @@ -809,7 +816,7 @@ int handle_receive_interrupt_nodma_rtail(struct hfi1_ctxtdata *rcd, int thread) last = RCV_PKT_DONE; process_rcv_update(last, &packet); } - process_rcv_qp_work(rcd); + process_rcv_qp_work(&packet); rcd->head = packet.rhqoff; bail: finish_packet(&packet); @@ -838,7 +845,7 @@ int handle_receive_interrupt_dma_rtail(struct hfi1_ctxtdata *rcd, int thread) last = RCV_PKT_DONE; process_rcv_update(last, &packet); } - process_rcv_qp_work(rcd); + process_rcv_qp_work(&packet); rcd->head = packet.rhqoff; bail: finish_packet(&packet); @@ -1068,7 +1075,7 @@ int handle_receive_interrupt(struct hfi1_ctxtdata *rcd, int thread) process_rcv_update(last, &packet); } - process_rcv_qp_work(rcd); + process_rcv_qp_work(&packet); rcd->head = packet.rhqoff; bail: @@ -1438,8 +1445,8 @@ static int hfi1_setup_9B_packet(struct hfi1_packet *packet) packet->sc = hfi1_9B_get_sc5(hdr, packet->rhf); packet->pad = ib_bth_get_pad(packet->ohdr); packet->extra_byte = 0; - packet->fecn = ib_bth_get_fecn(packet->ohdr); - packet->becn = ib_bth_get_becn(packet->ohdr); + packet->pkey = ib_bth_get_pkey(packet->ohdr); + packet->migrated = ib_bth_is_migration(packet->ohdr); return 0; drop: @@ -1492,8 +1499,10 @@ static int hfi1_setup_bypass_packet(struct hfi1_packet *packet) /* Query commonly used fields from packet header */ packet->opcode = ib_bth_get_opcode(packet->ohdr); - packet->hlen = hdr_len_by_opcode[packet->opcode] + 8 + grh_len; - packet->payload = packet->ebuf + packet->hlen - (4 * sizeof(u32)); + /* hdr_len_by_opcode already has an IB LRH factored in */ + packet->hlen = hdr_len_by_opcode[packet->opcode] + + (LRH_16B_BYTES - LRH_9B_BYTES) + grh_len; + packet->payload = packet->ebuf + packet->hlen - LRH_16B_BYTES; packet->slid = hfi1_16B_get_slid(packet->hdr); packet->dlid = hfi1_16B_get_dlid(packet->hdr); if (unlikely(hfi1_is_16B_mcast(packet->dlid))) @@ -1504,8 +1513,8 @@ static int hfi1_setup_bypass_packet(struct hfi1_packet *packet) packet->sl = ibp->sc_to_sl[packet->sc]; packet->pad = hfi1_16B_bth_get_pad(packet->ohdr); packet->extra_byte = SIZE_OF_LT; - packet->fecn = hfi1_16B_get_fecn(packet->hdr); - packet->becn = hfi1_16B_get_becn(packet->hdr); + packet->pkey = hfi1_16B_get_pkey(packet->hdr); + packet->migrated = opa_bth_is_migration(packet->ohdr); if (hfi1_bypass_ingress_pkt_check(packet)) goto drop; @@ -1550,19 +1559,7 @@ int process_receive_ib(struct hfi1_packet *packet) if (hfi1_setup_9B_packet(packet)) return RHF_RCV_CONTINUE; - trace_hfi1_rcvhdr(packet->rcd->ppd->dd, - packet->rcd->ctxt, - rhf_err_flags(packet->rhf), - RHF_RCV_TYPE_IB, - packet->hlen, - packet->tlen, - packet->updegr, - rhf_egr_index(packet->rhf)); - - if (unlikely( - (hfi1_dbg_fault_suppress_err(&packet->rcd->dd->verbs_dev) && - (packet->rhf & RHF_DC_ERR)))) - return RHF_RCV_CONTINUE; + trace_hfi1_rcvhdr(packet); if (unlikely(rhf_err_flags(packet->rhf))) { handle_eflags(packet); @@ -1598,6 +1595,8 @@ int process_receive_bypass(struct hfi1_packet *packet) if (hfi1_setup_bypass_packet(packet)) return RHF_RCV_CONTINUE; + trace_hfi1_rcvhdr(packet); + if (unlikely(rhf_err_flags(packet->rhf))) { handle_eflags(packet); return RHF_RCV_CONTINUE; diff --git a/drivers/infiniband/hw/hfi1/file_ops.c b/drivers/infiniband/hw/hfi1/file_ops.c index bd6f03cc5ee0..d9a0f2590294 100644 --- a/drivers/infiniband/hw/hfi1/file_ops.c +++ b/drivers/infiniband/hw/hfi1/file_ops.c @@ -196,9 +196,6 @@ static int hfi1_file_open(struct inode *inode, struct file *fp) if (!atomic_inc_not_zero(&dd->user_refcount)) return -ENXIO; - /* Just take a ref now. Not all opens result in a context assign */ - kobject_get(&dd->kobj); - /* The real work is performed later in assign_ctxt() */ fd = kzalloc(sizeof(*fd), GFP_KERNEL); @@ -208,6 +205,7 @@ static int hfi1_file_open(struct inode *inode, struct file *fp) fd->mm = current->mm; mmgrab(fd->mm); fd->dd = dd; + kobject_get(&fd->dd->kobj); fp->private_data = fd; } else { fp->private_data = NULL; diff --git a/drivers/infiniband/hw/hfi1/hfi.h b/drivers/infiniband/hw/hfi1/hfi.h index b42c22292597..90bc8c76d2ca 100644 --- a/drivers/infiniband/hw/hfi1/hfi.h +++ b/drivers/infiniband/hw/hfi1/hfi.h @@ -341,6 +341,7 @@ struct hfi1_packet { u32 slid; u16 tlen; s16 etail; + u16 pkey; u8 hlen; u8 numpkt; u8 rsize; @@ -351,8 +352,7 @@ struct hfi1_packet { u8 sc; u8 sl; u8 opcode; - bool becn; - bool fecn; + bool migrated; }; /* Packet types */ @@ -858,6 +858,13 @@ struct hfi1_pportdata { struct work_struct linkstate_active_work; /* Does this port need to prescan for FECNs */ bool cc_prescan; + /* + * Sample sendWaitCnt & sendWaitVlCnt during link transition + * and counter request. + */ + u64 port_vl_xmit_wait_last[C_VL_COUNT + 1]; + u16 prev_link_width; + u64 vl_xmit_flit_cnt[C_VL_COUNT + 1]; }; typedef int (*rhf_rcv_function_ptr)(struct hfi1_packet *packet); @@ -1779,19 +1786,15 @@ void hfi1_process_ecn_slowpath(struct rvt_qp *qp, struct hfi1_packet *pkt, static inline bool process_ecn(struct rvt_qp *qp, struct hfi1_packet *pkt, bool do_cnp) { - struct ib_other_headers *ohdr = pkt->ohdr; - - u32 bth1; - bool becn = false; - bool fecn = false; + bool becn; + bool fecn; if (pkt->etype == RHF_RCV_TYPE_BYPASS) { fecn = hfi1_16B_get_fecn(pkt->hdr); becn = hfi1_16B_get_becn(pkt->hdr); } else { - bth1 = be32_to_cpu(ohdr->bth[1]); - fecn = bth1 & IB_FECN_SMASK; - becn = bth1 & IB_BECN_SMASK; + fecn = ib_bth_get_fecn(pkt->ohdr); + becn = ib_bth_get_becn(pkt->ohdr); } if (unlikely(fecn || becn)) { hfi1_process_ecn_slowpath(qp, pkt, do_cnp); @@ -1957,6 +1960,7 @@ void hfi1_verbs_unregister_sysfs(struct hfi1_devdata *dd); int qsfp_dump(struct hfi1_pportdata *ppd, char *buf, int len); int hfi1_pcie_init(struct pci_dev *pdev, const struct pci_device_id *ent); +void hfi1_clean_up_interrupts(struct hfi1_devdata *dd); void hfi1_pcie_cleanup(struct pci_dev *pdev); int hfi1_pcie_ddinit(struct hfi1_devdata *dd, struct pci_dev *pdev); void hfi1_pcie_ddcleanup(struct hfi1_devdata *); @@ -2416,7 +2420,7 @@ static inline void hfi1_make_ib_hdr(struct ib_header *hdr, static inline void hfi1_make_16b_hdr(struct hfi1_16b_header *hdr, u32 slid, u32 dlid, u16 len, u16 pkey, - u8 becn, u8 fecn, u8 l4, + bool becn, bool fecn, u8 l4, u8 sc) { u32 lrh0 = 0; diff --git a/drivers/infiniband/hw/hfi1/init.c b/drivers/infiniband/hw/hfi1/init.c index 9b128268fb28..33eba2356742 100644 --- a/drivers/infiniband/hw/hfi1/init.c +++ b/drivers/infiniband/hw/hfi1/init.c @@ -172,7 +172,7 @@ int hfi1_create_kctxts(struct hfi1_devdata *dd) u16 i; int ret; - dd->rcd = kzalloc_node(dd->num_rcv_contexts * sizeof(*dd->rcd), + dd->rcd = kcalloc_node(dd->num_rcv_contexts, sizeof(*dd->rcd), GFP_KERNEL, dd->node); if (!dd->rcd) return -ENOMEM; @@ -439,15 +439,16 @@ int hfi1_create_ctxtdata(struct hfi1_pportdata *ppd, int numa, * The resulting value will be rounded down to the closest * multiple of dd->rcv_entries.group_size. */ - rcd->egrbufs.buffers = kzalloc_node( - rcd->egrbufs.count * sizeof(*rcd->egrbufs.buffers), - GFP_KERNEL, numa); + rcd->egrbufs.buffers = + kcalloc_node(rcd->egrbufs.count, + sizeof(*rcd->egrbufs.buffers), + GFP_KERNEL, numa); if (!rcd->egrbufs.buffers) goto bail; - rcd->egrbufs.rcvtids = kzalloc_node( - rcd->egrbufs.count * - sizeof(*rcd->egrbufs.rcvtids), - GFP_KERNEL, numa); + rcd->egrbufs.rcvtids = + kcalloc_node(rcd->egrbufs.count, + sizeof(*rcd->egrbufs.rcvtids), + GFP_KERNEL, numa); if (!rcd->egrbufs.rcvtids) goto bail; rcd->egrbufs.size = eager_buffer_size; @@ -637,6 +638,15 @@ void hfi1_init_pportdata(struct pci_dev *pdev, struct hfi1_pportdata *ppd, ppd->dd = dd; ppd->hw_pidx = hw_pidx; ppd->port = port; /* IB port number, not index */ + ppd->prev_link_width = LINK_WIDTH_DEFAULT; + /* + * There are C_VL_COUNT number of PortVLXmitWait counters. + * Adding 1 to C_VL_COUNT to include the PortXmitWait counter. + */ + for (i = 0; i < C_VL_COUNT + 1; i++) { + ppd->port_vl_xmit_wait_last[i] = 0; + ppd->vl_xmit_flit_cnt[i] = 0; + } default_pkey_idx = 1; @@ -1058,8 +1068,9 @@ static void shutdown_device(struct hfi1_devdata *dd) } dd->flags &= ~HFI1_INITTED; - /* mask interrupts, but not errors */ + /* mask and clean up interrupts, but not errors */ set_intr_state(dd, 0); + hfi1_clean_up_interrupts(dd); for (pidx = 0; pidx < dd->num_pports; ++pidx) { ppd = dd->pport + pidx; @@ -1218,6 +1229,7 @@ static void __hfi1_free_devdata(struct kobject *kobj) free_percpu(dd->rcv_limit); free_percpu(dd->send_schedule); free_percpu(dd->tx_opstats); + sdma_clean(dd, dd->num_sdma); rvt_dealloc_device(&dd->verbs_dev.rdi); } @@ -1704,6 +1716,7 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent) dd_dev_err(dd, "Failed to create /dev devices: %d\n", -j); if (initfail || ret) { + hfi1_clean_up_interrupts(dd); stop_timers(dd); flush_workqueue(ib_wq); for (pidx = 0; pidx < dd->num_pports; ++pidx) { diff --git a/drivers/infiniband/hw/hfi1/iowait.h b/drivers/infiniband/hw/hfi1/iowait.h index 591697d85eed..3d9c32c7c340 100644 --- a/drivers/infiniband/hw/hfi1/iowait.h +++ b/drivers/infiniband/hw/hfi1/iowait.h @@ -371,4 +371,13 @@ static inline void iowait_starve_find_max(struct iowait *w, u8 *max, } } +/** + * iowait_packet_queued() - determine if a packet is already built + * @wait: the wait structure + */ +static inline bool iowait_packet_queued(struct iowait *wait) +{ + return !list_empty(&wait->tx_head); +} + #endif diff --git a/drivers/infiniband/hw/hfi1/mad.c b/drivers/infiniband/hw/hfi1/mad.c index 34547a48a445..e9962c65c68f 100644 --- a/drivers/infiniband/hw/hfi1/mad.c +++ b/drivers/infiniband/hw/hfi1/mad.c @@ -2649,6 +2649,79 @@ static void a0_portstatus(struct hfi1_pportdata *ppd, } } +/** + * tx_link_width - convert link width bitmask to integer + * value representing actual link width. + * @link_width: width of active link + * @return: return index of the bit set in link_width var + * + * The function convert and return the index of bit set + * that indicate the current link width. + */ +u16 tx_link_width(u16 link_width) +{ + int n = LINK_WIDTH_DEFAULT; + u16 tx_width = n; + + while (link_width && n) { + if (link_width & (1 << (n - 1))) { + tx_width = n; + break; + } + n--; + } + + return tx_width; +} + +/** + * get_xmit_wait_counters - Convert HFI 's SendWaitCnt/SendWaitVlCnt + * counter in unit of TXE cycle times to flit times. + * @ppd: info of physical Hfi port + * @link_width: width of active link + * @link_speed: speed of active link + * @vl: represent VL0-VL7, VL15 for PortVLXmitWait counters request + * and if vl value is C_VL_COUNT, it represent SendWaitCnt + * counter request + * @return: return SendWaitCnt/SendWaitVlCnt counter value per vl. + * + * Convert SendWaitCnt/SendWaitVlCnt counter from TXE cycle times to + * flit times. Call this function to samples these counters. This + * function will calculate for previous state transition and update + * current state at end of function using ppd->prev_link_width and + * ppd->port_vl_xmit_wait_last to port_vl_xmit_wait_curr and link_width. + */ +u64 get_xmit_wait_counters(struct hfi1_pportdata *ppd, + u16 link_width, u16 link_speed, int vl) +{ + u64 port_vl_xmit_wait_curr; + u64 delta_vl_xmit_wait; + u64 xmit_wait_val; + + if (vl > C_VL_COUNT) + return 0; + if (vl < C_VL_COUNT) + port_vl_xmit_wait_curr = + read_port_cntr(ppd, C_TX_WAIT_VL, vl); + else + port_vl_xmit_wait_curr = + read_port_cntr(ppd, C_TX_WAIT, CNTR_INVALID_VL); + + xmit_wait_val = + port_vl_xmit_wait_curr - + ppd->port_vl_xmit_wait_last[vl]; + delta_vl_xmit_wait = + convert_xmit_counter(xmit_wait_val, + ppd->prev_link_width, + link_speed); + + ppd->vl_xmit_flit_cnt[vl] += delta_vl_xmit_wait; + ppd->port_vl_xmit_wait_last[vl] = port_vl_xmit_wait_curr; + ppd->prev_link_width = link_width; + + return ppd->vl_xmit_flit_cnt[vl]; +} + static int pma_get_opa_portstatus(struct opa_pma_mad *pmp, struct ib_device *ibdev, u8 port, u32 *resp_len) @@ -2668,6 +2741,8 @@ static int pma_get_opa_portstatus(struct opa_pma_mad *pmp, struct hfi1_pportdata *ppd = ppd_from_ibp(ibp); int vfi; u64 tmp, tmp2; + u16 link_width; + u16 link_speed; response_data_size = sizeof(struct opa_port_status_rsp) + num_vls * sizeof(struct _vls_pctrs); @@ -2711,8 +2786,16 @@ static int pma_get_opa_portstatus(struct opa_pma_mad *pmp, rsp->port_multicast_rcv_pkts = cpu_to_be64(read_dev_cntr(dd, C_DC_MC_RCV_PKTS, CNTR_INVALID_VL)); + /* + * Convert PortXmitWait counter from TXE cycle times + * to flit times. + */ + link_width = + tx_link_width(ppd->link_width_downgrade_tx_active); + link_speed = get_link_speed(ppd->link_speed_active); rsp->port_xmit_wait = - cpu_to_be64(read_port_cntr(ppd, C_TX_WAIT, CNTR_INVALID_VL)); + cpu_to_be64(get_xmit_wait_counters(ppd, link_width, + link_speed, C_VL_COUNT)); rsp->port_rcv_fecn = cpu_to_be64(read_dev_cntr(dd, C_DC_RCV_FCN, CNTR_INVALID_VL)); rsp->port_rcv_becn = @@ -2777,10 +2860,14 @@ static int pma_get_opa_portstatus(struct opa_pma_mad *pmp, rsp->vls[vfi].port_vl_xmit_pkts = cpu_to_be64(read_port_cntr(ppd, C_TX_PKT_VL, idx_from_vl(vl))); - + /* + * Convert PortVlXmitWait counter from TXE cycle + * times to flit times. + */ rsp->vls[vfi].port_vl_xmit_wait = - cpu_to_be64(read_port_cntr(ppd, C_TX_WAIT_VL, - idx_from_vl(vl))); + cpu_to_be64(get_xmit_wait_counters(ppd, link_width, + link_speed, + idx_from_vl(vl))); rsp->vls[vfi].port_vl_rcv_fecn = cpu_to_be64(read_dev_cntr(dd, C_DC_RCV_FCN_VL, @@ -2910,6 +2997,8 @@ static int pma_get_opa_datacounters(struct opa_pma_mad *pmp, unsigned long vl; u32 vl_select_mask; int vfi; + u16 link_width; + u16 link_speed; num_ports = be32_to_cpu(pmp->mad_hdr.attr_mod) >> 24; num_vls = hweight32(be32_to_cpu(req->vl_select_mask)); @@ -2959,8 +3048,16 @@ static int pma_get_opa_datacounters(struct opa_pma_mad *pmp, rsp->link_quality_indicator = cpu_to_be32((u32)lq); pma_get_opa_port_dctrs(ibdev, rsp); + /* + * Convert PortXmitWait counter from TXE + * cycle times to flit times. + */ + link_width = + tx_link_width(ppd->link_width_downgrade_tx_active); + link_speed = get_link_speed(ppd->link_speed_active); rsp->port_xmit_wait = - cpu_to_be64(read_port_cntr(ppd, C_TX_WAIT, CNTR_INVALID_VL)); + cpu_to_be64(get_xmit_wait_counters(ppd, link_width, + link_speed, C_VL_COUNT)); rsp->port_rcv_fecn = cpu_to_be64(read_dev_cntr(dd, C_DC_RCV_FCN, CNTR_INVALID_VL)); rsp->port_rcv_becn = @@ -2996,9 +3093,14 @@ static int pma_get_opa_datacounters(struct opa_pma_mad *pmp, cpu_to_be64(read_dev_cntr(dd, C_DC_RX_PKT_VL, idx_from_vl(vl))); + /* + * Convert PortVlXmitWait counter from TXE + * cycle times to flit times. + */ rsp->vls[vfi].port_vl_xmit_wait = - cpu_to_be64(read_port_cntr(ppd, C_TX_WAIT_VL, - idx_from_vl(vl))); + cpu_to_be64(get_xmit_wait_counters(ppd, link_width, + link_speed, + idx_from_vl(vl))); rsp->vls[vfi].port_vl_rcv_fecn = cpu_to_be64(read_dev_cntr(dd, C_DC_RCV_FCN_VL, @@ -3416,9 +3518,11 @@ static int pma_set_opa_portstatus(struct opa_pma_mad *pmp, if (counter_select & CS_PORT_MCAST_RCV_PKTS) write_dev_cntr(dd, C_DC_MC_RCV_PKTS, CNTR_INVALID_VL, 0); - if (counter_select & CS_PORT_XMIT_WAIT) + if (counter_select & CS_PORT_XMIT_WAIT) { write_port_cntr(ppd, C_TX_WAIT, CNTR_INVALID_VL, 0); - + ppd->port_vl_xmit_wait_last[C_VL_COUNT] = 0; + ppd->vl_xmit_flit_cnt[C_VL_COUNT] = 0; + } /* ignore cs_sw_portCongestion for HFIs */ if (counter_select & CS_PORT_RCV_FECN) @@ -3491,8 +3595,11 @@ static int pma_set_opa_portstatus(struct opa_pma_mad *pmp, if (counter_select & CS_PORT_RCV_PKTS) write_dev_cntr(dd, C_DC_RX_PKT_VL, idx_from_vl(vl), 0); - if (counter_select & CS_PORT_XMIT_WAIT) + if (counter_select & CS_PORT_XMIT_WAIT) { write_port_cntr(ppd, C_TX_WAIT_VL, idx_from_vl(vl), 0); + ppd->port_vl_xmit_wait_last[idx_from_vl(vl)] = 0; + ppd->vl_xmit_flit_cnt[idx_from_vl(vl)] = 0; + } /* sw_port_vl_congestion is 0 for HFIs */ if (counter_select & CS_PORT_RCV_FECN) diff --git a/drivers/infiniband/hw/hfi1/mad.h b/drivers/infiniband/hw/hfi1/mad.h index c4938f3d97c8..2f48e6953629 100644 --- a/drivers/infiniband/hw/hfi1/mad.h +++ b/drivers/infiniband/hw/hfi1/mad.h @@ -180,6 +180,15 @@ struct opa_mad_notice_attr { #define OPA_VLARB_PREEMPT_MATRIX 3 #define IB_PMA_PORT_COUNTERS_CONG cpu_to_be16(0xFF00) +#define LINK_SPEED_25G 1 +#define LINK_SPEED_12_5G 2 +#define LINK_WIDTH_DEFAULT 4 +#define DECIMAL_FACTORING 1000 +/* + * The default link width is multiplied by 1000 + * to get accurate value after division. + */ +#define FACTOR_LINK_WIDTH (LINK_WIDTH_DEFAULT * DECIMAL_FACTORING) struct ib_pma_portcounters_cong { u8 reserved; @@ -429,5 +438,41 @@ struct sc2vlnt { void hfi1_event_pkey_change(struct hfi1_devdata *dd, u8 port); void hfi1_handle_trap_timer(struct timer_list *t); - +u16 tx_link_width(u16 link_width); +u64 get_xmit_wait_counters(struct hfi1_pportdata *ppd, u16 link_width, + u16 link_speed, int vl); +/** + * get_link_speed - determine whether 12.5G or 25G speed + * @link_speed: the speed of active link + * @return: Return 2 if link speed identified as 12.5G + * or return 1 if link speed is 25G. + * + * The function indirectly calculate required link speed + * value for convert_xmit_counter function. If the link + * speed is 25G, the function return as 1 as it is required + * by xmit counter conversion formula :-( 25G / link_speed). + * This conversion will provide value 1 if current + * link speed is 25G or 2 if 12.5G.This is done to avoid + * 12.5 float number conversion. + */ +static inline u16 get_link_speed(u16 link_speed) +{ + return (link_speed == 1) ? + LINK_SPEED_12_5G : LINK_SPEED_25G; +} + +/** + * convert_xmit_counter - calculate flit times for given xmit counter + * value + * @xmit_wait_val: current xmit counter value + * @link_width: width of active link + * @link_speed: speed of active link + * @return: return xmit counter value in flit times. + */ +static inline u64 convert_xmit_counter(u64 xmit_wait_val, u16 link_width, + u16 link_speed) +{ + return (xmit_wait_val * 2 * (FACTOR_LINK_WIDTH / link_width) + * link_speed) / DECIMAL_FACTORING; +} #endif /* _HFI1_MAD_H */ diff --git a/drivers/infiniband/hw/hfi1/pcie.c b/drivers/infiniband/hw/hfi1/pcie.c index 8c7e7a60b715..83d66e862207 100644 --- a/drivers/infiniband/hw/hfi1/pcie.c +++ b/drivers/infiniband/hw/hfi1/pcie.c @@ -1034,6 +1034,7 @@ int do_pcie_gen3_transition(struct hfi1_devdata *dd) int do_retry, retry_count = 0; int intnum = 0; uint default_pset; + uint pset = pcie_pset; u16 target_vector, target_speed; u16 lnkctl2, vendor; u8 div; @@ -1201,16 +1202,16 @@ retry: * * Set Gen3EqPsetReqVec, leave other fields 0. */ - if (pcie_pset == UNSET_PSET) - pcie_pset = default_pset; - if (pcie_pset > 10) { /* valid range is 0-10, inclusive */ + if (pset == UNSET_PSET) + pset = default_pset; + if (pset > 10) { /* valid range is 0-10, inclusive */ dd_dev_err(dd, "%s: Invalid Eq Pset %u, setting to %d\n", - __func__, pcie_pset, default_pset); - pcie_pset = default_pset; + __func__, pset, default_pset); + pset = default_pset; } - dd_dev_info(dd, "%s: using EQ Pset %u\n", __func__, pcie_pset); + dd_dev_info(dd, "%s: using EQ Pset %u\n", __func__, pset); pci_write_config_dword(dd->pcidev, PCIE_CFG_REG_PL106, - ((1 << pcie_pset) << + ((1 << pset) << PCIE_CFG_REG_PL106_GEN3_EQ_PSET_REQ_VEC_SHIFT) | PCIE_CFG_REG_PL106_GEN3_EQ_EVAL2MS_DISABLE_SMASK | PCIE_CFG_REG_PL106_GEN3_EQ_PHASE23_EXIT_MODE_SMASK); @@ -1240,10 +1241,10 @@ retry: /* apply static CTLE tunings */ u8 pcie_dc, pcie_lf, pcie_hf, pcie_bw; - pcie_dc = ctle_tunings[pcie_pset][0]; - pcie_lf = ctle_tunings[pcie_pset][1]; - pcie_hf = ctle_tunings[pcie_pset][2]; - pcie_bw = ctle_tunings[pcie_pset][3]; + pcie_dc = ctle_tunings[pset][0]; + pcie_lf = ctle_tunings[pset][1]; + pcie_hf = ctle_tunings[pset][2]; + pcie_bw = ctle_tunings[pset][3]; write_gasket_interrupt(dd, intnum++, 0x0026, 0x0200 | pcie_dc); write_gasket_interrupt(dd, intnum++, 0x0026, 0x0100 | pcie_lf); write_gasket_interrupt(dd, intnum++, 0x0026, 0x0000 | pcie_hf); diff --git a/drivers/infiniband/hw/hfi1/pio.c b/drivers/infiniband/hw/hfi1/pio.c index 4c1198bc5e70..40dac4d16eb8 100644 --- a/drivers/infiniband/hw/hfi1/pio.c +++ b/drivers/infiniband/hw/hfi1/pio.c @@ -455,8 +455,8 @@ int init_send_contexts(struct hfi1_devdata *dd) dd->hw_to_sw = kmalloc_array(TXE_NUM_CONTEXTS, sizeof(u8), GFP_KERNEL); dd->send_contexts = kcalloc(dd->num_send_contexts, - sizeof(struct send_context_info), - GFP_KERNEL); + sizeof(struct send_context_info), + GFP_KERNEL); if (!dd->send_contexts || !dd->hw_to_sw) { kfree(dd->hw_to_sw); kfree(dd->send_contexts); @@ -856,8 +856,9 @@ struct send_context *sc_alloc(struct hfi1_devdata *dd, int type, * so head == tail can mean empty. */ sc->sr_size = sci->credits + 1; - sc->sr = kzalloc_node(sizeof(union pio_shadow_ring) * - sc->sr_size, GFP_KERNEL, numa); + sc->sr = kcalloc_node(sc->sr_size, + sizeof(union pio_shadow_ring), + GFP_KERNEL, numa); if (!sc->sr) { sc_free(sc); return NULL; @@ -1958,9 +1959,9 @@ int init_pervl_scs(struct hfi1_devdata *dd) hfi1_init_ctxt(dd->vld[15].sc); dd->vld[15].mtu = enum_to_mtu(OPA_MTU_2048); - dd->kernel_send_context = kzalloc_node(dd->num_send_contexts * - sizeof(struct send_context *), - GFP_KERNEL, dd->node); + dd->kernel_send_context = kcalloc_node(dd->num_send_contexts, + sizeof(struct send_context *), + GFP_KERNEL, dd->node); if (!dd->kernel_send_context) goto freesc15; diff --git a/drivers/infiniband/hw/hfi1/qp.c b/drivers/infiniband/hw/hfi1/qp.c index 5507910e8b8a..d30dd1a5b0a6 100644 --- a/drivers/infiniband/hw/hfi1/qp.c +++ b/drivers/infiniband/hw/hfi1/qp.c @@ -565,7 +565,7 @@ void qp_iter_print(struct seq_file *s, struct rvt_qp_iter *iter) if (qp->s_ack_queue) e = &qp->s_ack_queue[qp->s_tail_ack_queue]; seq_printf(s, - "N %d %s QP %x R %u %s %u %u %u f=%x %u %u %u %u %u %u SPSN %x %x %x %x %x RPSN %x S(%u %u %u %u %u %u %u) R(%u %u %u) RQP %x LID %x SL %u MTU %u %u %u %u %u SDE %p,%u SC %p,%u SCQ %u %u PID %d OS %x %x E %x %x %x RNR %d %s %d\n", + "N %d %s QP %x R %u %s %u %u f=%x %u %u %u %u %u %u SPSN %x %x %x %x %x RPSN %x S(%u %u %u %u %u %u %u) R(%u %u %u) RQP %x LID %x SL %u MTU %u %u %u %u %u SDE %p,%u SC %p,%u SCQ %u %u PID %d OS %x %x E %x %x %x RNR %d %s %d\n", iter->n, qp_idle(qp) ? "I" : "B", qp->ibqp.qp_num, @@ -573,7 +573,6 @@ void qp_iter_print(struct seq_file *s, struct rvt_qp_iter *iter) qp_type_str[qp->ibqp.qp_type], qp->state, wqe ? wqe->wr.opcode : 0, - qp->s_hdrwords, qp->s_flags, iowait_sdma_pending(&priv->s_iowait), iowait_pio_pending(&priv->s_iowait), @@ -795,7 +794,6 @@ void notify_error_qp(struct rvt_qp *qp) } if (!(qp->s_flags & RVT_S_BUSY)) { - qp->s_hdrwords = 0; if (qp->s_rdma_mr) { rvt_put_mr(qp->s_rdma_mr); qp->s_rdma_mr = NULL; diff --git a/drivers/infiniband/hw/hfi1/qp.h b/drivers/infiniband/hw/hfi1/qp.h index c06d2f8348e0..b2d4cba8d15b 100644 --- a/drivers/infiniband/hw/hfi1/qp.h +++ b/drivers/infiniband/hw/hfi1/qp.h @@ -51,12 +51,25 @@ #include <rdma/rdmavt_qp.h> #include "verbs.h" #include "sdma.h" +#include "verbs_txreq.h" extern unsigned int hfi1_qp_table_size; extern const struct rvt_operation_params hfi1_post_parms[]; /* + * Send if not busy or waiting for I/O and either + * a RC response is pending or we can process send work requests. + */ +static inline int hfi1_send_ok(struct rvt_qp *qp) +{ + return !(qp->s_flags & (RVT_S_BUSY | RVT_S_ANY_WAIT_IO)) && + (verbs_txreq_queued(qp) || + (qp->s_flags & RVT_S_RESP_PENDING) || + !(qp->s_flags & RVT_S_ANY_WAIT_SEND)); +} + +/* * free_ahg - clear ahg from QP */ static inline void clear_ahg(struct rvt_qp *qp) diff --git a/drivers/infiniband/hw/hfi1/rc.c b/drivers/infiniband/hw/hfi1/rc.c index 14cc212a21c7..da58046a02ea 100644 --- a/drivers/infiniband/hw/hfi1/rc.c +++ b/drivers/infiniband/hw/hfi1/rc.c @@ -226,12 +226,10 @@ normal: bth2 = mask_psn(qp->s_ack_psn); } qp->s_rdma_ack_cnt++; - qp->s_hdrwords = hwords; ps->s_txreq->sde = priv->s_sde; ps->s_txreq->s_cur_size = len; + ps->s_txreq->hdr_dwords = hwords; hfi1_make_ruc_header(qp, ohdr, bth0, bth2, middle, ps); - /* pbc */ - ps->s_txreq->hdr_dwords = qp->s_hdrwords + 2; return 1; bail: @@ -385,7 +383,6 @@ int hfi1_make_rc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps) : IB_WC_SUCCESS); if (local_ops) atomic_dec(&qp->local_ops_pending); - qp->s_hdrwords = 0; goto done_free_tx; } @@ -688,7 +685,7 @@ no_flow_control: bth2 |= IB_BTH_REQ_ACK; } qp->s_len -= len; - qp->s_hdrwords = hwords; + ps->s_txreq->hdr_dwords = hwords; ps->s_txreq->sde = priv->s_sde; ps->s_txreq->ss = ss; ps->s_txreq->s_cur_size = len; @@ -699,8 +696,6 @@ no_flow_control: bth2, middle, ps); - /* pbc */ - ps->s_txreq->hdr_dwords = qp->s_hdrwords + 2; return 1; done_free_tx: @@ -714,7 +709,6 @@ bail: bail_no_tx: ps->s_txreq = NULL; qp->s_flags &= ~RVT_S_BUSY; - qp->s_hdrwords = 0; return 0; } @@ -734,14 +728,16 @@ static inline void hfi1_make_bth_aeth(struct rvt_qp *qp, ohdr->bth[2] = cpu_to_be32(mask_psn(qp->r_ack_psn)); } -static inline void hfi1_queue_rc_ack(struct rvt_qp *qp, bool is_fecn) +static inline void hfi1_queue_rc_ack(struct hfi1_packet *packet, bool is_fecn) { - struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); + struct rvt_qp *qp = packet->qp; + struct hfi1_ibport *ibp; unsigned long flags; spin_lock_irqsave(&qp->s_lock, flags); if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) goto unlock; + ibp = rcd_to_iport(packet->rcd); this_cpu_inc(*ibp->rvp.rc_qacks); qp->s_flags |= RVT_S_ACK_PENDING | RVT_S_RESP_PENDING; qp->s_nak_state = qp->r_nak_state; @@ -755,13 +751,14 @@ unlock: spin_unlock_irqrestore(&qp->s_lock, flags); } -static inline void hfi1_make_rc_ack_9B(struct rvt_qp *qp, +static inline void hfi1_make_rc_ack_9B(struct hfi1_packet *packet, struct hfi1_opa_header *opa_hdr, u8 sc5, bool is_fecn, u64 *pbc_flags, u32 *hwords, u32 *nwords) { - struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); + struct rvt_qp *qp = packet->qp; + struct hfi1_ibport *ibp = rcd_to_iport(packet->rcd); struct hfi1_pportdata *ppd = ppd_from_ibp(ibp); struct ib_header *hdr = &opa_hdr->ibh; struct ib_other_headers *ohdr; @@ -802,19 +799,20 @@ static inline void hfi1_make_rc_ack_9B(struct rvt_qp *qp, hfi1_make_bth_aeth(qp, ohdr, bth0, bth1); } -static inline void hfi1_make_rc_ack_16B(struct rvt_qp *qp, +static inline void hfi1_make_rc_ack_16B(struct hfi1_packet *packet, struct hfi1_opa_header *opa_hdr, u8 sc5, bool is_fecn, u64 *pbc_flags, u32 *hwords, u32 *nwords) { - struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); + struct rvt_qp *qp = packet->qp; + struct hfi1_ibport *ibp = rcd_to_iport(packet->rcd); struct hfi1_pportdata *ppd = ppd_from_ibp(ibp); struct hfi1_16b_header *hdr = &opa_hdr->opah; struct ib_other_headers *ohdr; u32 bth0, bth1 = 0; u16 len, pkey; - u8 becn = !!is_fecn; + bool becn = is_fecn; u8 l4 = OPA_16B_L4_IB_LOCAL; u8 extra_bytes; @@ -854,7 +852,7 @@ static inline void hfi1_make_rc_ack_16B(struct rvt_qp *qp, hfi1_make_bth_aeth(qp, ohdr, bth0, bth1); } -typedef void (*hfi1_make_rc_ack)(struct rvt_qp *qp, +typedef void (*hfi1_make_rc_ack)(struct hfi1_packet *packet, struct hfi1_opa_header *opa_hdr, u8 sc5, bool is_fecn, u64 *pbc_flags, u32 *hwords, @@ -874,9 +872,10 @@ static const hfi1_make_rc_ack hfi1_make_rc_ack_tbl[2] = { * Note that RDMA reads and atomics are handled in the * send side QP state and send engine. */ -void hfi1_send_rc_ack(struct hfi1_ctxtdata *rcd, - struct rvt_qp *qp, bool is_fecn) +void hfi1_send_rc_ack(struct hfi1_packet *packet, bool is_fecn) { + struct hfi1_ctxtdata *rcd = packet->rcd; + struct rvt_qp *qp = packet->qp; struct hfi1_ibport *ibp = rcd_to_iport(rcd); struct hfi1_qp_priv *priv = qp->priv; struct hfi1_pportdata *ppd = ppd_from_ibp(ibp); @@ -893,13 +892,13 @@ void hfi1_send_rc_ack(struct hfi1_ctxtdata *rcd, /* Don't send ACK or NAK if a RDMA read or atomic is pending. */ if (qp->s_flags & RVT_S_RESP_PENDING) { - hfi1_queue_rc_ack(qp, is_fecn); + hfi1_queue_rc_ack(packet, is_fecn); return; } /* Ensure s_rdma_ack_cnt changes are committed */ if (qp->s_rdma_ack_cnt) { - hfi1_queue_rc_ack(qp, is_fecn); + hfi1_queue_rc_ack(packet, is_fecn); return; } @@ -908,7 +907,7 @@ void hfi1_send_rc_ack(struct hfi1_ctxtdata *rcd, return; /* Make the appropriate header */ - hfi1_make_rc_ack_tbl[priv->hdr_type](qp, &opa_hdr, sc5, is_fecn, + hfi1_make_rc_ack_tbl[priv->hdr_type](packet, &opa_hdr, sc5, is_fecn, &pbc_flags, &hwords, &nwords); plen = 2 /* PBC */ + hwords + nwords; @@ -922,7 +921,7 @@ void hfi1_send_rc_ack(struct hfi1_ctxtdata *rcd, * so that when enough buffer space becomes available, * the ACK is sent ahead of other outgoing packets. */ - hfi1_queue_rc_ack(qp, is_fecn); + hfi1_queue_rc_ack(packet, is_fecn); return; } trace_ack_output_ibhdr(dd_from_ibdev(qp->ibqp.device), @@ -1540,7 +1539,7 @@ static void rc_rcv_resp(struct hfi1_packet *packet) void *data = packet->payload; u32 tlen = packet->tlen; struct rvt_qp *qp = packet->qp; - struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); + struct hfi1_ibport *ibp; struct ib_other_headers *ohdr = packet->ohdr; struct rvt_swqe *wqe; enum ib_wc_status status; @@ -1697,6 +1696,7 @@ ack_op_err: goto ack_err; ack_seq_err: + ibp = rcd_to_iport(rcd); rdma_seq_err(qp, ibp, psn, rcd); goto ack_done; @@ -2037,7 +2037,6 @@ void hfi1_rc_rcv(struct hfi1_packet *packet) struct rvt_qp *qp = packet->qp; struct hfi1_ibport *ibp = rcd_to_iport(rcd); struct ib_other_headers *ohdr = packet->ohdr; - u32 bth0 = be32_to_cpu(ohdr->bth[0]); u32 opcode = packet->opcode; u32 hdrsize = packet->hlen; u32 psn = ib_bth_get_psn(packet->ohdr); @@ -2235,7 +2234,7 @@ send_last: wc.port_num = 0; /* Signal completion event if the solicited bit is set. */ rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, - (bth0 & IB_BTH_SOLICITED) != 0); + ib_bth_is_solicited(ohdr)); break; case OP(RDMA_WRITE_ONLY): @@ -2479,7 +2478,7 @@ nack_acc: qp->r_nak_state = IB_NAK_REMOTE_ACCESS_ERROR; qp->r_ack_psn = qp->r_psn; send_ack: - hfi1_send_rc_ack(rcd, qp, is_fecn); + hfi1_send_rc_ack(packet, is_fecn); } void hfi1_rc_hdrerr( diff --git a/drivers/infiniband/hw/hfi1/ruc.c b/drivers/infiniband/hw/hfi1/ruc.c index 13b994738f41..3daa94bdae3a 100644 --- a/drivers/infiniband/hw/hfi1/ruc.c +++ b/drivers/infiniband/hw/hfi1/ruc.c @@ -225,19 +225,8 @@ int hfi1_ruc_check_hdr(struct hfi1_ibport *ibp, struct hfi1_packet *packet) u32 dlid = packet->dlid; u32 slid = packet->slid; u32 sl = packet->sl; - int migrated; - u32 bth0, bth1; - u16 pkey; - - bth0 = be32_to_cpu(packet->ohdr->bth[0]); - bth1 = be32_to_cpu(packet->ohdr->bth[1]); - if (packet->etype == RHF_RCV_TYPE_BYPASS) { - pkey = hfi1_16B_get_pkey(packet->hdr); - migrated = bth1 & OPA_BTH_MIG_REQ; - } else { - pkey = ib_bth_get_pkey(packet->ohdr); - migrated = bth0 & IB_BTH_MIG_REQ; - } + bool migrated = packet->migrated; + u16 pkey = packet->pkey; if (qp->s_mig_state == IB_MIG_ARMED && migrated) { if (!packet->grh) { @@ -756,19 +745,18 @@ static inline void hfi1_make_ruc_header_16B(struct rvt_qp *qp, u32 slid; u16 pkey = hfi1_get_pkey(ibp, qp->s_pkey_index); u8 l4 = OPA_16B_L4_IB_LOCAL; - u8 extra_bytes = hfi1_get_16b_padding((qp->s_hdrwords << 2), - ps->s_txreq->s_cur_size); + u8 extra_bytes = hfi1_get_16b_padding( + (ps->s_txreq->hdr_dwords << 2), + ps->s_txreq->s_cur_size); u32 nwords = SIZE_OF_CRC + ((ps->s_txreq->s_cur_size + extra_bytes + SIZE_OF_LT) >> 2); - u8 becn = 0; + bool becn = false; if (unlikely(rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH) && hfi1_check_mcast(rdma_ah_get_dlid(&qp->remote_ah_attr))) { struct ib_grh *grh; struct ib_global_route *grd = rdma_ah_retrieve_grh(&qp->remote_ah_attr); - int hdrwords; - /* * Ensure OPA GIDs are transformed to IB gids * before creating the GRH. @@ -777,9 +765,10 @@ static inline void hfi1_make_ruc_header_16B(struct rvt_qp *qp, grd->sgid_index = 0; grh = &ps->s_txreq->phdr.hdr.opah.u.l.grh; l4 = OPA_16B_L4_IB_GLOBAL; - hdrwords = qp->s_hdrwords - 4; - qp->s_hdrwords += hfi1_make_grh(ibp, grh, grd, - hdrwords, nwords); + ps->s_txreq->hdr_dwords += + hfi1_make_grh(ibp, grh, grd, + ps->s_txreq->hdr_dwords - LRH_16B_DWORDS, + nwords); middle = 0; } @@ -798,7 +787,7 @@ static inline void hfi1_make_ruc_header_16B(struct rvt_qp *qp, if (qp->s_flags & RVT_S_ECN) { qp->s_flags &= ~RVT_S_ECN; /* we recently received a FECN, so return a BECN */ - becn = 1; + becn = true; } hfi1_make_ruc_bth(qp, ohdr, bth0, bth1, bth2); @@ -813,7 +802,7 @@ static inline void hfi1_make_ruc_header_16B(struct rvt_qp *qp, slid, opa_get_lid(rdma_ah_get_dlid(&qp->remote_ah_attr), 16B), - (qp->s_hdrwords + nwords) >> 1, + (ps->s_txreq->hdr_dwords + nwords) >> 1, pkey, becn, 0, l4, priv->s_sc); } @@ -833,13 +822,13 @@ static inline void hfi1_make_ruc_header_9B(struct rvt_qp *qp, if (unlikely(rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH)) { struct ib_grh *grh = &ps->s_txreq->phdr.hdr.ibh.u.l.grh; - int hdrwords = qp->s_hdrwords - 2; lrh0 = HFI1_LRH_GRH; - qp->s_hdrwords += + ps->s_txreq->hdr_dwords += hfi1_make_grh(ibp, grh, rdma_ah_read_grh(&qp->remote_ah_attr), - hdrwords, nwords); + ps->s_txreq->hdr_dwords - LRH_9B_DWORDS, + nwords); middle = 0; } lrh0 |= (priv->s_sc & 0xf) << 12 | @@ -865,7 +854,7 @@ static inline void hfi1_make_ruc_header_9B(struct rvt_qp *qp, hfi1_make_ruc_bth(qp, ohdr, bth0, bth1, bth2); hfi1_make_ib_hdr(&ps->s_txreq->phdr.hdr.ibh, lrh0, - qp->s_hdrwords + nwords, + ps->s_txreq->hdr_dwords + nwords, opa_get_lid(rdma_ah_get_dlid(&qp->remote_ah_attr), 9B), ppd_from_ibp(ibp)->lid | rdma_ah_get_path_bits(&qp->remote_ah_attr)); @@ -1030,7 +1019,7 @@ void hfi1_do_send(struct rvt_qp *qp, bool in_thread) ps.s_txreq = get_waiting_verbs_txreq(qp); do { /* Check for a constructed packet to be sent. */ - if (qp->s_hdrwords != 0) { + if (ps.s_txreq) { spin_unlock_irqrestore(&qp->s_lock, ps.flags); /* * If the packet cannot be sent now, return and @@ -1038,8 +1027,6 @@ void hfi1_do_send(struct rvt_qp *qp, bool in_thread) */ if (hfi1_verbs_send(qp, &ps)) return; - /* Record that s_ahg is empty. */ - qp->s_hdrwords = 0; /* allow other tasks to run */ if (schedule_send_yield(qp, &ps)) return; diff --git a/drivers/infiniband/hw/hfi1/sdma.c b/drivers/infiniband/hw/hfi1/sdma.c index 61c130dbed10..1f203309cf24 100644 --- a/drivers/infiniband/hw/hfi1/sdma.c +++ b/drivers/infiniband/hw/hfi1/sdma.c @@ -1275,13 +1275,15 @@ bail: return -ENOMEM; } -/* - * Clean up allocated memory. - * - * This routine is can be called regardless of the success of sdma_init() +/** + * sdma_clean() Clean up allocated memory + * @dd: struct hfi1_devdata + * @num_engines: num sdma engines * + * This routine can be called regardless of the success of + * sdma_init() */ -static void sdma_clean(struct hfi1_devdata *dd, size_t num_engines) +void sdma_clean(struct hfi1_devdata *dd, size_t num_engines) { size_t i; struct sdma_engine *sde; @@ -1386,7 +1388,8 @@ int sdma_init(struct hfi1_devdata *dd, u8 port) num_engines, descq_cnt); /* alloc memory for array of send engines */ - dd->per_sdma = kcalloc(num_engines, sizeof(*dd->per_sdma), GFP_KERNEL); + dd->per_sdma = kcalloc_node(num_engines, sizeof(*dd->per_sdma), + GFP_KERNEL, dd->node); if (!dd->per_sdma) return ret; @@ -1617,7 +1620,6 @@ void sdma_exit(struct hfi1_devdata *dd) */ sdma_finalput(&sde->state); } - sdma_clean(dd, dd->num_sdma); } /* diff --git a/drivers/infiniband/hw/hfi1/sdma.h b/drivers/infiniband/hw/hfi1/sdma.h index 374c59784950..46c775f255d1 100644 --- a/drivers/infiniband/hw/hfi1/sdma.h +++ b/drivers/infiniband/hw/hfi1/sdma.h @@ -420,6 +420,7 @@ struct sdma_engine { int sdma_init(struct hfi1_devdata *dd, u8 port); void sdma_start(struct hfi1_devdata *dd); void sdma_exit(struct hfi1_devdata *dd); +void sdma_clean(struct hfi1_devdata *dd, size_t num_engines); void sdma_all_running(struct hfi1_devdata *dd); void sdma_all_idle(struct hfi1_devdata *dd); void sdma_freeze_notify(struct hfi1_devdata *dd, int go_idle); diff --git a/drivers/infiniband/hw/hfi1/trace.c b/drivers/infiniband/hw/hfi1/trace.c index 959a80429ee9..89bd9851065b 100644 --- a/drivers/infiniband/hw/hfi1/trace.c +++ b/drivers/infiniband/hw/hfi1/trace.c @@ -138,7 +138,7 @@ static const char *parse_syndrome(u8 syndrome) } void hfi1_trace_parse_9b_bth(struct ib_other_headers *ohdr, - u8 *ack, u8 *becn, u8 *fecn, u8 *mig, + u8 *ack, bool *becn, bool *fecn, u8 *mig, u8 *se, u8 *pad, u8 *opcode, u8 *tver, u16 *pkey, u32 *psn, u32 *qpn) { @@ -184,7 +184,7 @@ void hfi1_trace_parse_9b_hdr(struct ib_header *hdr, bool sc5, } void hfi1_trace_parse_16b_hdr(struct hfi1_16b_header *hdr, - u8 *age, u8 *becn, u8 *fecn, + u8 *age, bool *becn, bool *fecn, u8 *l4, u8 *rc, u8 *sc, u16 *entropy, u16 *len, u16 *pkey, u32 *dlid, u32 *slid) @@ -207,7 +207,7 @@ void hfi1_trace_parse_16b_hdr(struct hfi1_16b_header *hdr, #define LRH_16B_PRN "age:%d becn:%d fecn:%d l4:%d " \ "rc:%d sc:%d pkey:0x%.4x entropy:0x%.4x" const char *hfi1_trace_fmt_lrh(struct trace_seq *p, bool bypass, - u8 age, u8 becn, u8 fecn, u8 l4, + u8 age, bool becn, bool fecn, u8 l4, u8 lnh, const char *lnh_name, u8 lver, u8 rc, u8 sc, u8 sl, u16 entropy, u16 len, u16 pkey, u32 dlid, u32 slid) @@ -235,7 +235,7 @@ const char *hfi1_trace_fmt_lrh(struct trace_seq *p, bool bypass, "op:0x%.2x,%s se:%d m:%d pad:%d tver:%d " \ "qpn:0x%.6x a:%d psn:0x%.8x" const char *hfi1_trace_fmt_bth(struct trace_seq *p, bool bypass, - u8 ack, u8 becn, u8 fecn, u8 mig, + u8 ack, bool becn, bool fecn, u8 mig, u8 se, u8 pad, u8 opcode, const char *opname, u8 tver, u16 pkey, u32 psn, u32 qpn) { diff --git a/drivers/infiniband/hw/hfi1/trace_ibhdrs.h b/drivers/infiniband/hw/hfi1/trace_ibhdrs.h index fb631278eccd..2847626d3819 100644 --- a/drivers/infiniband/hw/hfi1/trace_ibhdrs.h +++ b/drivers/infiniband/hw/hfi1/trace_ibhdrs.h @@ -101,7 +101,7 @@ u8 hfi1_trace_opa_hdr_len(struct hfi1_opa_header *opah); u8 hfi1_trace_packet_hdr_len(struct hfi1_packet *packet); const char *hfi1_trace_get_packet_l4_str(u8 l4); void hfi1_trace_parse_9b_bth(struct ib_other_headers *ohdr, - u8 *ack, u8 *becn, u8 *fecn, u8 *mig, + u8 *ack, bool *becn, bool *fecn, u8 *mig, u8 *se, u8 *pad, u8 *opcode, u8 *tver, u16 *pkey, u32 *psn, u32 *qpn); void hfi1_trace_parse_9b_hdr(struct ib_header *hdr, bool sc5, @@ -112,19 +112,19 @@ void hfi1_trace_parse_16b_bth(struct ib_other_headers *ohdr, u8 *pad, u8 *se, u8 *tver, u32 *psn, u32 *qpn); void hfi1_trace_parse_16b_hdr(struct hfi1_16b_header *hdr, - u8 *age, u8 *becn, u8 *fecn, + u8 *age, bool *becn, bool *fecn, u8 *l4, u8 *rc, u8 *sc, u16 *entropy, u16 *len, u16 *pkey, u32 *dlid, u32 *slid); const char *hfi1_trace_fmt_lrh(struct trace_seq *p, bool bypass, - u8 age, u8 becn, u8 fecn, u8 l4, + u8 age, bool becn, bool fecn, u8 l4, u8 lnh, const char *lnh_name, u8 lver, u8 rc, u8 sc, u8 sl, u16 entropy, u16 len, u16 pkey, u32 dlid, u32 slid); const char *hfi1_trace_fmt_bth(struct trace_seq *p, bool bypass, - u8 ack, u8 becn, u8 fecn, u8 mig, + u8 ack, bool becn, bool fecn, u8 mig, u8 se, u8 pad, u8 opcode, const char *opname, u8 tver, u16 pkey, u32 psn, u32 qpn); @@ -148,8 +148,8 @@ DECLARE_EVENT_CLASS(hfi1_input_ibhdr_template, __field(u8, etype) __field(u8, ack) __field(u8, age) - __field(u8, becn) - __field(u8, fecn) + __field(bool, becn) + __field(bool, fecn) __field(u8, l2) __field(u8, l4) __field(u8, lnh) @@ -290,8 +290,8 @@ DECLARE_EVENT_CLASS(hfi1_output_ibhdr_template, __field(u8, hdr_type) __field(u8, ack) __field(u8, age) - __field(u8, becn) - __field(u8, fecn) + __field(bool, becn) + __field(bool, fecn) __field(u8, l4) __field(u8, lnh) __field(u8, lver) diff --git a/drivers/infiniband/hw/hfi1/trace_rx.h b/drivers/infiniband/hw/hfi1/trace_rx.h index 4d487fee105d..7eceb57e0415 100644 --- a/drivers/infiniband/hw/hfi1/trace_rx.h +++ b/drivers/infiniband/hw/hfi1/trace_rx.h @@ -63,17 +63,9 @@ __print_symbolic(type, \ #define TRACE_SYSTEM hfi1_rx TRACE_EVENT(hfi1_rcvhdr, - TP_PROTO(struct hfi1_devdata *dd, - u32 ctxt, - u64 eflags, - u32 etype, - u32 hlen, - u32 tlen, - u32 updegr, - u32 etail - ), - TP_ARGS(dd, ctxt, eflags, etype, hlen, tlen, updegr, etail), - TP_STRUCT__entry(DD_DEV_ENTRY(dd) + TP_PROTO(struct hfi1_packet *packet), + TP_ARGS(packet), + TP_STRUCT__entry(DD_DEV_ENTRY(packet->rcd->dd) __field(u64, eflags) __field(u32, ctxt) __field(u32, etype) @@ -82,14 +74,14 @@ TRACE_EVENT(hfi1_rcvhdr, __field(u32, updegr) __field(u32, etail) ), - TP_fast_assign(DD_DEV_ASSIGN(dd); - __entry->eflags = eflags; - __entry->ctxt = ctxt; - __entry->etype = etype; - __entry->hlen = hlen; - __entry->tlen = tlen; - __entry->updegr = updegr; - __entry->etail = etail; + TP_fast_assign(DD_DEV_ASSIGN(packet->rcd->dd); + __entry->eflags = rhf_err_flags(packet->rhf); + __entry->ctxt = packet->rcd->ctxt; + __entry->etype = packet->etype; + __entry->hlen = packet->hlen; + __entry->tlen = packet->tlen; + __entry->updegr = packet->updegr; + __entry->etail = rhf_egr_index(packet->rhf); ), TP_printk( "[%s] ctxt %d eflags 0x%llx etype %d,%s hlen %d tlen %d updegr %d etail %d", diff --git a/drivers/infiniband/hw/hfi1/uc.c b/drivers/infiniband/hw/hfi1/uc.c index 132b63e787d1..9d7a3110c14c 100644 --- a/drivers/infiniband/hw/hfi1/uc.c +++ b/drivers/infiniband/hw/hfi1/uc.c @@ -144,7 +144,6 @@ int hfi1_make_uc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps) : IB_WC_SUCCESS); if (local_ops) atomic_dec(&qp->local_ops_pending); - qp->s_hdrwords = 0; goto done_free_tx; } /* @@ -267,14 +266,12 @@ int hfi1_make_uc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps) break; } qp->s_len -= len; - qp->s_hdrwords = hwords; + ps->s_txreq->hdr_dwords = hwords; ps->s_txreq->sde = priv->s_sde; ps->s_txreq->ss = &qp->s_sge; ps->s_txreq->s_cur_size = len; hfi1_make_ruc_header(qp, ohdr, bth0 | (qp->s_state << 24), mask_psn(qp->s_psn++), middle, ps); - /* pbc */ - ps->s_txreq->hdr_dwords = qp->s_hdrwords + 2; return 1; done_free_tx: @@ -288,7 +285,6 @@ bail: bail_no_tx: ps->s_txreq = NULL; qp->s_flags &= ~RVT_S_BUSY; - qp->s_hdrwords = 0; return 0; } @@ -480,8 +476,7 @@ last_imm: wc.port_num = 0; /* Signal completion event if the solicited bit is set. */ rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, - (ohdr->bth[0] & - cpu_to_be32(IB_BTH_SOLICITED)) != 0); + ib_bth_is_solicited(ohdr)); break; case OP(RDMA_WRITE_FIRST): diff --git a/drivers/infiniband/hw/hfi1/ud.c b/drivers/infiniband/hw/hfi1/ud.c index deb184574395..bcf3b0bebac8 100644 --- a/drivers/infiniband/hw/hfi1/ud.c +++ b/drivers/infiniband/hw/hfi1/ud.c @@ -340,15 +340,16 @@ void hfi1_make_ud_req_9B(struct rvt_qp *qp, struct hfi1_pkt_state *ps, extra_bytes = -wqe->length & 3; nwords = ((wqe->length + extra_bytes) >> 2) + SIZE_OF_CRC; /* header size in dwords LRH+BTH+DETH = (8+12+8)/4. */ - qp->s_hdrwords = 7; + ps->s_txreq->hdr_dwords = 7; if (wqe->wr.opcode == IB_WR_SEND_WITH_IMM) - qp->s_hdrwords++; + ps->s_txreq->hdr_dwords++; if (rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH) { grh = &ps->s_txreq->phdr.hdr.ibh.u.l.grh; - qp->s_hdrwords += hfi1_make_grh(ibp, grh, - rdma_ah_read_grh(ah_attr), - qp->s_hdrwords - 2, nwords); + ps->s_txreq->hdr_dwords += + hfi1_make_grh(ibp, grh, rdma_ah_read_grh(ah_attr), + ps->s_txreq->hdr_dwords - LRH_9B_DWORDS, + nwords); lrh0 = HFI1_LRH_GRH; ohdr = &ps->s_txreq->phdr.hdr.ibh.u.l.oth; } else { @@ -381,7 +382,7 @@ void hfi1_make_ud_req_9B(struct rvt_qp *qp, struct hfi1_pkt_state *ps, } } hfi1_make_bth_deth(qp, wqe, ohdr, &pkey, extra_bytes, false); - len = qp->s_hdrwords + nwords; + len = ps->s_txreq->hdr_dwords + nwords; /* Setup the packet */ ps->s_txreq->phdr.hdr.hdr_type = HFI1_PKT_TYPE_9B; @@ -405,12 +406,12 @@ void hfi1_make_ud_req_16B(struct rvt_qp *qp, struct hfi1_pkt_state *ps, ppd = ppd_from_ibp(ibp); ah_attr = &ibah_to_rvtah(wqe->ud_wr.ah)->attr; /* header size in dwords 16B LRH+BTH+DETH = (16+12+8)/4. */ - qp->s_hdrwords = 9; + ps->s_txreq->hdr_dwords = 9; if (wqe->wr.opcode == IB_WR_SEND_WITH_IMM) - qp->s_hdrwords++; + ps->s_txreq->hdr_dwords++; /* SW provides space for CRC and LT for bypass packets. */ - extra_bytes = hfi1_get_16b_padding((qp->s_hdrwords << 2), + extra_bytes = hfi1_get_16b_padding((ps->s_txreq->hdr_dwords << 2), wqe->length); nwords = ((wqe->length + extra_bytes + SIZE_OF_LT) >> 2) + SIZE_OF_CRC; @@ -428,8 +429,10 @@ void hfi1_make_ud_req_16B(struct rvt_qp *qp, struct hfi1_pkt_state *ps, grd->sgid_index = 0; } grh = &ps->s_txreq->phdr.hdr.opah.u.l.grh; - qp->s_hdrwords += hfi1_make_grh(ibp, grh, grd, - qp->s_hdrwords - 4, nwords); + ps->s_txreq->hdr_dwords += hfi1_make_grh( + ibp, grh, grd, + ps->s_txreq->hdr_dwords - LRH_16B_DWORDS, + nwords); ohdr = &ps->s_txreq->phdr.hdr.opah.u.l.oth; l4 = OPA_16B_L4_IB_GLOBAL; } else { @@ -452,7 +455,7 @@ void hfi1_make_ud_req_16B(struct rvt_qp *qp, struct hfi1_pkt_state *ps, hfi1_make_bth_deth(qp, wqe, ohdr, &pkey, extra_bytes, true); /* Convert dwords to flits */ - len = (qp->s_hdrwords + nwords) >> 1; + len = (ps->s_txreq->hdr_dwords + nwords) >> 1; /* Setup the packet */ ps->s_txreq->phdr.hdr.hdr_type = HFI1_PKT_TYPE_16B; @@ -562,8 +565,6 @@ int hfi1_make_ud_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps) priv->s_ahg->ahgcount = 0; priv->s_ahg->ahgidx = 0; priv->s_ahg->tx_flags = 0; - /* pbc */ - ps->s_txreq->hdr_dwords = qp->s_hdrwords + 2; return 1; @@ -578,7 +579,6 @@ bail: bail_no_tx: ps->s_txreq = NULL; qp->s_flags &= ~RVT_S_BUSY; - qp->s_hdrwords = 0; return 0; } @@ -649,7 +649,8 @@ void return_cnp_16B(struct hfi1_ibport *ibp, struct rvt_qp *qp, struct ib_grh *grh = &hdr.u.l.grh; grh->version_tclass_flow = old_grh->version_tclass_flow; - grh->paylen = cpu_to_be16((hwords - 4 + nwords) << 2); + grh->paylen = cpu_to_be16( + (hwords - LRH_16B_DWORDS + nwords) << 2); grh->hop_limit = 0xff; grh->sgid = old_grh->dgid; grh->dgid = old_grh->sgid; @@ -703,7 +704,8 @@ void return_cnp(struct hfi1_ibport *ibp, struct rvt_qp *qp, u32 remote_qpn, struct ib_grh *grh = &hdr.u.l.grh; grh->version_tclass_flow = old_grh->version_tclass_flow; - grh->paylen = cpu_to_be16((hwords - 2 + SIZE_OF_CRC) << 2); + grh->paylen = cpu_to_be16( + (hwords - LRH_9B_DWORDS + SIZE_OF_CRC) << 2); grh->hop_limit = 0xff; grh->sgid = old_grh->dgid; grh->dgid = old_grh->sgid; @@ -1046,8 +1048,7 @@ void hfi1_ud_rcv(struct hfi1_packet *packet) wc.port_num = qp->port_num; /* Signal completion event if the solicited bit is set. */ rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, - (ohdr->bth[0] & - cpu_to_be32(IB_BTH_SOLICITED)) != 0); + ib_bth_is_solicited(ohdr)); return; drop: diff --git a/drivers/infiniband/hw/hfi1/verbs.c b/drivers/infiniband/hw/hfi1/verbs.c index b8776a362a91..471d55c50066 100644 --- a/drivers/infiniband/hw/hfi1/verbs.c +++ b/drivers/infiniband/hw/hfi1/verbs.c @@ -835,7 +835,7 @@ static int build_verbs_tx_desc( { int ret = 0; struct hfi1_sdma_header *phdr = &tx->phdr; - u16 hdrbytes = tx->hdr_dwords << 2; + u16 hdrbytes = (tx->hdr_dwords + sizeof(pbc) / 4) << 2; u8 extra_bytes = 0; if (tx->phdr.hdr.hdr_type) { @@ -901,7 +901,7 @@ int hfi1_verbs_send_dma(struct rvt_qp *qp, struct hfi1_pkt_state *ps, { struct hfi1_qp_priv *priv = qp->priv; struct hfi1_ahg_info *ahg_info = priv->s_ahg; - u32 hdrwords = qp->s_hdrwords; + u32 hdrwords = ps->s_txreq->hdr_dwords; u32 len = ps->s_txreq->s_cur_size; u32 plen; struct hfi1_ibdev *dev = ps->dev; @@ -919,7 +919,7 @@ int hfi1_verbs_send_dma(struct rvt_qp *qp, struct hfi1_pkt_state *ps, } else { dwords = (len + 3) >> 2; } - plen = hdrwords + dwords + 2; + plen = hdrwords + dwords + sizeof(pbc) / 4; tx = ps->s_txreq; if (!sdma_txreq_built(&tx->txreq)) { @@ -1038,7 +1038,7 @@ int hfi1_verbs_send_pio(struct rvt_qp *qp, struct hfi1_pkt_state *ps, u64 pbc) { struct hfi1_qp_priv *priv = qp->priv; - u32 hdrwords = qp->s_hdrwords; + u32 hdrwords = ps->s_txreq->hdr_dwords; struct rvt_sge_state *ss = ps->s_txreq->ss; u32 len = ps->s_txreq->s_cur_size; u32 dwords; @@ -1064,7 +1064,7 @@ int hfi1_verbs_send_pio(struct rvt_qp *qp, struct hfi1_pkt_state *ps, dwords = (len + 3) >> 2; hdr = (u32 *)&ps->s_txreq->phdr.hdr.ibh; } - plen = hdrwords + dwords + 2; + plen = hdrwords + dwords + sizeof(pbc) / 4; /* only RC/UC use complete */ switch (qp->ibqp.qp_type) { diff --git a/drivers/infiniband/hw/hfi1/verbs.h b/drivers/infiniband/hw/hfi1/verbs.h index 87d1285a3340..2d787b8346ca 100644 --- a/drivers/infiniband/hw/hfi1/verbs.h +++ b/drivers/infiniband/hw/hfi1/verbs.h @@ -105,6 +105,11 @@ enum { HFI1_HAS_GRH = (1 << 0), }; +#define LRH_16B_BYTES (FIELD_SIZEOF(struct hfi1_16b_header, lrh)) +#define LRH_16B_DWORDS (LRH_16B_BYTES / sizeof(u32)) +#define LRH_9B_BYTES (FIELD_SIZEOF(struct ib_header, lrh)) +#define LRH_9B_DWORDS (LRH_9B_BYTES / sizeof(u32)) + struct hfi1_16b_header { u32 lrh[4]; union { @@ -246,17 +251,6 @@ static inline struct rvt_qp *iowait_to_qp(struct iowait *s_iowait) } /* - * Send if not busy or waiting for I/O and either - * a RC response is pending or we can process send work requests. - */ -static inline int hfi1_send_ok(struct rvt_qp *qp) -{ - return !(qp->s_flags & (RVT_S_BUSY | RVT_S_ANY_WAIT_IO)) && - (qp->s_hdrwords || (qp->s_flags & RVT_S_RESP_PENDING) || - !(qp->s_flags & RVT_S_ANY_WAIT_SEND)); -} - -/* * This must be called with s_lock held. */ void hfi1_bad_pkey(struct hfi1_ibport *ibp, u32 key, u32 sl, @@ -369,8 +363,7 @@ void hfi1_do_send(struct rvt_qp *qp, bool in_thread); void hfi1_send_complete(struct rvt_qp *qp, struct rvt_swqe *wqe, enum ib_wc_status status); -void hfi1_send_rc_ack(struct hfi1_ctxtdata *rcd, struct rvt_qp *qp, - bool is_fecn); +void hfi1_send_rc_ack(struct hfi1_packet *packet, bool is_fecn); int hfi1_make_rc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps); @@ -416,6 +409,11 @@ static inline void cacheless_memcpy(void *dst, void *src, size_t n) __copy_user_nocache(dst, (void __user *)src, n, 0); } +static inline bool opa_bth_is_migration(struct ib_other_headers *ohdr) +{ + return ohdr->bth[1] & cpu_to_be32(OPA_BTH_MIG_REQ); +} + extern const enum ib_wc_opcode ib_hfi1_wc_opcode[]; extern const u8 hdr_len_by_opcode[]; diff --git a/drivers/infiniband/hw/hfi1/verbs_txreq.h b/drivers/infiniband/hw/hfi1/verbs_txreq.h index cec7a4b34d16..729244c3086c 100644 --- a/drivers/infiniband/hw/hfi1/verbs_txreq.h +++ b/drivers/infiniband/hw/hfi1/verbs_txreq.h @@ -113,6 +113,13 @@ static inline struct verbs_txreq *get_waiting_verbs_txreq(struct rvt_qp *qp) return NULL; } +static inline bool verbs_txreq_queued(struct rvt_qp *qp) +{ + struct hfi1_qp_priv *priv = qp->priv; + + return iowait_packet_queued(&priv->s_iowait); +} + void hfi1_put_txreq(struct verbs_txreq *tx); int verbs_txreq_init(struct hfi1_ibdev *dev); void verbs_txreq_exit(struct hfi1_ibdev *dev); diff --git a/drivers/infiniband/hw/hns/hns_roce_common.h b/drivers/infiniband/hw/hns/hns_roce_common.h index dd67fafd0c40..319cb74aebaf 100644 --- a/drivers/infiniband/hw/hns/hns_roce_common.h +++ b/drivers/infiniband/hw/hns/hns_roce_common.h @@ -43,15 +43,15 @@ __raw_writel((__force u32)cpu_to_le32(value), (addr)) #define roce_get_field(origin, mask, shift) \ - (((origin) & (mask)) >> (shift)) + (((le32_to_cpu(origin)) & (mask)) >> (shift)) #define roce_get_bit(origin, shift) \ roce_get_field((origin), (1ul << (shift)), (shift)) #define roce_set_field(origin, mask, shift, val) \ do { \ - (origin) &= (~(mask)); \ - (origin) |= (((u32)(val) << (shift)) & (mask)); \ + (origin) &= ~cpu_to_le32(mask); \ + (origin) |= cpu_to_le32(((u32)(val) << (shift)) & (mask)); \ } while (0) #define roce_set_bit(origin, shift, val) \ diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h index 42c3b5a2d441..165a09b314f6 100644 --- a/drivers/infiniband/hw/hns/hns_roce_device.h +++ b/drivers/infiniband/hw/hns/hns_roce_device.h @@ -345,8 +345,8 @@ struct hns_roce_cq { struct hns_roce_cq_buf hr_buf; spinlock_t lock; struct ib_umem *umem; - void (*comp)(struct hns_roce_cq *); - void (*event)(struct hns_roce_cq *, enum hns_roce_event); + void (*comp)(struct hns_roce_cq *cq); + void (*event)(struct hns_roce_cq *cq, enum hns_roce_event event_type); struct hns_roce_uar *uar; u32 cq_depth; @@ -466,7 +466,7 @@ struct hns_roce_qp { struct ib_qp ibqp; struct hns_roce_buf hr_buf; struct hns_roce_wq rq; - __le64 doorbell_qpn; + u32 doorbell_qpn; __le32 sq_signal_bits; u32 sq_next_wqe; int sq_max_wqes_per_wr; @@ -486,8 +486,8 @@ struct hns_roce_qp { u32 atomic_rd_en; u32 pkey_index; u32 qkey; - void (*event)(struct hns_roce_qp *, - enum hns_roce_event); + void (*event)(struct hns_roce_qp *qp, + enum hns_roce_event event_type); unsigned long qpn; atomic_t refcount; diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c index 21ca9fa7c9d1..da13bd7c3ca9 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c @@ -195,23 +195,47 @@ static int hns_roce_v1_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, memcpy(&ud_sq_wqe->dgid[0], &ah->av.dgid[0], GID_LEN); - ud_sq_wqe->va0_l = (u32)wr->sg_list[0].addr; - ud_sq_wqe->va0_h = (wr->sg_list[0].addr) >> 32; - ud_sq_wqe->l_key0 = wr->sg_list[0].lkey; - - ud_sq_wqe->va1_l = (u32)wr->sg_list[1].addr; - ud_sq_wqe->va1_h = (wr->sg_list[1].addr) >> 32; - ud_sq_wqe->l_key1 = wr->sg_list[1].lkey; + ud_sq_wqe->va0_l = + cpu_to_le32((u32)wr->sg_list[0].addr); + ud_sq_wqe->va0_h = + cpu_to_le32((wr->sg_list[0].addr) >> 32); + ud_sq_wqe->l_key0 = + cpu_to_le32(wr->sg_list[0].lkey); + + ud_sq_wqe->va1_l = + cpu_to_le32((u32)wr->sg_list[1].addr); + ud_sq_wqe->va1_h = + cpu_to_le32((wr->sg_list[1].addr) >> 32); + ud_sq_wqe->l_key1 = + cpu_to_le32(wr->sg_list[1].lkey); ind++; } else if (ibqp->qp_type == IB_QPT_RC) { + u32 tmp_len = 0; + ctrl = wqe; memset(ctrl, 0, sizeof(struct hns_roce_wqe_ctrl_seg)); for (i = 0; i < wr->num_sge; i++) - ctrl->msg_length += wr->sg_list[i].length; + tmp_len += wr->sg_list[i].length; + + ctrl->msg_length = + cpu_to_le32(le32_to_cpu(ctrl->msg_length) + tmp_len); ctrl->sgl_pa_h = 0; ctrl->flag = 0; - ctrl->imm_data = send_ieth(wr); + + switch (wr->opcode) { + case IB_WR_SEND_WITH_IMM: + case IB_WR_RDMA_WRITE_WITH_IMM: + ctrl->imm_data = wr->ex.imm_data; + break; + case IB_WR_SEND_WITH_INV: + ctrl->inv_key = + cpu_to_le32(wr->ex.invalidate_rkey); + break; + default: + ctrl->imm_data = 0; + break; + } /*Ctrl field, ctrl set type: sig, solic, imm, fence */ /* SO wait for conforming application scenarios */ @@ -258,8 +282,8 @@ static int hns_roce_v1_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, dseg = wqe; if (wr->send_flags & IB_SEND_INLINE && wr->num_sge) { - if (ctrl->msg_length > - hr_dev->caps.max_sq_inline) { + if (le32_to_cpu(ctrl->msg_length) > + hr_dev->caps.max_sq_inline) { ret = -EINVAL; *bad_wr = wr; dev_err(dev, "inline len(1-%d)=%d, illegal", @@ -273,7 +297,7 @@ static int hns_roce_v1_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, wr->sg_list[i].length); wqe += wr->sg_list[i].length; } - ctrl->flag |= HNS_ROCE_WQE_INLINE; + ctrl->flag |= cpu_to_le32(HNS_ROCE_WQE_INLINE); } else { /*sqe num is two */ for (i = 0; i < wr->num_sge; i++) @@ -306,8 +330,8 @@ out: SQ_DOORBELL_U32_8_QPN_S, qp->doorbell_qpn); roce_set_bit(sq_db.u32_8, SQ_DOORBELL_HW_SYNC_S, 1); - doorbell[0] = sq_db.u32_4; - doorbell[1] = sq_db.u32_8; + doorbell[0] = le32_to_cpu(sq_db.u32_4); + doorbell[1] = le32_to_cpu(sq_db.u32_8); hns_roce_write64_k(doorbell, qp->sq.db_reg_l); qp->sq_next_wqe = ind; @@ -403,8 +427,8 @@ out: roce_set_bit(rq_db.u32_8, RQ_DOORBELL_U32_8_HW_SYNC_S, 1); - doorbell[0] = rq_db.u32_4; - doorbell[1] = rq_db.u32_8; + doorbell[0] = le32_to_cpu(rq_db.u32_4); + doorbell[1] = le32_to_cpu(rq_db.u32_8); hns_roce_write64_k(doorbell, hr_qp->rq.db_reg_l); } @@ -2261,7 +2285,7 @@ static int hns_roce_v1_poll_one(struct hns_roce_cq *hr_cq, CQE_BYTE_4_WQE_INDEX_M, CQE_BYTE_4_WQE_INDEX_S)& ((*cur_qp)->sq.wqe_cnt-1)); - switch (sq_wqe->flag & HNS_ROCE_WQE_OPCODE_MASK) { + switch (le32_to_cpu(sq_wqe->flag) & HNS_ROCE_WQE_OPCODE_MASK) { case HNS_ROCE_WQE_OPCODE_SEND: wc->opcode = IB_WC_SEND; break; @@ -2282,7 +2306,7 @@ static int hns_roce_v1_poll_one(struct hns_roce_cq *hr_cq, wc->status = IB_WC_GENERAL_ERR; break; } - wc->wc_flags = (sq_wqe->flag & HNS_ROCE_WQE_IMM ? + wc->wc_flags = (le32_to_cpu(sq_wqe->flag) & HNS_ROCE_WQE_IMM ? IB_WC_WITH_IMM : 0); wq = &(*cur_qp)->sq; diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v1.h b/drivers/infiniband/hw/hns/hns_roce_hw_v1.h index b44ddd239060..e9a2717ea7cd 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v1.h +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v1.h @@ -200,14 +200,14 @@ #define HNS_ROCE_AEQE_EVENT_CE_EVENT_CEQE_CEQN_M GENMASK(4, 0) struct hns_roce_cq_context { - u32 cqc_byte_4; - u32 cq_bt_l; - u32 cqc_byte_12; - u32 cur_cqe_ba0_l; - u32 cqc_byte_20; - u32 cqe_tptr_addr_l; - u32 cur_cqe_ba1_l; - u32 cqc_byte_32; + __le32 cqc_byte_4; + __le32 cq_bt_l; + __le32 cqc_byte_12; + __le32 cur_cqe_ba0_l; + __le32 cqc_byte_20; + __le32 cqe_tptr_addr_l; + __le32 cur_cqe_ba1_l; + __le32 cqc_byte_32; }; #define CQ_CONTEXT_CQC_BYTE_4_CQC_STATE_S 0 @@ -257,17 +257,17 @@ struct hns_roce_cq_context { (((1UL << 16) - 1) << CQ_CONTEXT_CQC_BYTE_32_CQ_CONS_IDX_S) struct hns_roce_cqe { - u32 cqe_byte_4; + __le32 cqe_byte_4; union { - u32 r_key; - u32 immediate_data; + __le32 r_key; + __be32 immediate_data; }; - u32 byte_cnt; - u32 cqe_byte_16; - u32 cqe_byte_20; - u32 s_mac_l; - u32 cqe_byte_28; - u32 reserved; + __le32 byte_cnt; + __le32 cqe_byte_16; + __le32 cqe_byte_20; + __le32 s_mac_l; + __le32 cqe_byte_28; + __le32 reserved; }; #define CQE_BYTE_4_OWNER_S 7 @@ -308,22 +308,22 @@ struct hns_roce_cqe { #define CQ_DB_REQ_NOT (1 << 16) struct hns_roce_v1_mpt_entry { - u32 mpt_byte_4; - u32 pbl_addr_l; - u32 mpt_byte_12; - u32 virt_addr_l; - u32 virt_addr_h; - u32 length; - u32 mpt_byte_28; - u32 pa0_l; - u32 mpt_byte_36; - u32 mpt_byte_40; - u32 mpt_byte_44; - u32 mpt_byte_48; - u32 pa4_l; - u32 mpt_byte_56; - u32 mpt_byte_60; - u32 mpt_byte_64; + __le32 mpt_byte_4; + __le32 pbl_addr_l; + __le32 mpt_byte_12; + __le32 virt_addr_l; + __le32 virt_addr_h; + __le32 length; + __le32 mpt_byte_28; + __le32 pa0_l; + __le32 mpt_byte_36; + __le32 mpt_byte_40; + __le32 mpt_byte_44; + __le32 mpt_byte_48; + __le32 pa4_l; + __le32 mpt_byte_56; + __le32 mpt_byte_60; + __le32 mpt_byte_64; }; #define MPT_BYTE_4_KEY_STATE_S 0 @@ -408,30 +408,32 @@ struct hns_roce_v1_mpt_entry { (((1UL << 8) - 1) << MPT_BYTE_64_L_KEY_IDX_H_S) struct hns_roce_wqe_ctrl_seg { - __be32 sgl_pa_h; - __be32 flag; - __be32 imm_data; - __be32 msg_length; + __le32 sgl_pa_h; + __le32 flag; + union { + __be32 imm_data; + __le32 inv_key; + }; + __le32 msg_length; }; struct hns_roce_wqe_data_seg { - __be64 addr; - __be32 lkey; - __be32 len; + __le64 addr; + __le32 lkey; + __le32 len; }; struct hns_roce_wqe_raddr_seg { - __be32 rkey; - __be32 len;/* reserved */ - __be64 raddr; + __le32 rkey; + __le32 len;/* reserved */ + __le64 raddr; }; struct hns_roce_rq_wqe_ctrl { - - u32 rwqe_byte_4; - u32 rocee_sgl_ba_l; - u32 rwqe_byte_12; - u32 reserved[5]; + __le32 rwqe_byte_4; + __le32 rocee_sgl_ba_l; + __le32 rwqe_byte_12; + __le32 reserved[5]; }; #define RQ_WQE_CTRL_RWQE_BYTE_12_RWQE_SGE_NUM_S 16 @@ -443,31 +445,31 @@ struct hns_roce_rq_wqe_ctrl { #define GID_LEN 16 struct hns_roce_ud_send_wqe { - u32 dmac_h; - u32 u32_8; - u32 immediate_data; + __le32 dmac_h; + __le32 u32_8; + __le32 immediate_data; - u32 u32_16; + __le32 u32_16; union { unsigned char dgid[GID_LEN]; struct { - u32 u32_20; - u32 u32_24; - u32 u32_28; - u32 u32_32; + __le32 u32_20; + __le32 u32_24; + __le32 u32_28; + __le32 u32_32; }; }; - u32 u32_36; - u32 u32_40; + __le32 u32_36; + __le32 u32_40; - u32 va0_l; - u32 va0_h; - u32 l_key0; + __le32 va0_l; + __le32 va0_h; + __le32 l_key0; - u32 va1_l; - u32 va1_h; - u32 l_key1; + __le32 va1_l; + __le32 va1_h; + __le32 l_key1; }; #define UD_SEND_WQE_U32_4_DMAC_0_S 0 @@ -535,16 +537,16 @@ struct hns_roce_ud_send_wqe { (((1UL << 8) - 1) << UD_SEND_WQE_U32_40_TRAFFIC_CLASS_S) struct hns_roce_sqp_context { - u32 qp1c_bytes_4; - u32 sq_rq_bt_l; - u32 qp1c_bytes_12; - u32 qp1c_bytes_16; - u32 qp1c_bytes_20; - u32 cur_rq_wqe_ba_l; - u32 qp1c_bytes_28; - u32 qp1c_bytes_32; - u32 cur_sq_wqe_ba_l; - u32 qp1c_bytes_40; + __le32 qp1c_bytes_4; + __le32 sq_rq_bt_l; + __le32 qp1c_bytes_12; + __le32 qp1c_bytes_16; + __le32 qp1c_bytes_20; + __le32 cur_rq_wqe_ba_l; + __le32 qp1c_bytes_28; + __le32 qp1c_bytes_32; + __le32 cur_sq_wqe_ba_l; + __le32 qp1c_bytes_40; }; #define QP1C_BYTES_4_QP_STATE_S 0 @@ -626,64 +628,64 @@ struct hns_roce_sqp_context { #define HNS_ROCE_WQE_OPCODE_MASK (15<<16) struct hns_roce_qp_context { - u32 qpc_bytes_4; - u32 qpc_bytes_8; - u32 qpc_bytes_12; - u32 qpc_bytes_16; - u32 sq_rq_bt_l; - u32 qpc_bytes_24; - u32 irrl_ba_l; - u32 qpc_bytes_32; - u32 qpc_bytes_36; - u32 dmac_l; - u32 qpc_bytes_44; - u32 qpc_bytes_48; - u8 dgid[16]; - u32 qpc_bytes_68; - u32 cur_rq_wqe_ba_l; - u32 qpc_bytes_76; - u32 rx_rnr_time; - u32 qpc_bytes_84; - u32 qpc_bytes_88; + __le32 qpc_bytes_4; + __le32 qpc_bytes_8; + __le32 qpc_bytes_12; + __le32 qpc_bytes_16; + __le32 sq_rq_bt_l; + __le32 qpc_bytes_24; + __le32 irrl_ba_l; + __le32 qpc_bytes_32; + __le32 qpc_bytes_36; + __le32 dmac_l; + __le32 qpc_bytes_44; + __le32 qpc_bytes_48; + u8 dgid[16]; + __le32 qpc_bytes_68; + __le32 cur_rq_wqe_ba_l; + __le32 qpc_bytes_76; + __le32 rx_rnr_time; + __le32 qpc_bytes_84; + __le32 qpc_bytes_88; union { - u32 rx_sge_len; - u32 dma_length; + __le32 rx_sge_len; + __le32 dma_length; }; union { - u32 rx_sge_num; - u32 rx_send_pktn; - u32 r_key; + __le32 rx_sge_num; + __le32 rx_send_pktn; + __le32 r_key; }; - u32 va_l; - u32 va_h; - u32 qpc_bytes_108; - u32 qpc_bytes_112; - u32 rx_cur_sq_wqe_ba_l; - u32 qpc_bytes_120; - u32 qpc_bytes_124; - u32 qpc_bytes_128; - u32 qpc_bytes_132; - u32 qpc_bytes_136; - u32 qpc_bytes_140; - u32 qpc_bytes_144; - u32 qpc_bytes_148; + __le32 va_l; + __le32 va_h; + __le32 qpc_bytes_108; + __le32 qpc_bytes_112; + __le32 rx_cur_sq_wqe_ba_l; + __le32 qpc_bytes_120; + __le32 qpc_bytes_124; + __le32 qpc_bytes_128; + __le32 qpc_bytes_132; + __le32 qpc_bytes_136; + __le32 qpc_bytes_140; + __le32 qpc_bytes_144; + __le32 qpc_bytes_148; union { - u32 rnr_retry; - u32 ack_time; + __le32 rnr_retry; + __le32 ack_time; }; - u32 qpc_bytes_156; - u32 pkt_use_len; - u32 qpc_bytes_164; - u32 qpc_bytes_168; + __le32 qpc_bytes_156; + __le32 pkt_use_len; + __le32 qpc_bytes_164; + __le32 qpc_bytes_168; union { - u32 sge_use_len; - u32 pa_use_len; + __le32 sge_use_len; + __le32 pa_use_len; }; - u32 qpc_bytes_176; - u32 qpc_bytes_180; - u32 tx_cur_sq_wqe_ba_l; - u32 qpc_bytes_188; - u32 rvd21; + __le32 qpc_bytes_176; + __le32 qpc_bytes_180; + __le32 tx_cur_sq_wqe_ba_l; + __le32 qpc_bytes_188; + __le32 rvd21; }; #define QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_S 0 @@ -996,8 +998,8 @@ struct hns_roce_qp_context { #define HCR_GO_BIT 15 struct hns_roce_rq_db { - u32 u32_4; - u32 u32_8; + __le32 u32_4; + __le32 u32_8; }; #define RQ_DOORBELL_U32_4_RQ_HEAD_S 0 @@ -1013,8 +1015,8 @@ struct hns_roce_rq_db { #define RQ_DOORBELL_U32_8_HW_SYNC_S 31 struct hns_roce_sq_db { - u32 u32_4; - u32 u32_8; + __le32 u32_4; + __le32 u32_8; }; #define SQ_DOORBELL_U32_4_SQ_HEAD_S 0 diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index 256fe110107a..db2ff352d75f 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -63,7 +63,8 @@ static int set_rwqe_data_seg(struct ib_qp *ibqp, struct ib_send_wr *wr, int i; if (wr->send_flags & IB_SEND_INLINE && wr->num_sge) { - if (rc_sq_wqe->msg_len > hr_dev->caps.max_sq_inline) { + if (le32_to_cpu(rc_sq_wqe->msg_len) > + hr_dev->caps.max_sq_inline) { *bad_wr = wr; dev_err(hr_dev->dev, "inline len(1-%d)=%d, illegal", rc_sq_wqe->msg_len, hr_dev->caps.max_sq_inline); @@ -136,6 +137,7 @@ static int hns_roce_v2_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, unsigned long flags; unsigned int ind; void *wqe = NULL; + u32 tmp_len = 0; bool loopback; int ret = 0; u8 *smac; @@ -218,9 +220,20 @@ static int hns_roce_v2_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, HNS_ROCE_V2_WQE_OP_SEND); for (i = 0; i < wr->num_sge; i++) - ud_sq_wqe->msg_len += wr->sg_list[i].length; + tmp_len += wr->sg_list[i].length; - ud_sq_wqe->immtdata = send_ieth(wr); + ud_sq_wqe->msg_len = + cpu_to_le32(le32_to_cpu(ud_sq_wqe->msg_len) + tmp_len); + + switch (wr->opcode) { + case IB_WR_SEND_WITH_IMM: + case IB_WR_RDMA_WRITE_WITH_IMM: + ud_sq_wqe->immtdata = wr->ex.imm_data; + break; + default: + ud_sq_wqe->immtdata = 0; + break; + } /* Set sig attr */ roce_set_bit(ud_sq_wqe->byte_4, @@ -254,8 +267,8 @@ static int hns_roce_v2_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, V2_UD_SEND_WQE_BYTE_24_UDPSPN_M, V2_UD_SEND_WQE_BYTE_24_UDPSPN_S, 0); ud_sq_wqe->qkey = - cpu_to_be32(ud_wr(wr)->remote_qkey & 0x80000000) ? - qp->qkey : ud_wr(wr)->remote_qkey; + cpu_to_le32(ud_wr(wr)->remote_qkey & 0x80000000 ? + qp->qkey : ud_wr(wr)->remote_qkey); roce_set_field(ud_sq_wqe->byte_32, V2_UD_SEND_WQE_BYTE_32_DQPN_M, V2_UD_SEND_WQE_BYTE_32_DQPN_S, @@ -264,7 +277,7 @@ static int hns_roce_v2_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, roce_set_field(ud_sq_wqe->byte_36, V2_UD_SEND_WQE_BYTE_36_VLAN_M, V2_UD_SEND_WQE_BYTE_36_VLAN_S, - ah->av.vlan); + le16_to_cpu(ah->av.vlan)); roce_set_field(ud_sq_wqe->byte_36, V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_M, V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_S, @@ -283,8 +296,8 @@ static int hns_roce_v2_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, roce_set_field(ud_sq_wqe->byte_40, V2_UD_SEND_WQE_BYTE_40_SL_M, V2_UD_SEND_WQE_BYTE_40_SL_S, - ah->av.sl_tclass_flowlabel >> - HNS_ROCE_SL_SHIFT); + le32_to_cpu(ah->av.sl_tclass_flowlabel) >> + HNS_ROCE_SL_SHIFT); roce_set_field(ud_sq_wqe->byte_40, V2_UD_SEND_WQE_BYTE_40_PORTN_M, V2_UD_SEND_WQE_BYTE_40_PORTN_S, @@ -311,9 +324,24 @@ static int hns_roce_v2_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, rc_sq_wqe = wqe; memset(rc_sq_wqe, 0, sizeof(*rc_sq_wqe)); for (i = 0; i < wr->num_sge; i++) - rc_sq_wqe->msg_len += wr->sg_list[i].length; + tmp_len += wr->sg_list[i].length; + + rc_sq_wqe->msg_len = + cpu_to_le32(le32_to_cpu(rc_sq_wqe->msg_len) + tmp_len); - rc_sq_wqe->inv_key_immtdata = send_ieth(wr); + switch (wr->opcode) { + case IB_WR_SEND_WITH_IMM: + case IB_WR_RDMA_WRITE_WITH_IMM: + rc_sq_wqe->immtdata = wr->ex.imm_data; + break; + case IB_WR_SEND_WITH_INV: + rc_sq_wqe->inv_key = + cpu_to_le32(wr->ex.invalidate_rkey); + break; + default: + rc_sq_wqe->immtdata = 0; + break; + } roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_FENCE_S, @@ -451,7 +479,7 @@ out: roce_set_field(sq_db.parameter, V2_DB_PARAMETER_SL_M, V2_DB_PARAMETER_SL_S, qp->sl); - hns_roce_write64_k((__be32 *)&sq_db, qp->sq.db_reg_l); + hns_roce_write64_k((__le32 *)&sq_db, qp->sq.db_reg_l); qp->sq_next_wqe = ind; qp->next_sge = sge_ind; @@ -513,7 +541,7 @@ static int hns_roce_v2_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr, } if (i < hr_qp->rq.max_gs) { - dseg[i].lkey = cpu_to_be32(HNS_ROCE_INVALID_LKEY); + dseg[i].lkey = cpu_to_le32(HNS_ROCE_INVALID_LKEY); dseg[i].addr = 0; } @@ -546,7 +574,7 @@ out: roce_set_field(rq_db.parameter, V2_DB_PARAMETER_CONS_IDX_M, V2_DB_PARAMETER_CONS_IDX_S, hr_qp->rq.head); - hns_roce_write64_k((__be32 *)&rq_db, hr_qp->rq.db_reg_l); + hns_roce_write64_k((__le32 *)&rq_db, hr_qp->rq.db_reg_l); } spin_unlock_irqrestore(&hr_qp->rq.lock, flags); @@ -2123,10 +2151,10 @@ static void set_access_flags(struct hns_roce_qp *hr_qp, u8 dest_rd_atomic; u32 access_flags; - dest_rd_atomic = !!(attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) ? + dest_rd_atomic = (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) ? attr->max_dest_rd_atomic : hr_qp->resp_depth; - access_flags = !!(attr_mask & IB_QP_ACCESS_FLAGS) ? + access_flags = (attr_mask & IB_QP_ACCESS_FLAGS) ? attr->qp_access_flags : hr_qp->atomic_rd_en; if (!dest_rd_atomic) diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h index 960df095392a..2bf8a47e3de3 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h @@ -224,22 +224,22 @@ enum hns_roce_sgid_type { }; struct hns_roce_v2_cq_context { - u32 byte_4_pg_ceqn; - u32 byte_8_cqn; - u32 cqe_cur_blk_addr; - u32 byte_16_hop_addr; - u32 cqe_nxt_blk_addr; - u32 byte_24_pgsz_addr; - u32 byte_28_cq_pi; - u32 byte_32_cq_ci; - u32 cqe_ba; - u32 byte_40_cqe_ba; - u32 byte_44_db_record; - u32 db_record_addr; - u32 byte_52_cqe_cnt; - u32 byte_56_cqe_period_maxcnt; - u32 cqe_report_timer; - u32 byte_64_se_cqe_idx; + __le32 byte_4_pg_ceqn; + __le32 byte_8_cqn; + __le32 cqe_cur_blk_addr; + __le32 byte_16_hop_addr; + __le32 cqe_nxt_blk_addr; + __le32 byte_24_pgsz_addr; + __le32 byte_28_cq_pi; + __le32 byte_32_cq_ci; + __le32 cqe_ba; + __le32 byte_40_cqe_ba; + __le32 byte_44_db_record; + __le32 db_record_addr; + __le32 byte_52_cqe_cnt; + __le32 byte_56_cqe_period_maxcnt; + __le32 cqe_report_timer; + __le32 byte_64_se_cqe_idx; }; #define HNS_ROCE_V2_CQ_DEFAULT_BURST_NUM 0x0 #define HNS_ROCE_V2_CQ_DEFAULT_INTERVAL 0x0 @@ -328,66 +328,66 @@ enum hns_roce_v2_qp_state { }; struct hns_roce_v2_qp_context { - u32 byte_4_sqpn_tst; - u32 wqe_sge_ba; - u32 byte_12_sq_hop; - u32 byte_16_buf_ba_pg_sz; - u32 byte_20_smac_sgid_idx; - u32 byte_24_mtu_tc; - u32 byte_28_at_fl; + __le32 byte_4_sqpn_tst; + __le32 wqe_sge_ba; + __le32 byte_12_sq_hop; + __le32 byte_16_buf_ba_pg_sz; + __le32 byte_20_smac_sgid_idx; + __le32 byte_24_mtu_tc; + __le32 byte_28_at_fl; u8 dgid[GID_LEN_V2]; - u32 dmac; - u32 byte_52_udpspn_dmac; - u32 byte_56_dqpn_err; - u32 byte_60_qpst_mapid; - u32 qkey_xrcd; - u32 byte_68_rq_db; - u32 rq_db_record_addr; - u32 byte_76_srqn_op_en; - u32 byte_80_rnr_rx_cqn; - u32 byte_84_rq_ci_pi; - u32 rq_cur_blk_addr; - u32 byte_92_srq_info; - u32 byte_96_rx_reqmsn; - u32 rq_nxt_blk_addr; - u32 byte_104_rq_sge; - u32 byte_108_rx_reqepsn; - u32 rq_rnr_timer; - u32 rx_msg_len; - u32 rx_rkey_pkt_info; - u64 rx_va; - u32 byte_132_trrl; - u32 trrl_ba; - u32 byte_140_raq; - u32 byte_144_raq; - u32 byte_148_raq; - u32 byte_152_raq; - u32 byte_156_raq; - u32 byte_160_sq_ci_pi; - u32 sq_cur_blk_addr; - u32 byte_168_irrl_idx; - u32 byte_172_sq_psn; - u32 byte_176_msg_pktn; - u32 sq_cur_sge_blk_addr; - u32 byte_184_irrl_idx; - u32 cur_sge_offset; - u32 byte_192_ext_sge; - u32 byte_196_sq_psn; - u32 byte_200_sq_max; - u32 irrl_ba; - u32 byte_208_irrl; - u32 byte_212_lsn; - u32 sq_timer; - u32 byte_220_retry_psn_msn; - u32 byte_224_retry_msg; - u32 rx_sq_cur_blk_addr; - u32 byte_232_irrl_sge; - u32 irrl_cur_sge_offset; - u32 byte_240_irrl_tail; - u32 byte_244_rnr_rxack; - u32 byte_248_ack_psn; - u32 byte_252_err_txcqn; - u32 byte_256_sqflush_rqcqe; + __le32 dmac; + __le32 byte_52_udpspn_dmac; + __le32 byte_56_dqpn_err; + __le32 byte_60_qpst_mapid; + __le32 qkey_xrcd; + __le32 byte_68_rq_db; + __le32 rq_db_record_addr; + __le32 byte_76_srqn_op_en; + __le32 byte_80_rnr_rx_cqn; + __le32 byte_84_rq_ci_pi; + __le32 rq_cur_blk_addr; + __le32 byte_92_srq_info; + __le32 byte_96_rx_reqmsn; + __le32 rq_nxt_blk_addr; + __le32 byte_104_rq_sge; + __le32 byte_108_rx_reqepsn; + __le32 rq_rnr_timer; + __le32 rx_msg_len; + __le32 rx_rkey_pkt_info; + __le64 rx_va; + __le32 byte_132_trrl; + __le32 trrl_ba; + __le32 byte_140_raq; + __le32 byte_144_raq; + __le32 byte_148_raq; + __le32 byte_152_raq; + __le32 byte_156_raq; + __le32 byte_160_sq_ci_pi; + __le32 sq_cur_blk_addr; + __le32 byte_168_irrl_idx; + __le32 byte_172_sq_psn; + __le32 byte_176_msg_pktn; + __le32 sq_cur_sge_blk_addr; + __le32 byte_184_irrl_idx; + __le32 cur_sge_offset; + __le32 byte_192_ext_sge; + __le32 byte_196_sq_psn; + __le32 byte_200_sq_max; + __le32 irrl_ba; + __le32 byte_208_irrl; + __le32 byte_212_lsn; + __le32 sq_timer; + __le32 byte_220_retry_psn_msn; + __le32 byte_224_retry_msg; + __le32 rx_sq_cur_blk_addr; + __le32 byte_232_irrl_sge; + __le32 irrl_cur_sge_offset; + __le32 byte_240_irrl_tail; + __le32 byte_244_rnr_rxack; + __le32 byte_248_ack_psn; + __le32 byte_252_err_txcqn; + __le32 byte_256_sqflush_rqcqe; }; #define V2_QPC_BYTE_4_TST_S 0 @@ -761,17 +761,17 @@ struct hns_roce_v2_qp_context { #define V2_QPC_BYTE_256_SQ_FLUSH_IDX_M GENMASK(31, 16) struct hns_roce_v2_cqe { - u32 byte_4; + __le32 byte_4; union { __le32 rkey; __be32 immtdata; }; - u32 byte_12; - u32 byte_16; - u32 byte_cnt; + __le32 byte_12; + __le32 byte_16; + __le32 byte_cnt; u8 smac[4]; - u32 byte_28; - u32 byte_32; + __le32 byte_28; + __le32 byte_32; }; #define V2_CQE_BYTE_4_OPCODE_S 0 @@ -901,8 +901,8 @@ struct hns_roce_v2_mpt_entry { #define V2_DB_PARAMETER_SL_M GENMASK(18, 16) struct hns_roce_v2_cq_db { - u32 byte_4; - u32 parameter; + __le32 byte_4; + __le32 parameter; }; #define V2_CQ_DB_BYTE_4_TAG_S 0 @@ -920,18 +920,18 @@ struct hns_roce_v2_cq_db { #define V2_CQ_DB_PARAMETER_NOTIFY_S 24 struct hns_roce_v2_ud_send_wqe { - u32 byte_4; - u32 msg_len; - u32 immtdata; - u32 byte_16; - u32 byte_20; - u32 byte_24; - u32 qkey; - u32 byte_32; - u32 byte_36; - u32 byte_40; - u32 dmac; - u32 byte_48; + __le32 byte_4; + __le32 msg_len; + __be32 immtdata; + __le32 byte_16; + __le32 byte_20; + __le32 byte_24; + __le32 qkey; + __le32 byte_32; + __le32 byte_36; + __le32 byte_40; + __le32 dmac; + __le32 byte_48; u8 dgid[GID_LEN_V2]; }; @@ -1004,13 +1004,16 @@ struct hns_roce_v2_ud_send_wqe { #define V2_UD_SEND_WQE_BYTE_48_SMAC_INDX_M GENMASK(31, 24) struct hns_roce_v2_rc_send_wqe { - u32 byte_4; - u32 msg_len; - u32 inv_key_immtdata; - u32 byte_16; - u32 byte_20; - u32 rkey; - u64 va; + __le32 byte_4; + __le32 msg_len; + union { + __le32 inv_key; + __be32 immtdata; + }; + __le32 byte_16; + __le32 byte_20; + __le32 rkey; + __le64 va; }; #define V2_RC_SEND_WQE_BYTE_4_OPCODE_S 0 @@ -1038,14 +1041,14 @@ struct hns_roce_v2_rc_send_wqe { #define V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M GENMASK(23, 0) struct hns_roce_v2_wqe_data_seg { - __be32 len; - __be32 lkey; - __be64 addr; + __le32 len; + __le32 lkey; + __le64 addr; }; struct hns_roce_v2_db { - u32 byte_4; - u32 parameter; + __le32 byte_4; + __le32 parameter; }; struct hns_roce_query_version { @@ -1105,12 +1108,12 @@ struct hns_roce_pf_res { #define PF_RES_DATA_5_PF_EQC_BT_NUM_M GENMASK(25, 16) struct hns_roce_vf_res_a { - u32 vf_id; - u32 vf_qpc_bt_idx_num; - u32 vf_srqc_bt_idx_num; - u32 vf_cqc_bt_idx_num; - u32 vf_mpt_bt_idx_num; - u32 vf_eqc_bt_idx_num; + __le32 vf_id; + __le32 vf_qpc_bt_idx_num; + __le32 vf_srqc_bt_idx_num; + __le32 vf_cqc_bt_idx_num; + __le32 vf_mpt_bt_idx_num; + __le32 vf_eqc_bt_idx_num; }; #define VF_RES_A_DATA_1_VF_QPC_BT_IDX_S 0 @@ -1144,11 +1147,11 @@ struct hns_roce_vf_res_a { #define VF_RES_A_DATA_5_VF_EQC_NUM_M GENMASK(25, 16) struct hns_roce_vf_res_b { - u32 rsv0; - u32 vf_smac_idx_num; - u32 vf_sgid_idx_num; - u32 vf_qid_idx_sl_num; - u32 rsv[2]; + __le32 rsv0; + __le32 vf_smac_idx_num; + __le32 vf_sgid_idx_num; + __le32 vf_qid_idx_sl_num; + __le32 rsv[2]; }; #define VF_RES_B_DATA_0_VF_ID_S 0 @@ -1180,11 +1183,11 @@ struct hns_roce_vf_res_b { #define ROCEE_VF_SGID_CFG4_SGID_TYPE_M GENMASK(1, 0) struct hns_roce_cfg_bt_attr { - u32 vf_qpc_cfg; - u32 vf_srqc_cfg; - u32 vf_cqc_cfg; - u32 vf_mpt_cfg; - u32 rsv[2]; + __le32 vf_qpc_cfg; + __le32 vf_srqc_cfg; + __le32 vf_cqc_cfg; + __le32 vf_mpt_cfg; + __le32 rsv[2]; }; #define CFG_BT_ATTR_DATA_0_VF_QPC_BA_PGSZ_S 0 @@ -1224,11 +1227,11 @@ struct hns_roce_cfg_bt_attr { #define CFG_BT_ATTR_DATA_3_VF_MPT_HOPNUM_M GENMASK(9, 8) struct hns_roce_cmq_desc { - u16 opcode; - u16 flag; - u16 retval; - u16 rsv; - u32 data[6]; + __le16 opcode; + __le16 flag; + __le16 retval; + __le16 rsv; + __le32 data[6]; }; #define HNS_ROCE_V2_GO_BIT_TIMEOUT_MSECS 10000 @@ -1274,18 +1277,18 @@ struct hns_roce_v2_priv { }; struct hns_roce_eq_context { - u32 byte_4; - u32 byte_8; - u32 byte_12; - u32 eqe_report_timer; - u32 eqe_ba0; - u32 eqe_ba1; - u32 byte_28; - u32 byte_32; - u32 byte_36; - u32 nxt_eqe_ba0; - u32 nxt_eqe_ba1; - u32 rsv[5]; + __le32 byte_4; + __le32 byte_8; + __le32 byte_12; + __le32 eqe_report_timer; + __le32 eqe_ba0; + __le32 eqe_ba1; + __le32 byte_28; + __le32 byte_32; + __le32 byte_36; + __le32 nxt_eqe_ba0; + __le32 nxt_eqe_ba1; + __le32 rsv[5]; }; #define HNS_ROCE_AEQ_DEFAULT_BURST_NUM 0x0 diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c index aa0c242ddc50..eb9a69fc7bec 100644 --- a/drivers/infiniband/hw/hns/hns_roce_main.c +++ b/drivers/infiniband/hw/hns/hns_roce_main.c @@ -200,7 +200,7 @@ static int hns_roce_query_device(struct ib_device *ib_dev, memset(props, 0, sizeof(*props)); - props->sys_image_guid = hr_dev->sys_image_guid; + props->sys_image_guid = cpu_to_be32(hr_dev->sys_image_guid); props->max_mr_size = (u64)(~(0ULL)); props->page_size_cap = hr_dev->caps.page_size_cap; props->vendor_id = hr_dev->vendor_id; @@ -636,7 +636,6 @@ err_unmap_dmpt: hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table); err_unmap_mtt: - hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtt_table); if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE)) hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtt_cqe_table); diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c index 4414cea9ef56..088973a05882 100644 --- a/drivers/infiniband/hw/hns/hns_roce_qp.c +++ b/drivers/infiniband/hw/hns/hns_roce_qp.c @@ -512,9 +512,9 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, hr_qp->ibqp.qp_type = init_attr->qp_type; if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) - hr_qp->sq_signal_bits = IB_SIGNAL_ALL_WR; + hr_qp->sq_signal_bits = cpu_to_le32(IB_SIGNAL_ALL_WR); else - hr_qp->sq_signal_bits = IB_SIGNAL_REQ_WR; + hr_qp->sq_signal_bits = cpu_to_le32(IB_SIGNAL_REQ_WR); ret = hns_roce_set_rq_size(hr_dev, &init_attr->cap, !!ib_pd->uobject, !!init_attr->srq, hr_qp); @@ -937,20 +937,6 @@ void hns_roce_unlock_cqs(struct hns_roce_cq *send_cq, } EXPORT_SYMBOL_GPL(hns_roce_unlock_cqs); -__be32 send_ieth(struct ib_send_wr *wr) -{ - switch (wr->opcode) { - case IB_WR_SEND_WITH_IMM: - case IB_WR_RDMA_WRITE_WITH_IMM: - return cpu_to_le32(wr->ex.imm_data); - case IB_WR_SEND_WITH_INV: - return cpu_to_le32(wr->ex.invalidate_rkey); - default: - return 0; - } -} -EXPORT_SYMBOL_GPL(send_ieth); - static void *get_wqe(struct hns_roce_qp *hr_qp, int offset) { diff --git a/drivers/infiniband/hw/qib/qib_rc.c b/drivers/infiniband/hw/qib/qib_rc.c index cfddff45413f..c9955d48c50f 100644 --- a/drivers/infiniband/hw/qib/qib_rc.c +++ b/drivers/infiniband/hw/qib/qib_rc.c @@ -1913,8 +1913,7 @@ send_last: wc.port_num = 0; /* Signal completion event if the solicited bit is set. */ rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, - (ohdr->bth[0] & - cpu_to_be32(IB_BTH_SOLICITED)) != 0); + ib_bth_is_solicited(ohdr)); break; case OP(RDMA_WRITE_FIRST): diff --git a/drivers/infiniband/hw/qib/qib_uc.c b/drivers/infiniband/hw/qib/qib_uc.c index 70c58b88192c..840eec6ebc33 100644 --- a/drivers/infiniband/hw/qib/qib_uc.c +++ b/drivers/infiniband/hw/qib/qib_uc.c @@ -401,8 +401,7 @@ last_imm: wc.port_num = 0; /* Signal completion event if the solicited bit is set. */ rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, - (ohdr->bth[0] & - cpu_to_be32(IB_BTH_SOLICITED)) != 0); + ib_bth_is_solicited(ohdr)); break; case OP(RDMA_WRITE_FIRST): diff --git a/drivers/infiniband/hw/qib/qib_ud.c b/drivers/infiniband/hw/qib/qib_ud.c index 386c3c4da0c7..3e4ff77260c2 100644 --- a/drivers/infiniband/hw/qib/qib_ud.c +++ b/drivers/infiniband/hw/qib/qib_ud.c @@ -579,8 +579,7 @@ void qib_ud_rcv(struct qib_ibport *ibp, struct ib_header *hdr, wc.port_num = qp->port_num; /* Signal completion event if the solicited bit is set. */ rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, - (ohdr->bth[0] & - cpu_to_be32(IB_BTH_SOLICITED)) != 0); + ib_bth_is_solicited(ohdr)); return; drop: diff --git a/drivers/infiniband/sw/rxe/rxe_av.c b/drivers/infiniband/sw/rxe/rxe_av.c index 1cc9e2e1365d..7522d1af3ae2 100644 --- a/drivers/infiniband/sw/rxe/rxe_av.c +++ b/drivers/infiniband/sw/rxe/rxe_av.c @@ -52,28 +52,24 @@ int rxe_av_chk_attr(struct rxe_dev *rxe, struct rdma_ah_attr *attr) return 0; } -int rxe_av_from_attr(struct rxe_dev *rxe, u8 port_num, - struct rxe_av *av, struct rdma_ah_attr *attr) +void rxe_av_from_attr(u8 port_num, struct rxe_av *av, + struct rdma_ah_attr *attr) { memset(av, 0, sizeof(*av)); memcpy(&av->grh, rdma_ah_read_grh(attr), sizeof(*rdma_ah_read_grh(attr))); av->port_num = port_num; - return 0; } -int rxe_av_to_attr(struct rxe_dev *rxe, struct rxe_av *av, - struct rdma_ah_attr *attr) +void rxe_av_to_attr(struct rxe_av *av, struct rdma_ah_attr *attr) { attr->type = RDMA_AH_ATTR_TYPE_ROCE; memcpy(rdma_ah_retrieve_grh(attr), &av->grh, sizeof(av->grh)); rdma_ah_set_ah_flags(attr, IB_AH_GRH); rdma_ah_set_port_num(attr, av->port_num); - return 0; } -int rxe_av_fill_ip_info(struct rxe_dev *rxe, - struct rxe_av *av, +void rxe_av_fill_ip_info(struct rxe_av *av, struct rdma_ah_attr *attr, struct ib_gid_attr *sgid_attr, union ib_gid *sgid) @@ -81,8 +77,6 @@ int rxe_av_fill_ip_info(struct rxe_dev *rxe, rdma_gid2ip(&av->sgid_addr._sockaddr, sgid); rdma_gid2ip(&av->dgid_addr._sockaddr, &rdma_ah_read_grh(attr)->dgid); av->network_type = ib_gid_to_network_type(sgid_attr->gid_type, sgid); - - return 0; } struct rxe_av *rxe_get_av(struct rxe_pkt_info *pkt) diff --git a/drivers/infiniband/sw/rxe/rxe_loc.h b/drivers/infiniband/sw/rxe/rxe_loc.h index 96c3a6c5c4b5..4ef75d5b729b 100644 --- a/drivers/infiniband/sw/rxe/rxe_loc.h +++ b/drivers/infiniband/sw/rxe/rxe_loc.h @@ -38,14 +38,12 @@ int rxe_av_chk_attr(struct rxe_dev *rxe, struct rdma_ah_attr *attr); -int rxe_av_from_attr(struct rxe_dev *rxe, u8 port_num, - struct rxe_av *av, struct rdma_ah_attr *attr); +void rxe_av_from_attr(u8 port_num, struct rxe_av *av, + struct rdma_ah_attr *attr); -int rxe_av_to_attr(struct rxe_dev *rxe, struct rxe_av *av, - struct rdma_ah_attr *attr); +void rxe_av_to_attr(struct rxe_av *av, struct rdma_ah_attr *attr); -int rxe_av_fill_ip_info(struct rxe_dev *rxe, - struct rxe_av *av, +void rxe_av_fill_ip_info(struct rxe_av *av, struct rdma_ah_attr *attr, struct ib_gid_attr *sgid_attr, union ib_gid *sgid); diff --git a/drivers/infiniband/sw/rxe/rxe_qp.c b/drivers/infiniband/sw/rxe/rxe_qp.c index 137d6c0c49d4..2fcf1cab7678 100644 --- a/drivers/infiniband/sw/rxe/rxe_qp.c +++ b/drivers/infiniband/sw/rxe/rxe_qp.c @@ -633,9 +633,8 @@ int rxe_qp_from_attr(struct rxe_qp *qp, struct ib_qp_attr *attr, int mask, ib_get_cached_gid(&rxe->ib_dev, 1, rdma_ah_read_grh(&attr->ah_attr)->sgid_index, &sgid, &sgid_attr); - rxe_av_from_attr(rxe, attr->port_num, &qp->pri_av, - &attr->ah_attr); - rxe_av_fill_ip_info(rxe, &qp->pri_av, &attr->ah_attr, + rxe_av_from_attr(attr->port_num, &qp->pri_av, &attr->ah_attr); + rxe_av_fill_ip_info(&qp->pri_av, &attr->ah_attr, &sgid_attr, &sgid); if (sgid_attr.ndev) dev_put(sgid_attr.ndev); @@ -648,9 +647,9 @@ int rxe_qp_from_attr(struct rxe_qp *qp, struct ib_qp_attr *attr, int mask, ib_get_cached_gid(&rxe->ib_dev, 1, sgid_index, &sgid, &sgid_attr); - rxe_av_from_attr(rxe, attr->alt_port_num, &qp->alt_av, + rxe_av_from_attr(attr->alt_port_num, &qp->alt_av, &attr->alt_ah_attr); - rxe_av_fill_ip_info(rxe, &qp->alt_av, &attr->alt_ah_attr, + rxe_av_fill_ip_info(&qp->alt_av, &attr->alt_ah_attr, &sgid_attr, &sgid); if (sgid_attr.ndev) dev_put(sgid_attr.ndev); @@ -765,8 +764,6 @@ int rxe_qp_from_attr(struct rxe_qp *qp, struct ib_qp_attr *attr, int mask, /* called by the query qp verb */ int rxe_qp_to_attr(struct rxe_qp *qp, struct ib_qp_attr *attr, int mask) { - struct rxe_dev *rxe = to_rdev(qp->ibqp.device); - *attr = qp->attr; attr->rq_psn = qp->resp.psn; @@ -781,8 +778,8 @@ int rxe_qp_to_attr(struct rxe_qp *qp, struct ib_qp_attr *attr, int mask) attr->cap.max_recv_sge = qp->rq.max_sge; } - rxe_av_to_attr(rxe, &qp->pri_av, &attr->ah_attr); - rxe_av_to_attr(rxe, &qp->alt_av, &attr->alt_ah_attr); + rxe_av_to_attr(&qp->pri_av, &attr->ah_attr); + rxe_av_to_attr(&qp->alt_av, &attr->alt_ah_attr); if (qp->req.state == QP_STATE_DRAIN) { attr->sq_draining = 1; diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c index 7210a784abb4..f4bab2cd0ec2 100644 --- a/drivers/infiniband/sw/rxe/rxe_verbs.c +++ b/drivers/infiniband/sw/rxe/rxe_verbs.c @@ -271,13 +271,12 @@ static int rxe_init_av(struct rxe_dev *rxe, struct rdma_ah_attr *attr, return err; } - err = rxe_av_from_attr(rxe, rdma_ah_get_port_num(attr), av, attr); - if (!err) - err = rxe_av_fill_ip_info(rxe, av, attr, &sgid_attr, &sgid); + rxe_av_from_attr(rdma_ah_get_port_num(attr), av, attr); + rxe_av_fill_ip_info(av, attr, &sgid_attr, &sgid); if (sgid_attr.ndev) dev_put(sgid_attr.ndev); - return err; + return 0; } static struct ib_ah *rxe_create_ah(struct ib_pd *ibpd, @@ -335,12 +334,11 @@ static int rxe_modify_ah(struct ib_ah *ibah, struct rdma_ah_attr *attr) static int rxe_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *attr) { - struct rxe_dev *rxe = to_rdev(ibah->device); struct rxe_ah *ah = to_rah(ibah); memset(attr, 0, sizeof(*attr)); attr->type = ibah->type; - rxe_av_to_attr(rxe, &ah->av, attr); + rxe_av_to_attr(&ah->av, attr); return 0; } diff --git a/drivers/infiniband/ulp/ipoib/ipoib_main.c b/drivers/infiniband/ulp/ipoib/ipoib_main.c index 5930c7d9a8fb..161ba8c76285 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib_main.c +++ b/drivers/infiniband/ulp/ipoib/ipoib_main.c @@ -2306,6 +2306,9 @@ static struct net_device *ipoib_add_port(const char *format, priv->ca, ipoib_event); ib_register_event_handler(&priv->event_handler); + /* call event handler to ensure pkey in sync */ + queue_work(ipoib_workqueue, &priv->flush_heavy); + result = register_netdev(priv->dev); if (result) { pr_warn("%s: couldn't register ipoib port %d; error %d\n", |