diff options
| -rw-r--r-- | drivers/ntb/ntb_hw.c | 121 | ||||
| -rw-r--r-- | drivers/ntb/ntb_hw.h | 7 | ||||
| -rw-r--r-- | drivers/ntb/ntb_regs.h | 16 | ||||
| -rw-r--r-- | drivers/ntb/ntb_transport.c | 77 |
4 files changed, 156 insertions, 65 deletions
diff --git a/drivers/ntb/ntb_hw.c b/drivers/ntb/ntb_hw.c index 1cb6e51e6bda..170e8e60cdb7 100644 --- a/drivers/ntb/ntb_hw.c +++ b/drivers/ntb/ntb_hw.c @@ -141,6 +141,24 @@ void ntb_unregister_event_callback(struct ntb_device *ndev) ndev->event_cb = NULL; } +static void ntb_irq_work(unsigned long data) +{ + struct ntb_db_cb *db_cb = (struct ntb_db_cb *)data; + int rc; + + rc = db_cb->callback(db_cb->data, db_cb->db_num); + if (rc) + tasklet_schedule(&db_cb->irq_work); + else { + struct ntb_device *ndev = db_cb->ndev; + unsigned long mask; + + mask = readw(ndev->reg_ofs.ldb_mask); + clear_bit(db_cb->db_num * ndev->bits_per_vector, &mask); + writew(mask, ndev->reg_ofs.ldb_mask); + } +} + /** * ntb_register_db_callback() - register a callback for doorbell interrupt * @ndev: pointer to ntb_device instance @@ -155,7 +173,7 @@ void ntb_unregister_event_callback(struct ntb_device *ndev) * RETURNS: An appropriate -ERRNO error value on error, or zero for success. */ int ntb_register_db_callback(struct ntb_device *ndev, unsigned int idx, - void *data, void (*func)(void *data, int db_num)) + void *data, int (*func)(void *data, int db_num)) { unsigned long mask; @@ -166,6 +184,10 @@ int ntb_register_db_callback(struct ntb_device *ndev, unsigned int idx, ndev->db_cb[idx].callback = func; ndev->db_cb[idx].data = data; + ndev->db_cb[idx].ndev = ndev; + + tasklet_init(&ndev->db_cb[idx].irq_work, ntb_irq_work, + (unsigned long) &ndev->db_cb[idx]); /* unmask interrupt */ mask = readw(ndev->reg_ofs.ldb_mask); @@ -194,6 +216,8 @@ void ntb_unregister_db_callback(struct ntb_device *ndev, unsigned int idx) set_bit(idx * ndev->bits_per_vector, &mask); writew(mask, ndev->reg_ofs.ldb_mask); + tasklet_disable(&ndev->db_cb[idx].irq_work); + ndev->db_cb[idx].callback = NULL; } @@ -678,6 +702,7 @@ static int ntb_xeon_setup(struct ntb_device *ndev) return -EINVAL; ndev->limits.max_mw = SNB_ERRATA_MAX_MW; + ndev->limits.max_db_bits = SNB_MAX_DB_BITS; ndev->reg_ofs.spad_write = ndev->mw[1].vbase + SNB_SPAD_OFFSET; ndev->reg_ofs.rdb = ndev->mw[1].vbase + @@ -688,8 +713,21 @@ static int ntb_xeon_setup(struct ntb_device *ndev) */ writeq(ndev->mw[1].bar_sz + 0x1000, ndev->reg_base + SNB_PBAR4LMT_OFFSET); + /* HW errata on the Limit registers. They can only be + * written when the base register is 4GB aligned and + * < 32bit. This should already be the case based on the + * driver defaults, but write the Limit registers first + * just in case. + */ } else { ndev->limits.max_mw = SNB_MAX_MW; + + /* HW Errata on bit 14 of b2bdoorbell register. Writes + * will not be mirrored to the remote system. Shrink + * the number of bits by one, since bit 14 is the last + * bit. + */ + ndev->limits.max_db_bits = SNB_MAX_DB_BITS - 1; ndev->reg_ofs.spad_write = ndev->reg_base + SNB_B2B_SPAD_OFFSET; ndev->reg_ofs.rdb = ndev->reg_base + @@ -699,6 +737,12 @@ static int ntb_xeon_setup(struct ntb_device *ndev) * something silly */ writeq(0, ndev->reg_base + SNB_PBAR4LMT_OFFSET); + /* HW errata on the Limit registers. They can only be + * written when the base register is 4GB aligned and + * < 32bit. This should already be the case based on the + * driver defaults, but write the Limit registers first + * just in case. + */ } /* The Xeon errata workaround requires setting SBAR Base @@ -769,6 +813,7 @@ static int ntb_xeon_setup(struct ntb_device *ndev) * have an equal amount. */ ndev->limits.max_spads = SNB_MAX_COMPAT_SPADS / 2; + ndev->limits.max_db_bits = SNB_MAX_DB_BITS; /* Note: The SDOORBELL is the cause of the errata. You REALLY * don't want to touch it. */ @@ -793,6 +838,7 @@ static int ntb_xeon_setup(struct ntb_device *ndev) * have an equal amount. */ ndev->limits.max_spads = SNB_MAX_COMPAT_SPADS / 2; + ndev->limits.max_db_bits = SNB_MAX_DB_BITS; ndev->reg_ofs.rdb = ndev->reg_base + SNB_PDOORBELL_OFFSET; ndev->reg_ofs.ldb = ndev->reg_base + SNB_SDOORBELL_OFFSET; ndev->reg_ofs.ldb_mask = ndev->reg_base + SNB_SDBMSK_OFFSET; @@ -819,7 +865,6 @@ static int ntb_xeon_setup(struct ntb_device *ndev) ndev->reg_ofs.lnk_stat = ndev->reg_base + SNB_SLINK_STATUS_OFFSET; ndev->reg_ofs.spci_cmd = ndev->reg_base + SNB_PCICMD_OFFSET; - ndev->limits.max_db_bits = SNB_MAX_DB_BITS; ndev->limits.msix_cnt = SNB_MSIX_CNT; ndev->bits_per_vector = SNB_DB_BITS_PER_VEC; @@ -934,12 +979,16 @@ static irqreturn_t bwd_callback_msix_irq(int irq, void *data) { struct ntb_db_cb *db_cb = data; struct ntb_device *ndev = db_cb->ndev; + unsigned long mask; dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for DB %d\n", irq, db_cb->db_num); - if (db_cb->callback) - db_cb->callback(db_cb->data, db_cb->db_num); + mask = readw(ndev->reg_ofs.ldb_mask); + set_bit(db_cb->db_num * ndev->bits_per_vector, &mask); + writew(mask, ndev->reg_ofs.ldb_mask); + + tasklet_schedule(&db_cb->irq_work); /* No need to check for the specific HB irq, any interrupt means * we're connected. @@ -955,12 +1004,16 @@ static irqreturn_t xeon_callback_msix_irq(int irq, void *data) { struct ntb_db_cb *db_cb = data; struct ntb_device *ndev = db_cb->ndev; + unsigned long mask; dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for DB %d\n", irq, db_cb->db_num); - if (db_cb->callback) - db_cb->callback(db_cb->data, db_cb->db_num); + mask = readw(ndev->reg_ofs.ldb_mask); + set_bit(db_cb->db_num * ndev->bits_per_vector, &mask); + writew(mask, ndev->reg_ofs.ldb_mask); + + tasklet_schedule(&db_cb->irq_work); /* On Sandybridge, there are 16 bits in the interrupt register * but only 4 vectors. So, 5 bits are assigned to the first 3 @@ -986,7 +1039,7 @@ static irqreturn_t xeon_event_msix_irq(int irq, void *dev) dev_err(&ndev->pdev->dev, "Error determining link status\n"); /* bit 15 is always the link bit */ - writew(1 << ndev->limits.max_db_bits, ndev->reg_ofs.ldb); + writew(1 << SNB_LINK_DB, ndev->reg_ofs.ldb); return IRQ_HANDLED; } @@ -1075,6 +1128,10 @@ static int ntb_setup_msix(struct ntb_device *ndev) "Only %d MSI-X vectors. Limiting the number of queues to that number.\n", rc); msix_entries = rc; + + rc = pci_enable_msix(pdev, ndev->msix_entries, msix_entries); + if (rc) + goto err1; } for (i = 0; i < msix_entries; i++) { @@ -1176,9 +1233,10 @@ static int ntb_setup_interrupts(struct ntb_device *ndev) */ if (ndev->hw_type == BWD_HW) writeq(~0, ndev->reg_ofs.ldb_mask); - else - writew(~(1 << ndev->limits.max_db_bits), - ndev->reg_ofs.ldb_mask); + else { + u16 var = 1 << SNB_LINK_DB; + writew(~var, ndev->reg_ofs.ldb_mask); + } rc = ntb_setup_msix(ndev); if (!rc) @@ -1286,6 +1344,39 @@ static void ntb_free_debugfs(struct ntb_device *ndev) } } +static void ntb_hw_link_up(struct ntb_device *ndev) +{ + if (ndev->conn_type == NTB_CONN_TRANSPARENT) + ntb_link_event(ndev, NTB_LINK_UP); + else { + u32 ntb_cntl; + + /* Let's bring the NTB link up */ + ntb_cntl = readl(ndev->reg_ofs.lnk_cntl); + ntb_cntl &= ~(NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK); + ntb_cntl |= NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP; + ntb_cntl |= NTB_CNTL_P2S_BAR45_SNOOP | NTB_CNTL_S2P_BAR45_SNOOP; + writel(ntb_cntl, ndev->reg_ofs.lnk_cntl); + } +} + +static void ntb_hw_link_down(struct ntb_device *ndev) +{ + u32 ntb_cntl; + + if (ndev->conn_type == NTB_CONN_TRANSPARENT) { + ntb_link_event(ndev, NTB_LINK_DOWN); + return; + } + + /* Bring NTB link down */ + ntb_cntl = readl(ndev->reg_ofs.lnk_cntl); + ntb_cntl &= ~(NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP); + ntb_cntl &= ~(NTB_CNTL_P2S_BAR45_SNOOP | NTB_CNTL_S2P_BAR45_SNOOP); + ntb_cntl |= NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK; + writel(ntb_cntl, ndev->reg_ofs.lnk_cntl); +} + static int ntb_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) { struct ntb_device *ndev; @@ -1374,9 +1465,7 @@ static int ntb_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (rc) goto err6; - /* Let's bring the NTB link up */ - writel(NTB_CNTL_BAR23_SNOOP | NTB_CNTL_BAR45_SNOOP, - ndev->reg_ofs.lnk_cntl); + ntb_hw_link_up(ndev); return 0; @@ -1406,12 +1495,8 @@ static void ntb_pci_remove(struct pci_dev *pdev) { struct ntb_device *ndev = pci_get_drvdata(pdev); int i; - u32 ntb_cntl; - /* Bring NTB link down */ - ntb_cntl = readl(ndev->reg_ofs.lnk_cntl); - ntb_cntl |= NTB_CNTL_LINK_DISABLE; - writel(ntb_cntl, ndev->reg_ofs.lnk_cntl); + ntb_hw_link_down(ndev); ntb_transport_free(ndev->ntb_transport); diff --git a/drivers/ntb/ntb_hw.h b/drivers/ntb/ntb_hw.h index 0a31cedae7d4..bbdb7edca10c 100644 --- a/drivers/ntb/ntb_hw.h +++ b/drivers/ntb/ntb_hw.h @@ -106,10 +106,11 @@ struct ntb_mw { }; struct ntb_db_cb { - void (*callback) (void *data, int db_num); + int (*callback)(void *data, int db_num); unsigned int db_num; void *data; struct ntb_device *ndev; + struct tasklet_struct irq_work; }; struct ntb_device { @@ -228,8 +229,8 @@ struct ntb_device *ntb_register_transport(struct pci_dev *pdev, void ntb_unregister_transport(struct ntb_device *ndev); void ntb_set_mw_addr(struct ntb_device *ndev, unsigned int mw, u64 addr); int ntb_register_db_callback(struct ntb_device *ndev, unsigned int idx, - void *data, void (*db_cb_func) (void *data, - int db_num)); + void *data, int (*db_cb_func)(void *data, + int db_num)); void ntb_unregister_db_callback(struct ntb_device *ndev, unsigned int idx); int ntb_register_event_callback(struct ntb_device *ndev, void (*event_cb_func) (void *handle, diff --git a/drivers/ntb/ntb_regs.h b/drivers/ntb/ntb_regs.h index aa4bdd393c58..9774506419d7 100644 --- a/drivers/ntb/ntb_regs.h +++ b/drivers/ntb/ntb_regs.h @@ -55,6 +55,7 @@ #define SNB_MAX_COMPAT_SPADS 16 /* Reserve the uppermost bit for link interrupt */ #define SNB_MAX_DB_BITS 15 +#define SNB_LINK_DB 15 #define SNB_DB_BITS_PER_VEC 5 #define SNB_MAX_MW 2 #define SNB_ERRATA_MAX_MW 1 @@ -75,9 +76,6 @@ #define SNB_SBAR2XLAT_OFFSET 0x0030 #define SNB_SBAR4XLAT_OFFSET 0x0038 #define SNB_SBAR0BASE_OFFSET 0x0040 -#define SNB_SBAR0BASE_OFFSET 0x0040 -#define SNB_SBAR2BASE_OFFSET 0x0048 -#define SNB_SBAR4BASE_OFFSET 0x0050 #define SNB_SBAR2BASE_OFFSET 0x0048 #define SNB_SBAR4BASE_OFFSET 0x0050 #define SNB_NTBCNTL_OFFSET 0x0058 @@ -145,11 +143,13 @@ #define BWD_LTSSMSTATEJMP_FORCEDETECT (1 << 2) #define BWD_IBIST_ERR_OFLOW 0x7FFF7FFF -#define NTB_CNTL_CFG_LOCK (1 << 0) -#define NTB_CNTL_LINK_DISABLE (1 << 1) -#define NTB_CNTL_BAR23_SNOOP (1 << 2) -#define NTB_CNTL_BAR45_SNOOP (1 << 6) -#define BWD_CNTL_LINK_DOWN (1 << 16) +#define NTB_CNTL_CFG_LOCK (1 << 0) +#define NTB_CNTL_LINK_DISABLE (1 << 1) +#define NTB_CNTL_S2P_BAR23_SNOOP (1 << 2) +#define NTB_CNTL_P2S_BAR23_SNOOP (1 << 4) +#define NTB_CNTL_S2P_BAR45_SNOOP (1 << 6) +#define NTB_CNTL_P2S_BAR45_SNOOP (1 << 8) +#define BWD_CNTL_LINK_DOWN (1 << 16) #define NTB_PPD_OFFSET 0x00D4 #define SNB_PPD_CONN_TYPE 0x0003 diff --git a/drivers/ntb/ntb_transport.c b/drivers/ntb/ntb_transport.c index d0222f13d154..3217f394d45b 100644 --- a/drivers/ntb/ntb_transport.c +++ b/drivers/ntb/ntb_transport.c @@ -119,7 +119,6 @@ struct ntb_transport_qp { void (*rx_handler) (struct ntb_transport_qp *qp, void *qp_data, void *data, int len); - struct tasklet_struct rx_work; struct list_head rx_pend_q; struct list_head rx_free_q; spinlock_t ntb_rx_pend_q_lock; @@ -584,11 +583,8 @@ static int ntb_set_mw(struct ntb_transport *nt, int num_mw, unsigned int size) return 0; } -static void ntb_qp_link_cleanup(struct work_struct *work) +static void ntb_qp_link_cleanup(struct ntb_transport_qp *qp) { - struct ntb_transport_qp *qp = container_of(work, - struct ntb_transport_qp, - link_cleanup); struct ntb_transport *nt = qp->transport; struct pci_dev *pdev = ntb_query_pdev(nt->ndev); @@ -602,6 +598,16 @@ static void ntb_qp_link_cleanup(struct work_struct *work) dev_info(&pdev->dev, "qp %d: Link Down\n", qp->qp_num); qp->qp_link = NTB_LINK_DOWN; +} + +static void ntb_qp_link_cleanup_work(struct work_struct *work) +{ + struct ntb_transport_qp *qp = container_of(work, + struct ntb_transport_qp, + link_cleanup); + struct ntb_transport *nt = qp->transport; + + ntb_qp_link_cleanup(qp); if (nt->transport_link == NTB_LINK_UP) schedule_delayed_work(&qp->link_work, @@ -613,22 +619,20 @@ static void ntb_qp_link_down(struct ntb_transport_qp *qp) schedule_work(&qp->link_cleanup); } -static void ntb_transport_link_cleanup(struct work_struct *work) +static void ntb_transport_link_cleanup(struct ntb_transport *nt) { - struct ntb_transport *nt = container_of(work, struct ntb_transport, - link_cleanup); int i; + /* Pass along the info to any clients */ + for (i = 0; i < nt->max_qps; i++) + if (!test_bit(i, &nt->qp_bitmap)) + ntb_qp_link_cleanup(&nt->qps[i]); + if (nt->transport_link == NTB_LINK_DOWN) cancel_delayed_work_sync(&nt->link_work); else nt->transport_link = NTB_LINK_DOWN; - /* Pass along the info to any clients */ - for (i = 0; i < nt->max_qps; i++) - if (!test_bit(i, &nt->qp_bitmap)) - ntb_qp_link_down(&nt->qps[i]); - /* The scratchpad registers keep the values if the remote side * goes down, blast them now to give them a sane value the next * time they are accessed @@ -637,6 +641,14 @@ static void ntb_transport_link_cleanup(struct work_struct *work) ntb_write_local_spad(nt->ndev, i, 0); } +static void ntb_transport_link_cleanup_work(struct work_struct *work) +{ + struct ntb_transport *nt = container_of(work, struct ntb_transport, + link_cleanup); + + ntb_transport_link_cleanup(nt); +} + static void ntb_transport_event_callback(void *data, enum ntb_hw_event event) { struct ntb_transport *nt = data; @@ -880,7 +892,7 @@ static int ntb_transport_init_queue(struct ntb_transport *nt, } INIT_DELAYED_WORK(&qp->link_work, ntb_qp_link_work); - INIT_WORK(&qp->link_cleanup, ntb_qp_link_cleanup); + INIT_WORK(&qp->link_cleanup, ntb_qp_link_cleanup_work); spin_lock_init(&qp->ntb_rx_pend_q_lock); spin_lock_init(&qp->ntb_rx_free_q_lock); @@ -936,7 +948,7 @@ int ntb_transport_init(struct pci_dev *pdev) } INIT_DELAYED_WORK(&nt->link_work, ntb_transport_link_work); - INIT_WORK(&nt->link_cleanup, ntb_transport_link_cleanup); + INIT_WORK(&nt->link_cleanup, ntb_transport_link_cleanup_work); rc = ntb_register_event_callback(nt->ndev, ntb_transport_event_callback); @@ -972,7 +984,7 @@ void ntb_transport_free(void *transport) struct ntb_device *ndev = nt->ndev; int i; - nt->transport_link = NTB_LINK_DOWN; + ntb_transport_link_cleanup(nt); /* verify that all the qp's are freed */ for (i = 0; i < nt->max_qps; i++) { @@ -1188,11 +1200,14 @@ err: goto out; } -static void ntb_transport_rx(unsigned long data) +static int ntb_transport_rxc_db(void *data, int db_num) { - struct ntb_transport_qp *qp = (struct ntb_transport_qp *)data; + struct ntb_transport_qp *qp = data; int rc, i; + dev_dbg(&ntb_query_pdev(qp->ndev)->dev, "%s: doorbell %d received\n", + __func__, db_num); + /* Limit the number of packets processed in a single interrupt to * provide fairness to others */ @@ -1204,16 +1219,8 @@ static void ntb_transport_rx(unsigned long data) if (qp->dma_chan) dma_async_issue_pending(qp->dma_chan); -} - -static void ntb_transport_rxc_db(void *data, int db_num) -{ - struct ntb_transport_qp *qp = data; - - dev_dbg(&ntb_query_pdev(qp->ndev)->dev, "%s: doorbell %d received\n", - __func__, db_num); - tasklet_schedule(&qp->rx_work); + return i; } static void ntb_tx_copy_callback(void *data) @@ -1432,11 +1439,12 @@ ntb_transport_create_queue(void *data, struct pci_dev *pdev, qp->tx_handler = handlers->tx_handler; qp->event_handler = handlers->event_handler; + dmaengine_get(); qp->dma_chan = dma_find_channel(DMA_MEMCPY); - if (!qp->dma_chan) + if (!qp->dma_chan) { + dmaengine_put(); dev_info(&pdev->dev, "Unable to allocate DMA channel, using CPU instead\n"); - else - dmaengine_get(); + } for (i = 0; i < NTB_QP_DEF_NUM_ENTRIES; i++) { entry = kzalloc(sizeof(struct ntb_queue_entry), GFP_ATOMIC); @@ -1458,25 +1466,23 @@ ntb_transport_create_queue(void *data, struct pci_dev *pdev, &qp->tx_free_q); } - tasklet_init(&qp->rx_work, ntb_transport_rx, (unsigned long) qp); - rc = ntb_register_db_callback(qp->ndev, free_queue, qp, ntb_transport_rxc_db); if (rc) - goto err3; + goto err2; dev_info(&pdev->dev, "NTB Transport QP %d created\n", qp->qp_num); return qp; -err3: - tasklet_disable(&qp->rx_work); err2: while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q))) kfree(entry); err1: while ((entry = ntb_list_rm(&qp->ntb_rx_free_q_lock, &qp->rx_free_q))) kfree(entry); + if (qp->dma_chan) + dmaengine_put(); set_bit(free_queue, &nt->qp_bitmap); err: return NULL; @@ -1515,7 +1521,6 @@ void ntb_transport_free_queue(struct ntb_transport_qp *qp) } ntb_unregister_db_callback(qp->ndev, qp->qp_num); - tasklet_disable(&qp->rx_work); cancel_delayed_work_sync(&qp->link_work); |