diff options
Diffstat (limited to 'drivers/net/ethernet/chelsio')
| -rw-r--r-- | drivers/net/ethernet/chelsio/cxgb4/cxgb4.h | 16 | ||||
| -rw-r--r-- | drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c | 214 | ||||
| -rw-r--r-- | drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h | 4 | ||||
| -rw-r--r-- | drivers/net/ethernet/chelsio/cxgb4/sge.c | 20 | ||||
| -rw-r--r-- | drivers/net/ethernet/chelsio/cxgb4/t4_regs.h | 53 | ||||
| -rw-r--r-- | drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h | 15 |
6 files changed, 280 insertions, 42 deletions
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h b/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h index 5f3c0a728e18..ec2dafe8ae5b 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h @@ -51,6 +51,8 @@ #define FW_VERSION_MINOR 1 #define FW_VERSION_MICRO 0 +#define CH_WARN(adap, fmt, ...) dev_warn(adap->pdev_dev, fmt, ## __VA_ARGS__) + enum { MAX_NPORTS = 4, /* max # of ports */ SERNUM_LEN = 24, /* Serial # length */ @@ -64,6 +66,15 @@ enum { MEM_MC }; +enum { + MEMWIN0_APERTURE = 65536, + MEMWIN0_BASE = 0x30000, + MEMWIN1_APERTURE = 32768, + MEMWIN1_BASE = 0x28000, + MEMWIN2_APERTURE = 2048, + MEMWIN2_BASE = 0x1b800, +}; + enum dev_master { MASTER_CANT, MASTER_MAY, @@ -403,6 +414,9 @@ struct sge_txq { struct tx_sw_desc *sdesc; /* address of SW Tx descriptor ring */ struct sge_qstat *stat; /* queue status entry */ dma_addr_t phys_addr; /* physical address of the ring */ + spinlock_t db_lock; + int db_disabled; + unsigned short db_pidx; }; struct sge_eth_txq { /* state for an SGE Ethernet Tx queue */ @@ -475,6 +489,7 @@ struct adapter { void __iomem *regs; struct pci_dev *pdev; struct device *pdev_dev; + unsigned int mbox; unsigned int fn; unsigned int flags; @@ -607,6 +622,7 @@ irqreturn_t t4_sge_intr_msix(int irq, void *cookie); void t4_sge_init(struct adapter *adap); void t4_sge_start(struct adapter *adap); void t4_sge_stop(struct adapter *adap); +extern int dbfifo_int_thresh; #define for_each_port(adapter, iter) \ for (iter = 0; iter < (adapter)->params.nports; ++iter) diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c index c243f932099e..e1f96fbb48c1 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c @@ -149,15 +149,6 @@ static unsigned int pfvfres_pmask(struct adapter *adapter, #endif enum { - MEMWIN0_APERTURE = 65536, - MEMWIN0_BASE = 0x30000, - MEMWIN1_APERTURE = 32768, - MEMWIN1_BASE = 0x28000, - MEMWIN2_APERTURE = 2048, - MEMWIN2_BASE = 0x1b800, -}; - -enum { MAX_TXQ_ENTRIES = 16384, MAX_CTRL_TXQ_ENTRIES = 1024, MAX_RSPQ_ENTRIES = 16384, @@ -371,6 +362,15 @@ static int set_addr_filters(const struct net_device *dev, bool sleep) uhash | mhash, sleep); } +int dbfifo_int_thresh = 10; /* 10 == 640 entry threshold */ +module_param(dbfifo_int_thresh, int, 0644); +MODULE_PARM_DESC(dbfifo_int_thresh, "doorbell fifo interrupt threshold"); + +int dbfifo_drain_delay = 1000; /* usecs to sleep while draining the dbfifo */ +module_param(dbfifo_drain_delay, int, 0644); +MODULE_PARM_DESC(dbfifo_drain_delay, + "usecs to sleep while draining the dbfifo"); + /* * Set Rx properties of a port, such as promiscruity, address filters, and MTU. * If @mtu is -1 it is left unchanged. @@ -389,6 +389,8 @@ static int set_rxmode(struct net_device *dev, int mtu, bool sleep_ok) return ret; } +static struct workqueue_struct *workq; + /** * link_start - enable a port * @dev: the port to enable @@ -2196,7 +2198,7 @@ static void cxgb4_queue_tid_release(struct tid_info *t, unsigned int chan, adap->tid_release_head = (void **)((uintptr_t)p | chan); if (!adap->tid_release_task_busy) { adap->tid_release_task_busy = true; - schedule_work(&adap->tid_release_task); + queue_work(workq, &adap->tid_release_task); } spin_unlock_bh(&adap->tid_release_lock); } @@ -2423,6 +2425,59 @@ void cxgb4_iscsi_init(struct net_device *dev, unsigned int tag_mask, } EXPORT_SYMBOL(cxgb4_iscsi_init); +int cxgb4_flush_eq_cache(struct net_device *dev) +{ + struct adapter *adap = netdev2adap(dev); + int ret; + + ret = t4_fwaddrspace_write(adap, adap->mbox, + 0xe1000000 + A_SGE_CTXT_CMD, 0x20000000); + return ret; +} +EXPORT_SYMBOL(cxgb4_flush_eq_cache); + +static int read_eq_indices(struct adapter *adap, u16 qid, u16 *pidx, u16 *cidx) +{ + u32 addr = t4_read_reg(adap, A_SGE_DBQ_CTXT_BADDR) + 24 * qid + 8; + __be64 indices; + int ret; + + ret = t4_mem_win_read_len(adap, addr, (__be32 *)&indices, 8); + if (!ret) { + indices = be64_to_cpu(indices); + *cidx = (indices >> 25) & 0xffff; + *pidx = (indices >> 9) & 0xffff; + } + return ret; +} + +int cxgb4_sync_txq_pidx(struct net_device *dev, u16 qid, u16 pidx, + u16 size) +{ + struct adapter *adap = netdev2adap(dev); + u16 hw_pidx, hw_cidx; + int ret; + + ret = read_eq_indices(adap, qid, &hw_pidx, &hw_cidx); + if (ret) + goto out; + + if (pidx != hw_pidx) { + u16 delta; + + if (pidx >= hw_pidx) + delta = pidx - hw_pidx; + else + delta = size - hw_pidx + pidx; + wmb(); + t4_write_reg(adap, MYPF_REG(A_SGE_PF_KDOORBELL), + V_QID(qid) | V_PIDX(delta)); + } +out: + return ret; +} +EXPORT_SYMBOL(cxgb4_sync_txq_pidx); + static struct pci_driver cxgb4_driver; static void check_neigh_update(struct neighbour *neigh) @@ -2456,6 +2511,95 @@ static struct notifier_block cxgb4_netevent_nb = { .notifier_call = netevent_cb }; +static void drain_db_fifo(struct adapter *adap, int usecs) +{ + u32 v; + + do { + set_current_state(TASK_UNINTERRUPTIBLE); + schedule_timeout(usecs_to_jiffies(usecs)); + v = t4_read_reg(adap, A_SGE_DBFIFO_STATUS); + if (G_LP_COUNT(v) == 0 && G_HP_COUNT(v) == 0) + break; + } while (1); +} + +static void disable_txq_db(struct sge_txq *q) +{ + spin_lock_irq(&q->db_lock); + q->db_disabled = 1; + spin_unlock_irq(&q->db_lock); +} + +static void enable_txq_db(struct sge_txq *q) +{ + spin_lock_irq(&q->db_lock); + q->db_disabled = 0; + spin_unlock_irq(&q->db_lock); +} + +static void disable_dbs(struct adapter *adap) +{ + int i; + + for_each_ethrxq(&adap->sge, i) + disable_txq_db(&adap->sge.ethtxq[i].q); + for_each_ofldrxq(&adap->sge, i) + disable_txq_db(&adap->sge.ofldtxq[i].q); + for_each_port(adap, i) + disable_txq_db(&adap->sge.ctrlq[i].q); +} + +static void enable_dbs(struct adapter *adap) +{ + int i; + + for_each_ethrxq(&adap->sge, i) + enable_txq_db(&adap->sge.ethtxq[i].q); + for_each_ofldrxq(&adap->sge, i) + enable_txq_db(&adap->sge.ofldtxq[i].q); + for_each_port(adap, i) + enable_txq_db(&adap->sge.ctrlq[i].q); +} + +static void sync_txq_pidx(struct adapter *adap, struct sge_txq *q) +{ + u16 hw_pidx, hw_cidx; + int ret; + + spin_lock_bh(&q->db_lock); + ret = read_eq_indices(adap, (u16)q->cntxt_id, &hw_pidx, &hw_cidx); + if (ret) + goto out; + if (q->db_pidx != hw_pidx) { + u16 delta; + + if (q->db_pidx >= hw_pidx) + delta = q->db_pidx - hw_pidx; + else + delta = q->size - hw_pidx + q->db_pidx; + wmb(); + t4_write_reg(adap, MYPF_REG(A_SGE_PF_KDOORBELL), + V_QID(q->cntxt_id) | V_PIDX(delta)); + } +out: + q->db_disabled = 0; + spin_unlock_bh(&q->db_lock); + if (ret) + CH_WARN(adap, "DB drop recovery failed.\n"); +} +static void recover_all_queues(struct adapter *adap) +{ + int i; + + for_each_ethrxq(&adap->sge, i) + sync_txq_pidx(adap, &adap->sge.ethtxq[i].q); + for_each_ofldrxq(&adap->sge, i) + sync_txq_pidx(adap, &adap->sge.ofldtxq[i].q); + for_each_port(adap, i) + sync_txq_pidx(adap, &adap->sge.ctrlq[i].q); +} + static void notify_rdma_uld(struct adapter *adap, enum cxgb4_control cmd) { mutex_lock(&uld_mutex); @@ -2468,55 +2612,41 @@ static void notify_rdma_uld(struct adapter *adap, enum cxgb4_control cmd) static void process_db_full(struct work_struct *work) { struct adapter *adap; - static int delay = 1000; - u32 v; adap = container_of(work, struct adapter, db_full_task); - - /* stop LLD queues */ - notify_rdma_uld(adap, CXGB4_CONTROL_DB_FULL); - do { - set_current_state(TASK_UNINTERRUPTIBLE); - schedule_timeout(usecs_to_jiffies(delay)); - v = t4_read_reg(adap, A_SGE_DBFIFO_STATUS); - if (G_LP_COUNT(v) == 0 && G_HP_COUNT(v) == 0) - break; - } while (1); + drain_db_fifo(adap, dbfifo_drain_delay); + t4_set_reg_field(adap, A_SGE_INT_ENABLE3, + F_DBFIFO_HP_INT | F_DBFIFO_LP_INT, + F_DBFIFO_HP_INT | F_DBFIFO_LP_INT); notify_rdma_uld(adap, CXGB4_CONTROL_DB_EMPTY); - - - /* - * The more we get db full interrupts, the more we'll delay - * in re-enabling db rings on queues, capped off at 200ms. - */ - delay = min(delay << 1, 200000); - - /* resume LLD queues */ } static void process_db_drop(struct work_struct *work) { struct adapter *adap; - adap = container_of(work, struct adapter, db_drop_task); + adap = container_of(work, struct adapter, db_drop_task); - /* - * sync the PIDX values in HW and SW for LLD queues. - */ - + t4_set_reg_field(adap, A_SGE_DOORBELL_CONTROL, F_DROPPED_DB, 0); + disable_dbs(adap); notify_rdma_uld(adap, CXGB4_CONTROL_DB_DROP); + drain_db_fifo(adap, 1); + recover_all_queues(adap); + enable_dbs(adap); } void t4_db_full(struct adapter *adap) { - schedule_work(&adap->db_full_task); + t4_set_reg_field(adap, A_SGE_INT_ENABLE3, + F_DBFIFO_HP_INT | F_DBFIFO_LP_INT, 0); + queue_work(workq, &adap->db_full_task); } void t4_db_dropped(struct adapter *adap) { - schedule_work(&adap->db_drop_task); + queue_work(workq, &adap->db_drop_task); } static void uld_attach(struct adapter *adap, unsigned int uld) @@ -2552,6 +2682,7 @@ static void uld_attach(struct adapter *adap, unsigned int uld) lli.gts_reg = adap->regs + MYPF_REG(SGE_PF_GTS); lli.db_reg = adap->regs + MYPF_REG(SGE_PF_KDOORBELL); lli.fw_vers = adap->params.fw_vers; + lli.dbfifo_int_thresh = dbfifo_int_thresh; handle = ulds[uld].add(&lli); if (IS_ERR(handle)) { @@ -3668,6 +3799,7 @@ static int __devinit init_one(struct pci_dev *pdev, adapter->pdev = pdev; adapter->pdev_dev = &pdev->dev; + adapter->mbox = func; adapter->fn = func; adapter->msg_enable = dflt_msg_enable; memset(adapter->chan_map, 0xff, sizeof(adapter->chan_map)); @@ -3865,6 +3997,10 @@ static int __init cxgb4_init_module(void) { int ret; + workq = create_singlethread_workqueue("cxgb4"); + if (!workq) + return -ENOMEM; + /* Debugfs support is optional, just warn if this fails */ cxgb4_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL); if (!cxgb4_debugfs_root) @@ -3880,6 +4016,8 @@ static void __exit cxgb4_cleanup_module(void) { pci_unregister_driver(&cxgb4_driver); debugfs_remove(cxgb4_debugfs_root); /* NULL ok */ + flush_workqueue(workq); + destroy_workqueue(workq); } module_init(cxgb4_init_module); diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h index 5cc2f27d60c7..d79980c5fc63 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h @@ -218,6 +218,7 @@ struct cxgb4_lld_info { unsigned short ucq_density; /* # of user CQs/page */ void __iomem *gts_reg; /* address of GTS register */ void __iomem *db_reg; /* address of kernel doorbell */ + int dbfifo_int_thresh; /* doorbell fifo int threshold */ }; struct cxgb4_uld_info { @@ -226,6 +227,7 @@ struct cxgb4_uld_info { int (*rx_handler)(void *handle, const __be64 *rsp, const struct pkt_gl *gl); int (*state_change)(void *handle, enum cxgb4_state new_state); + int (*control)(void *handle, enum cxgb4_control control, ...); }; int cxgb4_register_uld(enum cxgb4_uld type, const struct cxgb4_uld_info *p); @@ -243,4 +245,6 @@ void cxgb4_iscsi_init(struct net_device *dev, unsigned int tag_mask, const unsigned int *pgsz_order); struct sk_buff *cxgb4_pktgl_to_skb(const struct pkt_gl *gl, unsigned int skb_len, unsigned int pull_len); +int cxgb4_sync_txq_pidx(struct net_device *dev, u16 qid, u16 pidx, u16 size); +int cxgb4_flush_eq_cache(struct net_device *dev); #endif /* !__CXGB4_OFLD_H */ diff --git a/drivers/net/ethernet/chelsio/cxgb4/sge.c b/drivers/net/ethernet/chelsio/cxgb4/sge.c index 234c157a4879..e111d974afd8 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/sge.c +++ b/drivers/net/ethernet/chelsio/cxgb4/sge.c @@ -767,8 +767,13 @@ static void write_sgl(const struct sk_buff *skb, struct sge_txq *q, static inline void ring_tx_db(struct adapter *adap, struct sge_txq *q, int n) { wmb(); /* write descriptors before telling HW */ - t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL), - QID(q->cntxt_id) | PIDX(n)); + spin_lock(&q->db_lock); + if (!q->db_disabled) { + t4_write_reg(adap, MYPF_REG(A_SGE_PF_KDOORBELL), + V_QID(q->cntxt_id) | V_PIDX(n)); + } + q->db_pidx = q->pidx; + spin_unlock(&q->db_lock); } /** @@ -2081,6 +2086,7 @@ static void init_txq(struct adapter *adap, struct sge_txq *q, unsigned int id) q->stops = q->restarts = 0; q->stat = (void *)&q->desc[q->size]; q->cntxt_id = id; + spin_lock_init(&q->db_lock); adap->sge.egr_map[id - adap->sge.egr_start] = q; } @@ -2415,9 +2421,15 @@ void t4_sge_init(struct adapter *adap) RXPKTCPLMODE | (STAT_LEN == 128 ? EGRSTATUSPAGESIZE : 0)); + /* + * Set up to drop DOORBELL writes when the DOORBELL FIFO overflows + * and generate an interrupt when this occurs so we can recover. + */ t4_set_reg_field(adap, A_SGE_DBFIFO_STATUS, - V_HP_INT_THRESH(5) | V_LP_INT_THRESH(5), - V_HP_INT_THRESH(5) | V_LP_INT_THRESH(5)); + V_HP_INT_THRESH(M_HP_INT_THRESH) | + V_LP_INT_THRESH(M_LP_INT_THRESH), + V_HP_INT_THRESH(dbfifo_int_thresh) | + V_LP_INT_THRESH(dbfifo_int_thresh)); t4_set_reg_field(adap, A_SGE_DOORBELL_CONTROL, F_ENABLE_DROP, F_ENABLE_DROP); diff --git a/drivers/net/ethernet/chelsio/cxgb4/t4_regs.h b/drivers/net/ethernet/chelsio/cxgb4/t4_regs.h index 0adc5bcec7c4..111fc323f155 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/t4_regs.h +++ b/drivers/net/ethernet/chelsio/cxgb4/t4_regs.h @@ -190,6 +190,59 @@ #define SGE_DEBUG_DATA_LOW 0x10d4 #define SGE_INGRESS_QUEUES_PER_PAGE_PF 0x10f4 +#define S_LP_INT_THRESH 12 +#define V_LP_INT_THRESH(x) ((x) << S_LP_INT_THRESH) +#define S_HP_INT_THRESH 28 +#define V_HP_INT_THRESH(x) ((x) << S_HP_INT_THRESH) +#define A_SGE_DBFIFO_STATUS 0x10a4 + +#define S_ENABLE_DROP 13 +#define V_ENABLE_DROP(x) ((x) << S_ENABLE_DROP) +#define F_ENABLE_DROP V_ENABLE_DROP(1U) +#define A_SGE_DOORBELL_CONTROL 0x10a8 + +#define A_SGE_CTXT_CMD 0x11fc +#define A_SGE_DBQ_CTXT_BADDR 0x1084 + +#define A_SGE_PF_KDOORBELL 0x0 + +#define S_QID 15 +#define V_QID(x) ((x) << S_QID) + +#define S_PIDX 0 +#define V_PIDX(x) ((x) << S_PIDX) + +#define M_LP_COUNT 0x7ffU +#define S_LP_COUNT 0 +#define G_LP_COUNT(x) (((x) >> S_LP_COUNT) & M_LP_COUNT) + +#define M_HP_COUNT 0x7ffU +#define S_HP_COUNT 16 +#define G_HP_COUNT(x) (((x) >> S_HP_COUNT) & M_HP_COUNT) + +#define A_SGE_INT_ENABLE3 0x1040 + +#define S_DBFIFO_HP_INT 8 +#define V_DBFIFO_HP_INT(x) ((x) << S_DBFIFO_HP_INT) +#define F_DBFIFO_HP_INT V_DBFIFO_HP_INT(1U) + +#define S_DBFIFO_LP_INT 7 +#define V_DBFIFO_LP_INT(x) ((x) << S_DBFIFO_LP_INT) +#define F_DBFIFO_LP_INT V_DBFIFO_LP_INT(1U) + +#define S_DROPPED_DB 0 +#define V_DROPPED_DB(x) ((x) << S_DROPPED_DB) +#define F_DROPPED_DB V_DROPPED_DB(1U) + +#define S_ERR_DROPPED_DB 18 +#define V_ERR_DROPPED_DB(x) ((x) << S_ERR_DROPPED_DB) +#define F_ERR_DROPPED_DB V_ERR_DROPPED_DB(1U) + +#define A_PCIE_MEM_ACCESS_OFFSET 0x306c + +#define M_HP_INT_THRESH 0xfU +#define M_LP_INT_THRESH 0xfU + #define PCIE_PF_CLI 0x44 #define PCIE_INT_CAUSE 0x3004 #define UNXSPLCPLERR 0x20000000U diff --git a/drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h b/drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h index edcfd7ec7802..ad53f796b574 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h +++ b/drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h @@ -1620,4 +1620,19 @@ struct fw_hdr { #define FW_HDR_FW_VER_MINOR_GET(x) (((x) >> 16) & 0xff) #define FW_HDR_FW_VER_MICRO_GET(x) (((x) >> 8) & 0xff) #define FW_HDR_FW_VER_BUILD_GET(x) (((x) >> 0) & 0xff) + +#define S_FW_CMD_OP 24 +#define V_FW_CMD_OP(x) ((x) << S_FW_CMD_OP) + +#define S_FW_CMD_REQUEST 23 +#define V_FW_CMD_REQUEST(x) ((x) << S_FW_CMD_REQUEST) +#define F_FW_CMD_REQUEST V_FW_CMD_REQUEST(1U) + +#define S_FW_CMD_WRITE 21 +#define V_FW_CMD_WRITE(x) ((x) << S_FW_CMD_WRITE) +#define F_FW_CMD_WRITE V_FW_CMD_WRITE(1U) + +#define S_FW_LDST_CMD_ADDRSPACE 0 +#define V_FW_LDST_CMD_ADDRSPACE(x) ((x) << S_FW_LDST_CMD_ADDRSPACE) + #endif /* _T4FW_INTERFACE_H_ */ |