diff options
| -rw-r--r-- | Documentation/rapidio/tsi721.txt | 19 | ||||
| -rw-r--r-- | drivers/rapidio/devices/tsi721.h | 12 | ||||
| -rw-r--r-- | drivers/rapidio/devices/tsi721_dma.c | 718 |
3 files changed, 394 insertions, 355 deletions
diff --git a/Documentation/rapidio/tsi721.txt b/Documentation/rapidio/tsi721.txt index 335f3c6087dc..626052f403bb 100644 --- a/Documentation/rapidio/tsi721.txt +++ b/Documentation/rapidio/tsi721.txt @@ -20,13 +20,26 @@ II. Known problems None. -III. To do +III. DMA Engine Support - Add DMA data transfers (non-messaging). - Add inbound region (SRIO-to-PCIe) mapping. +Tsi721 mport driver supports DMA data transfers between local system memory and +remote RapidIO devices. This functionality is implemented according to SLAVE +mode API defined by common Linux kernel DMA Engine framework. + +Depending on system requirements RapidIO DMA operations can be included/excluded +by setting CONFIG_RAPIDIO_DMA_ENGINE option. Tsi721 miniport driver uses seven +out of eight available BDMA channels to support DMA data transfers. +One BDMA channel is reserved for generation of maintenance read/write requests. + +If Tsi721 mport driver have been built with RAPIDIO_DMA_ENGINE support included, +this driver will accept DMA-specific module parameter: + "dma_desc_per_channel" - defines number of hardware buffer descriptors used by + each BDMA channel of Tsi721 (by default - 128). IV. Version History + 1.1.0 - DMA operations re-worked to support data scatter/gather lists larger + than hardware buffer descriptors ring. 1.0.0 - Initial driver release. V. License diff --git a/drivers/rapidio/devices/tsi721.h b/drivers/rapidio/devices/tsi721.h index 0305675270ee..a7b42680a06a 100644 --- a/drivers/rapidio/devices/tsi721.h +++ b/drivers/rapidio/devices/tsi721.h @@ -644,27 +644,26 @@ enum tsi721_smsg_int_flag { #ifdef CONFIG_RAPIDIO_DMA_ENGINE -#define TSI721_BDMA_BD_RING_SZ 128 #define TSI721_BDMA_MAX_BCOUNT (TSI721_DMAD_BCOUNT1 + 1) struct tsi721_tx_desc { struct dma_async_tx_descriptor txd; - struct tsi721_dma_desc *hw_desc; u16 destid; /* low 64-bits of 66-bit RIO address */ u64 rio_addr; /* upper 2-bits of 66-bit RIO address */ u8 rio_addr_u; - u32 bcount; - bool interrupt; + enum dma_rtype rtype; struct list_head desc_node; - struct list_head tx_list; + struct scatterlist *sg; + unsigned int sg_len; + enum dma_status status; }; struct tsi721_bdma_chan { int id; void __iomem *regs; - int bd_num; /* number of buffer descriptors */ + int bd_num; /* number of HW buffer descriptors */ void *bd_base; /* start of DMA descriptors */ dma_addr_t bd_phys; void *sts_base; /* start of DMA BD status FIFO */ @@ -680,7 +679,6 @@ struct tsi721_bdma_chan { struct list_head active_list; struct list_head queue; struct list_head free_list; - dma_cookie_t completed_cookie; struct tasklet_struct tasklet; bool active; }; diff --git a/drivers/rapidio/devices/tsi721_dma.c b/drivers/rapidio/devices/tsi721_dma.c index 44341dc5b148..f64c5decb747 100644 --- a/drivers/rapidio/devices/tsi721_dma.c +++ b/drivers/rapidio/devices/tsi721_dma.c @@ -1,7 +1,7 @@ /* * DMA Engine support for Tsi721 PCIExpress-to-SRIO bridge * - * Copyright 2011 Integrated Device Technology, Inc. + * Copyright (c) 2011-2014 Integrated Device Technology, Inc. * Alexandre Bounine <alexandre.bounine@idt.com> * * This program is free software; you can redistribute it and/or modify it @@ -14,9 +14,8 @@ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * - * You should have received a copy of the GNU General Public License along with - * this program; if not, write to the Free Software Foundation, Inc., 59 - * Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * The full GNU General Public License is included in this distribution in the + * file called COPYING. */ #include <linux/io.h> @@ -32,9 +31,22 @@ #include <linux/interrupt.h> #include <linux/kfifo.h> #include <linux/delay.h> +#include "../../dma/dmaengine.h" #include "tsi721.h" +#define TSI721_DMA_TX_QUEUE_SZ 16 /* number of transaction descriptors */ + +#ifdef CONFIG_PCI_MSI +static irqreturn_t tsi721_bdma_msix(int irq, void *ptr); +#endif +static int tsi721_submit_sg(struct tsi721_tx_desc *desc); + +static unsigned int dma_desc_per_channel = 128; +module_param(dma_desc_per_channel, uint, S_IWUSR | S_IRUGO); +MODULE_PARM_DESC(dma_desc_per_channel, + "Number of DMA descriptors per channel (default: 128)"); + static inline struct tsi721_bdma_chan *to_tsi721_chan(struct dma_chan *chan) { return container_of(chan, struct tsi721_bdma_chan, dchan); @@ -59,7 +71,7 @@ struct tsi721_tx_desc *tsi721_dma_first_active( struct tsi721_tx_desc, desc_node); } -static int tsi721_bdma_ch_init(struct tsi721_bdma_chan *bdma_chan) +static int tsi721_bdma_ch_init(struct tsi721_bdma_chan *bdma_chan, int bd_num) { struct tsi721_dma_desc *bd_ptr; struct device *dev = bdma_chan->dchan.device->dev; @@ -67,17 +79,23 @@ static int tsi721_bdma_ch_init(struct tsi721_bdma_chan *bdma_chan) dma_addr_t bd_phys; dma_addr_t sts_phys; int sts_size; - int bd_num = bdma_chan->bd_num; +#ifdef CONFIG_PCI_MSI + struct tsi721_device *priv = to_tsi721(bdma_chan->dchan.device); +#endif dev_dbg(dev, "Init Block DMA Engine, CH%d\n", bdma_chan->id); - /* Allocate space for DMA descriptors */ + /* + * Allocate space for DMA descriptors + * (add an extra element for link descriptor) + */ bd_ptr = dma_zalloc_coherent(dev, - bd_num * sizeof(struct tsi721_dma_desc), + (bd_num + 1) * sizeof(struct tsi721_dma_desc), &bd_phys, GFP_KERNEL); if (!bd_ptr) return -ENOMEM; + bdma_chan->bd_num = bd_num; bdma_chan->bd_phys = bd_phys; bdma_chan->bd_base = bd_ptr; @@ -85,8 +103,8 @@ static int tsi721_bdma_ch_init(struct tsi721_bdma_chan *bdma_chan) bd_ptr, (unsigned long long)bd_phys); /* Allocate space for descriptor status FIFO */ - sts_size = (bd_num >= TSI721_DMA_MINSTSSZ) ? - bd_num : TSI721_DMA_MINSTSSZ; + sts_size = ((bd_num + 1) >= TSI721_DMA_MINSTSSZ) ? + (bd_num + 1) : TSI721_DMA_MINSTSSZ; sts_size = roundup_pow_of_two(sts_size); sts_ptr = dma_zalloc_coherent(dev, sts_size * sizeof(struct tsi721_dma_sts), @@ -94,7 +112,7 @@ static int tsi721_bdma_ch_init(struct tsi721_bdma_chan *bdma_chan) if (!sts_ptr) { /* Free space allocated for DMA descriptors */ dma_free_coherent(dev, - bd_num * sizeof(struct tsi721_dma_desc), + (bd_num + 1) * sizeof(struct tsi721_dma_desc), bd_ptr, bd_phys); bdma_chan->bd_base = NULL; return -ENOMEM; @@ -108,11 +126,11 @@ static int tsi721_bdma_ch_init(struct tsi721_bdma_chan *bdma_chan) "desc status FIFO @ %p (phys = %llx) size=0x%x\n", sts_ptr, (unsigned long long)sts_phys, sts_size); - /* Initialize DMA descriptors ring */ - bd_ptr[bd_num - 1].type_id = cpu_to_le32(DTYPE3 << 29); - bd_ptr[bd_num - 1].next_lo = cpu_to_le32((u64)bd_phys & + /* Initialize DMA descriptors ring using added link descriptor */ + bd_ptr[bd_num].type_id = cpu_to_le32(DTYPE3 << 29); + bd_ptr[bd_num].next_lo = cpu_to_le32((u64)bd_phys & TSI721_DMAC_DPTRL_MASK); - bd_ptr[bd_num - 1].next_hi = cpu_to_le32((u64)bd_phys >> 32); + bd_ptr[bd_num].next_hi = cpu_to_le32((u64)bd_phys >> 32); /* Setup DMA descriptor pointers */ iowrite32(((u64)bd_phys >> 32), @@ -134,6 +152,55 @@ static int tsi721_bdma_ch_init(struct tsi721_bdma_chan *bdma_chan) ioread32(bdma_chan->regs + TSI721_DMAC_INT); +#ifdef CONFIG_PCI_MSI + /* Request interrupt service if we are in MSI-X mode */ + if (priv->flags & TSI721_USING_MSIX) { + int rc, idx; + + idx = TSI721_VECT_DMA0_DONE + bdma_chan->id; + + rc = request_irq(priv->msix[idx].vector, tsi721_bdma_msix, 0, + priv->msix[idx].irq_name, (void *)bdma_chan); + + if (rc) { + dev_dbg(dev, "Unable to get MSI-X for BDMA%d-DONE\n", + bdma_chan->id); + goto err_out; + } + + idx = TSI721_VECT_DMA0_INT + bdma_chan->id; + + rc = request_irq(priv->msix[idx].vector, tsi721_bdma_msix, 0, + priv->msix[idx].irq_name, (void *)bdma_chan); + + if (rc) { + dev_dbg(dev, "Unable to get MSI-X for BDMA%d-INT\n", + bdma_chan->id); + free_irq( + priv->msix[TSI721_VECT_DMA0_DONE + + bdma_chan->id].vector, + (void *)bdma_chan); + } + +err_out: + if (rc) { + /* Free space allocated for DMA descriptors */ + dma_free_coherent(dev, + (bd_num + 1) * sizeof(struct tsi721_dma_desc), + bd_ptr, bd_phys); + bdma_chan->bd_base = NULL; + + /* Free space allocated for status descriptors */ + dma_free_coherent(dev, + sts_size * sizeof(struct tsi721_dma_sts), + sts_ptr, sts_phys); + bdma_chan->sts_base = NULL; + + return -EIO; + } + } +#endif /* CONFIG_PCI_MSI */ + /* Toggle DMA channel initialization */ iowrite32(TSI721_DMAC_CTL_INIT, bdma_chan->regs + TSI721_DMAC_CTL); ioread32(bdma_chan->regs + TSI721_DMAC_CTL); @@ -147,6 +214,9 @@ static int tsi721_bdma_ch_init(struct tsi721_bdma_chan *bdma_chan) static int tsi721_bdma_ch_free(struct tsi721_bdma_chan *bdma_chan) { u32 ch_stat; +#ifdef CONFIG_PCI_MSI + struct tsi721_device *priv = to_tsi721(bdma_chan->dchan.device); +#endif if (bdma_chan->bd_base == NULL) return 0; @@ -159,9 +229,18 @@ static int tsi721_bdma_ch_free(struct tsi721_bdma_chan *bdma_chan) /* Put DMA channel into init state */ iowrite32(TSI721_DMAC_CTL_INIT, bdma_chan->regs + TSI721_DMAC_CTL); +#ifdef CONFIG_PCI_MSI + if (priv->flags & TSI721_USING_MSIX) { + free_irq(priv->msix[TSI721_VECT_DMA0_DONE + + bdma_chan->id].vector, (void *)bdma_chan); + free_irq(priv->msix[TSI721_VECT_DMA0_INT + + bdma_chan->id].vector, (void *)bdma_chan); + } +#endif /* CONFIG_PCI_MSI */ + /* Free space allocated for DMA descriptors */ dma_free_coherent(bdma_chan->dchan.device->dev, - bdma_chan->bd_num * sizeof(struct tsi721_dma_desc), + (bdma_chan->bd_num + 1) * sizeof(struct tsi721_dma_desc), bdma_chan->bd_base, bdma_chan->bd_phys); bdma_chan->bd_base = NULL; @@ -243,8 +322,8 @@ static void tsi721_start_dma(struct tsi721_bdma_chan *bdma_chan) } dev_dbg(bdma_chan->dchan.device->dev, - "tx_chan: %p, chan: %d, regs: %p\n", - bdma_chan, bdma_chan->dchan.chan_id, bdma_chan->regs); + "%s: chan_%d (wrc=%d)\n", __func__, bdma_chan->id, + bdma_chan->wr_count_next); iowrite32(bdma_chan->wr_count_next, bdma_chan->regs + TSI721_DMAC_DWRCNT); @@ -253,72 +332,19 @@ static void tsi721_start_dma(struct tsi721_bdma_chan *bdma_chan) bdma_chan->wr_count = bdma_chan->wr_count_next; } -static void tsi721_desc_put(struct tsi721_bdma_chan *bdma_chan, - struct tsi721_tx_desc *desc) -{ - dev_dbg(bdma_chan->dchan.device->dev, - "Put desc: %p into free list\n", desc); - - if (desc) { - spin_lock_bh(&bdma_chan->lock); - list_splice_init(&desc->tx_list, &bdma_chan->free_list); - list_add(&desc->desc_node, &bdma_chan->free_list); - bdma_chan->wr_count_next = bdma_chan->wr_count; - spin_unlock_bh(&bdma_chan->lock); - } -} - -static -struct tsi721_tx_desc *tsi721_desc_get(struct tsi721_bdma_chan *bdma_chan) -{ - struct tsi721_tx_desc *tx_desc, *_tx_desc; - struct tsi721_tx_desc *ret = NULL; - int i; - - spin_lock_bh(&bdma_chan->lock); - list_for_each_entry_safe(tx_desc, _tx_desc, - &bdma_chan->free_list, desc_node) { - if (async_tx_test_ack(&tx_desc->txd)) { - list_del(&tx_desc->desc_node); - ret = tx_desc; - break; - } - dev_dbg(bdma_chan->dchan.device->dev, - "desc %p not ACKed\n", tx_desc); - } - - if (ret == NULL) { - dev_dbg(bdma_chan->dchan.device->dev, - "%s: unable to obtain tx descriptor\n", __func__); - goto err_out; - } - - i = bdma_chan->wr_count_next % bdma_chan->bd_num; - if (i == bdma_chan->bd_num - 1) { - i = 0; - bdma_chan->wr_count_next++; /* skip link descriptor */ - } - - bdma_chan->wr_count_next++; - tx_desc->txd.phys = bdma_chan->bd_phys + - i * sizeof(struct tsi721_dma_desc); - tx_desc->hw_desc = &((struct tsi721_dma_desc *)bdma_chan->bd_base)[i]; -err_out: - spin_unlock_bh(&bdma_chan->lock); - - return ret; -} - static int -tsi721_desc_fill_init(struct tsi721_tx_desc *desc, struct scatterlist *sg, - enum dma_rtype rtype, u32 sys_size) +tsi721_desc_fill_init(struct tsi721_tx_desc *desc, + struct tsi721_dma_desc *bd_ptr, + struct scatterlist *sg, u32 sys_size) { - struct tsi721_dma_desc *bd_ptr = desc->hw_desc; u64 rio_addr; + if (bd_ptr == NULL) + return -EINVAL; + /* Initialize DMA descriptor */ bd_ptr->type_id = cpu_to_le32((DTYPE1 << 29) | - (rtype << 19) | desc->destid); + (desc->rtype << 19) | desc->destid); bd_ptr->bcount = cpu_to_le32(((desc->rio_addr & 0x3) << 30) | (sys_size << 26)); rio_addr = (desc->rio_addr >> 2) | @@ -335,51 +361,32 @@ tsi721_desc_fill_init(struct tsi721_tx_desc *desc, struct scatterlist *sg, } static int -tsi721_desc_fill_end(struct tsi721_tx_desc *desc) +tsi721_desc_fill_end(struct tsi721_dma_desc *bd_ptr, u32 bcount, bool interrupt) { - struct tsi721_dma_desc *bd_ptr = desc->hw_desc; + if (bd_ptr == NULL) + return -EINVAL; /* Update DMA descriptor */ - if (desc->interrupt) + if (interrupt) bd_ptr->type_id |= cpu_to_le32(TSI721_DMAD_IOF); - bd_ptr->bcount |= cpu_to_le32(desc->bcount & TSI721_DMAD_BCOUNT1); + bd_ptr->bcount |= cpu_to_le32(bcount & TSI721_DMAD_BCOUNT1); return 0; } - -static void tsi721_dma_chain_complete(struct tsi721_bdma_chan *bdma_chan, - struct tsi721_tx_desc *desc) +static void tsi721_dma_tx_err(struct tsi721_bdma_chan *bdma_chan, + struct tsi721_tx_desc *desc) { struct dma_async_tx_descriptor *txd = &desc->txd; dma_async_tx_callback callback = txd->callback; void *param = txd->callback_param; - list_splice_init(&desc->tx_list, &bdma_chan->free_list); list_move(&desc->desc_node, &bdma_chan->free_list); - bdma_chan->completed_cookie = txd->cookie; if (callback) callback(param); } -static void tsi721_dma_complete_all(struct tsi721_bdma_chan *bdma_chan) -{ - struct tsi721_tx_desc *desc, *_d; - LIST_HEAD(list); - - BUG_ON(!tsi721_dma_is_idle(bdma_chan)); - - if (!list_empty(&bdma_chan->queue)) - tsi721_start_dma(bdma_chan); - - list_splice_init(&bdma_chan->active_list, &list); - list_splice_init(&bdma_chan->queue, &bdma_chan->active_list); - - list_for_each_entry_safe(desc, _d, &list, desc_node) - tsi721_dma_chain_complete(bdma_chan, desc); -} - static void tsi721_clr_stat(struct tsi721_bdma_chan *bdma_chan) { u32 srd_ptr; @@ -403,20 +410,159 @@ static void tsi721_clr_stat(struct tsi721_bdma_chan *bdma_chan) bdma_chan->sts_rdptr = srd_ptr; } +/* Must be called with the channel spinlock held */ +static int tsi721_submit_sg(struct tsi721_tx_desc *desc) +{ + struct dma_chan *dchan = desc->txd.chan; + struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan); + u32 sys_size; + u64 rio_addr; + dma_addr_t next_addr; + u32 bcount; + struct scatterlist *sg; + unsigned int i; + int err = 0; + struct tsi721_dma_desc *bd_ptr = NULL; + u32 idx, rd_idx; + u32 add_count = 0; + + if (!tsi721_dma_is_idle(bdma_chan)) { + dev_err(bdma_chan->dchan.device->dev, + "BUG: Attempt to use non-idle channel\n"); + return -EIO; + } + + /* + * Fill DMA channel's hardware buffer descriptors. + * (NOTE: RapidIO destination address is limited to 64 bits for now) + */ + rio_addr = desc->rio_addr; + next_addr = -1; + bcount = 0; + sys_size = dma_to_mport(bdma_chan->dchan.device)->sys_size; + + rd_idx = ioread32(bdma_chan->regs + TSI721_DMAC_DRDCNT); + rd_idx %= (bdma_chan->bd_num + 1); + + idx = bdma_chan->wr_count_next % (bdma_chan->bd_num + 1); + if (idx == bdma_chan->bd_num) { + /* wrap around link descriptor */ + idx = 0; + add_count++; + } + + dev_dbg(dchan->device->dev, "%s: BD ring status: rdi=%d wri=%d\n", + __func__, rd_idx, idx); + + for_each_sg(desc->sg, sg, desc->sg_len, i) { + + dev_dbg(dchan->device->dev, "sg%d/%d addr: 0x%llx len: %d\n", + i, desc->sg_len, + (unsigned long long)sg_dma_address(sg), sg_dma_len(sg)); + + if (sg_dma_len(sg) > TSI721_BDMA_MAX_BCOUNT) { + dev_err(dchan->device->dev, + "%s: SG entry %d is too large\n", __func__, i); + err = -EINVAL; + break; + } + + /* + * If this sg entry forms contiguous block with previous one, + * try to merge it into existing DMA descriptor + */ + if (next_addr == sg_dma_address(sg) && + bcount + sg_dma_len(sg) <= TSI721_BDMA_MAX_BCOUNT) { + /* Adjust byte count of the descriptor */ + bcount += sg_dma_len(sg); + goto entry_done; + } else if (next_addr != -1) { + /* Finalize descriptor using total byte count value */ + tsi721_desc_fill_end(bd_ptr, bcount, 0); + dev_dbg(dchan->device->dev, + "%s: prev desc final len: %d\n", + __func__, bcount); + } + + desc->rio_addr = rio_addr; + + if (i && idx == rd_idx) { + dev_dbg(dchan->device->dev, + "%s: HW descriptor ring is full @ %d\n", + __func__, i); + desc->sg = sg; + desc->sg_len -= i; + break; + } + + bd_ptr = &((struct tsi721_dma_desc *)bdma_chan->bd_base)[idx]; + err = tsi721_desc_fill_init(desc, bd_ptr, sg, sys_size); + if (err) { + dev_err(dchan->device->dev, + "Failed to build desc: err=%d\n", err); + break; + } + + dev_dbg(dchan->device->dev, "bd_ptr = %p did=%d raddr=0x%llx\n", + bd_ptr, desc->destid, desc->rio_addr); + + next_addr = sg_dma_address(sg); + bcount = sg_dma_len(sg); + + add_count++; + if (++idx == bdma_chan->bd_num) { + /* wrap around link descriptor */ + idx = 0; + add_count++; + } + +entry_done: + if (sg_is_last(sg)) { + tsi721_desc_fill_end(bd_ptr, bcount, 0); + dev_dbg(dchan->device->dev, "%s: last desc final len: %d\n", + __func__, bcount); + desc->sg_len = 0; + } else { + rio_addr += sg_dma_len(sg); + next_addr += sg_dma_len(sg); + } + } + + if (!err) + bdma_chan->wr_count_next += add_count; + + return err; +} + static void tsi721_advance_work(struct tsi721_bdma_chan *bdma_chan) { - if (list_empty(&bdma_chan->active_list) || - list_is_singular(&bdma_chan->active_list)) { - dev_dbg(bdma_chan->dchan.device->dev, - "%s: Active_list empty\n", __func__); - tsi721_dma_complete_all(bdma_chan); - } else { - dev_dbg(bdma_chan->dchan.device->dev, - "%s: Active_list NOT empty\n", __func__); - tsi721_dma_chain_complete(bdma_chan, - tsi721_dma_first_active(bdma_chan)); - tsi721_start_dma(bdma_chan); + struct tsi721_tx_desc *desc; + int err; + + dev_dbg(bdma_chan->dchan.device->dev, "%s: Enter\n", __func__); + + /* + * If there are any new transactions in the queue add them + * into the processing list + */ + if (!list_empty(&bdma_chan->queue)) + list_splice_init(&bdma_chan->queue, &bdma_chan->active_list); + + /* Start new transaction (if available) */ + if (!list_empty(&bdma_chan->active_list)) { + desc = tsi721_dma_first_active(bdma_chan); + err = tsi721_submit_sg(desc); + if (!err) + tsi721_start_dma(bdma_chan); + else { + tsi721_dma_tx_err(bdma_chan, desc); + dev_dbg(bdma_chan->dchan.device->dev, + "ERR: tsi721_submit_sg failed with err=%d\n", + err); + } } + + dev_dbg(bdma_chan->dchan.device->dev, "%s: Exit\n", __func__); } static void tsi721_dma_tasklet(unsigned long data) @@ -444,8 +590,29 @@ static void tsi721_dma_tasklet(unsigned long data) } if (dmac_int & (TSI721_DMAC_INT_DONE | TSI721_DMAC_INT_IOFDONE)) { + struct tsi721_tx_desc *desc; + tsi721_clr_stat(bdma_chan); spin_lock(&bdma_chan->lock); + desc = tsi721_dma_first_active(bdma_chan); + + if (desc->sg_len == 0) { + dma_async_tx_callback callback = NULL; + void *param = NULL; + + desc->status = DMA_COMPLETE; + dma_cookie_complete(&desc->txd); + if (desc->txd.flags & DMA_PREP_INTERRUPT) { + callback = desc->txd.callback; + param = desc->txd.callback_param; + } + list_move(&desc->desc_node, &bdma_chan->free_list); + spin_unlock(&bdma_chan->lock); + if (callback) + callback(param); + spin_lock(&bdma_chan->lock); + } + tsi721_advance_work(bdma_chan); spin_unlock(&bdma_chan->lock); } @@ -460,21 +627,24 @@ static dma_cookie_t tsi721_tx_submit(struct dma_async_tx_descriptor *txd) struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(txd->chan); dma_cookie_t cookie; - spin_lock_bh(&bdma_chan->lock); + /* Check if the descriptor is detached from any lists */ + if (!list_empty(&desc->desc_node)) { + dev_err(bdma_chan->dchan.device->dev, + "%s: wrong state of descriptor %p\n", __func__, txd); + return -EIO; + } - cookie = txd->chan->cookie; - if (++cookie < 0) - cookie = 1; - txd->chan->cookie = cookie; - txd->cookie = cookie; + spin_lock_bh(&bdma_chan->lock); - if (list_empty(&bdma_chan->active_list)) { - list_add_tail(&desc->desc_node, &bdma_chan->active_list); - tsi721_start_dma(bdma_chan); - } else { - list_add_tail(&desc->desc_node, &bdma_chan->queue); + if (!bdma_chan->active) { + spin_unlock_bh(&bdma_chan->lock); + return -ENODEV; } + cookie = dma_cookie_assign(txd); + desc->status = DMA_IN_PROGRESS; + list_add_tail(&desc->desc_node, &bdma_chan->queue); + spin_unlock_bh(&bdma_chan->lock); return cookie; } @@ -482,115 +652,52 @@ static dma_cookie_t tsi721_tx_submit(struct dma_async_tx_descriptor *txd) static int tsi721_alloc_chan_resources(struct dma_chan *dchan) { struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan); -#ifdef CONFIG_PCI_MSI - struct tsi721_device *priv = to_tsi721(dchan->device); -#endif struct tsi721_tx_desc *desc = NULL; - LIST_HEAD(tmp_list); int i; - int rc; + + dev_dbg(dchan->device->dev, "%s: for channel %d\n", + __func__, bdma_chan->id); if (bdma_chan->bd_base) - return bdma_chan->bd_num - 1; + return TSI721_DMA_TX_QUEUE_SZ; /* Initialize BDMA channel */ - if (tsi721_bdma_ch_init(bdma_chan)) { + if (tsi721_bdma_ch_init(bdma_chan, dma_desc_per_channel)) { dev_err(dchan->device->dev, "Unable to initialize data DMA" " channel %d, aborting\n", bdma_chan->id); - return -ENOMEM; + return -ENODEV; } - /* Alocate matching number of logical descriptors */ - desc = kcalloc((bdma_chan->bd_num - 1), sizeof(struct tsi721_tx_desc), + /* Allocate queue of transaction descriptors */ + desc = kcalloc(TSI721_DMA_TX_QUEUE_SZ, sizeof(struct tsi721_tx_desc), GFP_KERNEL); if (!desc) { dev_err(dchan->device->dev, "Failed to allocate logical descriptors\n"); - rc = -ENOMEM; - goto err_out; + tsi721_bdma_ch_free(bdma_chan); + return -ENOMEM; } bdma_chan->tx_desc = desc; - for (i = 0; i < bdma_chan->bd_num - 1; i++) { + for (i = 0; i < TSI721_DMA_TX_QUEUE_SZ; i++) { dma_async_tx_descriptor_init(&desc[i].txd, dchan); desc[i].txd.tx_submit = tsi721_tx_submit; desc[i].txd.flags = DMA_CTRL_ACK; - INIT_LIST_HEAD(&desc[i].tx_list); - list_add_tail(&desc[i].desc_node, &tmp_list); + list_add(&desc[i].desc_node, &bdma_chan->free_list); } - spin_lock_bh(&bdma_chan->lock); - list_splice(&tmp_list, &bdma_chan->free_list); - bdma_chan->completed_cookie = dchan->cookie = 1; - spin_unlock_bh(&bdma_chan->lock); - -#ifdef CONFIG_PCI_MSI - if (priv->flags & TSI721_USING_MSIX) { - /* Request interrupt service if we are in MSI-X mode */ - rc = request_irq( - priv->msix[TSI721_VECT_DMA0_DONE + - bdma_chan->id].vector, - tsi721_bdma_msix, 0, - priv->msix[TSI721_VECT_DMA0_DONE + - bdma_chan->id].irq_name, - (void *)bdma_chan); - - if (rc) { - dev_dbg(dchan->device->dev, - "Unable to allocate MSI-X interrupt for " - "BDMA%d-DONE\n", bdma_chan->id); - goto err_out; - } - - rc = request_irq(priv->msix[TSI721_VECT_DMA0_INT + - bdma_chan->id].vector, - tsi721_bdma_msix, 0, - priv->msix[TSI721_VECT_DMA0_INT + - bdma_chan->id].irq_name, - (void *)bdma_chan); - - if (rc) { - dev_dbg(dchan->device->dev, - "Unable to allocate MSI-X interrupt for " - "BDMA%d-INT\n", bdma_chan->id); - free_irq( - priv->msix[TSI721_VECT_DMA0_DONE + - bdma_chan->id].vector, - (void *)bdma_chan); - rc = -EIO; - goto err_out; - } - } -#endif /* CONFIG_PCI_MSI */ + dma_cookie_init(dchan); bdma_chan->active = true; tsi721_bdma_interrupt_enable(bdma_chan, 1); - return bdma_chan->bd_num - 1; - -err_out: - kfree(desc); - tsi721_bdma_ch_free(bdma_chan); - return rc; + return TSI721_DMA_TX_QUEUE_SZ; } -static void tsi721_free_chan_resources(struct dma_chan *dchan) +static void tsi721_sync_dma_irq(struct tsi721_bdma_chan *bdma_chan) { - struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan); - struct tsi721_device *priv = to_tsi721(dchan->device); - LIST_HEAD(list); - - dev_dbg(dchan->device->dev, "%s: Entry\n", __func__); - - if (bdma_chan->bd_base == NULL) - return; - - BUG_ON(!list_empty(&bdma_chan->active_list)); - BUG_ON(!list_empty(&bdma_chan->queue)); - - tsi721_bdma_interrupt_enable(bdma_chan, 0); - bdma_chan->active = false; + struct tsi721_device *priv = to_tsi721(bdma_chan->dchan.device); #ifdef CONFIG_PCI_MSI if (priv->flags & TSI721_USING_MSIX) { @@ -601,64 +708,48 @@ static void tsi721_free_chan_resources(struct dma_chan *dchan) } else #endif synchronize_irq(priv->pdev->irq); +} - tasklet_kill(&bdma_chan->tasklet); +static void tsi721_free_chan_resources(struct dma_chan *dchan) +{ + struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan); - spin_lock_bh(&bdma_chan->lock); - list_splice_init(&bdma_chan->free_list, &list); - spin_unlock_bh(&bdma_chan->lock); + dev_dbg(dchan->device->dev, "%s: for channel %d\n", + __func__, bdma_chan->id); -#ifdef CONFIG_PCI_MSI - if (priv->flags & TSI721_USING_MSIX) { - free_irq(priv->msix[TSI721_VECT_DMA0_DONE + - bdma_chan->id].vector, (void *)bdma_chan); - free_irq(priv->msix[TSI721_VECT_DMA0_INT + - bdma_chan->id].vector, (void *)bdma_chan); - } -#endif /* CONFIG_PCI_MSI */ + if (bdma_chan->bd_base == NULL) + return; - tsi721_bdma_ch_free(bdma_chan); + BUG_ON(!list_empty(&bdma_chan->active_list)); + BUG_ON(!list_empty(&bdma_chan->queue)); + + tsi721_bdma_interrupt_enable(bdma_chan, 0); + bdma_chan->active = false; + tsi721_sync_dma_irq(bdma_chan); + tasklet_kill(&bdma_chan->tasklet); + INIT_LIST_HEAD(&bdma_chan->free_list); kfree(bdma_chan->tx_desc); + tsi721_bdma_ch_free(bdma_chan); } static enum dma_status tsi721_tx_status(struct dma_chan *dchan, dma_cookie_t cookie, struct dma_tx_state *txstate) { - struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan); - dma_cookie_t last_used; - dma_cookie_t last_completed; - int ret; - - spin_lock_bh(&bdma_chan->lock); - last_completed = bdma_chan->completed_cookie; - last_used = dchan->cookie; - spin_unlock_bh(&bdma_chan->lock); - - ret = dma_async_is_complete(cookie, last_completed, last_used); - - dma_set_tx_state(txstate, last_completed, last_used, 0); - - dev_dbg(dchan->device->dev, - "%s: exit, ret: %d, last_completed: %d, last_used: %d\n", - __func__, ret, last_completed, last_used); - - return ret; + return dma_cookie_status(dchan, cookie, txstate); } static void tsi721_issue_pending(struct dma_chan *dchan) { struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan); - dev_dbg(dchan->device->dev, "%s: Entry\n", __func__); + dev_dbg(dchan->device->dev, "%s: Enter\n", __func__); - if (tsi721_dma_is_idle(bdma_chan)) { + if (tsi721_dma_is_idle(bdma_chan) && bdma_chan->active) { spin_lock_bh(&bdma_chan->lock); tsi721_advance_work(bdma_chan); spin_unlock_bh(&bdma_chan->lock); - } else - dev_dbg(dchan->device->dev, - "%s: DMA channel still busy\n", __func__); + } } static @@ -668,21 +759,19 @@ struct dma_async_tx_descriptor *tsi721_prep_rio_sg(struct dma_chan *dchan, void *tinfo) { struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan); - struct tsi721_tx_desc *desc = NULL; - struct tsi721_tx_desc *first = NULL; - struct scatterlist *sg; + struct tsi721_tx_desc *desc, *_d; struct rio_dma_ext *rext = tinfo; - u64 rio_addr = rext->rio_addr; /* limited to 64-bit rio_addr for now */ - unsigned int i; - u32 sys_size = dma_to_mport(dchan->device)->sys_size; enum dma_rtype rtype; - dma_addr_t next_addr = -1; + struct dma_async_tx_descriptor *txd = NULL; if (!sgl || !sg_len) { dev_err(dchan->device->dev, "%s: No SG list\n", __func__); return NULL; } + dev_dbg(dchan->device->dev, "%s: %s\n", __func__, + (dir == DMA_DEV_TO_MEM)?"READ":"WRITE"); + if (dir == DMA_DEV_TO_MEM) rtype = NREAD; else if (dir == DMA_MEM_TO_DEV) { @@ -704,97 +793,26 @@ struct dma_async_tx_descriptor *tsi721_prep_rio_sg(struct dma_chan *dchan, return NULL; } - for_each_sg(sgl, sg, sg_len, i) { - int err; - - if (sg_dma_len(sg) > TSI721_BDMA_MAX_BCOUNT) { - dev_err(dchan->device->dev, - "%s: SG entry %d is too large\n", __func__, i); - goto err_desc_put; - } - - /* - * If this sg entry forms contiguous block with previous one, - * try to merge it into existing DMA descriptor - */ - if (desc) { - if (next_addr == sg_dma_address(sg) && - desc->bcount + sg_dma_len(sg) <= - TSI721_BDMA_MAX_BCOUNT) { - /* Adjust byte count of the descriptor */ - desc->bcount += sg_dma_len(sg); - goto entry_done; - } - - /* - * Finalize this descriptor using total - * byte count value. - */ - tsi721_desc_fill_end(desc); - dev_dbg(dchan->device->dev, "%s: desc final len: %d\n", - __func__, desc->bcount); - } - - /* - * Obtain and initialize a new descriptor - */ - desc = tsi721_desc_get(bdma_chan); - if (!desc) { - dev_err(dchan->device->dev, - "%s: Failed to get new descriptor for SG %d\n", - __func__, i); - goto err_desc_put; - } - - desc->destid = rext->destid; - desc->rio_addr = rio_addr; - desc->rio_addr_u = 0; - desc->bcount = sg_dma_len(sg); - - dev_dbg(dchan->device->dev, - "sg%d desc: 0x%llx, addr: 0x%llx len: %d\n", - i, (u64)desc->txd.phys, - (unsigned long long)sg_dma_address(sg), - sg_dma_len(sg)); - - dev_dbg(dchan->device->dev, - "bd_ptr = %p did=%d raddr=0x%llx\n", - desc->hw_desc, desc->destid, desc->rio_addr); - - err = tsi721_desc_fill_init(desc, sg, rtype, sys_size); - if (err) { - dev_err(dchan->device->dev, - "Failed to build desc: %d\n", err); - goto err_desc_put; - } - - next_addr = sg_dma_address(sg); - - if (!first) - first = desc; - else - list_add_tail(&desc->desc_node, &first->tx_list); + spin_lock_bh(&bdma_chan->lock); -entry_done: - if (sg_is_last(sg)) { - desc->interrupt = (flags & DMA_PREP_INTERRUPT) != 0; - tsi721_desc_fill_end(desc); - dev_dbg(dchan->device->dev, "%s: desc final len: %d\n", - __func__, desc->bcount); - } else { - rio_addr += sg_dma_len(sg); - next_addr += sg_dma_len(sg); + list_for_each_entry_safe(desc, _d, &bdma_chan->free_list, desc_node) { + if (async_tx_test_ack(&desc->txd)) { + list_del_init(&desc->desc_node); + desc->destid = rext->destid; + desc->rio_addr = rext->rio_addr; + desc->rio_addr_u = 0; + desc->rtype = rtype; + desc->sg_len = sg_len; + desc->sg = sgl; + txd = &desc->txd; + txd->flags = flags; + break; } } - first->txd.cookie = -EBUSY; - desc->txd.flags = flags; - - return &first->txd; + spin_unlock_bh(&bdma_chan->lock); -err_desc_put: - tsi721_desc_put(bdma_chan, first); - return NULL; + return txd; } static int tsi721_device_control(struct dma_chan *dchan, enum dma_ctrl_cmd cmd, @@ -802,23 +820,34 @@ static int tsi721_device_control(struct dma_chan *dchan, enum dma_ctrl_cmd cmd, { struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan); struct tsi721_tx_desc *desc, *_d; + u32 dmac_int; LIST_HEAD(list); dev_dbg(dchan->device->dev, "%s: Entry\n", __func__); if (cmd != DMA_TERMINATE_ALL) - return -ENXIO; + return -ENOSYS; spin_lock_bh(&bdma_chan->lock); - /* make sure to stop the transfer */ - iowrite32(TSI721_DMAC_CTL_SUSP, bdma_chan->regs + TSI721_DMAC_CTL); + bdma_chan->active = false; + + if (!tsi721_dma_is_idle(bdma_chan)) { + /* make sure to stop the transfer */ + iowrite32(TSI721_DMAC_CTL_SUSP, + bdma_chan->regs + TSI721_DMAC_CTL); + + /* Wait until DMA channel stops */ + do { + dmac_int = ioread32(bdma_chan->regs + TSI721_DMAC_INT); + } while ((dmac_int & TSI721_DMAC_INT_SUSP) == 0); + } list_splice_init(&bdma_chan->active_list, &list); list_splice_init(&bdma_chan->queue, &list); list_for_each_entry_safe(desc, _d, &list, desc_node) - tsi721_dma_chain_complete(bdma_chan, desc); + tsi721_dma_tx_err(bdma_chan, desc); spin_unlock_bh(&bdma_chan->lock); @@ -828,22 +857,18 @@ static int tsi721_device_control(struct dma_chan *dchan, enum dma_ctrl_cmd cmd, int tsi721_register_dma(struct tsi721_device *priv) { int i; - int nr_channels = TSI721_DMA_MAXCH; + int nr_channels = 0; int err; struct rio_mport *mport = priv->mport; - mport->dma.dev = &priv->pdev->dev; - mport->dma.chancnt = nr_channels; - INIT_LIST_HEAD(&mport->dma.channels); - for (i = 0; i < nr_channels; i++) { + for (i = 0; i < TSI721_DMA_MAXCH; i++) { struct tsi721_bdma_chan *bdma_chan = &priv->bdma[i]; if (i == TSI721_DMACH_MAINT) continue; - bdma_chan->bd_num = TSI721_BDMA_BD_RING_SZ; bdma_chan->regs = priv->regs + TSI721_DMAC_BASE(i); bdma_chan->dchan.device = &mport->dma; @@ -862,12 +887,15 @@ int tsi721_register_dma(struct tsi721_device *priv) (unsigned long)bdma_chan); list_add_tail(&bdma_chan->dchan.device_node, &mport->dma.channels); + nr_channels++; } + mport->dma.chancnt = nr_channels; dma_cap_zero(mport->dma.cap_mask); dma_cap_set(DMA_PRIVATE, mport->dma.cap_mask); dma_cap_set(DMA_SLAVE, mport->dma.cap_mask); + mport->dma.dev = &priv->pdev->dev; mport->dma.device_alloc_chan_resources = tsi721_alloc_chan_resources; mport->dma.device_free_chan_resources = tsi721_free_chan_resources; mport->dma.device_tx_status = tsi721_tx_status; |