diff options
Diffstat (limited to 'drivers/usb/dwc2/gadget.c')
| -rw-r--r-- | drivers/usb/dwc2/gadget.c | 1126 |
1 files changed, 900 insertions, 226 deletions
diff --git a/drivers/usb/dwc2/gadget.c b/drivers/usb/dwc2/gadget.c index 24fbebc9b409..b95930f20d90 100644 --- a/drivers/usb/dwc2/gadget.c +++ b/drivers/usb/dwc2/gadget.c @@ -93,7 +93,18 @@ static void dwc2_hsotg_dump(struct dwc2_hsotg *hsotg); */ static inline bool using_dma(struct dwc2_hsotg *hsotg) { - return hsotg->g_using_dma; + return hsotg->params.g_dma; +} + +/* + * using_desc_dma - return the descriptor DMA status of the driver. + * @hsotg: The driver state. + * + * Return true if we're using descriptor DMA. + */ +static inline bool using_desc_dma(struct dwc2_hsotg *hsotg) +{ + return hsotg->params.g_dma_desc; } /** @@ -190,16 +201,17 @@ static void dwc2_hsotg_init_fifo(struct dwc2_hsotg *hsotg) unsigned int addr; int timeout; u32 val; + u32 *txfsz = hsotg->params.g_tx_fifo_size; /* Reset fifo map if not correctly cleared during previous session */ WARN_ON(hsotg->fifo_map); hsotg->fifo_map = 0; /* set RX/NPTX FIFO sizes */ - dwc2_writel(hsotg->g_rx_fifo_sz, hsotg->regs + GRXFSIZ); - dwc2_writel((hsotg->g_rx_fifo_sz << FIFOSIZE_STARTADDR_SHIFT) | - (hsotg->g_np_g_tx_fifo_sz << FIFOSIZE_DEPTH_SHIFT), - hsotg->regs + GNPTXFSIZ); + dwc2_writel(hsotg->params.g_rx_fifo_size, hsotg->regs + GRXFSIZ); + dwc2_writel((hsotg->params.g_rx_fifo_size << FIFOSIZE_STARTADDR_SHIFT) | + (hsotg->params.g_np_tx_fifo_size << FIFOSIZE_DEPTH_SHIFT), + hsotg->regs + GNPTXFSIZ); /* * arange all the rest of the TX FIFOs, as some versions of this @@ -209,7 +221,7 @@ static void dwc2_hsotg_init_fifo(struct dwc2_hsotg *hsotg) */ /* start at the end of the GNPTXFSIZ, rounded up */ - addr = hsotg->g_rx_fifo_sz + hsotg->g_np_g_tx_fifo_sz; + addr = hsotg->params.g_rx_fifo_size + hsotg->params.g_np_tx_fifo_size; /* * Configure fifos sizes from provided configuration and assign @@ -217,15 +229,16 @@ static void dwc2_hsotg_init_fifo(struct dwc2_hsotg *hsotg) * given endpoint. */ for (ep = 1; ep < MAX_EPS_CHANNELS; ep++) { - if (!hsotg->g_tx_fifo_sz[ep]) + if (!txfsz[ep]) continue; val = addr; - val |= hsotg->g_tx_fifo_sz[ep] << FIFOSIZE_DEPTH_SHIFT; - WARN_ONCE(addr + hsotg->g_tx_fifo_sz[ep] > hsotg->fifo_mem, + val |= txfsz[ep] << FIFOSIZE_DEPTH_SHIFT; + WARN_ONCE(addr + txfsz[ep] > hsotg->fifo_mem, "insufficient fifo memory"); - addr += hsotg->g_tx_fifo_sz[ep]; + addr += txfsz[ep]; dwc2_writel(val, hsotg->regs + DPTXFSIZN(ep)); + val = dwc2_readl(hsotg->regs + DPTXFSIZN(ep)); } /* @@ -303,12 +316,55 @@ static void dwc2_hsotg_unmap_dma(struct dwc2_hsotg *hsotg, struct dwc2_hsotg_req *hs_req) { struct usb_request *req = &hs_req->req; + usb_gadget_unmap_request(&hsotg->gadget, req, hs_ep->dir_in); +} - /* ignore this if we're not moving any data */ - if (hs_req->req.length == 0) - return; +/* + * dwc2_gadget_alloc_ctrl_desc_chains - allocate DMA descriptor chains + * for Control endpoint + * @hsotg: The device state. + * + * This function will allocate 4 descriptor chains for EP 0: 2 for + * Setup stage, per one for IN and OUT data/status transactions. + */ +static int dwc2_gadget_alloc_ctrl_desc_chains(struct dwc2_hsotg *hsotg) +{ + hsotg->setup_desc[0] = + dmam_alloc_coherent(hsotg->dev, + sizeof(struct dwc2_dma_desc), + &hsotg->setup_desc_dma[0], + GFP_KERNEL); + if (!hsotg->setup_desc[0]) + goto fail; + + hsotg->setup_desc[1] = + dmam_alloc_coherent(hsotg->dev, + sizeof(struct dwc2_dma_desc), + &hsotg->setup_desc_dma[1], + GFP_KERNEL); + if (!hsotg->setup_desc[1]) + goto fail; + + hsotg->ctrl_in_desc = + dmam_alloc_coherent(hsotg->dev, + sizeof(struct dwc2_dma_desc), + &hsotg->ctrl_in_desc_dma, + GFP_KERNEL); + if (!hsotg->ctrl_in_desc) + goto fail; + + hsotg->ctrl_out_desc = + dmam_alloc_coherent(hsotg->dev, + sizeof(struct dwc2_dma_desc), + &hsotg->ctrl_out_desc_dma, + GFP_KERNEL); + if (!hsotg->ctrl_out_desc) + goto fail; - usb_gadget_unmap_request(&hsotg->gadget, req, hs_ep->dir_in); + return 0; + +fail: + return -ENOMEM; } /** @@ -541,6 +597,273 @@ static u32 dwc2_hsotg_read_frameno(struct dwc2_hsotg *hsotg) } /** + * dwc2_gadget_get_chain_limit - get the maximum data payload value of the + * DMA descriptor chain prepared for specific endpoint + * @hs_ep: The endpoint + * + * Return the maximum data that can be queued in one go on a given endpoint + * depending on its descriptor chain capacity so that transfers that + * are too long can be split. + */ +static unsigned int dwc2_gadget_get_chain_limit(struct dwc2_hsotg_ep *hs_ep) +{ + int is_isoc = hs_ep->isochronous; + unsigned int maxsize; + + if (is_isoc) + maxsize = hs_ep->dir_in ? DEV_DMA_ISOC_TX_NBYTES_LIMIT : + DEV_DMA_ISOC_RX_NBYTES_LIMIT; + else + maxsize = DEV_DMA_NBYTES_LIMIT; + + /* Above size of one descriptor was chosen, multiple it */ + maxsize *= MAX_DMA_DESC_NUM_GENERIC; + + return maxsize; +} + +/* + * dwc2_gadget_get_desc_params - get DMA descriptor parameters. + * @hs_ep: The endpoint + * @mask: RX/TX bytes mask to be defined + * + * Returns maximum data payload for one descriptor after analyzing endpoint + * characteristics. + * DMA descriptor transfer bytes limit depends on EP type: + * Control out - MPS, + * Isochronous - descriptor rx/tx bytes bitfield limit, + * Control In/Bulk/Interrupt - multiple of mps. This will allow to not + * have concatenations from various descriptors within one packet. + * + * Selects corresponding mask for RX/TX bytes as well. + */ +static u32 dwc2_gadget_get_desc_params(struct dwc2_hsotg_ep *hs_ep, u32 *mask) +{ + u32 mps = hs_ep->ep.maxpacket; + int dir_in = hs_ep->dir_in; + u32 desc_size = 0; + + if (!hs_ep->index && !dir_in) { + desc_size = mps; + *mask = DEV_DMA_NBYTES_MASK; + } else if (hs_ep->isochronous) { + if (dir_in) { + desc_size = DEV_DMA_ISOC_TX_NBYTES_LIMIT; + *mask = DEV_DMA_ISOC_TX_NBYTES_MASK; + } else { + desc_size = DEV_DMA_ISOC_RX_NBYTES_LIMIT; + *mask = DEV_DMA_ISOC_RX_NBYTES_MASK; + } + } else { + desc_size = DEV_DMA_NBYTES_LIMIT; + *mask = DEV_DMA_NBYTES_MASK; + + /* Round down desc_size to be mps multiple */ + desc_size -= desc_size % mps; + } + + return desc_size; +} + +/* + * dwc2_gadget_config_nonisoc_xfer_ddma - prepare non ISOC DMA desc chain. + * @hs_ep: The endpoint + * @dma_buff: DMA address to use + * @len: Length of the transfer + * + * This function will iterate over descriptor chain and fill its entries + * with corresponding information based on transfer data. + */ +static void dwc2_gadget_config_nonisoc_xfer_ddma(struct dwc2_hsotg_ep *hs_ep, + dma_addr_t dma_buff, + unsigned int len) +{ + struct dwc2_hsotg *hsotg = hs_ep->parent; + int dir_in = hs_ep->dir_in; + struct dwc2_dma_desc *desc = hs_ep->desc_list; + u32 mps = hs_ep->ep.maxpacket; + u32 maxsize = 0; + u32 offset = 0; + u32 mask = 0; + int i; + + maxsize = dwc2_gadget_get_desc_params(hs_ep, &mask); + + hs_ep->desc_count = (len / maxsize) + + ((len % maxsize) ? 1 : 0); + if (len == 0) + hs_ep->desc_count = 1; + + for (i = 0; i < hs_ep->desc_count; ++i) { + desc->status = 0; + desc->status |= (DEV_DMA_BUFF_STS_HBUSY + << DEV_DMA_BUFF_STS_SHIFT); + + if (len > maxsize) { + if (!hs_ep->index && !dir_in) + desc->status |= (DEV_DMA_L | DEV_DMA_IOC); + + desc->status |= (maxsize << + DEV_DMA_NBYTES_SHIFT & mask); + desc->buf = dma_buff + offset; + + len -= maxsize; + offset += maxsize; + } else { + desc->status |= (DEV_DMA_L | DEV_DMA_IOC); + + if (dir_in) + desc->status |= (len % mps) ? DEV_DMA_SHORT : + ((hs_ep->send_zlp) ? DEV_DMA_SHORT : 0); + if (len > maxsize) + dev_err(hsotg->dev, "wrong len %d\n", len); + + desc->status |= + len << DEV_DMA_NBYTES_SHIFT & mask; + desc->buf = dma_buff + offset; + } + + desc->status &= ~DEV_DMA_BUFF_STS_MASK; + desc->status |= (DEV_DMA_BUFF_STS_HREADY + << DEV_DMA_BUFF_STS_SHIFT); + desc++; + } +} + +/* + * dwc2_gadget_fill_isoc_desc - fills next isochronous descriptor in chain. + * @hs_ep: The isochronous endpoint. + * @dma_buff: usb requests dma buffer. + * @len: usb request transfer length. + * + * Finds out index of first free entry either in the bottom or up half of + * descriptor chain depend on which is under SW control and not processed + * by HW. Then fills that descriptor with the data of the arrived usb request, + * frame info, sets Last and IOC bits increments next_desc. If filled + * descriptor is not the first one, removes L bit from the previous descriptor + * status. + */ +static int dwc2_gadget_fill_isoc_desc(struct dwc2_hsotg_ep *hs_ep, + dma_addr_t dma_buff, unsigned int len) +{ + struct dwc2_dma_desc *desc; + struct dwc2_hsotg *hsotg = hs_ep->parent; + u32 index; + u32 maxsize = 0; + u32 mask = 0; + + maxsize = dwc2_gadget_get_desc_params(hs_ep, &mask); + if (len > maxsize) { + dev_err(hsotg->dev, "wrong len %d\n", len); + return -EINVAL; + } + + /* + * If SW has already filled half of chain, then return and wait for + * the other chain to be processed by HW. + */ + if (hs_ep->next_desc == MAX_DMA_DESC_NUM_GENERIC / 2) + return -EBUSY; + + /* Increment frame number by interval for IN */ + if (hs_ep->dir_in) + dwc2_gadget_incr_frame_num(hs_ep); + + index = (MAX_DMA_DESC_NUM_GENERIC / 2) * hs_ep->isoc_chain_num + + hs_ep->next_desc; + + /* Sanity check of calculated index */ + if ((hs_ep->isoc_chain_num && index > MAX_DMA_DESC_NUM_GENERIC) || + (!hs_ep->isoc_chain_num && index > MAX_DMA_DESC_NUM_GENERIC / 2)) { + dev_err(hsotg->dev, "wrong index %d for iso chain\n", index); + return -EINVAL; + } + + desc = &hs_ep->desc_list[index]; + + /* Clear L bit of previous desc if more than one entries in the chain */ + if (hs_ep->next_desc) + hs_ep->desc_list[index - 1].status &= ~DEV_DMA_L; + + dev_dbg(hsotg->dev, "%s: Filling ep %d, dir %s isoc desc # %d\n", + __func__, hs_ep->index, hs_ep->dir_in ? "in" : "out", index); + + desc->status = 0; + desc->status |= (DEV_DMA_BUFF_STS_HBUSY << DEV_DMA_BUFF_STS_SHIFT); + + desc->buf = dma_buff; + desc->status |= (DEV_DMA_L | DEV_DMA_IOC | + ((len << DEV_DMA_NBYTES_SHIFT) & mask)); + + if (hs_ep->dir_in) { + desc->status |= ((hs_ep->mc << DEV_DMA_ISOC_PID_SHIFT) & + DEV_DMA_ISOC_PID_MASK) | + ((len % hs_ep->ep.maxpacket) ? + DEV_DMA_SHORT : 0) | + ((hs_ep->target_frame << + DEV_DMA_ISOC_FRNUM_SHIFT) & + DEV_DMA_ISOC_FRNUM_MASK); + } + + desc->status &= ~DEV_DMA_BUFF_STS_MASK; + desc->status |= (DEV_DMA_BUFF_STS_HREADY << DEV_DMA_BUFF_STS_SHIFT); + + /* Update index of last configured entry in the chain */ + hs_ep->next_desc++; + + return 0; +} + +/* + * dwc2_gadget_start_isoc_ddma - start isochronous transfer in DDMA + * @hs_ep: The isochronous endpoint. + * + * Prepare first descriptor chain for isochronous endpoints. Afterwards + * write DMA address to HW and enable the endpoint. + * + * Switch between descriptor chains via isoc_chain_num to give SW opportunity + * to prepare second descriptor chain while first one is being processed by HW. + */ +static void dwc2_gadget_start_isoc_ddma(struct dwc2_hsotg_ep *hs_ep) +{ + struct dwc2_hsotg *hsotg = hs_ep->parent; + struct dwc2_hsotg_req *hs_req, *treq; + int index = hs_ep->index; + int ret; + u32 dma_reg; + u32 depctl; + u32 ctrl; + + if (list_empty(&hs_ep->queue)) { + dev_dbg(hsotg->dev, "%s: No requests in queue\n", __func__); + return; + } + + list_for_each_entry_safe(hs_req, treq, &hs_ep->queue, queue) { + ret = dwc2_gadget_fill_isoc_desc(hs_ep, hs_req->req.dma, + hs_req->req.length); + if (ret) { + dev_dbg(hsotg->dev, "%s: desc chain full\n", __func__); + break; + } + } + + depctl = hs_ep->dir_in ? DIEPCTL(index) : DOEPCTL(index); + dma_reg = hs_ep->dir_in ? DIEPDMA(index) : DOEPDMA(index); + + /* write descriptor chain address to control register */ + dwc2_writel(hs_ep->desc_list_dma, hsotg->regs + dma_reg); + + ctrl = dwc2_readl(hsotg->regs + depctl); + ctrl |= DXEPCTL_EPENA | DXEPCTL_CNAK; + dwc2_writel(ctrl, hsotg->regs + depctl); + + /* Switch ISOC descriptor chain number being processed by SW*/ + hs_ep->isoc_chain_num = (hs_ep->isoc_chain_num ^ 1) & 0x1; + hs_ep->next_desc = 0; +} + +/** * dwc2_hsotg_start_req - start a USB request from an endpoint's queue * @hsotg: The controller state. * @hs_ep: The endpoint to process a request for @@ -565,6 +888,7 @@ static void dwc2_hsotg_start_req(struct dwc2_hsotg *hsotg, unsigned length; unsigned packets; unsigned maxreq; + unsigned int dma_reg; if (index != 0) { if (hs_ep->req && !continuing) { @@ -579,6 +903,7 @@ static void dwc2_hsotg_start_req(struct dwc2_hsotg *hsotg, } } + dma_reg = dir_in ? DIEPDMA(index) : DOEPDMA(index); epctrl_reg = dir_in ? DIEPCTL(index) : DOEPCTL(index); epsize_reg = dir_in ? DIEPTSIZ(index) : DOEPTSIZ(index); @@ -598,7 +923,11 @@ static void dwc2_hsotg_start_req(struct dwc2_hsotg *hsotg, dev_dbg(hsotg->dev, "ureq->length:%d ureq->actual:%d\n", ureq->length, ureq->actual); - maxreq = get_ep_limit(hs_ep); + if (!using_desc_dma(hsotg)) + maxreq = get_ep_limit(hs_ep); + else + maxreq = dwc2_gadget_get_chain_limit(hs_ep); + if (length > maxreq) { int round = maxreq % hs_ep->ep.maxpacket; @@ -650,22 +979,51 @@ static void dwc2_hsotg_start_req(struct dwc2_hsotg *hsotg, /* store the request as the current one we're doing */ hs_ep->req = hs_req; - /* write size / packets */ - dwc2_writel(epsize, hsotg->regs + epsize_reg); + if (using_desc_dma(hsotg)) { + u32 offset = 0; + u32 mps = hs_ep->ep.maxpacket; - if (using_dma(hsotg) && !continuing) { - unsigned int dma_reg; + /* Adjust length: EP0 - MPS, other OUT EPs - multiple of MPS */ + if (!dir_in) { + if (!index) + length = mps; + else if (length % mps) + length += (mps - (length % mps)); + } /* - * write DMA address to control register, buffer already - * synced by dwc2_hsotg_ep_queue(). + * If more data to send, adjust DMA for EP0 out data stage. + * ureq->dma stays unchanged, hence increment it by already + * passed passed data count before starting new transaction. */ + if (!index && hsotg->ep0_state == DWC2_EP0_DATA_OUT && + continuing) + offset = ureq->actual; - dma_reg = dir_in ? DIEPDMA(index) : DOEPDMA(index); - dwc2_writel(ureq->dma, hsotg->regs + dma_reg); + /* Fill DDMA chain entries */ + dwc2_gadget_config_nonisoc_xfer_ddma(hs_ep, ureq->dma + offset, + length); - dev_dbg(hsotg->dev, "%s: %pad => 0x%08x\n", - __func__, &ureq->dma, dma_reg); + /* write descriptor chain address to control register */ + dwc2_writel(hs_ep->desc_list_dma, hsotg->regs + dma_reg); + + dev_dbg(hsotg->dev, "%s: %08x pad => 0x%08x\n", + __func__, (u32)hs_ep->desc_list_dma, dma_reg); + } else { + /* write size / packets */ + dwc2_writel(epsize, hsotg->regs + epsize_reg); + + if (using_dma(hsotg) && !continuing && (length != 0)) { + /* + * write DMA address to control register, buffer + * already synced by dwc2_hsotg_ep_queue(). + */ + + dwc2_writel(ureq->dma, hsotg->regs + dma_reg); + + dev_dbg(hsotg->dev, "%s: %pad => 0x%08x\n", + __func__, &ureq->dma, dma_reg); + } } if (hs_ep->isochronous && hs_ep->interval == 1) { @@ -738,13 +1096,8 @@ static int dwc2_hsotg_map_dma(struct dwc2_hsotg *hsotg, struct dwc2_hsotg_ep *hs_ep, struct usb_request *req) { - struct dwc2_hsotg_req *hs_req = our_req(req); int ret; - /* if the length is zero, ignore the DMA data */ - if (hs_req->req.length == 0) - return 0; - ret = usb_gadget_map_request(&hsotg->gadget, req, hs_ep->dir_in); if (ret) goto dma_error; @@ -835,6 +1188,41 @@ static bool dwc2_gadget_target_frame_elapsed(struct dwc2_hsotg_ep *hs_ep) return false; } +/* + * dwc2_gadget_set_ep0_desc_chain - Set EP's desc chain pointers + * @hsotg: The driver state + * @hs_ep: the ep descriptor chain is for + * + * Called to update EP0 structure's pointers depend on stage of + * control transfer. + */ +static int dwc2_gadget_set_ep0_desc_chain(struct dwc2_hsotg *hsotg, + struct dwc2_hsotg_ep *hs_ep) +{ + switch (hsotg->ep0_state) { + case DWC2_EP0_SETUP: + case DWC2_EP0_STATUS_OUT: + hs_ep->desc_list = hsotg->setup_desc[0]; + hs_ep->desc_list_dma = hsotg->setup_desc_dma[0]; + break; + case DWC2_EP0_DATA_IN: + case DWC2_EP0_STATUS_IN: + hs_ep->desc_list = hsotg->ctrl_in_desc; + hs_ep->desc_list_dma = hsotg->ctrl_in_desc_dma; + break; + case DWC2_EP0_DATA_OUT: + hs_ep->desc_list = hsotg->ctrl_out_desc; + hs_ep->desc_list_dma = hsotg->ctrl_out_desc_dma; + break; + default: + dev_err(hsotg->dev, "invalid EP 0 state in queue %d\n", + hsotg->ep0_state); + return -EINVAL; + } + + return 0; +} + static int dwc2_hsotg_ep_queue(struct usb_ep *ep, struct usb_request *req, gfp_t gfp_flags) { @@ -870,10 +1258,32 @@ static int dwc2_hsotg_ep_queue(struct usb_ep *ep, struct usb_request *req, if (ret) return ret; } + /* If using descriptor DMA configure EP0 descriptor chain pointers */ + if (using_desc_dma(hs) && !hs_ep->index) { + ret = dwc2_gadget_set_ep0_desc_chain(hs, hs_ep); + if (ret) + return ret; + } first = list_empty(&hs_ep->queue); list_add_tail(&hs_req->queue, &hs_ep->queue); + /* + * Handle DDMA isochronous transfers separately - just add new entry + * to the half of descriptor chain that is not processed by HW. + * Transfer will be started once SW gets either one of NAK or + * OutTknEpDis interrupts. + */ + if (using_desc_dma(hs) && hs_ep->isochronous && + hs_ep->target_frame != TARGET_FRAME_INITIAL) { + ret = dwc2_gadget_fill_isoc_desc(hs_ep, hs_req->req.dma, + hs_req->req.length); + if (ret) + dev_dbg(hs->dev, "%s: ISO desc chain full\n", __func__); + + return 0; + } + if (first) { if (!hs_ep->isochronous) { dwc2_hsotg_start_req(hs, hs_ep, hs_req, false); @@ -1099,10 +1509,8 @@ static int dwc2_hsotg_ep_sethalt(struct usb_ep *ep, int value, bool now); */ static struct dwc2_hsotg_req *get_ep_head(struct dwc2_hsotg_ep *hs_ep) { - if (list_empty(&hs_ep->queue)) - return NULL; - - return list_first_entry(&hs_ep->queue, struct dwc2_hsotg_req, queue); + return list_first_entry_or_null(&hs_ep->queue, struct dwc2_hsotg_req, + queue); } /** @@ -1440,14 +1848,21 @@ static void dwc2_hsotg_program_zlp(struct dwc2_hsotg *hsotg, if (hs_ep->dir_in) dev_dbg(hsotg->dev, "Sending zero-length packet on ep%d\n", - index); + index); else dev_dbg(hsotg->dev, "Receiving zero-length packet on ep%d\n", - index); + index); + if (using_desc_dma(hsotg)) { + /* Not specific buffer needed for ep0 ZLP */ + dma_addr_t dma = hs_ep->desc_list_dma; - dwc2_writel(DXEPTSIZ_MC(1) | DXEPTSIZ_PKTCNT(1) | - DXEPTSIZ_XFERSIZE(0), hsotg->regs + - epsiz_reg); + dwc2_gadget_set_ep0_desc_chain(hsotg, hs_ep); + dwc2_gadget_config_nonisoc_xfer_ddma(hs_ep, dma, 0); + } else { + dwc2_writel(DXEPTSIZ_MC(1) | DXEPTSIZ_PKTCNT(1) | + DXEPTSIZ_XFERSIZE(0), hsotg->regs + + epsiz_reg); + } ctrl = dwc2_readl(hsotg->regs + epctl_reg); ctrl |= DXEPCTL_CNAK; /* clear NAK set by core */ @@ -1510,6 +1925,10 @@ static void dwc2_hsotg_complete_request(struct dwc2_hsotg *hsotg, spin_lock(&hsotg->lock); } + /* In DDMA don't need to proceed to starting of next ISOC request */ + if (using_desc_dma(hsotg) && hs_ep->isochronous) + return; + /* * Look to see if there is anything else to do. Note, the completion * of the previous request may have caused a new request to be started @@ -1521,6 +1940,115 @@ static void dwc2_hsotg_complete_request(struct dwc2_hsotg *hsotg, } } +/* + * dwc2_gadget_complete_isoc_request_ddma - complete an isoc request in DDMA + * @hs_ep: The endpoint the request was on. + * + * Get first request from the ep queue, determine descriptor on which complete + * happened. SW based on isoc_chain_num discovers which half of the descriptor + * chain is currently in use by HW, adjusts dma_address and calculates index + * of completed descriptor based on the value of DEPDMA register. Update actual + * length of request, giveback to gadget. + */ +static void dwc2_gadget_complete_isoc_request_ddma(struct dwc2_hsotg_ep *hs_ep) +{ + struct dwc2_hsotg *hsotg = hs_ep->parent; + struct dwc2_hsotg_req *hs_req; + struct usb_request *ureq; + int index; + dma_addr_t dma_addr; + u32 dma_reg; + u32 depdma; + u32 desc_sts; + u32 mask; + + hs_req = get_ep_head(hs_ep); + if (!hs_req) { + dev_warn(hsotg->dev, "%s: ISOC EP queue empty\n", __func__); + return; + } + ureq = &hs_req->req; + + dma_addr = hs_ep->desc_list_dma; + + /* + * If lower half of descriptor chain is currently use by SW, + * that means higher half is being processed by HW, so shift + * DMA address to higher half of descriptor chain. + */ + if (!hs_ep->isoc_chain_num) + dma_addr += sizeof(struct dwc2_dma_desc) * + (MAX_DMA_DESC_NUM_GENERIC / 2); + + dma_reg = hs_ep->dir_in ? DIEPDMA(hs_ep->index) : DOEPDMA(hs_ep->index); + depdma = dwc2_readl(hsotg->regs + dma_reg); + + index = (depdma - dma_addr) / sizeof(struct dwc2_dma_desc) - 1; + desc_sts = hs_ep->desc_list[index].status; + + mask = hs_ep->dir_in ? DEV_DMA_ISOC_TX_NBYTES_MASK : + DEV_DMA_ISOC_RX_NBYTES_MASK; + ureq->actual = ureq->length - + ((desc_sts & mask) >> DEV_DMA_ISOC_NBYTES_SHIFT); + + /* Adjust actual length for ISOC Out if length is not align of 4 */ + if (!hs_ep->dir_in && ureq->length & 0x3) + ureq->actual += 4 - (ureq->length & 0x3); + + dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, 0); +} + +/* + * dwc2_gadget_start_next_isoc_ddma - start next isoc request, if any. + * @hs_ep: The isochronous endpoint to be re-enabled. + * + * If ep has been disabled due to last descriptor servicing (IN endpoint) or + * BNA (OUT endpoint) check the status of other half of descriptor chain that + * was under SW control till HW was busy and restart the endpoint if needed. + */ +static void dwc2_gadget_start_next_isoc_ddma(struct dwc2_hsotg_ep *hs_ep) +{ + struct dwc2_hsotg *hsotg = hs_ep->parent; + u32 depctl; + u32 dma_reg; + u32 ctrl; + u32 dma_addr = hs_ep->desc_list_dma; + unsigned char index = hs_ep->index; + + dma_reg = hs_ep->dir_in ? DIEPDMA(index) : DOEPDMA(index); + depctl = hs_ep->dir_in ? DIEPCTL(index) : DOEPCTL(index); + + ctrl = dwc2_readl(hsotg->regs + depctl); + + /* + * EP was disabled if HW has processed last descriptor or BNA was set. + * So restart ep if SW has prepared new descriptor chain in ep_queue + * routine while HW was busy. + */ + if (!(ctrl & DXEPCTL_EPENA)) { + if (!hs_ep->next_desc) { + dev_dbg(hsotg->dev, "%s: No more ISOC requests\n", + __func__); + return; + } + + dma_addr += sizeof(struct dwc2_dma_desc) * + (MAX_DMA_DESC_NUM_GENERIC / 2) * + hs_ep->isoc_chain_num; + dwc2_writel(dma_addr, hsotg->regs + dma_reg); + + ctrl |= DXEPCTL_EPENA | DXEPCTL_CNAK; + dwc2_writel(ctrl, hsotg->regs + depctl); + + /* Switch ISOC descriptor chain number being processed by SW*/ + hs_ep->isoc_chain_num = (hs_ep->isoc_chain_num ^ 1) & 0x1; + hs_ep->next_desc = 0; + + dev_dbg(hsotg->dev, "%s: Restarted isochronous endpoint\n", + __func__); + } +} + /** * dwc2_hsotg_rx_data - receive data from the FIFO for an endpoint * @hsotg: The device state. @@ -1618,6 +2146,36 @@ static void dwc2_hsotg_change_ep_iso_parity(struct dwc2_hsotg *hsotg, dwc2_writel(ctrl, hsotg->regs + epctl_reg); } +/* + * dwc2_gadget_get_xfersize_ddma - get transferred bytes amount from desc + * @hs_ep - The endpoint on which transfer went + * + * Iterate over endpoints descriptor chain and get info on bytes remained + * in DMA descriptors after transfer has completed. Used for non isoc EPs. + */ +static unsigned int dwc2_gadget_get_xfersize_ddma(struct dwc2_hsotg_ep *hs_ep) +{ + struct dwc2_hsotg *hsotg = hs_ep->parent; + unsigned int bytes_rem = 0; + struct dwc2_dma_desc *desc = hs_ep->desc_list; + int i; + u32 status; + + if (!desc) + return -EINVAL; + + for (i = 0; i < hs_ep->desc_count; ++i) { + status = desc->status; + bytes_rem += status & DEV_DMA_NBYTES_MASK; + + if (status & DEV_DMA_STS_MASK) + dev_err(hsotg->dev, "descriptor %d closed with %x\n", + i, status & DEV_DMA_STS_MASK); + } + + return bytes_rem; +} + /** * dwc2_hsotg_handle_outdone - handle receiving OutDone/SetupDone from RXFIFO * @hsotg: The device instance @@ -1648,6 +2206,9 @@ static void dwc2_hsotg_handle_outdone(struct dwc2_hsotg *hsotg, int epnum) return; } + if (using_desc_dma(hsotg)) + size_left = dwc2_gadget_get_xfersize_ddma(hs_ep); + if (using_dma(hsotg)) { unsigned size_done; @@ -1682,7 +2243,9 @@ static void dwc2_hsotg_handle_outdone(struct dwc2_hsotg *hsotg, int epnum) */ } - if (epnum == 0 && hsotg->ep0_state == DWC2_EP0_DATA_OUT) { + /* DDMA IN status phase will start from StsPhseRcvd interrupt */ + if (!using_desc_dma(hsotg) && epnum == 0 && + hsotg->ep0_state == DWC2_EP0_DATA_OUT) { /* Move to STATUS IN */ dwc2_hsotg_ep0_zlp(hsotg, true); return; @@ -1812,17 +2375,17 @@ static u32 dwc2_hsotg_ep0_mps(unsigned int mps) * @hsotg: The driver state. * @ep: The index number of the endpoint * @mps: The maximum packet size in bytes + * @mc: The multicount value * * Configure the maximum packet size for the given endpoint, updating * the hardware control registers to reflect this. */ static void dwc2_hsotg_set_ep_maxpacket(struct dwc2_hsotg *hsotg, - unsigned int ep, unsigned int mps, unsigned int dir_in) + unsigned int ep, unsigned int mps, + unsigned int mc, unsigned int dir_in) { struct dwc2_hsotg_ep *hs_ep; void __iomem *regs = hsotg->regs; - u32 mpsval; - u32 mcval; u32 reg; hs_ep = index_to_ep(hsotg, ep, dir_in); @@ -1830,32 +2393,32 @@ static void dwc2_hsotg_set_ep_maxpacket(struct dwc2_hsotg *hsotg, return; if (ep == 0) { + u32 mps_bytes = mps; + /* EP0 is a special case */ - mpsval = dwc2_hsotg_ep0_mps(mps); - if (mpsval > 3) + mps = dwc2_hsotg_ep0_mps(mps_bytes); + if (mps > 3) goto bad_mps; - hs_ep->ep.maxpacket = mps; + hs_ep->ep.maxpacket = mps_bytes; hs_ep->mc = 1; } else { - mpsval = mps & DXEPCTL_MPS_MASK; - if (mpsval > 1024) + if (mps > 1024) goto bad_mps; - mcval = ((mps >> 11) & 0x3) + 1; - hs_ep->mc = mcval; - if (mcval > 3) + hs_ep->mc = mc; + if (mc > 3) goto bad_mps; - hs_ep->ep.maxpacket = mpsval; + hs_ep->ep.maxpacket = mps; } if (dir_in) { reg = dwc2_readl(regs + DIEPCTL(ep)); reg &= ~DXEPCTL_MPS_MASK; - reg |= mpsval; + reg |= mps; dwc2_writel(reg, regs + DIEPCTL(ep)); } else { reg = dwc2_readl(regs + DOEPCTL(ep)); reg &= ~DXEPCTL_MPS_MASK; - reg |= mpsval; + reg |= mps; dwc2_writel(reg, regs + DOEPCTL(ep)); } @@ -1954,6 +2517,13 @@ static void dwc2_hsotg_complete_in(struct dwc2_hsotg *hsotg, /* Finish ZLP handling for IN EP0 transactions */ if (hs_ep->index == 0 && hsotg->ep0_state == DWC2_EP0_STATUS_IN) { dev_dbg(hsotg->dev, "zlp packet sent\n"); + + /* + * While send zlp for DWC2_EP0_STATUS_IN EP direction was + * changed to IN. Change back to complete OUT transfer request + */ + hs_ep->dir_in = 0; + dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, 0); if (hsotg->test_mode) { int ret; @@ -1979,8 +2549,14 @@ static void dwc2_hsotg_complete_in(struct dwc2_hsotg *hsotg, * past the end of the buffer (DMA transfers are always 32bit * aligned). */ - - size_left = DXEPTSIZ_XFERSIZE_GET(epsize); + if (using_desc_dma(hsotg)) { + size_left = dwc2_gadget_get_xfersize_ddma(hs_ep); + if (size_left < 0) + dev_err(hsotg->dev, "error parsing DDMA results %d\n", + size_left); + } else { + size_left = DXEPTSIZ_XFERSIZE_GET(epsize); + } size_done = hs_ep->size_loaded - size_left; size_done += hs_ep->last_load; @@ -2128,12 +2704,28 @@ static void dwc2_gadget_handle_out_token_ep_disabled(struct dwc2_hsotg_ep *ep) struct dwc2_hsotg *hsotg = ep->parent; int dir_in = ep->dir_in; u32 doepmsk; + u32 tmp; if (dir_in || !ep->isochronous) return; + /* + * Store frame in which irq was asserted here, as + * it can change while completing request below. + */ + tmp = dwc2_hsotg_read_frameno(hsotg); + dwc2_hsotg_complete_request(hsotg, ep, get_ep_head(ep), -ENODATA); + if (using_desc_dma(hsotg)) { + if (ep->target_frame == TARGET_FRAME_INITIAL) { + /* Start first ISO Out */ + ep->target_frame = tmp; + dwc2_gadget_start_isoc_ddma(ep); + } + return; + } + if (ep->interval > 1 && ep->target_frame == TARGET_FRAME_INITIAL) { u32 dsts; @@ -2182,6 +2774,12 @@ static void dwc2_gadget_handle_nak(struct dwc2_hsotg_ep *hs_ep) if (hs_ep->target_frame == TARGET_FRAME_INITIAL) { hs_ep->target_frame = dwc2_hsotg_read_frameno(hsotg); + + if (using_desc_dma(hsotg)) { + dwc2_gadget_start_isoc_ddma(hs_ep); + return; + } + if (hs_ep->interval > 1) { u32 ctrl = dwc2_readl(hsotg->regs + DIEPCTL(hs_ep->index)); @@ -2237,8 +2835,15 @@ static void dwc2_hsotg_epint(struct dwc2_hsotg *hsotg, unsigned int idx, if (idx == 0 && (ints & (DXEPINT_SETUP | DXEPINT_SETUP_RCVD))) ints &= ~DXEPINT_XFERCOMPL; - if (ints & DXEPINT_STSPHSERCVD) - dev_dbg(hsotg->dev, "%s: StsPhseRcvd asserted\n", __func__); + /* + * Don't process XferCompl interrupt in DDMA if EP0 is still in SETUP + * stage and xfercomplete was generated without SETUP phase done + * interrupt. SW should parse received setup packet only after host's + * exit from setup phase of control transfer. + */ + if (using_desc_dma(hsotg) && idx == 0 && !hs_ep->dir_in && + hsotg->ep0_state == DWC2_EP0_SETUP && !(ints & DXEPINT_SETUP)) + ints &= ~DXEPINT_XFERCOMPL; if (ints & DXEPINT_XFERCOMPL) { dev_dbg(hsotg->dev, @@ -2246,11 +2851,17 @@ static void dwc2_hsotg_epint(struct dwc2_hsotg *hsotg, unsigned int idx, __func__, dwc2_readl(hsotg->regs + epctl_reg), dwc2_readl(hsotg->regs + epsiz_reg)); - /* - * we get OutDone from the FIFO, so we only need to look - * at completing IN requests here - */ - if (dir_in) { + /* In DDMA handle isochronous requests separately */ + if (using_desc_dma(hsotg) && hs_ep->isochronous) { + dwc2_gadget_complete_isoc_request_ddma(hs_ep); + /* Try to start next isoc request */ + dwc2_gadget_start_next_isoc_ddma(hs_ep); + } else if (dir_in) { + /* + * We get OutDone from the FIFO, so we only + * need to look at completing IN requests here + * if operating slave mode + */ if (hs_ep->isochronous && hs_ep->interval > 1) dwc2_gadget_incr_frame_num(hs_ep); @@ -2302,9 +2913,30 @@ static void dwc2_hsotg_epint(struct dwc2_hsotg *hsotg, unsigned int idx, } } + if (ints & DXEPINT_STSPHSERCVD) { + dev_dbg(hsotg->dev, "%s: StsPhseRcvd\n", __func__); + + /* Move to STATUS IN for DDMA */ + if (using_desc_dma(hsotg)) + dwc2_hsotg_ep0_zlp(hsotg, true); + } + if (ints & DXEPINT_BACK2BACKSETUP) dev_dbg(hsotg->dev, "%s: B2BSetup/INEPNakEff\n", __func__); + if (ints & DXEPINT_BNAINTR) { + dev_dbg(hsotg->dev, "%s: BNA interrupt\n", __func__); + + /* + * Try to start next isoc request, if any. + * Sometimes the endpoint remains enabled after BNA interrupt + * assertion, which is not expected, hence we can enter here + * couple of times. + */ + if (hs_ep->isochronous) + dwc2_gadget_start_next_isoc_ddma(hs_ep); + } + if (dir_in && !hs_ep->isochronous) { /* not sure if this is important, but we'll clear it anyway */ if (ints & DXEPINT_INTKNTXFEMP) { @@ -2372,6 +3004,8 @@ static void dwc2_hsotg_irq_enumdone(struct dwc2_hsotg *hsotg) case DSTS_ENUMSPD_LS: hsotg->gadget.speed = USB_SPEED_LOW; + ep0_mps = 8; + ep_mps = 8; /* * note, we don't actually support LS in this driver at the * moment, and the documentation seems to imply that it isn't @@ -2390,13 +3024,15 @@ static void dwc2_hsotg_irq_enumdone(struct dwc2_hsotg *hsotg) if (ep0_mps) { int i; /* Initialize ep0 for both in and out directions */ - dwc2_hsotg_set_ep_maxpacket(hsotg, 0, ep0_mps, 1); - dwc2_hsotg_set_ep_maxpacket(hsotg, 0, ep0_mps, 0); + dwc2_hsotg_set_ep_maxpacket(hsotg, 0, ep0_mps, 0, 1); + dwc2_hsotg_set_ep_maxpacket(hsotg, 0, ep0_mps, 0, 0); for (i = 1; i < hsotg->num_of_eps; i++) { if (hsotg->eps_in[i]) - dwc2_hsotg_set_ep_maxpacket(hsotg, i, ep_mps, 1); + dwc2_hsotg_set_ep_maxpacket(hsotg, i, ep_mps, + 0, 1); if (hsotg->eps_out[i]) - dwc2_hsotg_set_ep_maxpacket(hsotg, i, ep_mps, 0); + dwc2_hsotg_set_ep_maxpacket(hsotg, i, ep_mps, + 0, 0); } } @@ -2516,6 +3152,7 @@ void dwc2_hsotg_core_init_disconnected(struct dwc2_hsotg *hsotg, u32 intmsk; u32 val; u32 usbcfg; + u32 dcfg = 0; /* Kill any ep0 requests as controller will be reinitialized */ kill_all_requests(hsotg, hsotg->eps_out[0], -ECONNRESET); @@ -2534,10 +3171,17 @@ void dwc2_hsotg_core_init_disconnected(struct dwc2_hsotg *hsotg, usbcfg &= ~(GUSBCFG_TOUTCAL_MASK | GUSBCFG_PHYIF16 | GUSBCFG_SRPCAP | GUSBCFG_HNPCAP); - /* set the PLL on, remove the HNP/SRP and set the PHY */ - val = (hsotg->phyif == GUSBCFG_PHYIF8) ? 9 : 5; - usbcfg |= hsotg->phyif | GUSBCFG_TOUTCAL(7) | - (val << GUSBCFG_USBTRDTIM_SHIFT); + if (hsotg->params.phy_type == DWC2_PHY_TYPE_PARAM_FS && + (hsotg->params.speed == DWC2_SPEED_PARAM_FULL || + hsotg->params.speed == DWC2_SPEED_PARAM_LOW)) { + /* FS/LS Dedicated Transceiver Interface */ + usbcfg |= GUSBCFG_PHYSEL; + } else { + /* set the PLL on, remove the HNP/SRP and set the PHY */ + val = (hsotg->phyif == GUSBCFG_PHYIF8) ? 9 : 5; + usbcfg |= hsotg->phyif | GUSBCFG_TOUTCAL(7) | + (val << GUSBCFG_USBTRDTIM_SHIFT); + } dwc2_writel(usbcfg, hsotg->regs + GUSBCFG); dwc2_hsotg_init_fifo(hsotg); @@ -2545,7 +3189,23 @@ void dwc2_hsotg_core_init_disconnected(struct dwc2_hsotg *hsotg, if (!is_usb_reset) __orr32(hsotg->regs + DCTL, DCTL_SFTDISCON); - dwc2_writel(DCFG_EPMISCNT(1) | DCFG_DEVSPD_HS, hsotg->regs + DCFG); + dcfg |= DCFG_EPMISCNT(1); + + switch (hsotg->params.speed) { + case DWC2_SPEED_PARAM_LOW: + dcfg |= DCFG_DEVSPD_LS; + break; + case DWC2_SPEED_PARAM_FULL: + if (hsotg->params.phy_type == DWC2_PHY_TYPE_PARAM_FS) + dcfg |= DCFG_DEVSPD_FS48; + else + dcfg |= DCFG_DEVSPD_FS; + break; + default: + dcfg |= DCFG_DEVSPD_HS; + } + + dwc2_writel(dcfg, hsotg->regs + DCFG); /* Clear any pending OTG interrupts */ dwc2_writel(0xffffffff, hsotg->regs + GOTGINT); @@ -2556,23 +3216,31 @@ void dwc2_hsotg_core_init_disconnected(struct dwc2_hsotg *hsotg, GINTSTS_GOUTNAKEFF | GINTSTS_GINNAKEFF | GINTSTS_USBRST | GINTSTS_RESETDET | GINTSTS_ENUMDONE | GINTSTS_OTGINT | - GINTSTS_USBSUSP | GINTSTS_WKUPINT | - GINTSTS_INCOMPL_SOIN | GINTSTS_INCOMPL_SOOUT; + GINTSTS_USBSUSP | GINTSTS_WKUPINT; - if (hsotg->core_params->external_id_pin_ctl <= 0) + if (!using_desc_dma(hsotg)) + intmsk |= GINTSTS_INCOMPL_SOIN | GINTSTS_INCOMPL_SOOUT; + + if (hsotg->params.external_id_pin_ctl <= 0) intmsk |= GINTSTS_CONIDSTSCHNG; dwc2_writel(intmsk, hsotg->regs + GINTMSK); - if (using_dma(hsotg)) + if (using_dma(hsotg)) { dwc2_writel(GAHBCFG_GLBL_INTR_EN | GAHBCFG_DMA_EN | (GAHBCFG_HBSTLEN_INCR4 << GAHBCFG_HBSTLEN_SHIFT), hsotg->regs + GAHBCFG); - else + + /* Set DDMA mode support in the core if needed */ + if (using_desc_dma(hsotg)) + __orr32(hsotg->regs + DCFG, DCFG_DESCDMA_EN); + + } else { dwc2_writel(((hsotg->dedicated_fifos) ? (GAHBCFG_NP_TXF_EMP_LVL | GAHBCFG_P_TXF_EMP_LVL) : 0) | GAHBCFG_GLBL_INTR_EN, hsotg->regs + GAHBCFG); + } /* * If INTknTXFEmpMsk is enabled, it's important to disable ep interrupts @@ -2588,13 +3256,18 @@ void dwc2_hsotg_core_init_disconnected(struct dwc2_hsotg *hsotg, /* * don't need XferCompl, we get that from RXFIFO in slave mode. In - * DMA mode we may need this. + * DMA mode we may need this and StsPhseRcvd. */ - dwc2_writel((using_dma(hsotg) ? (DIEPMSK_XFERCOMPLMSK) : 0) | + dwc2_writel((using_dma(hsotg) ? (DIEPMSK_XFERCOMPLMSK | + DOEPMSK_STSPHSERCVDMSK) : 0) | DOEPMSK_EPDISBLDMSK | DOEPMSK_AHBERRMSK | - DOEPMSK_SETUPMSK | DOEPMSK_STSPHSERCVDMSK, + DOEPMSK_SETUPMSK, hsotg->regs + DOEPMSK); + /* Enable BNA interrupt for DDMA */ + if (using_desc_dma(hsotg)) + __orr32(hsotg->regs + DOEPMSK, DOEPMSK_BNAMSK); + dwc2_writel(0, hsotg->regs + DAINTMSK); dev_dbg(hsotg->dev, "EP0: DIEPCTL0=0x%08x, DOEPCTL0=0x%08x\n", @@ -2935,6 +3608,95 @@ irq_retry: return IRQ_HANDLED; } +static int dwc2_hsotg_wait_bit_set(struct dwc2_hsotg *hs_otg, u32 reg, + u32 bit, u32 timeout) +{ + u32 i; + + for (i = 0; i < timeout; i++) { + if (dwc2_readl(hs_otg->regs + reg) & bit) + return 0; + udelay(1); + } + + return -ETIMEDOUT; +} + +static void dwc2_hsotg_ep_stop_xfr(struct dwc2_hsotg *hsotg, + struct dwc2_hsotg_ep *hs_ep) +{ + u32 epctrl_reg; + u32 epint_reg; + + epctrl_reg = hs_ep->dir_in ? DIEPCTL(hs_ep->index) : + DOEPCTL(hs_ep->index); + epint_reg = hs_ep->dir_in ? DIEPINT(hs_ep->index) : + DOEPINT(hs_ep->index); + + dev_dbg(hsotg->dev, "%s: stopping transfer on %s\n", __func__, + hs_ep->name); + + if (hs_ep->dir_in) { + if (hsotg->dedicated_fifos || hs_ep->periodic) { + __orr32(hsotg->regs + epctrl_reg, DXEPCTL_SNAK); + /* Wait for Nak effect */ + if (dwc2_hsotg_wait_bit_set(hsotg, epint_reg, + DXEPINT_INEPNAKEFF, 100)) + dev_warn(hsotg->dev, + "%s: timeout DIEPINT.NAKEFF\n", + __func__); + } else { + __orr32(hsotg->regs + DCTL, DCTL_SGNPINNAK); + /* Wait for Nak effect */ + if (dwc2_hsotg_wait_bit_set(hsotg, GINTSTS, + GINTSTS_GINNAKEFF, 100)) + dev_warn(hsotg->dev, + "%s: timeout GINTSTS.GINNAKEFF\n", + __func__); + } + } else { + if (!(dwc2_readl(hsotg->regs + GINTSTS) & GINTSTS_GOUTNAKEFF)) + __orr32(hsotg->regs + DCTL, DCTL_SGOUTNAK); + + /* Wait for global nak to take effect */ + if (dwc2_hsotg_wait_bit_set(hsotg, GINTSTS, + GINTSTS_GOUTNAKEFF, 100)) + dev_warn(hsotg->dev, "%s: timeout GINTSTS.GOUTNAKEFF\n", + __func__); + } + + /* Disable ep */ + __orr32(hsotg->regs + epctrl_reg, DXEPCTL_EPDIS | DXEPCTL_SNAK); + + /* Wait for ep to be disabled */ + if (dwc2_hsotg_wait_bit_set(hsotg, epint_reg, DXEPINT_EPDISBLD, 100)) + dev_warn(hsotg->dev, + "%s: timeout DOEPCTL.EPDisable\n", __func__); + + /* Clear EPDISBLD interrupt */ + __orr32(hsotg->regs + epint_reg, DXEPINT_EPDISBLD); + + if (hs_ep->dir_in) { + unsigned short fifo_index; + + if (hsotg->dedicated_fifos || hs_ep->periodic) + fifo_index = hs_ep->fifo_index; + else + fifo_index = 0; + + /* Flush TX FIFO */ + dwc2_flush_tx_fifo(hsotg, fifo_index); + + /* Clear Global In NP NAK in Shared FIFO for non periodic ep */ + if (!hsotg->dedicated_fifos && !hs_ep->periodic) + __orr32(hsotg->regs + DCTL, DCTL_CGNPINNAK); + + } else { + /* Remove global NAKs */ + __orr32(hsotg->regs + DCTL, DCTL_CGOUTNAK); + } +} + /** * dwc2_hsotg_ep_enable - enable the given endpoint * @ep: The USB endpint to configure @@ -2952,6 +3714,7 @@ static int dwc2_hsotg_ep_enable(struct usb_ep *ep, u32 epctrl_reg; u32 epctrl; u32 mps; + u32 mc; u32 mask; unsigned int dir_in; unsigned int i, val, size; @@ -2975,6 +3738,7 @@ static int dwc2_hsotg_ep_enable(struct usb_ep *ep, } mps = usb_endpoint_maxp(desc); + mc = usb_endpoint_maxp_mult(desc); /* note, we handle this here instead of dwc2_hsotg_set_ep_maxpacket */ @@ -2984,6 +3748,18 @@ static int dwc2_hsotg_ep_enable(struct usb_ep *ep, dev_dbg(hsotg->dev, "%s: read DxEPCTL=0x%08x from 0x%08x\n", __func__, epctrl, epctrl_reg); + /* Allocate DMA descriptor chain for non-ctrl endpoints */ + if (using_desc_dma(hsotg)) { + hs_ep->desc_list = dma_alloc_coherent(hsotg->dev, + MAX_DMA_DESC_NUM_GENERIC * + sizeof(struct dwc2_dma_desc), + &hs_ep->desc_list_dma, GFP_KERNEL); + if (!hs_ep->desc_list) { + ret = -ENOMEM; + goto error2; + } + } + spin_lock_irqsave(&hsotg->lock, flags); epctrl &= ~(DXEPCTL_EPTYPE_MASK | DXEPCTL_MPS_MASK); @@ -2996,7 +3772,7 @@ static int dwc2_hsotg_ep_enable(struct usb_ep *ep, epctrl |= DXEPCTL_USBACTEP; /* update the endpoint state */ - dwc2_hsotg_set_ep_maxpacket(hsotg, hs_ep->index, mps, dir_in); + dwc2_hsotg_set_ep_maxpacket(hsotg, hs_ep->index, mps, mc, dir_in); /* default, set to non-periodic */ hs_ep->isochronous = 0; @@ -3011,6 +3787,8 @@ static int dwc2_hsotg_ep_enable(struct usb_ep *ep, hs_ep->isochronous = 1; hs_ep->interval = 1 << (desc->bInterval - 1); hs_ep->target_frame = TARGET_FRAME_INITIAL; + hs_ep->isoc_chain_num = 0; + hs_ep->next_desc = 0; if (dir_in) { hs_ep->periodic = 1; mask = dwc2_readl(hsotg->regs + DIEPMSK); @@ -3067,7 +3845,7 @@ static int dwc2_hsotg_ep_enable(struct usb_ep *ep, dev_err(hsotg->dev, "%s: No suitable fifo found\n", __func__); ret = -ENOMEM; - goto error; + goto error1; } hsotg->fifo_map |= 1 << fifo_index; epctrl |= DXEPCTL_TXFNUM(fifo_index); @@ -3089,8 +3867,17 @@ static int dwc2_hsotg_ep_enable(struct usb_ep *ep, /* enable the endpoint interrupt */ dwc2_hsotg_ctrl_epint(hsotg, index, dir_in, 1); -error: +error1: spin_unlock_irqrestore(&hsotg->lock, flags); + +error2: + if (ret && using_desc_dma(hsotg) && hs_ep->desc_list) { + dma_free_coherent(hsotg->dev, MAX_DMA_DESC_NUM_GENERIC * + sizeof(struct dwc2_dma_desc), + hs_ep->desc_list, hs_ep->desc_list_dma); + hs_ep->desc_list = NULL; + } + return ret; } @@ -3115,11 +3902,23 @@ static int dwc2_hsotg_ep_disable(struct usb_ep *ep) return -EINVAL; } + /* Remove DMA memory allocated for non-control Endpoints */ + if (using_desc_dma(hsotg)) { + dma_free_coherent(hsotg->dev, MAX_DMA_DESC_NUM_GENERIC * + sizeof(struct dwc2_dma_desc), + hs_ep->desc_list, hs_ep->desc_list_dma); + hs_ep->desc_list = NULL; + } + epctrl_reg = dir_in ? DIEPCTL(index) : DOEPCTL(index); spin_lock_irqsave(&hsotg->lock, flags); ctrl = dwc2_readl(hsotg->regs + epctrl_reg); + + if (ctrl & DXEPCTL_EPENA) + dwc2_hsotg_ep_stop_xfr(hsotg, hs_ep); + ctrl &= ~DXEPCTL_EPENA; ctrl &= ~DXEPCTL_USBACTEP; ctrl |= DXEPCTL_SNAK; @@ -3158,77 +3957,6 @@ static bool on_list(struct dwc2_hsotg_ep *ep, struct dwc2_hsotg_req *test) return false; } -static int dwc2_hsotg_wait_bit_set(struct dwc2_hsotg *hs_otg, u32 reg, - u32 bit, u32 timeout) -{ - u32 i; - - for (i = 0; i < timeout; i++) { - if (dwc2_readl(hs_otg->regs + reg) & bit) - return 0; - udelay(1); - } - - return -ETIMEDOUT; -} - -static void dwc2_hsotg_ep_stop_xfr(struct dwc2_hsotg *hsotg, - struct dwc2_hsotg_ep *hs_ep) -{ - u32 epctrl_reg; - u32 epint_reg; - - epctrl_reg = hs_ep->dir_in ? DIEPCTL(hs_ep->index) : - DOEPCTL(hs_ep->index); - epint_reg = hs_ep->dir_in ? DIEPINT(hs_ep->index) : - DOEPINT(hs_ep->index); - - dev_dbg(hsotg->dev, "%s: stopping transfer on %s\n", __func__, - hs_ep->name); - if (hs_ep->dir_in) { - __orr32(hsotg->regs + epctrl_reg, DXEPCTL_SNAK); - /* Wait for Nak effect */ - if (dwc2_hsotg_wait_bit_set(hsotg, epint_reg, - DXEPINT_INEPNAKEFF, 100)) - dev_warn(hsotg->dev, - "%s: timeout DIEPINT.NAKEFF\n", __func__); - } else { - if (!(dwc2_readl(hsotg->regs + GINTSTS) & GINTSTS_GOUTNAKEFF)) - __orr32(hsotg->regs + DCTL, DCTL_SGOUTNAK); - - /* Wait for global nak to take effect */ - if (dwc2_hsotg_wait_bit_set(hsotg, GINTSTS, - GINTSTS_GOUTNAKEFF, 100)) - dev_warn(hsotg->dev, - "%s: timeout GINTSTS.GOUTNAKEFF\n", __func__); - } - - /* Disable ep */ - __orr32(hsotg->regs + epctrl_reg, DXEPCTL_EPDIS | DXEPCTL_SNAK); - - /* Wait for ep to be disabled */ - if (dwc2_hsotg_wait_bit_set(hsotg, epint_reg, DXEPINT_EPDISBLD, 100)) - dev_warn(hsotg->dev, - "%s: timeout DOEPCTL.EPDisable\n", __func__); - - if (hs_ep->dir_in) { - if (hsotg->dedicated_fifos) { - dwc2_writel(GRSTCTL_TXFNUM(hs_ep->fifo_index) | - GRSTCTL_TXFFLSH, hsotg->regs + GRSTCTL); - /* Wait for fifo flush */ - if (dwc2_hsotg_wait_bit_set(hsotg, GRSTCTL, - GRSTCTL_TXFFLSH, 100)) - dev_warn(hsotg->dev, - "%s: timeout flushing fifos\n", - __func__); - } - /* TODO: Flush shared tx fifo */ - } else { - /* Remove global NAKs */ - __bic32(hsotg->regs + DCTL, DCTL_SGOUTNAK); - } -} - /** * dwc2_hsotg_ep_dequeue - dequeue given endpoint * @ep: The endpoint to dequeue. @@ -3665,14 +4393,21 @@ static void dwc2_hsotg_initep(struct dwc2_hsotg *hsotg, hs_ep->parent = hsotg; hs_ep->ep.name = hs_ep->name; - usb_ep_set_maxpacket_limit(&hs_ep->ep, epnum ? 1024 : EP0_MPS_LIMIT); + + if (hsotg->params.speed == DWC2_SPEED_PARAM_LOW) + usb_ep_set_maxpacket_limit(&hs_ep->ep, 8); + else + usb_ep_set_maxpacket_limit(&hs_ep->ep, + epnum ? 1024 : EP0_MPS_LIMIT); hs_ep->ep.ops = &dwc2_hsotg_ep_ops; if (epnum == 0) { hs_ep->ep.caps.type_control = true; } else { - hs_ep->ep.caps.type_iso = true; - hs_ep->ep.caps.type_bulk = true; + if (hsotg->params.speed != DWC2_SPEED_PARAM_LOW) { + hs_ep->ep.caps.type_iso = true; + hs_ep->ep.caps.type_bulk = true; + } hs_ep->ep.caps.type_int = true; } @@ -3802,51 +4537,6 @@ static void dwc2_hsotg_dump(struct dwc2_hsotg *hsotg) #endif } -#ifdef CONFIG_OF -static void dwc2_hsotg_of_probe(struct dwc2_hsotg *hsotg) -{ - struct device_node *np = hsotg->dev->of_node; - u32 len = 0; - u32 i = 0; - - /* Enable dma if requested in device tree */ - hsotg->g_using_dma = of_property_read_bool(np, "g-use-dma"); - - /* - * Register TX periodic fifo size per endpoint. - * EP0 is excluded since it has no fifo configuration. - */ - if (!of_find_property(np, "g-tx-fifo-size", &len)) - goto rx_fifo; - - len /= sizeof(u32); - - /* Read tx fifo sizes other than ep0 */ - if (of_property_read_u32_array(np, "g-tx-fifo-size", - &hsotg->g_tx_fifo_sz[1], len)) - goto rx_fifo; - - /* Add ep0 */ - len++; - - /* Make remaining TX fifos unavailable */ - if (len < MAX_EPS_CHANNELS) { - for (i = len; i < MAX_EPS_CHANNELS; i++) - hsotg->g_tx_fifo_sz[i] = 0; - } - -rx_fifo: - /* Register RX fifo size */ - of_property_read_u32(np, "g-rx-fifo-size", &hsotg->g_rx_fifo_sz); - - /* Register NPTX fifo size */ - of_property_read_u32(np, "g-np-tx-fifo-size", - &hsotg->g_np_g_tx_fifo_sz); -} -#else -static inline void dwc2_hsotg_of_probe(struct dwc2_hsotg *hsotg) { } -#endif - /** * dwc2_gadget_init - init function for gadget * @dwc2: The data structure for the DWC2 driver. @@ -3857,33 +4547,11 @@ int dwc2_gadget_init(struct dwc2_hsotg *hsotg, int irq) struct device *dev = hsotg->dev; int epnum; int ret; - int i; - u32 p_tx_fifo[] = DWC2_G_P_LEGACY_TX_FIFO_SIZE; - - /* Initialize to legacy fifo configuration values */ - hsotg->g_rx_fifo_sz = 2048; - hsotg->g_np_g_tx_fifo_sz = 1024; - memcpy(&hsotg->g_tx_fifo_sz[1], p_tx_fifo, sizeof(p_tx_fifo)); - /* Device tree specific probe */ - dwc2_hsotg_of_probe(hsotg); - - /* Check against largest possible value. */ - if (hsotg->g_np_g_tx_fifo_sz > - hsotg->hw_params.dev_nperio_tx_fifo_size) { - dev_warn(dev, "Specified GNPTXFDEP=%d > %d\n", - hsotg->g_np_g_tx_fifo_sz, - hsotg->hw_params.dev_nperio_tx_fifo_size); - hsotg->g_np_g_tx_fifo_sz = - hsotg->hw_params.dev_nperio_tx_fifo_size; - } /* Dump fifo information */ dev_dbg(dev, "NonPeriodic TXFIFO size: %d\n", - hsotg->g_np_g_tx_fifo_sz); - dev_dbg(dev, "RXFIFO size: %d\n", hsotg->g_rx_fifo_sz); - for (i = 0; i < MAX_EPS_CHANNELS; i++) - dev_dbg(dev, "Periodic TXFIFO%2d size: %d\n", i, - hsotg->g_tx_fifo_sz[i]); + hsotg->params.g_np_tx_fifo_size); + dev_dbg(dev, "RXFIFO size: %d\n", hsotg->params.g_rx_fifo_size); hsotg->gadget.max_speed = USB_SPEED_HIGH; hsotg->gadget.ops = &dwc2_hsotg_gadget_ops; @@ -3909,6 +4577,12 @@ int dwc2_gadget_init(struct dwc2_hsotg *hsotg, int irq) if (!hsotg->ep0_buff) return -ENOMEM; + if (using_desc_dma(hsotg)) { + ret = dwc2_gadget_alloc_ctrl_desc_chains(hsotg); + if (ret < 0) + return ret; + } + ret = devm_request_irq(hsotg->dev, irq, dwc2_hsotg_irq, IRQF_SHARED, dev_name(hsotg->dev), hsotg); if (ret < 0) { |