LCOV - code coverage report
Current view: top level - drivers/usb/host - ehci-q.c (source / functions) Hit Total Coverage
Test: coverage.info Lines: 601 601 100.0 %
Date: 2017-01-25 Functions: 17 17 100.0 %

          Line data    Source code
       1             : /*
       2             :  * Copyright (C) 2001-2004 by David Brownell
       3             :  *
       4             :  * This program is free software; you can redistribute it and/or modify it
       5             :  * under the terms of the GNU General Public License as published by the
       6             :  * Free Software Foundation; either version 2 of the License, or (at your
       7             :  * option) any later version.
       8             :  *
       9             :  * This program is distributed in the hope that it will be useful, but
      10             :  * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
      11             :  * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
      12             :  * for more details.
      13             :  *
      14             :  * You should have received a copy of the GNU General Public License
      15             :  * along with this program; if not, write to the Free Software Foundation,
      16             :  * Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
      17             :  */
      18             : 
      19             : /* this file is part of ehci-hcd.c */
      20             : 
      21             : /*-------------------------------------------------------------------------*/
      22             : 
      23             : /*
      24             :  * EHCI hardware queue manipulation ... the core.  QH/QTD manipulation.
      25             :  *
      26             :  * Control, bulk, and interrupt traffic all use "qh" lists.  They list "qtd"
      27             :  * entries describing USB transactions, max 16-20kB/entry (with 4kB-aligned
      28             :  * buffers needed for the larger number).  We use one QH per endpoint, queue
      29             :  * multiple urbs (all three types) per endpoint.  URBs may need several qtds.
      30             :  *
      31             :  * ISO traffic uses "ISO TD" (itd, and sitd) records, and (along with
      32             :  * interrupts) needs careful scheduling.  Performance improvements can be
      33             :  * an ongoing challenge.  That's in "ehci-sched.c".
      34             :  *
      35             :  * USB 1.1 devices are handled (a) by "companion" OHCI or UHCI root hubs,
      36             :  * or otherwise through transaction translators (TTs) in USB 2.0 hubs using
      37             :  * (b) special fields in qh entries or (c) split iso entries.  TTs will
      38             :  * buffer low/full speed data so the host collects it at high speed.
      39             :  */
      40             : 
      41             : /*-------------------------------------------------------------------------*/
      42             : 
      43             : /* fill a qtd, returning how much of the buffer we were able to queue up */
      44             : 
      45             : static int
      46             : qtd_fill(struct ehci_hcd *ehci, struct ehci_qtd *qtd, dma_addr_t buf,
      47             :                   size_t len, int token, int maxpacket)
      48             : {
      49           6 :         int     i, count;
      50          12 :         u64     addr = buf;
      51           6 : 
      52           6 :         /* one buffer entry per 4K ... first might be short or unaligned */
      53          12 :         qtd->hw_buf[0] = cpu_to_hc32(ehci, (u32)addr);
      54          12 :         qtd->hw_buf_hi[0] = cpu_to_hc32(ehci, (u32)(addr >> 32));
      55           6 :         count = 0x1000 - (buf & 0x0fff);    /* rest of that page */
      56          24 :         if (likely (len < count))            /* ... iff needed */
      57           6 :                 count = len;
      58             :         else {
      59           6 :                 buf +=  0x1000;
      60           6 :                 buf &= ~0x0fff;
      61             : 
      62             :                 /* per-qtd limit: from 16K to 20K (best alignment) */
      63          42 :                 for (i = 1; count < len && i < 5; i++) {
      64          18 :                         addr = buf;
      65          18 :                         qtd->hw_buf[i] = cpu_to_hc32(ehci, (u32)addr);
      66          12 :                         qtd->hw_buf_hi[i] = cpu_to_hc32(ehci,
      67             :                                         (u32)(addr >> 32));
      68           6 :                         buf += 0x1000;
      69          12 :                         if ((count + 0x1000) < len)
      70           6 :                                 count += 0x1000;
      71             :                         else
      72           6 :                                 count = len;
      73             :                 }
      74             : 
      75             :                 /* short packets may only terminate transfers */
      76          12 :                 if (count != len)
      77           6 :                         count -= (count % maxpacket);
      78             :         }
      79          24 :         qtd->hw_token = cpu_to_hc32(ehci, (count << 16) | token);
      80           6 :         qtd->length = count;
      81             : 
      82           6 :         return count;
      83             : }
      84             : 
      85             : /*-------------------------------------------------------------------------*/
      86             : 
      87             : static inline void
      88             : qh_update (struct ehci_hcd *ehci, struct ehci_qh *qh, struct ehci_qtd *qtd)
      89             : {
      90         684 :         struct ehci_qh_hw *hw = qh->hw;
      91         342 : 
      92         342 :         /* writes to an active overlay are unsafe */
      93        2736 :         BUG_ON(qh->qh_state != QH_STATE_IDLE);
      94         342 : 
      95        1368 :         hw->hw_qtd_next = QTD_NEXT(ehci, qtd->qtd_dma);
      96        1026 :         hw->hw_alt_next = EHCI_LIST_END(ehci);
      97         342 : 
      98         342 :         /* Except for control endpoints, we make hardware maintain data
      99             :          * toggle (like OHCI) ... here (re)initialize the toggle in the QH,
     100             :          * and set the pseudo-toggle in udev. Only usb_clear_halt() will
     101             :          * ever clear it.
     102             :          */
     103        1368 :         if (!(hw->hw_info1 & cpu_to_hc32(ehci, 1 << 14))) {
     104             :                 unsigned        is_out, epnum;
     105             : 
     106         342 :                 is_out = qh->is_out;
     107        1026 :                 epnum = (hc32_to_cpup(ehci, &hw->hw_info1) >> 8) & 0x0f;
     108        1368 :                 if (unlikely (!usb_gettoggle (qh->dev, epnum, is_out))) {
     109        1026 :                         hw->hw_token &= ~cpu_to_hc32(ehci, QTD_TOGGLE);
     110         342 :                         usb_settoggle (qh->dev, epnum, is_out, 1);
     111             :                 }
     112             :         }
     113             : 
     114             :         /* HC must see latest qtd and qh data before we clear ACTIVE+HALT */
     115        1026 :         wmb ();
     116        2394 :         hw->hw_token &= cpu_to_hc32(ehci, QTD_TOGGLE | QTD_STS_PING);
     117         342 : }
     118             : 
     119             : /* if it weren't for a common silicon quirk (writing the dummy into the qh
     120             :  * overlay, so qh->hw_token wrongly becomes inactive/halted), only fault
     121             :  * recovery (including urb dequeue) would need software changes to a QH...
     122             :  */
     123             : static void
     124             : qh_refresh (struct ehci_hcd *ehci, struct ehci_qh *qh)
     125             : {
     126         171 :         struct ehci_qtd *qtd;
     127         171 : 
     128         855 :         if (list_empty (&qh->qtd_list))
     129         342 :                 qtd = qh->dummy;
     130             :         else {
     131         342 :                 qtd = list_entry (qh->qtd_list.next,
     132             :                                 struct ehci_qtd, qtd_list);
     133             :                 /* first qtd may already be partially processed */
     134         855 :                 if (cpu_to_hc32(ehci, qtd->qtd_dma) == qh->hw->hw_current)
     135         171 :                         qtd = NULL;
     136             :         }
     137             : 
     138         684 :         if (qtd)
     139         684 :                 qh_update (ehci, qh, qtd);
     140         513 : }
     141             : 
     142             : /*-------------------------------------------------------------------------*/
     143             : 
     144             : static void qh_link_async(struct ehci_hcd *ehci, struct ehci_qh *qh);
     145             : 
     146             : static void ehci_clear_tt_buffer_complete(struct usb_hcd *hcd,
     147             :                 struct usb_host_endpoint *ep)
     148           1 : {
     149           4 :         struct ehci_hcd         *ehci = hcd_to_ehci(hcd);
     150           3 :         struct ehci_qh          *qh = ep->hcpriv;
     151           1 :         unsigned long           flags;
     152           1 : 
     153           4 :         spin_lock_irqsave(&ehci->lock, flags);
     154           1 :         qh->clearing_tt = 0;
     155           9 :         if (qh->qh_state == QH_STATE_IDLE && !list_empty(&qh->qtd_list)
     156             :                         && HC_IS_RUNNING(hcd->state))
     157           3 :                 qh_link_async(ehci, qh);
     158           6 :         spin_unlock_irqrestore(&ehci->lock, flags);
     159           1 : }
     160             : 
     161             : static void ehci_clear_tt_buffer(struct ehci_hcd *ehci, struct ehci_qh *qh,
     162             :                 struct urb *urb, u32 token)
     163             : {
     164          80 : 
     165          80 :         /* If an async split transaction gets an error or is unlinked,
     166          80 :          * the TT buffer may be left in an indeterminate state.  We
     167             :          * have to clear the TT buffer.
     168             :          *
     169             :          * Note: this routine is never called for Isochronous transfers.
     170             :          */
     171         560 :         if (urb->dev->tt && !usb_pipeint(urb->pipe) && !qh->clearing_tt) {
     172             : #ifdef DEBUG
     173             :                 struct usb_device *tt = urb->dev->tt->hub;
     174             :                 dev_dbg(&tt->dev,
     175             :                         "clear tt buffer port %d, a%d ep%d t%08x\n",
     176             :                         urb->dev->ttport, urb->dev->devnum,
     177             :                         usb_pipeendpoint(urb->pipe), token);
     178             : #endif /* DEBUG */
     179         800 :                 if (!ehci_is_TDI(ehci)
     180          80 :                                 || urb->dev->tt->hub !=
     181             :                                    ehci_to_hcd(ehci)->self.root_hub) {
     182         480 :                         if (usb_hub_clear_tt_buffer(urb) == 0)
     183         160 :                                 qh->clearing_tt = 1;
     184             :                 } else {
     185             : 
     186             :                         /* REVISIT ARC-derived cores don't clear the root
     187             :                          * hub TT buffer in this way...
     188         240 :                          */
     189             :                 }
     190             :         }
     191             : }
     192             : 
     193             : static int qtd_copy_status (
     194             :         struct ehci_hcd *ehci,
     195             :         struct urb *urb,
     196         200 :         size_t length,
     197         200 :         u32 token
     198         200 : )
     199         200 : {
     200         400 :         int     status = -EINPROGRESS;
     201             : 
     202             :         /* count IN/OUT bytes, not SETUP (even short packets) */
     203         800 :         if (likely (QTD_PID (token) != 2))
     204         200 :                 urb->actual_length += length - QTD_LENGTH (token);
     205             : 
     206             :         /* don't modify error codes */
     207         800 :         if (unlikely(urb->unlinked))
     208         200 :                 return status;
     209             : 
     210             :         /* force cleanup after short read; not always an error */
     211        1600 :         if (unlikely (IS_SHORT_READ (token)))
     212         200 :                 status = -EREMOTEIO;
     213             : 
     214             :         /* serious "can't proceed" faults reported by the hardware */
     215         800 :         if (token & QTD_STS_HALT) {
     216         800 :                 if (token & QTD_STS_BABBLE) {
     217             :                         /* FIXME "must" disable babbling device's port too */
     218         400 :                         status = -EOVERFLOW;
     219             :                 /* CERR nonzero + halt --> stall */
     220         800 :                 } else if (QTD_CERR(token)) {
     221         400 :                         status = -EPIPE;
     222             : 
     223             :                 /* In theory, more than one of the following bits can be set
     224             :                  * since they are sticky and the transaction is retried.
     225             :                  * Which to test first is rather arbitrary.
     226             :                  */
     227         800 :                 } else if (token & QTD_STS_MMF) {
     228             :                         /* fs/ls interrupt xfer missed the complete-split */
     229         400 :                         status = -EPROTO;
     230         800 :                 } else if (token & QTD_STS_DBE) {
     231        2400 :                         status = (QTD_PID (token) == 1) /* IN ? */
     232             :                                 ? -ENOSR  /* hc couldn't read data */
     233             :                                 : -ECOMM; /* hc couldn't write data */
     234         800 :                 } else if (token & QTD_STS_XACT) {
     235             :                         /* timeout, bad CRC, wrong PID, etc */
     236             :                         ehci_dbg(ehci, "devpath %s ep%d%s 3strikes\n",
     237             :                                 urb->dev->devpath,
     238             :                                 usb_pipeendpoint(urb->pipe),
     239             :                                 usb_pipein(urb->pipe) ? "in" : "out");
     240         400 :                         status = -EPROTO;
     241             :                 } else {        /* unknown */
     242         400 :                         status = -EPROTO;
     243             :                 }
     244             : 
     245             :                 ehci_vdbg (ehci,
     246             :                         "dev%d ep%d%s qtd token %08x --> status %d\n",
     247             :                         usb_pipedevice (urb->pipe),
     248             :                         usb_pipeendpoint (urb->pipe),
     249             :                         usb_pipein (urb->pipe) ? "in" : "out",
     250             :                         token, status);
     251             :         }
     252             : 
     253         400 :         return status;
     254             : }
     255             : 
     256             : static void
     257             : ehci_urb_done(struct ehci_hcd *ehci, struct urb *urb, int status)
     258             : __releases(ehci->lock)
     259          86 : __acquires(ehci->lock)
     260          86 : {
     261         516 :         if (likely (urb->hcpriv != NULL)) {
     262         258 :                 struct ehci_qh  *qh = (struct ehci_qh *) urb->hcpriv;
     263          86 : 
     264          86 :                 /* S-mask in a QH means it's an interrupt urb */
     265         430 :                 if ((qh->hw->hw_info2 & cpu_to_hc32(ehci, QH_SMASK)) != 0) {
     266             : 
     267             :                         /* ... update hc-wide periodic stats (for usbfs) */
     268         258 :                         ehci_to_hcd(ehci)->self.bandwidth_int_reqs--;
     269             :                 }
     270         516 :                 qh_put (qh);
     271             :         }
     272             : 
     273         516 :         if (unlikely(urb->unlinked)) {
     274             :                 COUNT(ehci->stats.unlink);
     275             :         } else {
     276             :                 /* report non-error and short read status as zero */
     277         344 :                 if (status == -EINPROGRESS || status == -EREMOTEIO)
     278          86 :                         status = 0;
     279             :                 COUNT(ehci->stats.complete);
     280             :         }
     281             : 
     282             : #ifdef EHCI_URB_TRACE
     283             :         ehci_dbg (ehci,
     284             :                 "%s %s urb %p ep%d%s status %d len %d/%d\n",
     285             :                 __func__, urb->dev->devpath, urb,
     286             :                 usb_pipeendpoint (urb->pipe),
     287             :                 usb_pipein (urb->pipe) ? "in" : "out",
     288             :                 status,
     289             :                 urb->actual_length, urb->transfer_buffer_length);
     290             : #endif
     291             : 
     292             :         /* complete() can reenter this HCD */
     293         258 :         usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb);
     294         172 :         spin_unlock (&ehci->lock);
     295         258 :         usb_hcd_giveback_urb(ehci_to_hcd(ehci), urb, status);
     296         172 :         spin_lock (&ehci->lock);
     297          86 : }
     298             : 
     299             : static void start_unlink_async (struct ehci_hcd *ehci, struct ehci_qh *qh);
     300             : static void unlink_async (struct ehci_hcd *ehci, struct ehci_qh *qh);
     301             : 
     302             : static int qh_schedule (struct ehci_hcd *ehci, struct ehci_qh *qh);
     303             : 
     304             : /*
     305             :  * Process and free completed qtds for a qh, returning URBs to drivers.
     306             :  * Chases up to qh->hw_current.  Returns number of completions called,
     307             :  * indicating how much "real" work we did.
     308             :  */
     309             : static unsigned
     310             : qh_completions (struct ehci_hcd *ehci, struct ehci_qh *qh)
     311             : {
     312          80 :         struct ehci_qtd         *last, *end = qh->dummy;
     313          40 :         struct list_head        *entry, *tmp;
     314          40 :         int                     last_status;
     315          40 :         int                     stopped;
     316          80 :         unsigned                count = 0;
     317          40 :         u8                      state;
     318          80 :         struct ehci_qh_hw       *hw = qh->hw;
     319          40 : 
     320         280 :         if (unlikely (list_empty (&qh->qtd_list)))
     321          80 :                 return count;
     322          40 : 
     323          40 :         /* completions (or tasks on other cpus) must never clobber HALT
     324          40 :          * till we've gone through and cleaned everything up, even when
     325          40 :          * they add urbs to this qh's queue or mark them for unlinking.
     326          40 :          *
     327          40 :          * NOTE:  unlinking expects to be done in queue order.
     328          40 :          *
     329          40 :          * It's a bug for qh->qh_state to be anything other than
     330          40 :          * QH_STATE_IDLE, unless our caller is scan_async() or
     331          40 :          * scan_periodic().
     332          40 :          */
     333          80 :         state = qh->qh_state;
     334          80 :         qh->qh_state = QH_STATE_COMPLETING;
     335          80 :         stopped = (state == QH_STATE_IDLE);
     336          80 : 
     337          40 :  rescan:
     338          80 :         last = NULL;
     339          80 :         last_status = -EINPROGRESS;
     340          40 :         qh->needs_rescan = 0;
     341             : 
     342             :         /* remove de-activated QTDs from front of queue.
     343             :          * after faults (including short reads), cleanup this urb
     344             :          * then let the queue advance.
     345             :          * if queue is stopped, handles unlinks.
     346             :          */
     347         360 :         list_for_each_safe (entry, tmp, &qh->qtd_list) {
     348         120 :                 struct ehci_qtd *qtd;
     349          40 :                 struct urb      *urb;
     350          40 :                 u32             token = 0;
     351             : 
     352          80 :                 qtd = list_entry (entry, struct ehci_qtd, qtd_list);
     353          80 :                 urb = qtd->urb;
     354             : 
     355             :                 /* clean up any state from previous QTD ...*/
     356          80 :                 if (last) {
     357         200 :                         if (likely (last->urb != urb)) {
     358          80 :                                 ehci_urb_done(ehci, last->urb, last_status);
     359          40 :                                 count++;
     360          40 :                                 last_status = -EINPROGRESS;
     361             :                         }
     362         160 :                         ehci_qtd_free (ehci, last);
     363          40 :                         last = NULL;
     364             :                 }
     365             : 
     366             :                 /* ignore urbs submitted during completions we reported */
     367         160 :                 if (qtd == end)
     368          80 :                         break;
     369             : 
     370             :                 /* hardware copies qtd out of qh overlay */
     371          80 :                 rmb ();
     372         160 :                 token = hc32_to_cpu(ehci, qtd->hw_token);
     373             : 
     374          40 :                 /* always clean up qtds the hc de-activated */
     375             :  retry_xacterr:
     376          80 :                 if ((token & QTD_STS_ACTIVE) == 0) {
     377             : 
     378             :                         /* on STALL, error, and short reads this urb must
     379             :                          * complete and all its qtds must be recycled.
     380             :                          */
     381          80 :                         if ((token & QTD_STS_HALT) != 0) {
     382             : 
     383             :                                 /* retry transaction errors until we
     384             :                                  * reach the software xacterr limit
     385             :                                  */
     386         440 :                                 if ((token & QTD_STS_XACT) &&
     387             :                                                 QTD_CERR(token) == 0 &&
     388             :                                                 ++qh->xacterrs < QH_XACTERR_MAX &&
     389             :                                                 !urb->unlinked) {
     390             :                                         ehci_dbg(ehci,
     391             :         "detected XactErr len %zu/%zu retry %d\n",
     392             :         qtd->length - QTD_LENGTH(token), qtd->length, qh->xacterrs);
     393             : 
     394             :                                         /* reset the token in the qtd and the
     395             :                                          * qh overlay (which still contains
     396             :                                          * the qtd) so that we pick up from
     397             :                                          * where we left off
     398             :                                          */
     399          40 :                                         token &= ~QTD_STS_HALT;
     400          40 :                                         token |= QTD_STS_ACTIVE |
     401             :                                                         (EHCI_TUNE_CERR << 10);
     402          80 :                                         qtd->hw_token = cpu_to_hc32(ehci,
     403             :                                                         token);
     404          40 :                                         wmb();
     405          80 :                                         hw->hw_token = cpu_to_hc32(ehci,
     406             :                                                         token);
     407          40 :                                         goto retry_xacterr;
     408             :                                 }
     409          40 :                                 stopped = 1;
     410             : 
     411             :                         /* magic dummy for some short reads; qh won't advance.
     412             :                          * that silicon quirk can kick in with this dummy too.
     413             :                          *
     414             :                          * other short reads won't stop the queue, including
     415             :                          * control transfers (status stage handles that) or
     416             :                          * most other single-qtd reads ... the queue stops if
     417             :                          * URB_SHORT_NOT_OK was set so the driver submitting
     418             :                          * the urbs could clean it up.
     419             :                          */
     420         320 :                         } else if (IS_SHORT_READ (token)
     421             :                                         && !(qtd->hw_alt_next
     422             :                                                 & EHCI_LIST_END(ehci))) {
     423          40 :                                 stopped = 1;
     424             :                         }
     425             : 
     426             :                 /* stop scanning when we reach qtds the hc is using */
     427         480 :                 } else if (likely (!stopped
     428             :                                 && HC_IS_RUNNING (ehci_to_hcd(ehci)->state))) {
     429          40 :                         break;
     430          40 : 
     431             :                 /* scan the whole queue for unlinks whenever it stops */
     432             :                 } else {
     433          80 :                         stopped = 1;
     434             : 
     435             :                         /* cancel everything if we halt, suspend, etc */
     436         240 :                         if (!HC_IS_RUNNING(ehci_to_hcd(ehci)->state))
     437          40 :                                 last_status = -ESHUTDOWN;
     438             : 
     439             :                         /* this qtd is active; skip it unless a previous qtd
     440             :                          * for its urb faulted, or its urb was canceled.
     441             :                          */
     442         160 :                         else if (last_status == -EINPROGRESS && !urb->unlinked)
     443          40 :                                 continue;
     444             : 
     445             :                         /* qh unlinked; token in overlay may be most current */
     446         280 :                         if (state == QH_STATE_IDLE
     447             :                                         && cpu_to_hc32(ehci, qtd->qtd_dma)
     448             :                                                 == hw->hw_current) {
     449          80 :                                 token = hc32_to_cpu(ehci, hw->hw_token);
     450             : 
     451             :                                 /* An unlink may leave an incomplete
     452             :                                  * async transaction in the TT buffer.
     453             :                                  * We have to clear it.
     454             :                                  */
     455         160 :                                 ehci_clear_tt_buffer(ehci, qh, urb, token);
     456             :                         }
     457             :                 }
     458             : 
     459             :                 /* unless we already know the urb's status, collect qtd status
     460             :                  * and update count of bytes transferred.  in common short read
     461             :                  * cases with only one data qtd (including control transfers),
     462             :                  * queue processing won't halt.  but with two or more qtds (for
     463             :                  * example, with a 32 KB transfer), when the first qtd gets a
     464             :                  * short read the second must be removed by hand.
     465             :                  */
     466         400 :                 if (last_status == -EINPROGRESS) {
     467         800 :                         last_status = qtd_copy_status(ehci, urb,
     468             :                                         qtd->length, token);
     469         240 :                         if (last_status == -EREMOTEIO
     470             :                                         && (qtd->hw_alt_next
     471             :                                                 & EHCI_LIST_END(ehci)))
     472          40 :                                 last_status = -EINPROGRESS;
     473             : 
     474             :                         /* As part of low/full-speed endpoint-halt processing
     475             :                          * we must clear the TT buffer (11.17.5).
     476             :                          */
     477         640 :                         if (unlikely(last_status != -EINPROGRESS &&
     478             :                                         last_status != -EREMOTEIO)) {
     479             :                                 /* The TT's in some hubs malfunction when they
     480             :                                  * receive this request following a STALL (they
     481             :                                  * stop sending isochronous packets).  Since a
     482             :                                  * STALL can't leave the TT buffer in a busy
     483             :                                  * state (if you believe Figures 11-48 - 11-51
     484             :                                  * in the USB 2.0 spec), we won't clear the TT
     485             :                                  * buffer in this case.  Strictly speaking this
     486             :                                  * is a violation of the spec.
     487             :                                  */
     488          80 :                                 if (last_status != -EPIPE)
     489         160 :                                         ehci_clear_tt_buffer(ehci, qh, urb,
     490             :                                                         token);
     491             :                         }
     492             :                 }
     493             : 
     494             :                 /* if we're removing something not at the queue head,
     495             :                  * patch the hardware queue pointer.
     496             :                  */
     497        1120 :                 if (stopped && qtd->qtd_list.prev != &qh->qtd_list) {
     498         560 :                         last = list_entry (qtd->qtd_list.prev,
     499             :                                         struct ehci_qtd, qtd_list);
     500         280 :                         last->hw_next = qtd->hw_next;
     501             :                 }
     502             : 
     503             :                 /* remove qtd; it's recycled after possible urb completion */
     504         560 :                 list_del (&qtd->qtd_list);
     505          40 :                 last = qtd;
     506             : 
     507             :                 /* reinit the xacterr counter for the next qtd */
     508          40 :                 qh->xacterrs = 0;
     509             :         }
     510          40 : 
     511             :         /* last urb's completion might still need calling */
     512         320 :         if (likely (last != NULL)) {
     513          80 :                 ehci_urb_done(ehci, last->urb, last_status);
     514          40 :                 count++;
     515          80 :                 ehci_qtd_free (ehci, last);
     516             :         }
     517             : 
     518             :         /* Do we need to rescan for URBs dequeued during a giveback? */
     519         320 :         if (unlikely(qh->needs_rescan)) {
     520             :                 /* If the QH is already unlinked, do the rescan now. */
     521          80 :                 if (state == QH_STATE_IDLE)
     522          40 :                         goto rescan;
     523             : 
     524             :                 /* Otherwise we have to wait until the QH is fully unlinked.
     525             :                  * Our caller will start an unlink if qh->needs_rescan is
     526             :                  * set.  But if an unlink has already started, nothing needs
     527             :                  * to be done.
     528             :                  */
     529          80 :                 if (state != QH_STATE_LINKED)
     530          40 :                         qh->needs_rescan = 0;
     531             :         }
     532             : 
     533             :         /* restore original state; caller must unlink or relink */
     534          40 :         qh->qh_state = state;
     535             : 
     536             :         /* be sure the hardware's done with the qh before refreshing
     537             :          * it after fault cleanup, or recovering from silicon wrongly
     538             :          * overlaying the dummy qtd (which reduces DMA chatter).
     539             :          */
     540         280 :         if (stopped != 0 || hw->hw_qtd_next == EHCI_LIST_END(ehci)) {
     541         120 :                 switch (state) {
     542         320 :                 case QH_STATE_IDLE:
     543         320 :                         qh_refresh(ehci, qh);
     544          40 :                         break;
     545         320 :                 case QH_STATE_LINKED:
     546             :                         /* We won't refresh a QH that's linked (after the HC
     547             :                          * stopped the queue).  That avoids a race:
     548             :                          *  - HC reads first part of QH;
     549             :                          *  - CPU updates that first part and the token;
     550             :                          *  - HC reads rest of that QH, including token
     551             :                          * Result:  HC gets an inconsistent image, and then
     552             :                          * DMAs to/from the wrong memory (corrupting it).
     553             :                          *
     554             :                          * That should be rare for interrupt transfers,
     555             :                          * except maybe high bandwidth ...
     556             :                          */
     557             : 
     558             :                         /* Tell the caller to start an unlink */
     559          80 :                         qh->needs_rescan = 1;
     560          80 :                         break;
     561          80 :                 /* otherwise, unlink already started */
     562             :                 }
     563          80 :         }
     564             : 
     565         120 :         return count;
     566             : }
     567             : 
     568             : /*-------------------------------------------------------------------------*/
     569             : 
     570             : // high bandwidth multiplier, as encoded in highspeed endpoint descriptors
     571             : #define hb_mult(wMaxPacketSize) (1 + (((wMaxPacketSize) >> 11) & 0x03))
     572             : // ... and packet size, for any kind of endpoint descriptor
     573             : #define max_packet(wMaxPacketSize) ((wMaxPacketSize) & 0x07ff)
     574             : 
     575             : /*
     576             :  * reverse of qh_urb_transaction:  free a list of TDs.
     577             :  * used for cleanup after errors, before HC sees an URB's TDs.
     578             :  */
     579             : static void qtd_list_free (
     580             :         struct ehci_hcd         *ehci,
     581           8 :         struct urb              *urb,
     582           8 :         struct list_head        *qtd_list
     583           8 : ) {
     584           8 :         struct list_head        *entry, *temp;
     585             : 
     586          56 :         list_for_each_safe (entry, temp, qtd_list) {
     587           8 :                 struct ehci_qtd *qtd;
     588          16 : 
     589          16 :                 qtd = list_entry (entry, struct ehci_qtd, qtd_list);
     590          16 :                 list_del (&qtd->qtd_list);
     591          16 :                 ehci_qtd_free (ehci, qtd);
     592             :         }
     593           8 : }
     594             : 
     595             : /*
     596             :  * create a list of filled qtds for this URB; won't link into qh.
     597             :  */
     598             : static struct list_head *
     599             : qh_urb_transaction (
     600             :         struct ehci_hcd         *ehci,
     601             :         struct urb              *urb,
     602           2 :         struct list_head        *head,
     603           2 :         gfp_t                   flags
     604           2 : ) {
     605           2 :         struct ehci_qtd         *qtd, *qtd_prev;
     606           2 :         dma_addr_t              buf;
     607           2 :         int                     len, this_sg_len, maxpacket;
     608           2 :         int                     is_input;
     609           2 :         u32                     token;
     610           2 :         int                     i;
     611           2 :         struct scatterlist      *sg;
     612           2 : 
     613           2 :         /*
     614           2 :          * URBs map to sequences of QTDs:  one logical transaction
     615           2 :          */
     616           8 :         qtd = ehci_qtd_alloc (ehci, flags);
     617          10 :         if (unlikely (!qtd))
     618           4 :                 return NULL;
     619           6 :         list_add_tail (&qtd->qtd_list, head);
     620           4 :         qtd->urb = urb;
     621           2 : 
     622           4 :         token = QTD_STS_ACTIVE;
     623           4 :         token |= (EHCI_TUNE_CERR << 10);
     624           2 :         /* for split transactions, SplitXState initialized to zero */
     625           2 : 
     626           6 :         len = urb->transfer_buffer_length;
     627           6 :         is_input = usb_pipein (urb->pipe);
     628           6 :         if (usb_pipecontrol (urb->pipe)) {
     629             :                 /* SETUP pid */
     630           4 :                 qtd_fill(ehci, qtd, urb->setup_dma,
     631             :                                 sizeof (struct usb_ctrlrequest),
     632             :                                 token | (2 /* "setup" */ << 8), 8);
     633             : 
     634             :                 /* ... and always at least one more pid */
     635           2 :                 token ^= QTD_TOGGLE;
     636           2 :                 qtd_prev = qtd;
     637           6 :                 qtd = ehci_qtd_alloc (ehci, flags);
     638           8 :                 if (unlikely (!qtd))
     639           2 :                         goto cleanup;
     640           2 :                 qtd->urb = urb;
     641           6 :                 qtd_prev->hw_next = QTD_NEXT(ehci, qtd->qtd_dma);
     642           4 :                 list_add_tail (&qtd->qtd_list, head);
     643             : 
     644             :                 /* for zero length DATA stages, STATUS is always IN */
     645           4 :                 if (len == 0)
     646           2 :                         token |= (1 /* "in" */ << 8);
     647             :         }
     648             : 
     649             :         /*
     650             :          * data transfer stage:  buffer setup
     651             :          */
     652           4 :         i = urb->num_sgs;
     653          16 :         if (len > 0 && i > 0) {
     654           4 :                 sg = urb->sg->sg;
     655           4 :                 buf = sg_dma_address(sg);
     656             : 
     657             :                 /* urb->transfer_buffer_length may be smaller than the
     658             :                  * size of the scatterlist (or vice versa)
     659             :                  */
     660          36 :                 this_sg_len = min_t(int, sg_dma_len(sg), len);
     661             :         } else {
     662           4 :                 sg = NULL;
     663           4 :                 buf = urb->transfer_dma;
     664           4 :                 this_sg_len = len;
     665             :         }
     666             : 
     667           8 :         if (is_input)
     668           4 :                 token |= (1 /* "in" */ << 8);
     669             :         /* else it's already initted to "out" pid (0 << 8) */
     670             : 
     671          18 :         maxpacket = max_packet(usb_maxpacket(urb->dev, urb->pipe, !is_input));
     672             : 
     673           2 :         /*
     674             :          * buffer gets wrapped in one or more qtds;
     675             :          * last one may be "short" (including zero len)
     676             :          * and may serve as a control status ack
     677             :          */
     678             :         for (;;) {
     679             :                 int this_qtd_len;
     680             : 
     681           4 :                 this_qtd_len = qtd_fill(ehci, qtd, buf, this_sg_len, token,
     682             :                                 maxpacket);
     683           2 :                 this_sg_len -= this_qtd_len;
     684           2 :                 len -= this_qtd_len;
     685           2 :                 buf += this_qtd_len;
     686             : 
     687             :                 /*
     688             :                  * short reads advance to a "magic" dummy instead of the next
     689             :                  * qtd ... that forces the queue to stop, for manual cleanup.
     690             :                  * (this will usually be overridden later.)
     691             :                  */
     692           4 :                 if (is_input)
     693           2 :                         qtd->hw_alt_next = ehci->async->hw->hw_alt_next;
     694             : 
     695             :                 /* qh makes control packets use qtd toggle; maybe switch it */
     696           4 :                 if ((maxpacket & (this_qtd_len + (maxpacket - 1))) == 0)
     697           2 :                         token ^= QTD_TOGGLE;
     698             : 
     699           8 :                 if (likely(this_sg_len <= 0)) {
     700          10 :                         if (--i <= 0 || len <= 0)
     701           2 :                                 break;
     702           2 :                         sg = sg_next(sg);
     703           2 :                         buf = sg_dma_address(sg);
     704          18 :                         this_sg_len = min_t(int, sg_dma_len(sg), len);
     705             :                 }
     706             : 
     707           2 :                 qtd_prev = qtd;
     708           6 :                 qtd = ehci_qtd_alloc (ehci, flags);
     709           8 :                 if (unlikely (!qtd))
     710           2 :                         goto cleanup;
     711           2 :                 qtd->urb = urb;
     712           6 :                 qtd_prev->hw_next = QTD_NEXT(ehci, qtd->qtd_dma);
     713           4 :                 list_add_tail (&qtd->qtd_list, head);
     714           2 :         }
     715           2 : 
     716             :         /*
     717             :          * unless the caller requires manual cleanup after short reads,
     718             :          * have the alt_next mechanism keep the queue running after the
     719             :          * last data qtd (the only one, for control and most other cases).
     720             :          */
     721          16 :         if (likely ((urb->transfer_flags & URB_SHORT_NOT_OK) == 0
     722             :                                 || usb_pipecontrol (urb->pipe)))
     723           8 :                 qtd->hw_alt_next = EHCI_LIST_END(ehci);
     724             : 
     725             :         /*
     726             :          * control requests may need a terminating data "status" ack;
     727             :          * bulk ones may need a terminating short packet (zero length).
     728             :          */
     729          16 :         if (likely (urb->transfer_buffer_length != 0)) {
     730           2 :                 int     one_more = 0;
     731             : 
     732           4 :                 if (usb_pipecontrol (urb->pipe)) {
     733           2 :                         one_more = 1;
     734           2 :                         token ^= 0x0100;        /* "in" <--> "out"  */
     735           2 :                         token |= QTD_TOGGLE;    /* force DATA1 */
     736          12 :                 } else if (usb_pipebulk (urb->pipe)
     737             :                                 && (urb->transfer_flags & URB_ZERO_PACKET)
     738             :                                 && !(urb->transfer_buffer_length % maxpacket)) {
     739           2 :                         one_more = 1;
     740             :                 }
     741           4 :                 if (one_more) {
     742           2 :                         qtd_prev = qtd;
     743           6 :                         qtd = ehci_qtd_alloc (ehci, flags);
     744           8 :                         if (unlikely (!qtd))
     745           2 :                                 goto cleanup;
     746           2 :                         qtd->urb = urb;
     747           6 :                         qtd_prev->hw_next = QTD_NEXT(ehci, qtd->qtd_dma);
     748           4 :                         list_add_tail (&qtd->qtd_list, head);
     749             : 
     750             :                         /* never any data in such packets */
     751           4 :                         qtd_fill(ehci, qtd, 0, 0, token, 0);
     752             :                 }
     753             :         }
     754             : 
     755             :         /* by default, enable interrupt on urb completion */
     756          12 :         if (likely (!(urb->transfer_flags & URB_NO_INTERRUPT)))
     757           6 :                 qtd->hw_token |= cpu_to_hc32(ehci, QTD_IOC);
     758           4 :         return head;
     759           6 : 
     760             : cleanup:
     761          12 :         qtd_list_free (ehci, urb, head);
     762           2 :         return NULL;
     763             : }
     764             : 
     765             : /*-------------------------------------------------------------------------*/
     766             : 
     767             : // Would be best to create all qh's from config descriptors,
     768             : // when each interface/altsetting is established.  Unlink
     769             : // any previous qh and cancel its urbs first; endpoints are
     770             : // implicitly reset then (data toggle too).
     771             : // That'd mean updating how usbcore talks to HCDs. (2.7?)
     772             : 
     773             : 
     774             : /*
     775             :  * Each QH holds a qtd list; a QH is used for everything except iso.
     776             :  *
     777             :  * For interrupt urbs, the scheduler must set the microframe scheduling
     778             :  * mask(s) each time the QH gets scheduled.  For highspeed, that's
     779             :  * just one microframe in the s-mask.  For split interrupt transactions
     780             :  * there are additional complications: c-mask, maybe FSTNs.
     781             :  */
     782             : static struct ehci_qh *
     783             : qh_make (
     784             :         struct ehci_hcd         *ehci,
     785           4 :         struct urb              *urb,
     786           4 :         gfp_t                   flags
     787           4 : ) {
     788          20 :         struct ehci_qh          *qh = ehci_qh_alloc (ehci, flags);
     789          12 :         u32                     info1 = 0, info2 = 0;
     790           4 :         int                     is_input, type;
     791           8 :         int                     maxp = 0;
     792           8 :         struct usb_tt           *tt = urb->dev->tt;
     793           4 :         struct ehci_qh_hw       *hw;
     794           4 : 
     795          12 :         if (!qh)
     796           8 :                 return qh;
     797           4 : 
     798           4 :         /*
     799           4 :          * init endpoint/device data for this QH
     800             :          */
     801           4 :         info1 |= usb_pipeendpoint (urb->pipe) << 8;
     802           4 :         info1 |= usb_pipedevice (urb->pipe) << 0;
     803             : 
     804           8 :         is_input = usb_pipein (urb->pipe);
     805           4 :         type = usb_pipetype (urb->pipe);
     806          20 :         maxp = usb_maxpacket (urb->dev, urb->pipe, !is_input);
     807             : 
     808             :         /* 1024 byte maxpacket is a hardware ceiling.  High bandwidth
     809             :          * acts like up to 3KB, but is built from smaller packets.
     810             :          */
     811           8 :         if (max_packet(maxp) > 1024) {
     812             :                 ehci_dbg(ehci, "bogus qh maxpacket %d\n", max_packet(maxp));
     813           4 :                 goto done;
     814             :         }
     815             : 
     816             :         /* Compute interrupt scheduling parameters just once, and save.
     817             :          * - allowing for high bandwidth, how many nsec/uframe are used?
     818             :          * - split transactions need a second CSPLIT uframe; same question
     819             :          * - splits also need a schedule gap (for full/low speed I/O)
     820             :          * - qh has a polling interval
     821             :          *
     822             :          * For control/bulk requests, the HC or TT handles these.
     823             :          */
     824           8 :         if (type == PIPE_INTERRUPT) {
     825           8 :                 qh->usecs = NS_TO_US(usb_calc_bus_time(USB_SPEED_HIGH,
     826             :                                 is_input, 0,
     827             :                                 hb_mult(maxp) * max_packet(maxp)));
     828           4 :                 qh->start = NO_FRAME;
     829             : 
     830          12 :                 if (urb->dev->speed == USB_SPEED_HIGH) {
     831           4 :                         qh->c_usecs = 0;
     832           4 :                         qh->gap_uf = 0;
     833             : 
     834           4 :                         qh->period = urb->interval >> 3;
     835          20 :                         if (qh->period == 0 && urb->interval != 1) {
     836             :                                 /* NOTE interval 2 or 4 uframes could work.
     837             :                                  * But interval 1 scheduling is simpler, and
     838             :                                  * includes high bandwidth.
     839             :                                  */
     840           4 :                                 urb->interval = 1;
     841          12 :                         } else if (qh->period > ehci->periodic_size) {
     842           8 :                                 qh->period = ehci->periodic_size;
     843           8 :                                 urb->interval = qh->period << 3;
     844             :                         }
     845             :                 } else {
     846             :                         int             think_time;
     847             : 
     848             :                         /* gap is f(FS/LS transfer times) */
     849          12 :                         qh->gap_uf = 1 + usb_calc_bus_time (urb->dev->speed,
     850             :                                         is_input, 0, maxp) / (125 * 1000);
     851             : 
     852             :                         /* FIXME this just approximates SPLIT/CSPLIT times */
     853           8 :                         if (is_input) {         // SPLIT, gap, CSPLIT+DATA
     854           8 :                                 qh->c_usecs = qh->usecs + HS_USECS (0);
     855           4 :                                 qh->usecs = HS_USECS (1);
     856             :                         } else {                // SPLIT+DATA, gap, CSPLIT
     857           8 :                                 qh->usecs += HS_USECS (1);
     858           4 :                                 qh->c_usecs = HS_USECS (0);
     859             :                         }
     860             : 
     861          28 :                         think_time = tt ? tt->think_time : 0;
     862          12 :                         qh->tt_usecs = NS_TO_US (think_time +
     863             :                                         usb_calc_bus_time (urb->dev->speed,
     864             :                                         is_input, 0, max_packet (maxp)));
     865           8 :                         qh->period = urb->interval;
     866          12 :                         if (qh->period > ehci->periodic_size) {
     867           8 :                                 qh->period = ehci->periodic_size;
     868           8 :                                 urb->interval = qh->period;
     869             :                         }
     870             :                 }
     871             :         }
     872             : 
     873             :         /* support for tt scheduling, and access to toggles */
     874           4 :         qh->dev = urb->dev;
     875             : 
     876             :         /* using TT? */
     877             :         switch (urb->dev->speed) {
     878          16 :         case USB_SPEED_LOW:
     879           4 :                 info1 |= (1 << 12);       /* EPS "low" */
     880           4 :                 /* FALL THROUGH */
     881             : 
     882          16 :         case USB_SPEED_FULL:
     883             :                 /* EPS 0 means "full" */
     884           8 :                 if (type != PIPE_INTERRUPT)
     885           4 :                         info1 |= (EHCI_TUNE_RL_TT << 28);
     886           8 :                 if (type == PIPE_CONTROL) {
     887           4 :                         info1 |= (1 << 27);       /* for TT */
     888           4 :                         info1 |= 1 << 14; /* toggle from qtd */
     889             :                 }
     890           4 :                 info1 |= maxp << 16;
     891             : 
     892           4 :                 info2 |= (EHCI_TUNE_MULT_TT << 30);
     893             : 
     894             :                 /* Some Freescale processors have an erratum in which the
     895             :                  * port number in the queue head was 0..N-1 instead of 1..N.
     896             :                  */
     897             :                 if (ehci_has_fsl_portno_bug(ehci))
     898             :                         info2 |= (urb->dev->ttport-1) << 23;
     899             :                 else
     900           4 :                         info2 |= urb->dev->ttport << 23;
     901             : 
     902             :                 /* set the address of the TT; for TDI's integrated
     903             :                  * root hub tt, leave it zeroed.
     904             :                  */
     905          28 :                 if (tt && tt->hub != ehci_to_hcd(ehci)->self.root_hub)
     906           4 :                         info2 |= tt->hub->devnum << 16;
     907             : 
     908             :                 /* NOTE:  if (PIPE_INTERRUPT) { scheduler sets c-mask } */
     909             : 
     910           8 :                 break;
     911           4 : 
     912          16 :         case USB_SPEED_HIGH:            /* no TT involved */
     913           4 :                 info1 |= (2 << 12);       /* EPS "high" */
     914           8 :                 if (type == PIPE_CONTROL) {
     915           4 :                         info1 |= (EHCI_TUNE_RL_HS << 28);
     916           4 :                         info1 |= 64 << 16;        /* usb2 fixed maxpacket */
     917           4 :                         info1 |= 1 << 14; /* toggle from qtd */
     918           4 :                         info2 |= (EHCI_TUNE_MULT_HS << 30);
     919           8 :                 } else if (type == PIPE_BULK) {
     920           4 :                         info1 |= (EHCI_TUNE_RL_HS << 28);
     921             :                         /* The USB spec says that high speed bulk endpoints
     922             :                          * always use 512 byte maxpacket.  But some device
     923             :                          * vendors decided to ignore that, and MSFT is happy
     924             :                          * to help them do so.  So now people expect to use
     925             :                          * such nonconformant devices with Linux too; sigh.
     926             :                          */
     927           4 :                         info1 |= max_packet(maxp) << 16;
     928           4 :                         info2 |= (EHCI_TUNE_MULT_HS << 30);
     929             :                 } else {                /* PIPE_INTERRUPT */
     930           4 :                         info1 |= max_packet (maxp) << 16;
     931           4 :                         info2 |= hb_mult (maxp) << 30;
     932             :                 }
     933           4 :                 break;
     934           8 :         default:
     935           8 :                 dbg ("bogus dev %p speed %d", urb->dev, urb->dev->speed);
     936             : done:
     937          12 :                 qh_put (qh);
     938           4 :                 return NULL;
     939             :         }
     940             : 
     941             :         /* NOTE:  if (PIPE_INTERRUPT) { scheduler sets s-mask } */
     942             : 
     943             :         /* init as live, toggle clear, advance to dummy */
     944           8 :         qh->qh_state = QH_STATE_IDLE;
     945           8 :         hw = qh->hw;
     946          16 :         hw->hw_info1 = cpu_to_hc32(ehci, info1);
     947           8 :         hw->hw_info2 = cpu_to_hc32(ehci, info2);
     948           4 :         qh->is_out = !is_input;
     949           4 :         usb_settoggle (urb->dev, usb_pipeendpoint (urb->pipe), !is_input, 1);
     950          16 :         qh_refresh (ehci, qh);
     951           4 :         return qh;
     952             : }
     953             : 
     954             : /*-------------------------------------------------------------------------*/
     955             : 
     956             : /* move qh (and its qtds) onto async queue; maybe enable queue.  */
     957             : 
     958             : static void qh_link_async (struct ehci_hcd *ehci, struct ehci_qh *qh)
     959             : {
     960         130 :         __hc32          dma = QH_NEXT(ehci, qh->qh_dma);
     961          26 :         struct ehci_qh  *head;
     962          26 : 
     963          26 :         /* Don't link a QH if there's a Clear-TT-Buffer pending */
     964         130 :         if (unlikely(qh->clearing_tt))
     965          52 :                 return;
     966          26 : 
     967         260 :         WARN_ON(qh->qh_state != QH_STATE_IDLE);
     968          26 : 
     969             :         /* (re)start the async schedule? */
     970          26 :         head = ehci->async;
     971          52 :         timer_action_done (ehci, TIMER_ASYNC_OFF);
     972          52 :         if (!head->qh_next.qh) {
     973          78 :                 u32     cmd = ehci_readl(ehci, &ehci->regs->command);
     974             : 
     975          52 :                 if (!(cmd & CMD_ASE)) {
     976             :                         /* in case a clear of CMD_ASE didn't take yet */
     977          52 :                         (void)handshake(ehci, &ehci->regs->status,
     978             :                                         STS_ASS, 0, 150);
     979          26 :                         cmd |= CMD_ASE | CMD_RUN;
     980          52 :                         ehci_writel(ehci, cmd, &ehci->regs->command);
     981          78 :                         ehci_to_hcd(ehci)->state = HC_STATE_RUNNING;
     982             :                         /* posted write need not be known to HC yet ... */
     983             :                 }
     984             :         }
     985             : 
     986             :         /* clear halt and/or toggle; and maybe recover from silicon quirk */
     987         312 :         qh_refresh(ehci, qh);
     988             : 
     989             :         /* splice right after start */
     990          26 :         qh->qh_next = head->qh_next;
     991          26 :         qh->hw->hw_next = head->hw->hw_next;
     992          26 :         wmb ();
     993             : 
     994          26 :         head->qh_next.qh = qh;
     995          26 :         head->hw->hw_next = dma;
     996             : 
     997          52 :         qh_get(qh);
     998          26 :         qh->xacterrs = 0;
     999          26 :         qh->qh_state = QH_STATE_LINKED;
    1000          26 :         /* qtd completions reported later by interrupt */
    1001             : }
    1002             : 
    1003             : /*-------------------------------------------------------------------------*/
    1004             : 
    1005             : /*
    1006             :  * For control/bulk/interrupt, return QH with these TDs appended.
    1007             :  * Allocates and initializes the QH if necessary.
    1008             :  * Returns null if it can't allocate a QH it needs to.
    1009             :  * If the QH has TDs (urbs) already, that's great.
    1010             :  */
    1011             : static struct ehci_qh *qh_append_tds (
    1012             :         struct ehci_hcd         *ehci,
    1013             :         struct urb              *urb,
    1014           4 :         struct list_head        *qtd_list,
    1015           4 :         int                     epnum,
    1016           4 :         void                    **ptr
    1017           4 : )
    1018           4 : {
    1019           8 :         struct ehci_qh          *qh = NULL;
    1020          16 :         __hc32                  qh_addr_mask = cpu_to_hc32(ehci, 0x7f);
    1021           4 : 
    1022           8 :         qh = (struct ehci_qh *) *ptr;
    1023          20 :         if (unlikely (qh == NULL)) {
    1024           4 :                 /* can't sleep here, we have ehci->lock... */
    1025          20 :                 qh = qh_make (ehci, urb, GFP_ATOMIC);
    1026           8 :                 *ptr = qh;
    1027           4 :         }
    1028          28 :         if (likely (qh != NULL)) {
    1029           4 :                 struct ehci_qtd *qtd;
    1030             : 
    1031          24 :                 if (unlikely (list_empty (qtd_list)))
    1032           4 :                         qtd = NULL;
    1033             :                 else
    1034          12 :                         qtd = list_entry (qtd_list->next, struct ehci_qtd,
    1035             :                                         qtd_list);
    1036             : 
    1037             :                 /* control qh may need patching ... */
    1038          16 :                 if (unlikely (epnum == 0)) {
    1039             : 
    1040             :                         /* usb_reset_device() briefly reverts to address 0 */
    1041           8 :                         if (usb_pipedevice (urb->pipe) == 0)
    1042           4 :                                 qh->hw->hw_info1 &= ~qh_addr_mask;
    1043             :                 }
    1044             : 
    1045             :                 /* just one way to queue requests: swap with the dummy qtd.
    1046             :                  * only hc or qh_refresh() ever modify the overlay.
    1047             :                  */
    1048          16 :                 if (likely (qtd != NULL)) {
    1049             :                         struct ehci_qtd         *dummy;
    1050             :                         dma_addr_t              dma;
    1051             :                         __hc32                  token;
    1052             : 
    1053             :                         /* to avoid racing the HC, use the dummy td instead of
    1054             :                          * the first td of our list (becomes new dummy).  both
    1055             :                          * tds stay deactivated until we're done, when the
    1056             :                          * HC is allowed to fetch the old dummy (4.10.2).
    1057             :                          */
    1058           4 :                         token = qtd->hw_token;
    1059           8 :                         qtd->hw_token = HALT_BIT(ehci);
    1060           4 :                         wmb ();
    1061           4 :                         dummy = qh->dummy;
    1062             : 
    1063           4 :                         dma = dummy->qtd_dma;
    1064           4 :                         *dummy = *qtd;
    1065           4 :                         dummy->qtd_dma = dma;
    1066             : 
    1067           8 :                         list_del (&qtd->qtd_list);
    1068           8 :                         list_add (&dummy->qtd_list, qtd_list);
    1069          12 :                         list_splice_tail(qtd_list, &qh->qtd_list);
    1070             : 
    1071           8 :                         ehci_qtd_init(ehci, qtd, qtd->qtd_dma);
    1072           4 :                         qh->dummy = qtd;
    1073             : 
    1074             :                         /* hc must see the new dummy at list end */
    1075           4 :                         dma = qtd->qtd_dma;
    1076           8 :                         qtd = list_entry (qh->qtd_list.prev,
    1077             :                                         struct ehci_qtd, qtd_list);
    1078           8 :                         qtd->hw_next = QTD_NEXT(ehci, dma);
    1079             : 
    1080             :                         /* let the hc process these next qtds */
    1081           4 :                         wmb ();
    1082           4 :                         dummy->hw_token = token;
    1083             : 
    1084          12 :                         urb->hcpriv = qh_get (qh);
    1085             :                 }
    1086             :         }
    1087          12 :         return qh;
    1088             : }
    1089             : 
    1090             : /*-------------------------------------------------------------------------*/
    1091             : 
    1092             : static int
    1093             : submit_async (
    1094             :         struct ehci_hcd         *ehci,
    1095             :         struct urb              *urb,
    1096           1 :         struct list_head        *qtd_list,
    1097           1 :         gfp_t                   mem_flags
    1098           1 : ) {
    1099           1 :         struct ehci_qtd         *qtd;
    1100           1 :         int                     epnum;
    1101           1 :         unsigned long           flags;
    1102           2 :         struct ehci_qh          *qh = NULL;
    1103           1 :         int                     rc;
    1104           1 : 
    1105           4 :         qtd = list_entry (qtd_list->next, struct ehci_qtd, qtd_list);
    1106           2 :         epnum = urb->ep->desc.bEndpointAddress;
    1107           1 : 
    1108           1 : #ifdef EHCI_URB_TRACE
    1109           1 :         ehci_dbg (ehci,
    1110           1 :                 "%s %s urb %p ep%d%s len %d, qtd %p [qh %p]\n",
    1111           1 :                 __func__, urb->dev->devpath, urb,
    1112             :                 epnum & 0x0f, (epnum & USB_DIR_IN) ? "in" : "out",
    1113             :                 urb->transfer_buffer_length,
    1114             :                 qtd, urb->ep->hcpriv);
    1115             : #endif
    1116             : 
    1117           3 :         spin_lock_irqsave (&ehci->lock, flags);
    1118           8 :         if (unlikely(!test_bit(HCD_FLAG_HW_ACCESSIBLE,
    1119             :                                &ehci_to_hcd(ehci)->flags))) {
    1120           1 :                 rc = -ESHUTDOWN;
    1121           1 :                 goto done;
    1122             :         }
    1123           3 :         rc = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb);
    1124           4 :         if (unlikely(rc))
    1125           1 :                 goto done;
    1126             : 
    1127           4 :         qh = qh_append_tds(ehci, urb, qtd_list, epnum, &urb->ep->hcpriv);
    1128           4 :         if (unlikely(qh == NULL)) {
    1129           3 :                 usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb);
    1130           1 :                 rc = -ENOMEM;
    1131           1 :                 goto done;
    1132             :         }
    1133             : 
    1134             :         /* Control/bulk operations through TTs don't need scheduling,
    1135             :          * the HC and TT handle it when the TT has a buffer ready.
    1136             :          */
    1137           5 :         if (likely (qh->qh_state == QH_STATE_IDLE))
    1138           3 :                 qh_link_async(ehci, qh);
    1139             :  done:
    1140          10 :         spin_unlock_irqrestore (&ehci->lock, flags);
    1141           4 :         if (unlikely (qh == NULL))
    1142           2 :                 qtd_list_free (ehci, urb, qtd_list);
    1143           4 :         return rc;
    1144             : }
    1145             : 
    1146             : /*-------------------------------------------------------------------------*/
    1147             : 
    1148             : /* the async qh for the qtds being reclaimed are now unlinked from the HC */
    1149             : 
    1150             : static void end_unlink_async (struct ehci_hcd *ehci)
    1151             : {
    1152          48 :         struct ehci_qh          *qh = ehci->reclaim;
    1153          24 :         struct ehci_qh          *next;
    1154          24 : 
    1155          72 :         iaa_watchdog_done(ehci);
    1156          24 : 
    1157             :         // qh->hw_next = cpu_to_hc32(qh->qh_dma);
    1158          24 :         qh->qh_state = QH_STATE_IDLE;
    1159          24 :         qh->qh_next.qh = NULL;
    1160          72 :         qh_put (qh);                    // refcount from reclaim
    1161             : 
    1162             :         /* other unlink(s) may be pending (in QH_STATE_UNLINK_WAIT) */
    1163          24 :         next = qh->reclaim;
    1164          24 :         ehci->reclaim = next;
    1165          24 :         qh->reclaim = NULL;
    1166             : 
    1167         120 :         qh_completions (ehci, qh);
    1168             : 
    1169         216 :         if (!list_empty (&qh->qtd_list)
    1170             :                         && HC_IS_RUNNING (ehci_to_hcd(ehci)->state))
    1171          72 :                 qh_link_async (ehci, qh);
    1172          24 :         else {
    1173             :                 /* it's not free to turn the async schedule on/off; leave it
    1174             :                  * active but idle for a while once it empties.
    1175             :                  */
    1176         192 :                 if (HC_IS_RUNNING (ehci_to_hcd(ehci)->state)
    1177             :                                 && ehci->async->qh_next.qh == NULL)
    1178          96 :                         timer_action (ehci, TIMER_ASYNC_OFF);
    1179             :         }
    1180         216 :         qh_put(qh);                     /* refcount from async list */
    1181             : 
    1182          48 :         if (next) {
    1183          24 :                 ehci->reclaim = NULL;
    1184          60 :                 start_unlink_async (ehci, next);
    1185             :         }
    1186          48 : }
    1187             : 
    1188             : /* makes sure the async qh will become idle */
    1189             : /* caller must own ehci->lock */
    1190             : 
    1191             : static void start_unlink_async (struct ehci_hcd *ehci, struct ehci_qh *qh)
    1192             : {
    1193          96 :         int             cmd = ehci_readl(ehci, &ehci->regs->command);
    1194          24 :         struct ehci_qh  *prev;
    1195          24 : 
    1196          24 : #ifdef DEBUG
    1197          24 :         assert_spin_locked(&ehci->lock);
    1198          24 :         if (ehci->reclaim
    1199          24 :                         || (qh->qh_state != QH_STATE_LINKED
    1200             :                                 && qh->qh_state != QH_STATE_UNLINK_WAIT)
    1201             :                         )
    1202             :                 BUG ();
    1203             : #endif
    1204             : 
    1205             :         /* stop async schedule right now? */
    1206         120 :         if (unlikely (qh == ehci->async)) {
    1207             :                 /* can't get here without STS_ASS set */
    1208         168 :                 if (ehci_to_hcd(ehci)->state != HC_STATE_HALT
    1209             :                                 && !ehci->reclaim) {
    1210             :                         /* ... and CMD_IAAD clear */
    1211          48 :                         ehci_writel(ehci, cmd & ~CMD_ASE,
    1212             :                                     &ehci->regs->command);
    1213          24 :                         wmb ();
    1214             :                         // handshake later, if we need to
    1215          48 :                         timer_action_done (ehci, TIMER_ASYNC_OFF);
    1216             :                 }
    1217          48 :                 return;
    1218             :         }
    1219             : 
    1220          24 :         qh->qh_state = QH_STATE_UNLINK;
    1221          72 :         ehci->reclaim = qh = qh_get (qh);
    1222             : 
    1223          24 :         prev = ehci->async;
    1224          72 :         while (prev->qh_next.qh != qh)
    1225          48 :                 prev = prev->qh_next.qh;
    1226          48 : 
    1227          24 :         prev->hw->hw_next = qh->hw->hw_next;
    1228          24 :         prev->qh_next = qh->qh_next;
    1229          24 :         wmb ();
    1230             : 
    1231             :         /* If the controller isn't running, we don't have to wait for it */
    1232         144 :         if (unlikely(!HC_IS_RUNNING(ehci_to_hcd(ehci)->state))) {
    1233             :                 /* if (unlikely (qh->reclaim != 0))
    1234             :                  *      this will recurse, probably not much
    1235             :                  */
    1236          54 :                 end_unlink_async (ehci);
    1237          24 :                 return;
    1238             :         }
    1239             : 
    1240          24 :         cmd |= CMD_IAAD;
    1241          48 :         ehci_writel(ehci, cmd, &ehci->regs->command);
    1242          48 :         (void)ehci_readl(ehci, &ehci->regs->command);
    1243          48 :         iaa_watchdog_start(ehci);
    1244          24 : }
    1245             : 
    1246             : /*-------------------------------------------------------------------------*/
    1247             : 
    1248             : static void scan_async (struct ehci_hcd *ehci)
    1249             : {
    1250           3 :         bool                    stopped;
    1251           3 :         struct ehci_qh          *qh;
    1252           6 :         enum ehci_timer_action  action = TIMER_IO_WATCHDOG;
    1253           3 : 
    1254           9 :         ehci->stamp = ehci_readl(ehci, &ehci->regs->frame_index);
    1255           9 :         timer_action_done (ehci, TIMER_ASYNC_SHRINK);
    1256           3 : rescan:
    1257          15 :         stopped = !HC_IS_RUNNING(ehci_to_hcd(ehci)->state);
    1258           6 :         qh = ehci->async->qh_next.qh;
    1259          12 :         if (likely (qh != NULL)) {
    1260           3 :                 do {
    1261             :                         /* clean any finished work for this qh */
    1262          24 :                         if (!list_empty(&qh->qtd_list) && (stopped ||
    1263             :                                         qh->stamp != ehci->stamp)) {
    1264             :                                 int temp;
    1265             : 
    1266             :                                 /* unlinks could happen here; completion
    1267             :                                  * reporting drops the lock.  rescan using
    1268             :                                  * the latest schedule, but don't rescan
    1269             :                                  * qhs we already finished (no looping)
    1270             :                                  * unless the controller is stopped.
    1271             :                                  */
    1272           6 :                                 qh = qh_get (qh);
    1273           3 :                                 qh->stamp = ehci->stamp;
    1274          18 :                                 temp = qh_completions (ehci, qh);
    1275           9 :                                 if (qh->needs_rescan)
    1276          15 :                                         unlink_async(ehci, qh);
    1277          18 :                                 qh_put (qh);
    1278           6 :                                 if (temp != 0) {
    1279           3 :                                         goto rescan;
    1280             :                                 }
    1281             :                         }
    1282             : 
    1283             :                         /* unlink idle entries, reducing DMA usage as well
    1284             :                          * as HCD schedule-scanning costs.  delay for any qh
    1285             :                          * we just scanned, there's a not-unusual case that it
    1286             :                          * doesn't stay idle for long.
    1287             :                          * (plus, avoids some kind of re-activation race.)
    1288             :                          */
    1289          27 :                         if (list_empty(&qh->qtd_list)
    1290             :                                         && qh->qh_state == QH_STATE_LINKED) {
    1291          21 :                                 if (!ehci->reclaim && (stopped ||
    1292             :                                         ((ehci->stamp - qh->stamp) & 0x1fff)
    1293             :                                                 >= EHCI_SHRINK_FRAMES * 8))
    1294          15 :                                         start_unlink_async(ehci, qh);
    1295             :                                 else
    1296           3 :                                         action = TIMER_ASYNC_SHRINK;
    1297             :                         }
    1298             : 
    1299           6 :                         qh = qh->qh_next.qh;
    1300          12 :                 } while (qh);
    1301             :         }
    1302          24 :         if (action == TIMER_ASYNC_SHRINK)
    1303          36 :                 timer_action (ehci, TIMER_ASYNC_SHRINK);
    1304          12 : }

Generated by: LCOV version 1.10