Line data Source code
1 : /*
2 : * Copyright (C) 2001-2004 by David Brownell
3 : *
4 : * This program is free software; you can redistribute it and/or modify it
5 : * under the terms of the GNU General Public License as published by the
6 : * Free Software Foundation; either version 2 of the License, or (at your
7 : * option) any later version.
8 : *
9 : * This program is distributed in the hope that it will be useful, but
10 : * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
11 : * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 : * for more details.
13 : *
14 : * You should have received a copy of the GNU General Public License
15 : * along with this program; if not, write to the Free Software Foundation,
16 : * Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
17 : */
18 :
19 : /* this file is part of ehci-hcd.c */
20 :
21 : /*-------------------------------------------------------------------------*/
22 :
23 : /*
24 : * EHCI hardware queue manipulation ... the core. QH/QTD manipulation.
25 : *
26 : * Control, bulk, and interrupt traffic all use "qh" lists. They list "qtd"
27 : * entries describing USB transactions, max 16-20kB/entry (with 4kB-aligned
28 : * buffers needed for the larger number). We use one QH per endpoint, queue
29 : * multiple urbs (all three types) per endpoint. URBs may need several qtds.
30 : *
31 : * ISO traffic uses "ISO TD" (itd, and sitd) records, and (along with
32 : * interrupts) needs careful scheduling. Performance improvements can be
33 : * an ongoing challenge. That's in "ehci-sched.c".
34 : *
35 : * USB 1.1 devices are handled (a) by "companion" OHCI or UHCI root hubs,
36 : * or otherwise through transaction translators (TTs) in USB 2.0 hubs using
37 : * (b) special fields in qh entries or (c) split iso entries. TTs will
38 : * buffer low/full speed data so the host collects it at high speed.
39 : */
40 :
41 : /*-------------------------------------------------------------------------*/
42 :
43 : /* fill a qtd, returning how much of the buffer we were able to queue up */
44 :
45 : static int
46 : qtd_fill(struct ehci_hcd *ehci, struct ehci_qtd *qtd, dma_addr_t buf,
47 : size_t len, int token, int maxpacket)
48 : {
49 6 : int i, count;
50 12 : u64 addr = buf;
51 6 :
52 6 : /* one buffer entry per 4K ... first might be short or unaligned */
53 12 : qtd->hw_buf[0] = cpu_to_hc32(ehci, (u32)addr);
54 12 : qtd->hw_buf_hi[0] = cpu_to_hc32(ehci, (u32)(addr >> 32));
55 6 : count = 0x1000 - (buf & 0x0fff); /* rest of that page */
56 24 : if (likely (len < count)) /* ... iff needed */
57 6 : count = len;
58 : else {
59 6 : buf += 0x1000;
60 6 : buf &= ~0x0fff;
61 :
62 : /* per-qtd limit: from 16K to 20K (best alignment) */
63 42 : for (i = 1; count < len && i < 5; i++) {
64 18 : addr = buf;
65 18 : qtd->hw_buf[i] = cpu_to_hc32(ehci, (u32)addr);
66 12 : qtd->hw_buf_hi[i] = cpu_to_hc32(ehci,
67 : (u32)(addr >> 32));
68 6 : buf += 0x1000;
69 12 : if ((count + 0x1000) < len)
70 6 : count += 0x1000;
71 : else
72 6 : count = len;
73 : }
74 :
75 : /* short packets may only terminate transfers */
76 12 : if (count != len)
77 6 : count -= (count % maxpacket);
78 : }
79 24 : qtd->hw_token = cpu_to_hc32(ehci, (count << 16) | token);
80 6 : qtd->length = count;
81 :
82 6 : return count;
83 : }
84 :
85 : /*-------------------------------------------------------------------------*/
86 :
87 : static inline void
88 : qh_update (struct ehci_hcd *ehci, struct ehci_qh *qh, struct ehci_qtd *qtd)
89 : {
90 684 : struct ehci_qh_hw *hw = qh->hw;
91 342 :
92 342 : /* writes to an active overlay are unsafe */
93 2736 : BUG_ON(qh->qh_state != QH_STATE_IDLE);
94 342 :
95 1368 : hw->hw_qtd_next = QTD_NEXT(ehci, qtd->qtd_dma);
96 1026 : hw->hw_alt_next = EHCI_LIST_END(ehci);
97 342 :
98 342 : /* Except for control endpoints, we make hardware maintain data
99 : * toggle (like OHCI) ... here (re)initialize the toggle in the QH,
100 : * and set the pseudo-toggle in udev. Only usb_clear_halt() will
101 : * ever clear it.
102 : */
103 1368 : if (!(hw->hw_info1 & cpu_to_hc32(ehci, 1 << 14))) {
104 : unsigned is_out, epnum;
105 :
106 342 : is_out = qh->is_out;
107 1026 : epnum = (hc32_to_cpup(ehci, &hw->hw_info1) >> 8) & 0x0f;
108 1368 : if (unlikely (!usb_gettoggle (qh->dev, epnum, is_out))) {
109 1026 : hw->hw_token &= ~cpu_to_hc32(ehci, QTD_TOGGLE);
110 342 : usb_settoggle (qh->dev, epnum, is_out, 1);
111 : }
112 : }
113 :
114 : /* HC must see latest qtd and qh data before we clear ACTIVE+HALT */
115 1026 : wmb ();
116 2394 : hw->hw_token &= cpu_to_hc32(ehci, QTD_TOGGLE | QTD_STS_PING);
117 342 : }
118 :
119 : /* if it weren't for a common silicon quirk (writing the dummy into the qh
120 : * overlay, so qh->hw_token wrongly becomes inactive/halted), only fault
121 : * recovery (including urb dequeue) would need software changes to a QH...
122 : */
123 : static void
124 : qh_refresh (struct ehci_hcd *ehci, struct ehci_qh *qh)
125 : {
126 171 : struct ehci_qtd *qtd;
127 171 :
128 855 : if (list_empty (&qh->qtd_list))
129 342 : qtd = qh->dummy;
130 : else {
131 342 : qtd = list_entry (qh->qtd_list.next,
132 : struct ehci_qtd, qtd_list);
133 : /* first qtd may already be partially processed */
134 855 : if (cpu_to_hc32(ehci, qtd->qtd_dma) == qh->hw->hw_current)
135 171 : qtd = NULL;
136 : }
137 :
138 684 : if (qtd)
139 684 : qh_update (ehci, qh, qtd);
140 513 : }
141 :
142 : /*-------------------------------------------------------------------------*/
143 :
144 : static void qh_link_async(struct ehci_hcd *ehci, struct ehci_qh *qh);
145 :
146 : static void ehci_clear_tt_buffer_complete(struct usb_hcd *hcd,
147 : struct usb_host_endpoint *ep)
148 1 : {
149 4 : struct ehci_hcd *ehci = hcd_to_ehci(hcd);
150 3 : struct ehci_qh *qh = ep->hcpriv;
151 1 : unsigned long flags;
152 1 :
153 4 : spin_lock_irqsave(&ehci->lock, flags);
154 1 : qh->clearing_tt = 0;
155 9 : if (qh->qh_state == QH_STATE_IDLE && !list_empty(&qh->qtd_list)
156 : && HC_IS_RUNNING(hcd->state))
157 3 : qh_link_async(ehci, qh);
158 6 : spin_unlock_irqrestore(&ehci->lock, flags);
159 1 : }
160 :
161 : static void ehci_clear_tt_buffer(struct ehci_hcd *ehci, struct ehci_qh *qh,
162 : struct urb *urb, u32 token)
163 : {
164 80 :
165 80 : /* If an async split transaction gets an error or is unlinked,
166 80 : * the TT buffer may be left in an indeterminate state. We
167 : * have to clear the TT buffer.
168 : *
169 : * Note: this routine is never called for Isochronous transfers.
170 : */
171 560 : if (urb->dev->tt && !usb_pipeint(urb->pipe) && !qh->clearing_tt) {
172 : #ifdef DEBUG
173 : struct usb_device *tt = urb->dev->tt->hub;
174 : dev_dbg(&tt->dev,
175 : "clear tt buffer port %d, a%d ep%d t%08x\n",
176 : urb->dev->ttport, urb->dev->devnum,
177 : usb_pipeendpoint(urb->pipe), token);
178 : #endif /* DEBUG */
179 800 : if (!ehci_is_TDI(ehci)
180 80 : || urb->dev->tt->hub !=
181 : ehci_to_hcd(ehci)->self.root_hub) {
182 480 : if (usb_hub_clear_tt_buffer(urb) == 0)
183 160 : qh->clearing_tt = 1;
184 : } else {
185 :
186 : /* REVISIT ARC-derived cores don't clear the root
187 : * hub TT buffer in this way...
188 240 : */
189 : }
190 : }
191 : }
192 :
193 : static int qtd_copy_status (
194 : struct ehci_hcd *ehci,
195 : struct urb *urb,
196 200 : size_t length,
197 200 : u32 token
198 200 : )
199 200 : {
200 400 : int status = -EINPROGRESS;
201 :
202 : /* count IN/OUT bytes, not SETUP (even short packets) */
203 800 : if (likely (QTD_PID (token) != 2))
204 200 : urb->actual_length += length - QTD_LENGTH (token);
205 :
206 : /* don't modify error codes */
207 800 : if (unlikely(urb->unlinked))
208 200 : return status;
209 :
210 : /* force cleanup after short read; not always an error */
211 1600 : if (unlikely (IS_SHORT_READ (token)))
212 200 : status = -EREMOTEIO;
213 :
214 : /* serious "can't proceed" faults reported by the hardware */
215 800 : if (token & QTD_STS_HALT) {
216 800 : if (token & QTD_STS_BABBLE) {
217 : /* FIXME "must" disable babbling device's port too */
218 400 : status = -EOVERFLOW;
219 : /* CERR nonzero + halt --> stall */
220 800 : } else if (QTD_CERR(token)) {
221 400 : status = -EPIPE;
222 :
223 : /* In theory, more than one of the following bits can be set
224 : * since they are sticky and the transaction is retried.
225 : * Which to test first is rather arbitrary.
226 : */
227 800 : } else if (token & QTD_STS_MMF) {
228 : /* fs/ls interrupt xfer missed the complete-split */
229 400 : status = -EPROTO;
230 800 : } else if (token & QTD_STS_DBE) {
231 2400 : status = (QTD_PID (token) == 1) /* IN ? */
232 : ? -ENOSR /* hc couldn't read data */
233 : : -ECOMM; /* hc couldn't write data */
234 800 : } else if (token & QTD_STS_XACT) {
235 : /* timeout, bad CRC, wrong PID, etc */
236 : ehci_dbg(ehci, "devpath %s ep%d%s 3strikes\n",
237 : urb->dev->devpath,
238 : usb_pipeendpoint(urb->pipe),
239 : usb_pipein(urb->pipe) ? "in" : "out");
240 400 : status = -EPROTO;
241 : } else { /* unknown */
242 400 : status = -EPROTO;
243 : }
244 :
245 : ehci_vdbg (ehci,
246 : "dev%d ep%d%s qtd token %08x --> status %d\n",
247 : usb_pipedevice (urb->pipe),
248 : usb_pipeendpoint (urb->pipe),
249 : usb_pipein (urb->pipe) ? "in" : "out",
250 : token, status);
251 : }
252 :
253 400 : return status;
254 : }
255 :
256 : static void
257 : ehci_urb_done(struct ehci_hcd *ehci, struct urb *urb, int status)
258 : __releases(ehci->lock)
259 86 : __acquires(ehci->lock)
260 86 : {
261 516 : if (likely (urb->hcpriv != NULL)) {
262 258 : struct ehci_qh *qh = (struct ehci_qh *) urb->hcpriv;
263 86 :
264 86 : /* S-mask in a QH means it's an interrupt urb */
265 430 : if ((qh->hw->hw_info2 & cpu_to_hc32(ehci, QH_SMASK)) != 0) {
266 :
267 : /* ... update hc-wide periodic stats (for usbfs) */
268 258 : ehci_to_hcd(ehci)->self.bandwidth_int_reqs--;
269 : }
270 516 : qh_put (qh);
271 : }
272 :
273 516 : if (unlikely(urb->unlinked)) {
274 : COUNT(ehci->stats.unlink);
275 : } else {
276 : /* report non-error and short read status as zero */
277 344 : if (status == -EINPROGRESS || status == -EREMOTEIO)
278 86 : status = 0;
279 : COUNT(ehci->stats.complete);
280 : }
281 :
282 : #ifdef EHCI_URB_TRACE
283 : ehci_dbg (ehci,
284 : "%s %s urb %p ep%d%s status %d len %d/%d\n",
285 : __func__, urb->dev->devpath, urb,
286 : usb_pipeendpoint (urb->pipe),
287 : usb_pipein (urb->pipe) ? "in" : "out",
288 : status,
289 : urb->actual_length, urb->transfer_buffer_length);
290 : #endif
291 :
292 : /* complete() can reenter this HCD */
293 258 : usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb);
294 172 : spin_unlock (&ehci->lock);
295 258 : usb_hcd_giveback_urb(ehci_to_hcd(ehci), urb, status);
296 172 : spin_lock (&ehci->lock);
297 86 : }
298 :
299 : static void start_unlink_async (struct ehci_hcd *ehci, struct ehci_qh *qh);
300 : static void unlink_async (struct ehci_hcd *ehci, struct ehci_qh *qh);
301 :
302 : static int qh_schedule (struct ehci_hcd *ehci, struct ehci_qh *qh);
303 :
304 : /*
305 : * Process and free completed qtds for a qh, returning URBs to drivers.
306 : * Chases up to qh->hw_current. Returns number of completions called,
307 : * indicating how much "real" work we did.
308 : */
309 : static unsigned
310 : qh_completions (struct ehci_hcd *ehci, struct ehci_qh *qh)
311 : {
312 80 : struct ehci_qtd *last, *end = qh->dummy;
313 40 : struct list_head *entry, *tmp;
314 40 : int last_status;
315 40 : int stopped;
316 80 : unsigned count = 0;
317 40 : u8 state;
318 80 : struct ehci_qh_hw *hw = qh->hw;
319 40 :
320 280 : if (unlikely (list_empty (&qh->qtd_list)))
321 80 : return count;
322 40 :
323 40 : /* completions (or tasks on other cpus) must never clobber HALT
324 40 : * till we've gone through and cleaned everything up, even when
325 40 : * they add urbs to this qh's queue or mark them for unlinking.
326 40 : *
327 40 : * NOTE: unlinking expects to be done in queue order.
328 40 : *
329 40 : * It's a bug for qh->qh_state to be anything other than
330 40 : * QH_STATE_IDLE, unless our caller is scan_async() or
331 40 : * scan_periodic().
332 40 : */
333 80 : state = qh->qh_state;
334 80 : qh->qh_state = QH_STATE_COMPLETING;
335 80 : stopped = (state == QH_STATE_IDLE);
336 80 :
337 40 : rescan:
338 80 : last = NULL;
339 80 : last_status = -EINPROGRESS;
340 40 : qh->needs_rescan = 0;
341 :
342 : /* remove de-activated QTDs from front of queue.
343 : * after faults (including short reads), cleanup this urb
344 : * then let the queue advance.
345 : * if queue is stopped, handles unlinks.
346 : */
347 360 : list_for_each_safe (entry, tmp, &qh->qtd_list) {
348 120 : struct ehci_qtd *qtd;
349 40 : struct urb *urb;
350 40 : u32 token = 0;
351 :
352 80 : qtd = list_entry (entry, struct ehci_qtd, qtd_list);
353 80 : urb = qtd->urb;
354 :
355 : /* clean up any state from previous QTD ...*/
356 80 : if (last) {
357 200 : if (likely (last->urb != urb)) {
358 80 : ehci_urb_done(ehci, last->urb, last_status);
359 40 : count++;
360 40 : last_status = -EINPROGRESS;
361 : }
362 160 : ehci_qtd_free (ehci, last);
363 40 : last = NULL;
364 : }
365 :
366 : /* ignore urbs submitted during completions we reported */
367 160 : if (qtd == end)
368 80 : break;
369 :
370 : /* hardware copies qtd out of qh overlay */
371 80 : rmb ();
372 160 : token = hc32_to_cpu(ehci, qtd->hw_token);
373 :
374 40 : /* always clean up qtds the hc de-activated */
375 : retry_xacterr:
376 80 : if ((token & QTD_STS_ACTIVE) == 0) {
377 :
378 : /* on STALL, error, and short reads this urb must
379 : * complete and all its qtds must be recycled.
380 : */
381 80 : if ((token & QTD_STS_HALT) != 0) {
382 :
383 : /* retry transaction errors until we
384 : * reach the software xacterr limit
385 : */
386 440 : if ((token & QTD_STS_XACT) &&
387 : QTD_CERR(token) == 0 &&
388 : ++qh->xacterrs < QH_XACTERR_MAX &&
389 : !urb->unlinked) {
390 : ehci_dbg(ehci,
391 : "detected XactErr len %zu/%zu retry %d\n",
392 : qtd->length - QTD_LENGTH(token), qtd->length, qh->xacterrs);
393 :
394 : /* reset the token in the qtd and the
395 : * qh overlay (which still contains
396 : * the qtd) so that we pick up from
397 : * where we left off
398 : */
399 40 : token &= ~QTD_STS_HALT;
400 40 : token |= QTD_STS_ACTIVE |
401 : (EHCI_TUNE_CERR << 10);
402 80 : qtd->hw_token = cpu_to_hc32(ehci,
403 : token);
404 40 : wmb();
405 80 : hw->hw_token = cpu_to_hc32(ehci,
406 : token);
407 40 : goto retry_xacterr;
408 : }
409 40 : stopped = 1;
410 :
411 : /* magic dummy for some short reads; qh won't advance.
412 : * that silicon quirk can kick in with this dummy too.
413 : *
414 : * other short reads won't stop the queue, including
415 : * control transfers (status stage handles that) or
416 : * most other single-qtd reads ... the queue stops if
417 : * URB_SHORT_NOT_OK was set so the driver submitting
418 : * the urbs could clean it up.
419 : */
420 320 : } else if (IS_SHORT_READ (token)
421 : && !(qtd->hw_alt_next
422 : & EHCI_LIST_END(ehci))) {
423 40 : stopped = 1;
424 : }
425 :
426 : /* stop scanning when we reach qtds the hc is using */
427 480 : } else if (likely (!stopped
428 : && HC_IS_RUNNING (ehci_to_hcd(ehci)->state))) {
429 40 : break;
430 40 :
431 : /* scan the whole queue for unlinks whenever it stops */
432 : } else {
433 80 : stopped = 1;
434 :
435 : /* cancel everything if we halt, suspend, etc */
436 240 : if (!HC_IS_RUNNING(ehci_to_hcd(ehci)->state))
437 40 : last_status = -ESHUTDOWN;
438 :
439 : /* this qtd is active; skip it unless a previous qtd
440 : * for its urb faulted, or its urb was canceled.
441 : */
442 160 : else if (last_status == -EINPROGRESS && !urb->unlinked)
443 40 : continue;
444 :
445 : /* qh unlinked; token in overlay may be most current */
446 280 : if (state == QH_STATE_IDLE
447 : && cpu_to_hc32(ehci, qtd->qtd_dma)
448 : == hw->hw_current) {
449 80 : token = hc32_to_cpu(ehci, hw->hw_token);
450 :
451 : /* An unlink may leave an incomplete
452 : * async transaction in the TT buffer.
453 : * We have to clear it.
454 : */
455 160 : ehci_clear_tt_buffer(ehci, qh, urb, token);
456 : }
457 : }
458 :
459 : /* unless we already know the urb's status, collect qtd status
460 : * and update count of bytes transferred. in common short read
461 : * cases with only one data qtd (including control transfers),
462 : * queue processing won't halt. but with two or more qtds (for
463 : * example, with a 32 KB transfer), when the first qtd gets a
464 : * short read the second must be removed by hand.
465 : */
466 400 : if (last_status == -EINPROGRESS) {
467 800 : last_status = qtd_copy_status(ehci, urb,
468 : qtd->length, token);
469 240 : if (last_status == -EREMOTEIO
470 : && (qtd->hw_alt_next
471 : & EHCI_LIST_END(ehci)))
472 40 : last_status = -EINPROGRESS;
473 :
474 : /* As part of low/full-speed endpoint-halt processing
475 : * we must clear the TT buffer (11.17.5).
476 : */
477 640 : if (unlikely(last_status != -EINPROGRESS &&
478 : last_status != -EREMOTEIO)) {
479 : /* The TT's in some hubs malfunction when they
480 : * receive this request following a STALL (they
481 : * stop sending isochronous packets). Since a
482 : * STALL can't leave the TT buffer in a busy
483 : * state (if you believe Figures 11-48 - 11-51
484 : * in the USB 2.0 spec), we won't clear the TT
485 : * buffer in this case. Strictly speaking this
486 : * is a violation of the spec.
487 : */
488 80 : if (last_status != -EPIPE)
489 160 : ehci_clear_tt_buffer(ehci, qh, urb,
490 : token);
491 : }
492 : }
493 :
494 : /* if we're removing something not at the queue head,
495 : * patch the hardware queue pointer.
496 : */
497 1120 : if (stopped && qtd->qtd_list.prev != &qh->qtd_list) {
498 560 : last = list_entry (qtd->qtd_list.prev,
499 : struct ehci_qtd, qtd_list);
500 280 : last->hw_next = qtd->hw_next;
501 : }
502 :
503 : /* remove qtd; it's recycled after possible urb completion */
504 560 : list_del (&qtd->qtd_list);
505 40 : last = qtd;
506 :
507 : /* reinit the xacterr counter for the next qtd */
508 40 : qh->xacterrs = 0;
509 : }
510 40 :
511 : /* last urb's completion might still need calling */
512 320 : if (likely (last != NULL)) {
513 80 : ehci_urb_done(ehci, last->urb, last_status);
514 40 : count++;
515 80 : ehci_qtd_free (ehci, last);
516 : }
517 :
518 : /* Do we need to rescan for URBs dequeued during a giveback? */
519 320 : if (unlikely(qh->needs_rescan)) {
520 : /* If the QH is already unlinked, do the rescan now. */
521 80 : if (state == QH_STATE_IDLE)
522 40 : goto rescan;
523 :
524 : /* Otherwise we have to wait until the QH is fully unlinked.
525 : * Our caller will start an unlink if qh->needs_rescan is
526 : * set. But if an unlink has already started, nothing needs
527 : * to be done.
528 : */
529 80 : if (state != QH_STATE_LINKED)
530 40 : qh->needs_rescan = 0;
531 : }
532 :
533 : /* restore original state; caller must unlink or relink */
534 40 : qh->qh_state = state;
535 :
536 : /* be sure the hardware's done with the qh before refreshing
537 : * it after fault cleanup, or recovering from silicon wrongly
538 : * overlaying the dummy qtd (which reduces DMA chatter).
539 : */
540 280 : if (stopped != 0 || hw->hw_qtd_next == EHCI_LIST_END(ehci)) {
541 120 : switch (state) {
542 320 : case QH_STATE_IDLE:
543 320 : qh_refresh(ehci, qh);
544 40 : break;
545 320 : case QH_STATE_LINKED:
546 : /* We won't refresh a QH that's linked (after the HC
547 : * stopped the queue). That avoids a race:
548 : * - HC reads first part of QH;
549 : * - CPU updates that first part and the token;
550 : * - HC reads rest of that QH, including token
551 : * Result: HC gets an inconsistent image, and then
552 : * DMAs to/from the wrong memory (corrupting it).
553 : *
554 : * That should be rare for interrupt transfers,
555 : * except maybe high bandwidth ...
556 : */
557 :
558 : /* Tell the caller to start an unlink */
559 80 : qh->needs_rescan = 1;
560 80 : break;
561 80 : /* otherwise, unlink already started */
562 : }
563 80 : }
564 :
565 120 : return count;
566 : }
567 :
568 : /*-------------------------------------------------------------------------*/
569 :
570 : // high bandwidth multiplier, as encoded in highspeed endpoint descriptors
571 : #define hb_mult(wMaxPacketSize) (1 + (((wMaxPacketSize) >> 11) & 0x03))
572 : // ... and packet size, for any kind of endpoint descriptor
573 : #define max_packet(wMaxPacketSize) ((wMaxPacketSize) & 0x07ff)
574 :
575 : /*
576 : * reverse of qh_urb_transaction: free a list of TDs.
577 : * used for cleanup after errors, before HC sees an URB's TDs.
578 : */
579 : static void qtd_list_free (
580 : struct ehci_hcd *ehci,
581 8 : struct urb *urb,
582 8 : struct list_head *qtd_list
583 8 : ) {
584 8 : struct list_head *entry, *temp;
585 :
586 56 : list_for_each_safe (entry, temp, qtd_list) {
587 8 : struct ehci_qtd *qtd;
588 16 :
589 16 : qtd = list_entry (entry, struct ehci_qtd, qtd_list);
590 16 : list_del (&qtd->qtd_list);
591 16 : ehci_qtd_free (ehci, qtd);
592 : }
593 8 : }
594 :
595 : /*
596 : * create a list of filled qtds for this URB; won't link into qh.
597 : */
598 : static struct list_head *
599 : qh_urb_transaction (
600 : struct ehci_hcd *ehci,
601 : struct urb *urb,
602 2 : struct list_head *head,
603 2 : gfp_t flags
604 2 : ) {
605 2 : struct ehci_qtd *qtd, *qtd_prev;
606 2 : dma_addr_t buf;
607 2 : int len, this_sg_len, maxpacket;
608 2 : int is_input;
609 2 : u32 token;
610 2 : int i;
611 2 : struct scatterlist *sg;
612 2 :
613 2 : /*
614 2 : * URBs map to sequences of QTDs: one logical transaction
615 2 : */
616 8 : qtd = ehci_qtd_alloc (ehci, flags);
617 10 : if (unlikely (!qtd))
618 4 : return NULL;
619 6 : list_add_tail (&qtd->qtd_list, head);
620 4 : qtd->urb = urb;
621 2 :
622 4 : token = QTD_STS_ACTIVE;
623 4 : token |= (EHCI_TUNE_CERR << 10);
624 2 : /* for split transactions, SplitXState initialized to zero */
625 2 :
626 6 : len = urb->transfer_buffer_length;
627 6 : is_input = usb_pipein (urb->pipe);
628 6 : if (usb_pipecontrol (urb->pipe)) {
629 : /* SETUP pid */
630 4 : qtd_fill(ehci, qtd, urb->setup_dma,
631 : sizeof (struct usb_ctrlrequest),
632 : token | (2 /* "setup" */ << 8), 8);
633 :
634 : /* ... and always at least one more pid */
635 2 : token ^= QTD_TOGGLE;
636 2 : qtd_prev = qtd;
637 6 : qtd = ehci_qtd_alloc (ehci, flags);
638 8 : if (unlikely (!qtd))
639 2 : goto cleanup;
640 2 : qtd->urb = urb;
641 6 : qtd_prev->hw_next = QTD_NEXT(ehci, qtd->qtd_dma);
642 4 : list_add_tail (&qtd->qtd_list, head);
643 :
644 : /* for zero length DATA stages, STATUS is always IN */
645 4 : if (len == 0)
646 2 : token |= (1 /* "in" */ << 8);
647 : }
648 :
649 : /*
650 : * data transfer stage: buffer setup
651 : */
652 4 : i = urb->num_sgs;
653 16 : if (len > 0 && i > 0) {
654 4 : sg = urb->sg->sg;
655 4 : buf = sg_dma_address(sg);
656 :
657 : /* urb->transfer_buffer_length may be smaller than the
658 : * size of the scatterlist (or vice versa)
659 : */
660 36 : this_sg_len = min_t(int, sg_dma_len(sg), len);
661 : } else {
662 4 : sg = NULL;
663 4 : buf = urb->transfer_dma;
664 4 : this_sg_len = len;
665 : }
666 :
667 8 : if (is_input)
668 4 : token |= (1 /* "in" */ << 8);
669 : /* else it's already initted to "out" pid (0 << 8) */
670 :
671 18 : maxpacket = max_packet(usb_maxpacket(urb->dev, urb->pipe, !is_input));
672 :
673 2 : /*
674 : * buffer gets wrapped in one or more qtds;
675 : * last one may be "short" (including zero len)
676 : * and may serve as a control status ack
677 : */
678 : for (;;) {
679 : int this_qtd_len;
680 :
681 4 : this_qtd_len = qtd_fill(ehci, qtd, buf, this_sg_len, token,
682 : maxpacket);
683 2 : this_sg_len -= this_qtd_len;
684 2 : len -= this_qtd_len;
685 2 : buf += this_qtd_len;
686 :
687 : /*
688 : * short reads advance to a "magic" dummy instead of the next
689 : * qtd ... that forces the queue to stop, for manual cleanup.
690 : * (this will usually be overridden later.)
691 : */
692 4 : if (is_input)
693 2 : qtd->hw_alt_next = ehci->async->hw->hw_alt_next;
694 :
695 : /* qh makes control packets use qtd toggle; maybe switch it */
696 4 : if ((maxpacket & (this_qtd_len + (maxpacket - 1))) == 0)
697 2 : token ^= QTD_TOGGLE;
698 :
699 8 : if (likely(this_sg_len <= 0)) {
700 10 : if (--i <= 0 || len <= 0)
701 2 : break;
702 2 : sg = sg_next(sg);
703 2 : buf = sg_dma_address(sg);
704 18 : this_sg_len = min_t(int, sg_dma_len(sg), len);
705 : }
706 :
707 2 : qtd_prev = qtd;
708 6 : qtd = ehci_qtd_alloc (ehci, flags);
709 8 : if (unlikely (!qtd))
710 2 : goto cleanup;
711 2 : qtd->urb = urb;
712 6 : qtd_prev->hw_next = QTD_NEXT(ehci, qtd->qtd_dma);
713 4 : list_add_tail (&qtd->qtd_list, head);
714 2 : }
715 2 :
716 : /*
717 : * unless the caller requires manual cleanup after short reads,
718 : * have the alt_next mechanism keep the queue running after the
719 : * last data qtd (the only one, for control and most other cases).
720 : */
721 16 : if (likely ((urb->transfer_flags & URB_SHORT_NOT_OK) == 0
722 : || usb_pipecontrol (urb->pipe)))
723 8 : qtd->hw_alt_next = EHCI_LIST_END(ehci);
724 :
725 : /*
726 : * control requests may need a terminating data "status" ack;
727 : * bulk ones may need a terminating short packet (zero length).
728 : */
729 16 : if (likely (urb->transfer_buffer_length != 0)) {
730 2 : int one_more = 0;
731 :
732 4 : if (usb_pipecontrol (urb->pipe)) {
733 2 : one_more = 1;
734 2 : token ^= 0x0100; /* "in" <--> "out" */
735 2 : token |= QTD_TOGGLE; /* force DATA1 */
736 12 : } else if (usb_pipebulk (urb->pipe)
737 : && (urb->transfer_flags & URB_ZERO_PACKET)
738 : && !(urb->transfer_buffer_length % maxpacket)) {
739 2 : one_more = 1;
740 : }
741 4 : if (one_more) {
742 2 : qtd_prev = qtd;
743 6 : qtd = ehci_qtd_alloc (ehci, flags);
744 8 : if (unlikely (!qtd))
745 2 : goto cleanup;
746 2 : qtd->urb = urb;
747 6 : qtd_prev->hw_next = QTD_NEXT(ehci, qtd->qtd_dma);
748 4 : list_add_tail (&qtd->qtd_list, head);
749 :
750 : /* never any data in such packets */
751 4 : qtd_fill(ehci, qtd, 0, 0, token, 0);
752 : }
753 : }
754 :
755 : /* by default, enable interrupt on urb completion */
756 12 : if (likely (!(urb->transfer_flags & URB_NO_INTERRUPT)))
757 6 : qtd->hw_token |= cpu_to_hc32(ehci, QTD_IOC);
758 4 : return head;
759 6 :
760 : cleanup:
761 12 : qtd_list_free (ehci, urb, head);
762 2 : return NULL;
763 : }
764 :
765 : /*-------------------------------------------------------------------------*/
766 :
767 : // Would be best to create all qh's from config descriptors,
768 : // when each interface/altsetting is established. Unlink
769 : // any previous qh and cancel its urbs first; endpoints are
770 : // implicitly reset then (data toggle too).
771 : // That'd mean updating how usbcore talks to HCDs. (2.7?)
772 :
773 :
774 : /*
775 : * Each QH holds a qtd list; a QH is used for everything except iso.
776 : *
777 : * For interrupt urbs, the scheduler must set the microframe scheduling
778 : * mask(s) each time the QH gets scheduled. For highspeed, that's
779 : * just one microframe in the s-mask. For split interrupt transactions
780 : * there are additional complications: c-mask, maybe FSTNs.
781 : */
782 : static struct ehci_qh *
783 : qh_make (
784 : struct ehci_hcd *ehci,
785 4 : struct urb *urb,
786 4 : gfp_t flags
787 4 : ) {
788 20 : struct ehci_qh *qh = ehci_qh_alloc (ehci, flags);
789 12 : u32 info1 = 0, info2 = 0;
790 4 : int is_input, type;
791 8 : int maxp = 0;
792 8 : struct usb_tt *tt = urb->dev->tt;
793 4 : struct ehci_qh_hw *hw;
794 4 :
795 12 : if (!qh)
796 8 : return qh;
797 4 :
798 4 : /*
799 4 : * init endpoint/device data for this QH
800 : */
801 4 : info1 |= usb_pipeendpoint (urb->pipe) << 8;
802 4 : info1 |= usb_pipedevice (urb->pipe) << 0;
803 :
804 8 : is_input = usb_pipein (urb->pipe);
805 4 : type = usb_pipetype (urb->pipe);
806 20 : maxp = usb_maxpacket (urb->dev, urb->pipe, !is_input);
807 :
808 : /* 1024 byte maxpacket is a hardware ceiling. High bandwidth
809 : * acts like up to 3KB, but is built from smaller packets.
810 : */
811 8 : if (max_packet(maxp) > 1024) {
812 : ehci_dbg(ehci, "bogus qh maxpacket %d\n", max_packet(maxp));
813 4 : goto done;
814 : }
815 :
816 : /* Compute interrupt scheduling parameters just once, and save.
817 : * - allowing for high bandwidth, how many nsec/uframe are used?
818 : * - split transactions need a second CSPLIT uframe; same question
819 : * - splits also need a schedule gap (for full/low speed I/O)
820 : * - qh has a polling interval
821 : *
822 : * For control/bulk requests, the HC or TT handles these.
823 : */
824 8 : if (type == PIPE_INTERRUPT) {
825 8 : qh->usecs = NS_TO_US(usb_calc_bus_time(USB_SPEED_HIGH,
826 : is_input, 0,
827 : hb_mult(maxp) * max_packet(maxp)));
828 4 : qh->start = NO_FRAME;
829 :
830 12 : if (urb->dev->speed == USB_SPEED_HIGH) {
831 4 : qh->c_usecs = 0;
832 4 : qh->gap_uf = 0;
833 :
834 4 : qh->period = urb->interval >> 3;
835 20 : if (qh->period == 0 && urb->interval != 1) {
836 : /* NOTE interval 2 or 4 uframes could work.
837 : * But interval 1 scheduling is simpler, and
838 : * includes high bandwidth.
839 : */
840 4 : urb->interval = 1;
841 12 : } else if (qh->period > ehci->periodic_size) {
842 8 : qh->period = ehci->periodic_size;
843 8 : urb->interval = qh->period << 3;
844 : }
845 : } else {
846 : int think_time;
847 :
848 : /* gap is f(FS/LS transfer times) */
849 12 : qh->gap_uf = 1 + usb_calc_bus_time (urb->dev->speed,
850 : is_input, 0, maxp) / (125 * 1000);
851 :
852 : /* FIXME this just approximates SPLIT/CSPLIT times */
853 8 : if (is_input) { // SPLIT, gap, CSPLIT+DATA
854 8 : qh->c_usecs = qh->usecs + HS_USECS (0);
855 4 : qh->usecs = HS_USECS (1);
856 : } else { // SPLIT+DATA, gap, CSPLIT
857 8 : qh->usecs += HS_USECS (1);
858 4 : qh->c_usecs = HS_USECS (0);
859 : }
860 :
861 28 : think_time = tt ? tt->think_time : 0;
862 12 : qh->tt_usecs = NS_TO_US (think_time +
863 : usb_calc_bus_time (urb->dev->speed,
864 : is_input, 0, max_packet (maxp)));
865 8 : qh->period = urb->interval;
866 12 : if (qh->period > ehci->periodic_size) {
867 8 : qh->period = ehci->periodic_size;
868 8 : urb->interval = qh->period;
869 : }
870 : }
871 : }
872 :
873 : /* support for tt scheduling, and access to toggles */
874 4 : qh->dev = urb->dev;
875 :
876 : /* using TT? */
877 : switch (urb->dev->speed) {
878 16 : case USB_SPEED_LOW:
879 4 : info1 |= (1 << 12); /* EPS "low" */
880 4 : /* FALL THROUGH */
881 :
882 16 : case USB_SPEED_FULL:
883 : /* EPS 0 means "full" */
884 8 : if (type != PIPE_INTERRUPT)
885 4 : info1 |= (EHCI_TUNE_RL_TT << 28);
886 8 : if (type == PIPE_CONTROL) {
887 4 : info1 |= (1 << 27); /* for TT */
888 4 : info1 |= 1 << 14; /* toggle from qtd */
889 : }
890 4 : info1 |= maxp << 16;
891 :
892 4 : info2 |= (EHCI_TUNE_MULT_TT << 30);
893 :
894 : /* Some Freescale processors have an erratum in which the
895 : * port number in the queue head was 0..N-1 instead of 1..N.
896 : */
897 : if (ehci_has_fsl_portno_bug(ehci))
898 : info2 |= (urb->dev->ttport-1) << 23;
899 : else
900 4 : info2 |= urb->dev->ttport << 23;
901 :
902 : /* set the address of the TT; for TDI's integrated
903 : * root hub tt, leave it zeroed.
904 : */
905 28 : if (tt && tt->hub != ehci_to_hcd(ehci)->self.root_hub)
906 4 : info2 |= tt->hub->devnum << 16;
907 :
908 : /* NOTE: if (PIPE_INTERRUPT) { scheduler sets c-mask } */
909 :
910 8 : break;
911 4 :
912 16 : case USB_SPEED_HIGH: /* no TT involved */
913 4 : info1 |= (2 << 12); /* EPS "high" */
914 8 : if (type == PIPE_CONTROL) {
915 4 : info1 |= (EHCI_TUNE_RL_HS << 28);
916 4 : info1 |= 64 << 16; /* usb2 fixed maxpacket */
917 4 : info1 |= 1 << 14; /* toggle from qtd */
918 4 : info2 |= (EHCI_TUNE_MULT_HS << 30);
919 8 : } else if (type == PIPE_BULK) {
920 4 : info1 |= (EHCI_TUNE_RL_HS << 28);
921 : /* The USB spec says that high speed bulk endpoints
922 : * always use 512 byte maxpacket. But some device
923 : * vendors decided to ignore that, and MSFT is happy
924 : * to help them do so. So now people expect to use
925 : * such nonconformant devices with Linux too; sigh.
926 : */
927 4 : info1 |= max_packet(maxp) << 16;
928 4 : info2 |= (EHCI_TUNE_MULT_HS << 30);
929 : } else { /* PIPE_INTERRUPT */
930 4 : info1 |= max_packet (maxp) << 16;
931 4 : info2 |= hb_mult (maxp) << 30;
932 : }
933 4 : break;
934 8 : default:
935 8 : dbg ("bogus dev %p speed %d", urb->dev, urb->dev->speed);
936 : done:
937 12 : qh_put (qh);
938 4 : return NULL;
939 : }
940 :
941 : /* NOTE: if (PIPE_INTERRUPT) { scheduler sets s-mask } */
942 :
943 : /* init as live, toggle clear, advance to dummy */
944 8 : qh->qh_state = QH_STATE_IDLE;
945 8 : hw = qh->hw;
946 16 : hw->hw_info1 = cpu_to_hc32(ehci, info1);
947 8 : hw->hw_info2 = cpu_to_hc32(ehci, info2);
948 4 : qh->is_out = !is_input;
949 4 : usb_settoggle (urb->dev, usb_pipeendpoint (urb->pipe), !is_input, 1);
950 16 : qh_refresh (ehci, qh);
951 4 : return qh;
952 : }
953 :
954 : /*-------------------------------------------------------------------------*/
955 :
956 : /* move qh (and its qtds) onto async queue; maybe enable queue. */
957 :
958 : static void qh_link_async (struct ehci_hcd *ehci, struct ehci_qh *qh)
959 : {
960 130 : __hc32 dma = QH_NEXT(ehci, qh->qh_dma);
961 26 : struct ehci_qh *head;
962 26 :
963 26 : /* Don't link a QH if there's a Clear-TT-Buffer pending */
964 130 : if (unlikely(qh->clearing_tt))
965 52 : return;
966 26 :
967 260 : WARN_ON(qh->qh_state != QH_STATE_IDLE);
968 26 :
969 : /* (re)start the async schedule? */
970 26 : head = ehci->async;
971 52 : timer_action_done (ehci, TIMER_ASYNC_OFF);
972 52 : if (!head->qh_next.qh) {
973 78 : u32 cmd = ehci_readl(ehci, &ehci->regs->command);
974 :
975 52 : if (!(cmd & CMD_ASE)) {
976 : /* in case a clear of CMD_ASE didn't take yet */
977 52 : (void)handshake(ehci, &ehci->regs->status,
978 : STS_ASS, 0, 150);
979 26 : cmd |= CMD_ASE | CMD_RUN;
980 52 : ehci_writel(ehci, cmd, &ehci->regs->command);
981 78 : ehci_to_hcd(ehci)->state = HC_STATE_RUNNING;
982 : /* posted write need not be known to HC yet ... */
983 : }
984 : }
985 :
986 : /* clear halt and/or toggle; and maybe recover from silicon quirk */
987 312 : qh_refresh(ehci, qh);
988 :
989 : /* splice right after start */
990 26 : qh->qh_next = head->qh_next;
991 26 : qh->hw->hw_next = head->hw->hw_next;
992 26 : wmb ();
993 :
994 26 : head->qh_next.qh = qh;
995 26 : head->hw->hw_next = dma;
996 :
997 52 : qh_get(qh);
998 26 : qh->xacterrs = 0;
999 26 : qh->qh_state = QH_STATE_LINKED;
1000 26 : /* qtd completions reported later by interrupt */
1001 : }
1002 :
1003 : /*-------------------------------------------------------------------------*/
1004 :
1005 : /*
1006 : * For control/bulk/interrupt, return QH with these TDs appended.
1007 : * Allocates and initializes the QH if necessary.
1008 : * Returns null if it can't allocate a QH it needs to.
1009 : * If the QH has TDs (urbs) already, that's great.
1010 : */
1011 : static struct ehci_qh *qh_append_tds (
1012 : struct ehci_hcd *ehci,
1013 : struct urb *urb,
1014 4 : struct list_head *qtd_list,
1015 4 : int epnum,
1016 4 : void **ptr
1017 4 : )
1018 4 : {
1019 8 : struct ehci_qh *qh = NULL;
1020 16 : __hc32 qh_addr_mask = cpu_to_hc32(ehci, 0x7f);
1021 4 :
1022 8 : qh = (struct ehci_qh *) *ptr;
1023 20 : if (unlikely (qh == NULL)) {
1024 4 : /* can't sleep here, we have ehci->lock... */
1025 20 : qh = qh_make (ehci, urb, GFP_ATOMIC);
1026 8 : *ptr = qh;
1027 4 : }
1028 28 : if (likely (qh != NULL)) {
1029 4 : struct ehci_qtd *qtd;
1030 :
1031 24 : if (unlikely (list_empty (qtd_list)))
1032 4 : qtd = NULL;
1033 : else
1034 12 : qtd = list_entry (qtd_list->next, struct ehci_qtd,
1035 : qtd_list);
1036 :
1037 : /* control qh may need patching ... */
1038 16 : if (unlikely (epnum == 0)) {
1039 :
1040 : /* usb_reset_device() briefly reverts to address 0 */
1041 8 : if (usb_pipedevice (urb->pipe) == 0)
1042 4 : qh->hw->hw_info1 &= ~qh_addr_mask;
1043 : }
1044 :
1045 : /* just one way to queue requests: swap with the dummy qtd.
1046 : * only hc or qh_refresh() ever modify the overlay.
1047 : */
1048 16 : if (likely (qtd != NULL)) {
1049 : struct ehci_qtd *dummy;
1050 : dma_addr_t dma;
1051 : __hc32 token;
1052 :
1053 : /* to avoid racing the HC, use the dummy td instead of
1054 : * the first td of our list (becomes new dummy). both
1055 : * tds stay deactivated until we're done, when the
1056 : * HC is allowed to fetch the old dummy (4.10.2).
1057 : */
1058 4 : token = qtd->hw_token;
1059 8 : qtd->hw_token = HALT_BIT(ehci);
1060 4 : wmb ();
1061 4 : dummy = qh->dummy;
1062 :
1063 4 : dma = dummy->qtd_dma;
1064 4 : *dummy = *qtd;
1065 4 : dummy->qtd_dma = dma;
1066 :
1067 8 : list_del (&qtd->qtd_list);
1068 8 : list_add (&dummy->qtd_list, qtd_list);
1069 12 : list_splice_tail(qtd_list, &qh->qtd_list);
1070 :
1071 8 : ehci_qtd_init(ehci, qtd, qtd->qtd_dma);
1072 4 : qh->dummy = qtd;
1073 :
1074 : /* hc must see the new dummy at list end */
1075 4 : dma = qtd->qtd_dma;
1076 8 : qtd = list_entry (qh->qtd_list.prev,
1077 : struct ehci_qtd, qtd_list);
1078 8 : qtd->hw_next = QTD_NEXT(ehci, dma);
1079 :
1080 : /* let the hc process these next qtds */
1081 4 : wmb ();
1082 4 : dummy->hw_token = token;
1083 :
1084 12 : urb->hcpriv = qh_get (qh);
1085 : }
1086 : }
1087 12 : return qh;
1088 : }
1089 :
1090 : /*-------------------------------------------------------------------------*/
1091 :
1092 : static int
1093 : submit_async (
1094 : struct ehci_hcd *ehci,
1095 : struct urb *urb,
1096 1 : struct list_head *qtd_list,
1097 1 : gfp_t mem_flags
1098 1 : ) {
1099 1 : struct ehci_qtd *qtd;
1100 1 : int epnum;
1101 1 : unsigned long flags;
1102 2 : struct ehci_qh *qh = NULL;
1103 1 : int rc;
1104 1 :
1105 4 : qtd = list_entry (qtd_list->next, struct ehci_qtd, qtd_list);
1106 2 : epnum = urb->ep->desc.bEndpointAddress;
1107 1 :
1108 1 : #ifdef EHCI_URB_TRACE
1109 1 : ehci_dbg (ehci,
1110 1 : "%s %s urb %p ep%d%s len %d, qtd %p [qh %p]\n",
1111 1 : __func__, urb->dev->devpath, urb,
1112 : epnum & 0x0f, (epnum & USB_DIR_IN) ? "in" : "out",
1113 : urb->transfer_buffer_length,
1114 : qtd, urb->ep->hcpriv);
1115 : #endif
1116 :
1117 3 : spin_lock_irqsave (&ehci->lock, flags);
1118 8 : if (unlikely(!test_bit(HCD_FLAG_HW_ACCESSIBLE,
1119 : &ehci_to_hcd(ehci)->flags))) {
1120 1 : rc = -ESHUTDOWN;
1121 1 : goto done;
1122 : }
1123 3 : rc = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb);
1124 4 : if (unlikely(rc))
1125 1 : goto done;
1126 :
1127 4 : qh = qh_append_tds(ehci, urb, qtd_list, epnum, &urb->ep->hcpriv);
1128 4 : if (unlikely(qh == NULL)) {
1129 3 : usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb);
1130 1 : rc = -ENOMEM;
1131 1 : goto done;
1132 : }
1133 :
1134 : /* Control/bulk operations through TTs don't need scheduling,
1135 : * the HC and TT handle it when the TT has a buffer ready.
1136 : */
1137 5 : if (likely (qh->qh_state == QH_STATE_IDLE))
1138 3 : qh_link_async(ehci, qh);
1139 : done:
1140 10 : spin_unlock_irqrestore (&ehci->lock, flags);
1141 4 : if (unlikely (qh == NULL))
1142 2 : qtd_list_free (ehci, urb, qtd_list);
1143 4 : return rc;
1144 : }
1145 :
1146 : /*-------------------------------------------------------------------------*/
1147 :
1148 : /* the async qh for the qtds being reclaimed are now unlinked from the HC */
1149 :
1150 : static void end_unlink_async (struct ehci_hcd *ehci)
1151 : {
1152 48 : struct ehci_qh *qh = ehci->reclaim;
1153 24 : struct ehci_qh *next;
1154 24 :
1155 72 : iaa_watchdog_done(ehci);
1156 24 :
1157 : // qh->hw_next = cpu_to_hc32(qh->qh_dma);
1158 24 : qh->qh_state = QH_STATE_IDLE;
1159 24 : qh->qh_next.qh = NULL;
1160 72 : qh_put (qh); // refcount from reclaim
1161 :
1162 : /* other unlink(s) may be pending (in QH_STATE_UNLINK_WAIT) */
1163 24 : next = qh->reclaim;
1164 24 : ehci->reclaim = next;
1165 24 : qh->reclaim = NULL;
1166 :
1167 120 : qh_completions (ehci, qh);
1168 :
1169 216 : if (!list_empty (&qh->qtd_list)
1170 : && HC_IS_RUNNING (ehci_to_hcd(ehci)->state))
1171 72 : qh_link_async (ehci, qh);
1172 24 : else {
1173 : /* it's not free to turn the async schedule on/off; leave it
1174 : * active but idle for a while once it empties.
1175 : */
1176 192 : if (HC_IS_RUNNING (ehci_to_hcd(ehci)->state)
1177 : && ehci->async->qh_next.qh == NULL)
1178 96 : timer_action (ehci, TIMER_ASYNC_OFF);
1179 : }
1180 216 : qh_put(qh); /* refcount from async list */
1181 :
1182 48 : if (next) {
1183 24 : ehci->reclaim = NULL;
1184 60 : start_unlink_async (ehci, next);
1185 : }
1186 48 : }
1187 :
1188 : /* makes sure the async qh will become idle */
1189 : /* caller must own ehci->lock */
1190 :
1191 : static void start_unlink_async (struct ehci_hcd *ehci, struct ehci_qh *qh)
1192 : {
1193 96 : int cmd = ehci_readl(ehci, &ehci->regs->command);
1194 24 : struct ehci_qh *prev;
1195 24 :
1196 24 : #ifdef DEBUG
1197 24 : assert_spin_locked(&ehci->lock);
1198 24 : if (ehci->reclaim
1199 24 : || (qh->qh_state != QH_STATE_LINKED
1200 : && qh->qh_state != QH_STATE_UNLINK_WAIT)
1201 : )
1202 : BUG ();
1203 : #endif
1204 :
1205 : /* stop async schedule right now? */
1206 120 : if (unlikely (qh == ehci->async)) {
1207 : /* can't get here without STS_ASS set */
1208 168 : if (ehci_to_hcd(ehci)->state != HC_STATE_HALT
1209 : && !ehci->reclaim) {
1210 : /* ... and CMD_IAAD clear */
1211 48 : ehci_writel(ehci, cmd & ~CMD_ASE,
1212 : &ehci->regs->command);
1213 24 : wmb ();
1214 : // handshake later, if we need to
1215 48 : timer_action_done (ehci, TIMER_ASYNC_OFF);
1216 : }
1217 48 : return;
1218 : }
1219 :
1220 24 : qh->qh_state = QH_STATE_UNLINK;
1221 72 : ehci->reclaim = qh = qh_get (qh);
1222 :
1223 24 : prev = ehci->async;
1224 72 : while (prev->qh_next.qh != qh)
1225 48 : prev = prev->qh_next.qh;
1226 48 :
1227 24 : prev->hw->hw_next = qh->hw->hw_next;
1228 24 : prev->qh_next = qh->qh_next;
1229 24 : wmb ();
1230 :
1231 : /* If the controller isn't running, we don't have to wait for it */
1232 144 : if (unlikely(!HC_IS_RUNNING(ehci_to_hcd(ehci)->state))) {
1233 : /* if (unlikely (qh->reclaim != 0))
1234 : * this will recurse, probably not much
1235 : */
1236 54 : end_unlink_async (ehci);
1237 24 : return;
1238 : }
1239 :
1240 24 : cmd |= CMD_IAAD;
1241 48 : ehci_writel(ehci, cmd, &ehci->regs->command);
1242 48 : (void)ehci_readl(ehci, &ehci->regs->command);
1243 48 : iaa_watchdog_start(ehci);
1244 24 : }
1245 :
1246 : /*-------------------------------------------------------------------------*/
1247 :
1248 : static void scan_async (struct ehci_hcd *ehci)
1249 : {
1250 3 : bool stopped;
1251 3 : struct ehci_qh *qh;
1252 6 : enum ehci_timer_action action = TIMER_IO_WATCHDOG;
1253 3 :
1254 9 : ehci->stamp = ehci_readl(ehci, &ehci->regs->frame_index);
1255 9 : timer_action_done (ehci, TIMER_ASYNC_SHRINK);
1256 3 : rescan:
1257 15 : stopped = !HC_IS_RUNNING(ehci_to_hcd(ehci)->state);
1258 6 : qh = ehci->async->qh_next.qh;
1259 12 : if (likely (qh != NULL)) {
1260 3 : do {
1261 : /* clean any finished work for this qh */
1262 24 : if (!list_empty(&qh->qtd_list) && (stopped ||
1263 : qh->stamp != ehci->stamp)) {
1264 : int temp;
1265 :
1266 : /* unlinks could happen here; completion
1267 : * reporting drops the lock. rescan using
1268 : * the latest schedule, but don't rescan
1269 : * qhs we already finished (no looping)
1270 : * unless the controller is stopped.
1271 : */
1272 6 : qh = qh_get (qh);
1273 3 : qh->stamp = ehci->stamp;
1274 18 : temp = qh_completions (ehci, qh);
1275 9 : if (qh->needs_rescan)
1276 15 : unlink_async(ehci, qh);
1277 18 : qh_put (qh);
1278 6 : if (temp != 0) {
1279 3 : goto rescan;
1280 : }
1281 : }
1282 :
1283 : /* unlink idle entries, reducing DMA usage as well
1284 : * as HCD schedule-scanning costs. delay for any qh
1285 : * we just scanned, there's a not-unusual case that it
1286 : * doesn't stay idle for long.
1287 : * (plus, avoids some kind of re-activation race.)
1288 : */
1289 27 : if (list_empty(&qh->qtd_list)
1290 : && qh->qh_state == QH_STATE_LINKED) {
1291 21 : if (!ehci->reclaim && (stopped ||
1292 : ((ehci->stamp - qh->stamp) & 0x1fff)
1293 : >= EHCI_SHRINK_FRAMES * 8))
1294 15 : start_unlink_async(ehci, qh);
1295 : else
1296 3 : action = TIMER_ASYNC_SHRINK;
1297 : }
1298 :
1299 6 : qh = qh->qh_next.qh;
1300 12 : } while (qh);
1301 : }
1302 24 : if (action == TIMER_ASYNC_SHRINK)
1303 36 : timer_action (ehci, TIMER_ASYNC_SHRINK);
1304 12 : }
|