1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37static int ehci_get_frame (struct usb_hcd *hcd);
38
39
40
41
42
43
44
45
46static union ehci_shadow *
47periodic_next_shadow(struct ehci_hcd *ehci, union ehci_shadow *periodic,
48 __hc32 tag)
49{
50 switch (hc32_to_cpu(ehci, tag)) {
51 case Q_TYPE_QH:
52 return &periodic->qh->qh_next;
53 case Q_TYPE_FSTN:
54 return &periodic->fstn->fstn_next;
55 case Q_TYPE_ITD:
56 return &periodic->itd->itd_next;
57
58 default:
59 return &periodic->sitd->sitd_next;
60 }
61}
62
63
64static void periodic_unlink (struct ehci_hcd *ehci, unsigned frame, void *ptr)
65{
66 union ehci_shadow *prev_p = &ehci->pshadow[frame];
67 __hc32 *hw_p = &ehci->periodic[frame];
68 union ehci_shadow here = *prev_p;
69
70
71 while (here.ptr && here.ptr != ptr) {
72 prev_p = periodic_next_shadow(ehci, prev_p,
73 Q_NEXT_TYPE(ehci, *hw_p));
74 hw_p = here.hw_next;
75 here = *prev_p;
76 }
77
78 if (!here.ptr)
79 return;
80
81
82
83
84 *prev_p = *periodic_next_shadow(ehci, &here,
85 Q_NEXT_TYPE(ehci, *hw_p));
86 *hw_p = *here.hw_next;
87}
88
89
90static unsigned short
91periodic_usecs (struct ehci_hcd *ehci, unsigned frame, unsigned uframe)
92{
93 __hc32 *hw_p = &ehci->periodic [frame];
94 union ehci_shadow *q = &ehci->pshadow [frame];
95 unsigned usecs = 0;
96
97 while (q->ptr) {
98 switch (hc32_to_cpu(ehci, Q_NEXT_TYPE(ehci, *hw_p))) {
99 case Q_TYPE_QH:
100
101 if (q->qh->hw_info2 & cpu_to_hc32(ehci, 1 << uframe))
102 usecs += q->qh->usecs;
103
104 if (q->qh->hw_info2 & cpu_to_hc32(ehci,
105 1 << (8 + uframe)))
106 usecs += q->qh->c_usecs;
107 hw_p = &q->qh->hw_next;
108 q = &q->qh->qh_next;
109 break;
110
111 default:
112
113
114
115 if (q->fstn->hw_prev != EHCI_LIST_END(ehci)) {
116 ehci_dbg (ehci, "ignoring FSTN cost ...\n");
117 }
118 hw_p = &q->fstn->hw_next;
119 q = &q->fstn->fstn_next;
120 break;
121 case Q_TYPE_ITD:
122 if (q->itd->hw_transaction[uframe])
123 usecs += q->itd->stream->usecs;
124 hw_p = &q->itd->hw_next;
125 q = &q->itd->itd_next;
126 break;
127 case Q_TYPE_SITD:
128
129 if (q->sitd->hw_uframe & cpu_to_hc32(ehci,
130 1 << uframe)) {
131 if (q->sitd->hw_fullspeed_ep &
132 cpu_to_hc32(ehci, 1<<31))
133 usecs += q->sitd->stream->usecs;
134 else
135 usecs += HS_USECS_ISO (188);
136 }
137
138
139 if (q->sitd->hw_uframe &
140 cpu_to_hc32(ehci, 1 << (8 + uframe))) {
141
142 usecs += q->sitd->stream->c_usecs;
143 }
144
145 hw_p = &q->sitd->hw_next;
146 q = &q->sitd->sitd_next;
147 break;
148 }
149 }
150#ifdef DEBUG
151 if (usecs > 100)
152 ehci_err (ehci, "uframe %d sched overrun: %d usecs\n",
153 frame * 8 + uframe, usecs);
154#endif
155 return usecs;
156}
157
158
159
160static int same_tt (struct usb_device *dev1, struct usb_device *dev2)
161{
162 if (!dev1->tt || !dev2->tt)
163 return 0;
164 if (dev1->tt != dev2->tt)
165 return 0;
166 if (dev1->tt->multi)
167 return dev1->ttport == dev2->ttport;
168 else
169 return 1;
170}
171
172#ifdef CONFIG_USB_EHCI_TT_NEWSCHED
173
174
175
176
177
178
179
180
181
182static inline unsigned char tt_start_uframe(struct ehci_hcd *ehci, __hc32 mask)
183{
184 unsigned char smask = QH_SMASK & hc32_to_cpu(ehci, mask);
185 if (!smask) {
186 ehci_err(ehci, "invalid empty smask!\n");
187
188 return 7;
189 }
190 return ffs(smask) - 1;
191}
192
193static const unsigned char
194max_tt_usecs[] = { 125, 125, 125, 125, 125, 125, 30, 0 };
195
196
197static inline void carryover_tt_bandwidth(unsigned short tt_usecs[8])
198{
199 int i;
200 for (i=0; i<7; i++) {
201 if (max_tt_usecs[i] < tt_usecs[i]) {
202 tt_usecs[i+1] += tt_usecs[i] - max_tt_usecs[i];
203 tt_usecs[i] = max_tt_usecs[i];
204 }
205 }
206}
207
208
209
210
211
212
213
214
215
216
217
218static void
219periodic_tt_usecs (
220 struct ehci_hcd *ehci,
221 struct usb_device *dev,
222 unsigned frame,
223 unsigned short tt_usecs[8]
224)
225{
226 __hc32 *hw_p = &ehci->periodic [frame];
227 union ehci_shadow *q = &ehci->pshadow [frame];
228 unsigned char uf;
229
230 memset(tt_usecs, 0, 16);
231
232 while (q->ptr) {
233 switch (hc32_to_cpu(ehci, Q_NEXT_TYPE(ehci, *hw_p))) {
234 case Q_TYPE_ITD:
235 hw_p = &q->itd->hw_next;
236 q = &q->itd->itd_next;
237 continue;
238 case Q_TYPE_QH:
239 if (same_tt(dev, q->qh->dev)) {
240 uf = tt_start_uframe(ehci, q->qh->hw_info2);
241 tt_usecs[uf] += q->qh->tt_usecs;
242 }
243 hw_p = &q->qh->hw_next;
244 q = &q->qh->qh_next;
245 continue;
246 case Q_TYPE_SITD:
247 if (same_tt(dev, q->sitd->urb->dev)) {
248 uf = tt_start_uframe(ehci, q->sitd->hw_uframe);
249 tt_usecs[uf] += q->sitd->stream->tt_usecs;
250 }
251 hw_p = &q->sitd->hw_next;
252 q = &q->sitd->sitd_next;
253 continue;
254
255 default:
256 ehci_dbg(ehci, "ignoring periodic frame %d FSTN\n",
257 frame);
258 hw_p = &q->fstn->hw_next;
259 q = &q->fstn->fstn_next;
260 }
261 }
262
263 carryover_tt_bandwidth(tt_usecs);
264
265 if (max_tt_usecs[7] < tt_usecs[7])
266 ehci_err(ehci, "frame %d tt sched overrun: %d usecs\n",
267 frame, tt_usecs[7] - max_tt_usecs[7]);
268}
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291static int tt_available (
292 struct ehci_hcd *ehci,
293 unsigned period,
294 struct usb_device *dev,
295 unsigned frame,
296 unsigned uframe,
297 u16 usecs
298)
299{
300 if ((period == 0) || (uframe >= 7))
301 return 0;
302
303 for (; frame < ehci->periodic_size; frame += period) {
304 unsigned short tt_usecs[8];
305
306 periodic_tt_usecs (ehci, dev, frame, tt_usecs);
307
308 ehci_vdbg(ehci, "tt frame %d check %d usecs start uframe %d in"
309 " schedule %d/%d/%d/%d/%d/%d/%d/%d\n",
310 frame, usecs, uframe,
311 tt_usecs[0], tt_usecs[1], tt_usecs[2], tt_usecs[3],
312 tt_usecs[4], tt_usecs[5], tt_usecs[6], tt_usecs[7]);
313
314 if (max_tt_usecs[uframe] <= tt_usecs[uframe]) {
315 ehci_vdbg(ehci, "frame %d uframe %d fully scheduled\n",
316 frame, uframe);
317 return 0;
318 }
319
320
321
322
323
324
325 if (125 < usecs) {
326 int ufs = (usecs / 125) - 1;
327 int i;
328 for (i = uframe; i < (uframe + ufs) && i < 8; i++)
329 if (0 < tt_usecs[i]) {
330 ehci_vdbg(ehci,
331 "multi-uframe xfer can't fit "
332 "in frame %d uframe %d\n",
333 frame, i);
334 return 0;
335 }
336 }
337
338 tt_usecs[uframe] += usecs;
339
340 carryover_tt_bandwidth(tt_usecs);
341
342
343 if (max_tt_usecs[7] < tt_usecs[7]) {
344 ehci_vdbg(ehci,
345 "tt unavailable usecs %d frame %d uframe %d\n",
346 usecs, frame, uframe);
347 return 0;
348 }
349 }
350
351 return 1;
352}
353
354#else
355
356
357
358
359
360static int tt_no_collision (
361 struct ehci_hcd *ehci,
362 unsigned period,
363 struct usb_device *dev,
364 unsigned frame,
365 u32 uf_mask
366)
367{
368 if (period == 0)
369 return 0;
370
371
372
373
374
375 for (; frame < ehci->periodic_size; frame += period) {
376 union ehci_shadow here;
377 __hc32 type;
378
379 here = ehci->pshadow [frame];
380 type = Q_NEXT_TYPE(ehci, ehci->periodic [frame]);
381 while (here.ptr) {
382 switch (hc32_to_cpu(ehci, type)) {
383 case Q_TYPE_ITD:
384 type = Q_NEXT_TYPE(ehci, here.itd->hw_next);
385 here = here.itd->itd_next;
386 continue;
387 case Q_TYPE_QH:
388 if (same_tt (dev, here.qh->dev)) {
389 u32 mask;
390
391 mask = hc32_to_cpu(ehci,
392 here.qh->hw_info2);
393
394 mask |= mask >> 8;
395 if (mask & uf_mask)
396 break;
397 }
398 type = Q_NEXT_TYPE(ehci, here.qh->hw_next);
399 here = here.qh->qh_next;
400 continue;
401 case Q_TYPE_SITD:
402 if (same_tt (dev, here.sitd->urb->dev)) {
403 u16 mask;
404
405 mask = hc32_to_cpu(ehci, here.sitd
406 ->hw_uframe);
407
408 mask |= mask >> 8;
409 if (mask & uf_mask)
410 break;
411 }
412 type = Q_NEXT_TYPE(ehci, here.sitd->hw_next);
413 here = here.sitd->sitd_next;
414 continue;
415
416 default:
417 ehci_dbg (ehci,
418 "periodic frame %d bogus type %d\n",
419 frame, type);
420 }
421
422
423 return 0;
424 }
425 }
426
427
428 return 1;
429}
430
431#endif
432
433
434
435static int enable_periodic (struct ehci_hcd *ehci)
436{
437 u32 cmd;
438 int status;
439
440 if (ehci->periodic_sched++)
441 return 0;
442
443
444
445
446 status = handshake_on_error_set_halt(ehci, &ehci->regs->status,
447 STS_PSS, 0, 9 * 125);
448 if (status)
449 return status;
450
451 cmd = ehci_readl(ehci, &ehci->regs->command) | CMD_PSE;
452 ehci_writel(ehci, cmd, &ehci->regs->command);
453
454 ehci_to_hcd(ehci)->state = HC_STATE_RUNNING;
455
456
457 ehci->next_uframe = ehci_readl(ehci, &ehci->regs->frame_index)
458 % (ehci->periodic_size << 3);
459 return 0;
460}
461
462static int disable_periodic (struct ehci_hcd *ehci)
463{
464 u32 cmd;
465 int status;
466
467 if (--ehci->periodic_sched)
468 return 0;
469
470
471
472
473 status = handshake_on_error_set_halt(ehci, &ehci->regs->status,
474 STS_PSS, STS_PSS, 9 * 125);
475 if (status)
476 return status;
477
478 cmd = ehci_readl(ehci, &ehci->regs->command) & ~CMD_PSE;
479 ehci_writel(ehci, cmd, &ehci->regs->command);
480
481
482 ehci->next_uframe = -1;
483 return 0;
484}
485
486
487
488
489
490
491
492
493
494static int qh_link_periodic (struct ehci_hcd *ehci, struct ehci_qh *qh)
495{
496 unsigned i;
497 unsigned period = qh->period;
498
499 dev_dbg (&qh->dev->dev,
500 "link qh%d-%04x/%p start %d [%d/%d us]\n",
501 period, hc32_to_cpup(ehci, &qh->hw_info2) & (QH_CMASK | QH_SMASK),
502 qh, qh->start, qh->usecs, qh->c_usecs);
503
504
505 if (period == 0)
506 period = 1;
507
508 for (i = qh->start; i < ehci->periodic_size; i += period) {
509 union ehci_shadow *prev = &ehci->pshadow[i];
510 __hc32 *hw_p = &ehci->periodic[i];
511 union ehci_shadow here = *prev;
512 __hc32 type = 0;
513
514
515 while (here.ptr) {
516 type = Q_NEXT_TYPE(ehci, *hw_p);
517 if (type == cpu_to_hc32(ehci, Q_TYPE_QH))
518 break;
519 prev = periodic_next_shadow(ehci, prev, type);
520 hw_p = &here.qh->hw_next;
521 here = *prev;
522 }
523
524
525
526
527 while (here.ptr && qh != here.qh) {
528 if (qh->period > here.qh->period)
529 break;
530 prev = &here.qh->qh_next;
531 hw_p = &here.qh->hw_next;
532 here = *prev;
533 }
534
535 if (qh != here.qh) {
536 qh->qh_next = here;
537 if (here.qh)
538 qh->hw_next = *hw_p;
539 wmb ();
540 prev->qh = qh;
541 *hw_p = QH_NEXT (ehci, qh->qh_dma);
542 }
543 }
544 qh->qh_state = QH_STATE_LINKED;
545 qh_get (qh);
546
547
548 ehci_to_hcd(ehci)->self.bandwidth_allocated += qh->period
549 ? ((qh->usecs + qh->c_usecs) / qh->period)
550 : (qh->usecs * 8);
551
552
553 return enable_periodic(ehci);
554}
555
556static int qh_unlink_periodic(struct ehci_hcd *ehci, struct ehci_qh *qh)
557{
558 unsigned i;
559 unsigned period;
560
561
562
563
564
565
566
567
568
569 if ((period = qh->period) == 0)
570 period = 1;
571
572 for (i = qh->start; i < ehci->periodic_size; i += period)
573 periodic_unlink (ehci, i, qh);
574
575
576 ehci_to_hcd(ehci)->self.bandwidth_allocated -= qh->period
577 ? ((qh->usecs + qh->c_usecs) / qh->period)
578 : (qh->usecs * 8);
579
580 dev_dbg (&qh->dev->dev,
581 "unlink qh%d-%04x/%p start %d [%d/%d us]\n",
582 qh->period,
583 hc32_to_cpup(ehci, &qh->hw_info2) & (QH_CMASK | QH_SMASK),
584 qh, qh->start, qh->usecs, qh->c_usecs);
585
586
587 qh->qh_state = QH_STATE_UNLINK;
588 qh->qh_next.ptr = NULL;
589 qh_put (qh);
590
591
592 return disable_periodic(ehci);
593}
594
595static void intr_deschedule (struct ehci_hcd *ehci, struct ehci_qh *qh)
596{
597 unsigned wait;
598
599 qh_unlink_periodic (ehci, qh);
600
601
602
603
604
605
606 if (list_empty (&qh->qtd_list)
607 || (cpu_to_hc32(ehci, QH_CMASK)
608 & qh->hw_info2) != 0)
609 wait = 2;
610 else
611 wait = 55;
612
613 udelay (wait);
614 qh->qh_state = QH_STATE_IDLE;
615 qh->hw_next = EHCI_LIST_END(ehci);
616 wmb ();
617}
618
619
620
621static int check_period (
622 struct ehci_hcd *ehci,
623 unsigned frame,
624 unsigned uframe,
625 unsigned period,
626 unsigned usecs
627) {
628 int claimed;
629
630
631
632
633 if (uframe >= 8)
634 return 0;
635
636
637
638
639
640 usecs = 100 - usecs;
641
642
643
644
645 if (unlikely (period == 0)) {
646 do {
647 for (uframe = 0; uframe < 7; uframe++) {
648 claimed = periodic_usecs (ehci, frame, uframe);
649 if (claimed > usecs)
650 return 0;
651 }
652 } while ((frame += 1) < ehci->periodic_size);
653
654
655 } else {
656 do {
657 claimed = periodic_usecs (ehci, frame, uframe);
658 if (claimed > usecs)
659 return 0;
660 } while ((frame += period) < ehci->periodic_size);
661 }
662
663
664 return 1;
665}
666
667static int check_intr_schedule (
668 struct ehci_hcd *ehci,
669 unsigned frame,
670 unsigned uframe,
671 const struct ehci_qh *qh,
672 __hc32 *c_maskp
673)
674{
675 int retval = -ENOSPC;
676 u8 mask = 0;
677
678 if (qh->c_usecs && uframe >= 6)
679 goto done;
680
681 if (!check_period (ehci, frame, uframe, qh->period, qh->usecs))
682 goto done;
683 if (!qh->c_usecs) {
684 retval = 0;
685 *c_maskp = 0;
686 goto done;
687 }
688
689#ifdef CONFIG_USB_EHCI_TT_NEWSCHED
690 if (tt_available (ehci, qh->period, qh->dev, frame, uframe,
691 qh->tt_usecs)) {
692 unsigned i;
693
694
695 for (i=uframe+1; i<8 && i<uframe+4; i++)
696 if (!check_period (ehci, frame, i,
697 qh->period, qh->c_usecs))
698 goto done;
699 else
700 mask |= 1 << i;
701
702 retval = 0;
703
704 *c_maskp = cpu_to_hc32(ehci, mask << 8);
705 }
706#else
707
708
709
710
711
712
713
714 mask = 0x03 << (uframe + qh->gap_uf);
715 *c_maskp = cpu_to_hc32(ehci, mask << 8);
716
717 mask |= 1 << uframe;
718 if (tt_no_collision (ehci, qh->period, qh->dev, frame, mask)) {
719 if (!check_period (ehci, frame, uframe + qh->gap_uf + 1,
720 qh->period, qh->c_usecs))
721 goto done;
722 if (!check_period (ehci, frame, uframe + qh->gap_uf,
723 qh->period, qh->c_usecs))
724 goto done;
725 retval = 0;
726 }
727#endif
728done:
729 return retval;
730}
731
732
733
734
735static int qh_schedule(struct ehci_hcd *ehci, struct ehci_qh *qh)
736{
737 int status;
738 unsigned uframe;
739 __hc32 c_mask;
740 unsigned frame;
741
742 qh_refresh(ehci, qh);
743 qh->hw_next = EHCI_LIST_END(ehci);
744 frame = qh->start;
745
746
747 if (frame < qh->period) {
748 uframe = ffs(hc32_to_cpup(ehci, &qh->hw_info2) & QH_SMASK);
749 status = check_intr_schedule (ehci, frame, --uframe,
750 qh, &c_mask);
751 } else {
752 uframe = 0;
753 c_mask = 0;
754 status = -ENOSPC;
755 }
756
757
758
759
760 if (status) {
761
762 if (qh->period) {
763 frame = qh->period - 1;
764 do {
765 for (uframe = 0; uframe < 8; uframe++) {
766 status = check_intr_schedule (ehci,
767 frame, uframe, qh,
768 &c_mask);
769 if (status == 0)
770 break;
771 }
772 } while (status && frame--);
773
774
775 } else {
776 frame = 0;
777 status = check_intr_schedule (ehci, 0, 0, qh, &c_mask);
778 }
779 if (status)
780 goto done;
781 qh->start = frame;
782
783
784 qh->hw_info2 &= cpu_to_hc32(ehci, ~(QH_CMASK | QH_SMASK));
785 qh->hw_info2 |= qh->period
786 ? cpu_to_hc32(ehci, 1 << uframe)
787 : cpu_to_hc32(ehci, QH_SMASK);
788 qh->hw_info2 |= c_mask;
789 } else
790 ehci_dbg (ehci, "reused qh %p schedule\n", qh);
791
792
793 status = qh_link_periodic (ehci, qh);
794done:
795 return status;
796}
797
798static int intr_submit (
799 struct ehci_hcd *ehci,
800 struct urb *urb,
801 struct list_head *qtd_list,
802 gfp_t mem_flags
803) {
804 unsigned epnum;
805 unsigned long flags;
806 struct ehci_qh *qh;
807 int status;
808 struct list_head empty;
809
810
811 epnum = urb->ep->desc.bEndpointAddress;
812
813 spin_lock_irqsave (&ehci->lock, flags);
814
815 if (unlikely(!test_bit(HCD_FLAG_HW_ACCESSIBLE,
816 &ehci_to_hcd(ehci)->flags))) {
817 status = -ESHUTDOWN;
818 goto done_not_linked;
819 }
820 status = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb);
821 if (unlikely(status))
822 goto done_not_linked;
823
824
825 INIT_LIST_HEAD (&empty);
826 qh = qh_append_tds(ehci, urb, &empty, epnum, &urb->ep->hcpriv);
827 if (qh == NULL) {
828 status = -ENOMEM;
829 goto done;
830 }
831 if (qh->qh_state == QH_STATE_IDLE) {
832 if ((status = qh_schedule (ehci, qh)) != 0)
833 goto done;
834 }
835
836
837 qh = qh_append_tds(ehci, urb, qtd_list, epnum, &urb->ep->hcpriv);
838 BUG_ON (qh == NULL);
839
840
841 ehci_to_hcd(ehci)->self.bandwidth_int_reqs++;
842
843done:
844 if (unlikely(status))
845 usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb);
846done_not_linked:
847 spin_unlock_irqrestore (&ehci->lock, flags);
848 if (status)
849 qtd_list_free (ehci, urb, qtd_list);
850
851 return status;
852}
853
854
855
856
857
858static struct ehci_iso_stream *
859iso_stream_alloc (gfp_t mem_flags)
860{
861 struct ehci_iso_stream *stream;
862
863 stream = kzalloc(sizeof *stream, mem_flags);
864 if (likely (stream != NULL)) {
865 INIT_LIST_HEAD(&stream->td_list);
866 INIT_LIST_HEAD(&stream->free_list);
867 stream->next_uframe = -1;
868 stream->refcount = 1;
869 }
870 return stream;
871}
872
873static void
874iso_stream_init (
875 struct ehci_hcd *ehci,
876 struct ehci_iso_stream *stream,
877 struct usb_device *dev,
878 int pipe,
879 unsigned interval
880)
881{
882 static const u8 smask_out [] = { 0x01, 0x03, 0x07, 0x0f, 0x1f, 0x3f };
883
884 u32 buf1;
885 unsigned epnum, maxp;
886 int is_input;
887 long bandwidth;
888
889
890
891
892
893 epnum = usb_pipeendpoint (pipe);
894 is_input = usb_pipein (pipe) ? USB_DIR_IN : 0;
895 maxp = usb_maxpacket(dev, pipe, !is_input);
896 if (is_input) {
897 buf1 = (1 << 11);
898 } else {
899 buf1 = 0;
900 }
901
902
903 if (dev->speed == USB_SPEED_HIGH) {
904 unsigned multi = hb_mult(maxp);
905
906 stream->highspeed = 1;
907
908 maxp = max_packet(maxp);
909 buf1 |= maxp;
910 maxp *= multi;
911
912 stream->buf0 = cpu_to_hc32(ehci, (epnum << 8) | dev->devnum);
913 stream->buf1 = cpu_to_hc32(ehci, buf1);
914 stream->buf2 = cpu_to_hc32(ehci, multi);
915
916
917
918
919 stream->usecs = HS_USECS_ISO (maxp);
920 bandwidth = stream->usecs * 8;
921 bandwidth /= interval;
922
923 } else {
924 u32 addr;
925 int think_time;
926 int hs_transfers;
927
928 addr = dev->ttport << 24;
929 if (!ehci_is_TDI(ehci)
930 || (dev->tt->hub !=
931 ehci_to_hcd(ehci)->self.root_hub))
932 addr |= dev->tt->hub->devnum << 16;
933 addr |= epnum << 8;
934 addr |= dev->devnum;
935 stream->usecs = HS_USECS_ISO (maxp);
936 think_time = dev->tt ? dev->tt->think_time : 0;
937 stream->tt_usecs = NS_TO_US (think_time + usb_calc_bus_time (
938 dev->speed, is_input, 1, maxp));
939 hs_transfers = max (1u, (maxp + 187) / 188);
940 if (is_input) {
941 u32 tmp;
942
943 addr |= 1 << 31;
944 stream->c_usecs = stream->usecs;
945 stream->usecs = HS_USECS_ISO (1);
946 stream->raw_mask = 1;
947
948
949 tmp = (1 << (hs_transfers + 2)) - 1;
950 stream->raw_mask |= tmp << (8 + 2);
951 } else
952 stream->raw_mask = smask_out [hs_transfers - 1];
953 bandwidth = stream->usecs + stream->c_usecs;
954 bandwidth /= interval << 3;
955
956
957 stream->address = cpu_to_hc32(ehci, addr);
958 }
959 stream->bandwidth = bandwidth;
960
961 stream->udev = dev;
962
963 stream->bEndpointAddress = is_input | epnum;
964 stream->interval = interval;
965 stream->maxp = maxp;
966}
967
968static void
969iso_stream_put(struct ehci_hcd *ehci, struct ehci_iso_stream *stream)
970{
971 stream->refcount--;
972
973
974
975
976 if (stream->refcount == 1) {
977 int is_in;
978
979
980
981 while (!list_empty (&stream->free_list)) {
982 struct list_head *entry;
983
984 entry = stream->free_list.next;
985 list_del (entry);
986
987
988 if (stream->highspeed) {
989 struct ehci_itd *itd;
990
991 itd = list_entry (entry, struct ehci_itd,
992 itd_list);
993 dma_pool_free (ehci->itd_pool, itd,
994 itd->itd_dma);
995 } else {
996 struct ehci_sitd *sitd;
997
998 sitd = list_entry (entry, struct ehci_sitd,
999 sitd_list);
1000 dma_pool_free (ehci->sitd_pool, sitd,
1001 sitd->sitd_dma);
1002 }
1003 }
1004
1005 is_in = (stream->bEndpointAddress & USB_DIR_IN) ? 0x10 : 0;
1006 stream->bEndpointAddress &= 0x0f;
1007 stream->ep->hcpriv = NULL;
1008
1009 if (stream->rescheduled) {
1010 ehci_info (ehci, "ep%d%s-iso rescheduled "
1011 "%lu times in %lu seconds\n",
1012 stream->bEndpointAddress, is_in ? "in" : "out",
1013 stream->rescheduled,
1014 ((jiffies - stream->start)/HZ)
1015 );
1016 }
1017
1018 kfree(stream);
1019 }
1020}
1021
1022static inline struct ehci_iso_stream *
1023iso_stream_get (struct ehci_iso_stream *stream)
1024{
1025 if (likely (stream != NULL))
1026 stream->refcount++;
1027 return stream;
1028}
1029
1030static struct ehci_iso_stream *
1031iso_stream_find (struct ehci_hcd *ehci, struct urb *urb)
1032{
1033 unsigned epnum;
1034 struct ehci_iso_stream *stream;
1035 struct usb_host_endpoint *ep;
1036 unsigned long flags;
1037
1038 epnum = usb_pipeendpoint (urb->pipe);
1039 if (usb_pipein(urb->pipe))
1040 ep = urb->dev->ep_in[epnum];
1041 else
1042 ep = urb->dev->ep_out[epnum];
1043
1044 spin_lock_irqsave (&ehci->lock, flags);
1045 stream = ep->hcpriv;
1046
1047 if (unlikely (stream == NULL)) {
1048 stream = iso_stream_alloc(GFP_ATOMIC);
1049 if (likely (stream != NULL)) {
1050
1051 ep->hcpriv = stream;
1052 stream->ep = ep;
1053 iso_stream_init(ehci, stream, urb->dev, urb->pipe,
1054 urb->interval);
1055 }
1056
1057
1058 } else if (unlikely (stream->hw_info1 != 0)) {
1059 ehci_dbg (ehci, "dev %s ep%d%s, not iso??\n",
1060 urb->dev->devpath, epnum,
1061 usb_pipein(urb->pipe) ? "in" : "out");
1062 stream = NULL;
1063 }
1064
1065
1066 stream = iso_stream_get (stream);
1067
1068 spin_unlock_irqrestore (&ehci->lock, flags);
1069 return stream;
1070}
1071
1072
1073
1074
1075
1076static struct ehci_iso_sched *
1077iso_sched_alloc (unsigned packets, gfp_t mem_flags)
1078{
1079 struct ehci_iso_sched *iso_sched;
1080 int size = sizeof *iso_sched;
1081
1082 size += packets * sizeof (struct ehci_iso_packet);
1083 iso_sched = kzalloc(size, mem_flags);
1084 if (likely (iso_sched != NULL)) {
1085 INIT_LIST_HEAD (&iso_sched->td_list);
1086 }
1087 return iso_sched;
1088}
1089
1090static inline void
1091itd_sched_init(
1092 struct ehci_hcd *ehci,
1093 struct ehci_iso_sched *iso_sched,
1094 struct ehci_iso_stream *stream,
1095 struct urb *urb
1096)
1097{
1098 unsigned i;
1099 dma_addr_t dma = urb->transfer_dma;
1100
1101
1102 iso_sched->span = urb->number_of_packets * stream->interval;
1103
1104
1105
1106
1107 for (i = 0; i < urb->number_of_packets; i++) {
1108 struct ehci_iso_packet *uframe = &iso_sched->packet [i];
1109 unsigned length;
1110 dma_addr_t buf;
1111 u32 trans;
1112
1113 length = urb->iso_frame_desc [i].length;
1114 buf = dma + urb->iso_frame_desc [i].offset;
1115
1116 trans = EHCI_ISOC_ACTIVE;
1117 trans |= buf & 0x0fff;
1118 if (unlikely (((i + 1) == urb->number_of_packets))
1119 && !(urb->transfer_flags & URB_NO_INTERRUPT))
1120 trans |= EHCI_ITD_IOC;
1121 trans |= length << 16;
1122 uframe->transaction = cpu_to_hc32(ehci, trans);
1123
1124
1125 uframe->bufp = (buf & ~(u64)0x0fff);
1126 buf += length;
1127 if (unlikely ((uframe->bufp != (buf & ~(u64)0x0fff))))
1128 uframe->cross = 1;
1129 }
1130}
1131
1132static void
1133iso_sched_free (
1134 struct ehci_iso_stream *stream,
1135 struct ehci_iso_sched *iso_sched
1136)
1137{
1138 if (!iso_sched)
1139 return;
1140
1141 list_splice (&iso_sched->td_list, &stream->free_list);
1142 kfree (iso_sched);
1143}
1144
1145static int
1146itd_urb_transaction (
1147 struct ehci_iso_stream *stream,
1148 struct ehci_hcd *ehci,
1149 struct urb *urb,
1150 gfp_t mem_flags
1151)
1152{
1153 struct ehci_itd *itd;
1154 dma_addr_t itd_dma;
1155 int i;
1156 unsigned num_itds;
1157 struct ehci_iso_sched *sched;
1158 unsigned long flags;
1159
1160 sched = iso_sched_alloc (urb->number_of_packets, mem_flags);
1161 if (unlikely (sched == NULL))
1162 return -ENOMEM;
1163
1164 itd_sched_init(ehci, sched, stream, urb);
1165
1166 if (urb->interval < 8)
1167 num_itds = 1 + (sched->span + 7) / 8;
1168 else
1169 num_itds = urb->number_of_packets;
1170
1171
1172 spin_lock_irqsave (&ehci->lock, flags);
1173 for (i = 0; i < num_itds; i++) {
1174
1175
1176
1177
1178
1179
1180 if (likely (!list_empty(&stream->free_list))) {
1181 itd = list_entry (stream->free_list.prev,
1182 struct ehci_itd, itd_list);
1183 list_del (&itd->itd_list);
1184 itd_dma = itd->itd_dma;
1185 } else {
1186 spin_unlock_irqrestore (&ehci->lock, flags);
1187 itd = dma_pool_alloc (ehci->itd_pool, mem_flags,
1188 &itd_dma);
1189 spin_lock_irqsave (&ehci->lock, flags);
1190 if (!itd) {
1191 iso_sched_free(stream, sched);
1192 spin_unlock_irqrestore(&ehci->lock, flags);
1193 return -ENOMEM;
1194 }
1195 }
1196
1197 memset (itd, 0, sizeof *itd);
1198 itd->itd_dma = itd_dma;
1199 list_add (&itd->itd_list, &sched->td_list);
1200 }
1201 spin_unlock_irqrestore (&ehci->lock, flags);
1202
1203
1204 urb->hcpriv = sched;
1205 urb->error_count = 0;
1206 return 0;
1207}
1208
1209
1210
1211static inline int
1212itd_slot_ok (
1213 struct ehci_hcd *ehci,
1214 u32 mod,
1215 u32 uframe,
1216 u8 usecs,
1217 u32 period
1218)
1219{
1220 uframe %= period;
1221 do {
1222
1223 if (periodic_usecs (ehci, uframe >> 3, uframe & 0x7)
1224 > (100 - usecs))
1225 return 0;
1226
1227
1228 uframe += period;
1229 } while (uframe < mod);
1230 return 1;
1231}
1232
1233static inline int
1234sitd_slot_ok (
1235 struct ehci_hcd *ehci,
1236 u32 mod,
1237 struct ehci_iso_stream *stream,
1238 u32 uframe,
1239 struct ehci_iso_sched *sched,
1240 u32 period_uframes
1241)
1242{
1243 u32 mask, tmp;
1244 u32 frame, uf;
1245
1246 mask = stream->raw_mask << (uframe & 7);
1247
1248
1249 if (mask & ~0xffff)
1250 return 0;
1251
1252
1253
1254
1255
1256
1257 uframe %= period_uframes;
1258 do {
1259 u32 max_used;
1260
1261 frame = uframe >> 3;
1262 uf = uframe & 7;
1263
1264#ifdef CONFIG_USB_EHCI_TT_NEWSCHED
1265
1266
1267
1268 if (!tt_available (ehci, period_uframes << 3,
1269 stream->udev, frame, uf, stream->tt_usecs))
1270 return 0;
1271#else
1272
1273
1274
1275 if (!tt_no_collision (ehci, period_uframes << 3,
1276 stream->udev, frame, mask))
1277 return 0;
1278#endif
1279
1280
1281 max_used = 100 - stream->usecs;
1282 for (tmp = stream->raw_mask & 0xff; tmp; tmp >>= 1, uf++) {
1283 if (periodic_usecs (ehci, frame, uf) > max_used)
1284 return 0;
1285 }
1286
1287
1288 if (stream->c_usecs) {
1289 uf = uframe & 7;
1290 max_used = 100 - stream->c_usecs;
1291 do {
1292 tmp = 1 << uf;
1293 tmp <<= 8;
1294 if ((stream->raw_mask & tmp) == 0)
1295 continue;
1296 if (periodic_usecs (ehci, frame, uf)
1297 > max_used)
1298 return 0;
1299 } while (++uf < 8);
1300 }
1301
1302
1303 uframe += period_uframes;
1304 } while (uframe < mod);
1305
1306 stream->splits = cpu_to_hc32(ehci, stream->raw_mask << (uframe & 7));
1307 return 1;
1308}
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321#define SCHEDULE_SLOP 10
1322
1323static int
1324iso_stream_schedule (
1325 struct ehci_hcd *ehci,
1326 struct urb *urb,
1327 struct ehci_iso_stream *stream
1328)
1329{
1330 u32 now, start, max, period;
1331 int status;
1332 unsigned mod = ehci->periodic_size << 3;
1333 struct ehci_iso_sched *sched = urb->hcpriv;
1334
1335 if (sched->span > (mod - 8 * SCHEDULE_SLOP)) {
1336 ehci_dbg (ehci, "iso request %p too long\n", urb);
1337 status = -EFBIG;
1338 goto fail;
1339 }
1340
1341 if ((stream->depth + sched->span) > mod) {
1342 ehci_dbg (ehci, "request %p would overflow (%d+%d>%d)\n",
1343 urb, stream->depth, sched->span, mod);
1344 status = -EFBIG;
1345 goto fail;
1346 }
1347
1348 now = ehci_readl(ehci, &ehci->regs->frame_index) % mod;
1349
1350
1351 max = now + mod;
1352
1353
1354
1355
1356
1357
1358 if (likely (!list_empty (&stream->td_list))) {
1359 start = stream->next_uframe;
1360 if (start < now)
1361 start += mod;
1362
1363
1364 if (start >= max - 2 * 8 * SCHEDULE_SLOP)
1365 start += stream->interval * DIV_ROUND_UP(
1366 max - start, stream->interval) - mod;
1367
1368
1369 if (unlikely((start + sched->span) >= max)) {
1370 status = -EFBIG;
1371 goto fail;
1372 }
1373 goto ready;
1374 }
1375
1376
1377
1378
1379
1380
1381
1382 start = SCHEDULE_SLOP * 8 + (now & ~0x07);
1383 start %= mod;
1384 stream->next_uframe = start;
1385
1386
1387
1388 period = urb->interval;
1389 if (!stream->highspeed)
1390 period <<= 3;
1391
1392
1393 for (; start < (stream->next_uframe + period); start++) {
1394 int enough_space;
1395
1396
1397 if (stream->highspeed)
1398 enough_space = itd_slot_ok (ehci, mod, start,
1399 stream->usecs, period);
1400 else {
1401 if ((start % 8) >= 6)
1402 continue;
1403 enough_space = sitd_slot_ok (ehci, mod, stream,
1404 start, sched, period);
1405 }
1406
1407
1408 if (enough_space) {
1409 stream->next_uframe = start % mod;
1410 goto ready;
1411 }
1412 }
1413
1414
1415 ehci_dbg (ehci, "iso %ssched full %p (now %d max %d)\n",
1416 list_empty (&stream->td_list) ? "" : "re",
1417 urb, now, max);
1418 status = -ENOSPC;
1419
1420fail:
1421 iso_sched_free (stream, sched);
1422 urb->hcpriv = NULL;
1423 return status;
1424
1425ready:
1426
1427 urb->start_frame = stream->next_uframe;
1428 if (!stream->highspeed)
1429 urb->start_frame >>= 3;
1430 return 0;
1431}
1432
1433
1434
1435static inline void
1436itd_init(struct ehci_hcd *ehci, struct ehci_iso_stream *stream,
1437 struct ehci_itd *itd)
1438{
1439 int i;
1440
1441
1442 itd->hw_next = EHCI_LIST_END(ehci);
1443 itd->hw_bufp [0] = stream->buf0;
1444 itd->hw_bufp [1] = stream->buf1;
1445 itd->hw_bufp [2] = stream->buf2;
1446
1447 for (i = 0; i < 8; i++)
1448 itd->index[i] = -1;
1449
1450
1451}
1452
1453static inline void
1454itd_patch(
1455 struct ehci_hcd *ehci,
1456 struct ehci_itd *itd,
1457 struct ehci_iso_sched *iso_sched,
1458 unsigned index,
1459 u16 uframe
1460)
1461{
1462 struct ehci_iso_packet *uf = &iso_sched->packet [index];
1463 unsigned pg = itd->pg;
1464
1465
1466
1467 uframe &= 0x07;
1468 itd->index [uframe] = index;
1469
1470 itd->hw_transaction[uframe] = uf->transaction;
1471 itd->hw_transaction[uframe] |= cpu_to_hc32(ehci, pg << 12);
1472 itd->hw_bufp[pg] |= cpu_to_hc32(ehci, uf->bufp & ~(u32)0);
1473 itd->hw_bufp_hi[pg] |= cpu_to_hc32(ehci, (u32)(uf->bufp >> 32));
1474
1475
1476 if (unlikely (uf->cross)) {
1477 u64 bufp = uf->bufp + 4096;
1478
1479 itd->pg = ++pg;
1480 itd->hw_bufp[pg] |= cpu_to_hc32(ehci, bufp & ~(u32)0);
1481 itd->hw_bufp_hi[pg] |= cpu_to_hc32(ehci, (u32)(bufp >> 32));
1482 }
1483}
1484
1485static inline void
1486itd_link (struct ehci_hcd *ehci, unsigned frame, struct ehci_itd *itd)
1487{
1488
1489 itd->itd_next = ehci->pshadow [frame];
1490 itd->hw_next = ehci->periodic [frame];
1491 ehci->pshadow [frame].itd = itd;
1492 itd->frame = frame;
1493 wmb ();
1494 ehci->periodic[frame] = cpu_to_hc32(ehci, itd->itd_dma | Q_TYPE_ITD);
1495}
1496
1497
1498static int
1499itd_link_urb (
1500 struct ehci_hcd *ehci,
1501 struct urb *urb,
1502 unsigned mod,
1503 struct ehci_iso_stream *stream
1504)
1505{
1506 int packet;
1507 unsigned next_uframe, uframe, frame;
1508 struct ehci_iso_sched *iso_sched = urb->hcpriv;
1509 struct ehci_itd *itd;
1510
1511 next_uframe = stream->next_uframe % mod;
1512
1513 if (unlikely (list_empty(&stream->td_list))) {
1514 ehci_to_hcd(ehci)->self.bandwidth_allocated
1515 += stream->bandwidth;
1516 ehci_vdbg (ehci,
1517 "schedule devp %s ep%d%s-iso period %d start %d.%d\n",
1518 urb->dev->devpath, stream->bEndpointAddress & 0x0f,
1519 (stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out",
1520 urb->interval,
1521 next_uframe >> 3, next_uframe & 0x7);
1522 stream->start = jiffies;
1523 }
1524 ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs++;
1525
1526
1527 for (packet = 0, itd = NULL; packet < urb->number_of_packets; ) {
1528 if (itd == NULL) {
1529
1530
1531
1532
1533
1534 itd = list_entry (iso_sched->td_list.next,
1535 struct ehci_itd, itd_list);
1536 list_move_tail (&itd->itd_list, &stream->td_list);
1537 itd->stream = iso_stream_get (stream);
1538 itd->urb = usb_get_urb (urb);
1539 itd_init (ehci, stream, itd);
1540 }
1541
1542 uframe = next_uframe & 0x07;
1543 frame = next_uframe >> 3;
1544
1545 itd_patch(ehci, itd, iso_sched, packet, uframe);
1546
1547 next_uframe += stream->interval;
1548 stream->depth += stream->interval;
1549 next_uframe %= mod;
1550 packet++;
1551
1552
1553 if (((next_uframe >> 3) != frame)
1554 || packet == urb->number_of_packets) {
1555 itd_link (ehci, frame % ehci->periodic_size, itd);
1556 itd = NULL;
1557 }
1558 }
1559 stream->next_uframe = next_uframe;
1560
1561
1562 iso_sched_free (stream, iso_sched);
1563 urb->hcpriv = NULL;
1564
1565 timer_action (ehci, TIMER_IO_WATCHDOG);
1566 return enable_periodic(ehci);
1567}
1568
1569#define ISO_ERRS (EHCI_ISOC_BUF_ERR | EHCI_ISOC_BABBLE | EHCI_ISOC_XACTERR)
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581static unsigned
1582itd_complete (
1583 struct ehci_hcd *ehci,
1584 struct ehci_itd *itd
1585) {
1586 struct urb *urb = itd->urb;
1587 struct usb_iso_packet_descriptor *desc;
1588 u32 t;
1589 unsigned uframe;
1590 int urb_index = -1;
1591 struct ehci_iso_stream *stream = itd->stream;
1592 struct usb_device *dev;
1593 unsigned retval = false;
1594
1595
1596 for (uframe = 0; uframe < 8; uframe++) {
1597 if (likely (itd->index[uframe] == -1))
1598 continue;
1599 urb_index = itd->index[uframe];
1600 desc = &urb->iso_frame_desc [urb_index];
1601
1602 t = hc32_to_cpup(ehci, &itd->hw_transaction [uframe]);
1603 itd->hw_transaction [uframe] = 0;
1604 stream->depth -= stream->interval;
1605
1606
1607 if (unlikely (t & ISO_ERRS)) {
1608 urb->error_count++;
1609 if (t & EHCI_ISOC_BUF_ERR)
1610 desc->status = usb_pipein (urb->pipe)
1611 ? -ENOSR
1612 : -ECOMM;
1613 else if (t & EHCI_ISOC_BABBLE)
1614 desc->status = -EOVERFLOW;
1615 else
1616 desc->status = -EPROTO;
1617
1618
1619 if (!(t & EHCI_ISOC_BABBLE))
1620 desc->actual_length = EHCI_ITD_LENGTH (t);
1621 } else if (likely ((t & EHCI_ISOC_ACTIVE) == 0)) {
1622 desc->status = 0;
1623 desc->actual_length = EHCI_ITD_LENGTH (t);
1624 } else {
1625
1626 desc->status = -EXDEV;
1627 }
1628 }
1629
1630
1631 if (likely ((urb_index + 1) != urb->number_of_packets))
1632 goto done;
1633
1634
1635
1636
1637
1638
1639
1640 dev = urb->dev;
1641 ehci_urb_done(ehci, urb, 0);
1642 retval = true;
1643 urb = NULL;
1644 (void) disable_periodic(ehci);
1645 ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs--;
1646
1647 if (unlikely (list_empty (&stream->td_list))) {
1648 ehci_to_hcd(ehci)->self.bandwidth_allocated
1649 -= stream->bandwidth;
1650 ehci_vdbg (ehci,
1651 "deschedule devp %s ep%d%s-iso\n",
1652 dev->devpath, stream->bEndpointAddress & 0x0f,
1653 (stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out");
1654 }
1655 iso_stream_put (ehci, stream);
1656
1657done:
1658 usb_put_urb(urb);
1659 itd->urb = NULL;
1660 itd->stream = NULL;
1661 list_move(&itd->itd_list, &stream->free_list);
1662 iso_stream_put(ehci, stream);
1663
1664 return retval;
1665}
1666
1667
1668
1669static int itd_submit (struct ehci_hcd *ehci, struct urb *urb,
1670 gfp_t mem_flags)
1671{
1672 int status = -EINVAL;
1673 unsigned long flags;
1674 struct ehci_iso_stream *stream;
1675
1676
1677 stream = iso_stream_find (ehci, urb);
1678 if (unlikely (stream == NULL)) {
1679 ehci_dbg (ehci, "can't get iso stream\n");
1680 return -ENOMEM;
1681 }
1682 if (unlikely (urb->interval != stream->interval)) {
1683 ehci_dbg (ehci, "can't change iso interval %d --> %d\n",
1684 stream->interval, urb->interval);
1685 goto done;
1686 }
1687
1688#ifdef EHCI_URB_TRACE
1689 ehci_dbg (ehci,
1690 "%s %s urb %p ep%d%s len %d, %d pkts %d uframes [%p]\n",
1691 __func__, urb->dev->devpath, urb,
1692 usb_pipeendpoint (urb->pipe),
1693 usb_pipein (urb->pipe) ? "in" : "out",
1694 urb->transfer_buffer_length,
1695 urb->number_of_packets, urb->interval,
1696 stream);
1697#endif
1698
1699
1700 status = itd_urb_transaction (stream, ehci, urb, mem_flags);
1701 if (unlikely (status < 0)) {
1702 ehci_dbg (ehci, "can't init itds\n");
1703 goto done;
1704 }
1705
1706
1707 spin_lock_irqsave (&ehci->lock, flags);
1708 if (unlikely(!test_bit(HCD_FLAG_HW_ACCESSIBLE,
1709 &ehci_to_hcd(ehci)->flags))) {
1710 status = -ESHUTDOWN;
1711 goto done_not_linked;
1712 }
1713 status = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb);
1714 if (unlikely(status))
1715 goto done_not_linked;
1716 status = iso_stream_schedule(ehci, urb, stream);
1717 if (likely (status == 0))
1718 itd_link_urb (ehci, urb, ehci->periodic_size << 3, stream);
1719 else
1720 usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb);
1721done_not_linked:
1722 spin_unlock_irqrestore (&ehci->lock, flags);
1723
1724done:
1725 if (unlikely (status < 0))
1726 iso_stream_put (ehci, stream);
1727 return status;
1728}
1729
1730
1731
1732
1733
1734
1735
1736
1737static inline void
1738sitd_sched_init(
1739 struct ehci_hcd *ehci,
1740 struct ehci_iso_sched *iso_sched,
1741 struct ehci_iso_stream *stream,
1742 struct urb *urb
1743)
1744{
1745 unsigned i;
1746 dma_addr_t dma = urb->transfer_dma;
1747
1748
1749 iso_sched->span = urb->number_of_packets * stream->interval;
1750
1751
1752
1753
1754 for (i = 0; i < urb->number_of_packets; i++) {
1755 struct ehci_iso_packet *packet = &iso_sched->packet [i];
1756 unsigned length;
1757 dma_addr_t buf;
1758 u32 trans;
1759
1760 length = urb->iso_frame_desc [i].length & 0x03ff;
1761 buf = dma + urb->iso_frame_desc [i].offset;
1762
1763 trans = SITD_STS_ACTIVE;
1764 if (((i + 1) == urb->number_of_packets)
1765 && !(urb->transfer_flags & URB_NO_INTERRUPT))
1766 trans |= SITD_IOC;
1767 trans |= length << 16;
1768 packet->transaction = cpu_to_hc32(ehci, trans);
1769
1770
1771 packet->bufp = buf;
1772 packet->buf1 = (buf + length) & ~0x0fff;
1773 if (packet->buf1 != (buf & ~(u64)0x0fff))
1774 packet->cross = 1;
1775
1776
1777 if (stream->bEndpointAddress & USB_DIR_IN)
1778 continue;
1779 length = (length + 187) / 188;
1780 if (length > 1)
1781 length |= 1 << 3;
1782 packet->buf1 |= length;
1783 }
1784}
1785
1786static int
1787sitd_urb_transaction (
1788 struct ehci_iso_stream *stream,
1789 struct ehci_hcd *ehci,
1790 struct urb *urb,
1791 gfp_t mem_flags
1792)
1793{
1794 struct ehci_sitd *sitd;
1795 dma_addr_t sitd_dma;
1796 int i;
1797 struct ehci_iso_sched *iso_sched;
1798 unsigned long flags;
1799
1800 iso_sched = iso_sched_alloc (urb->number_of_packets, mem_flags);
1801 if (iso_sched == NULL)
1802 return -ENOMEM;
1803
1804 sitd_sched_init(ehci, iso_sched, stream, urb);
1805
1806
1807 spin_lock_irqsave (&ehci->lock, flags);
1808 for (i = 0; i < urb->number_of_packets; i++) {
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820 if (!list_empty(&stream->free_list)) {
1821 sitd = list_entry (stream->free_list.prev,
1822 struct ehci_sitd, sitd_list);
1823 list_del (&sitd->sitd_list);
1824 sitd_dma = sitd->sitd_dma;
1825 } else {
1826 spin_unlock_irqrestore (&ehci->lock, flags);
1827 sitd = dma_pool_alloc (ehci->sitd_pool, mem_flags,
1828 &sitd_dma);
1829 spin_lock_irqsave (&ehci->lock, flags);
1830 if (!sitd) {
1831 iso_sched_free(stream, iso_sched);
1832 spin_unlock_irqrestore(&ehci->lock, flags);
1833 return -ENOMEM;
1834 }
1835 }
1836
1837 memset (sitd, 0, sizeof *sitd);
1838 sitd->sitd_dma = sitd_dma;
1839 list_add (&sitd->sitd_list, &iso_sched->td_list);
1840 }
1841
1842
1843 urb->hcpriv = iso_sched;
1844 urb->error_count = 0;
1845
1846 spin_unlock_irqrestore (&ehci->lock, flags);
1847 return 0;
1848}
1849
1850
1851
1852static inline void
1853sitd_patch(
1854 struct ehci_hcd *ehci,
1855 struct ehci_iso_stream *stream,
1856 struct ehci_sitd *sitd,
1857 struct ehci_iso_sched *iso_sched,
1858 unsigned index
1859)
1860{
1861 struct ehci_iso_packet *uf = &iso_sched->packet [index];
1862 u64 bufp = uf->bufp;
1863
1864 sitd->hw_next = EHCI_LIST_END(ehci);
1865 sitd->hw_fullspeed_ep = stream->address;
1866 sitd->hw_uframe = stream->splits;
1867 sitd->hw_results = uf->transaction;
1868 sitd->hw_backpointer = EHCI_LIST_END(ehci);
1869
1870 bufp = uf->bufp;
1871 sitd->hw_buf[0] = cpu_to_hc32(ehci, bufp);
1872 sitd->hw_buf_hi[0] = cpu_to_hc32(ehci, bufp >> 32);
1873
1874 sitd->hw_buf[1] = cpu_to_hc32(ehci, uf->buf1);
1875 if (uf->cross)
1876 bufp += 4096;
1877 sitd->hw_buf_hi[1] = cpu_to_hc32(ehci, bufp >> 32);
1878 sitd->index = index;
1879}
1880
1881static inline void
1882sitd_link (struct ehci_hcd *ehci, unsigned frame, struct ehci_sitd *sitd)
1883{
1884
1885 sitd->sitd_next = ehci->pshadow [frame];
1886 sitd->hw_next = ehci->periodic [frame];
1887 ehci->pshadow [frame].sitd = sitd;
1888 sitd->frame = frame;
1889 wmb ();
1890 ehci->periodic[frame] = cpu_to_hc32(ehci, sitd->sitd_dma | Q_TYPE_SITD);
1891}
1892
1893
1894static int
1895sitd_link_urb (
1896 struct ehci_hcd *ehci,
1897 struct urb *urb,
1898 unsigned mod,
1899 struct ehci_iso_stream *stream
1900)
1901{
1902 int packet;
1903 unsigned next_uframe;
1904 struct ehci_iso_sched *sched = urb->hcpriv;
1905 struct ehci_sitd *sitd;
1906
1907 next_uframe = stream->next_uframe;
1908
1909 if (list_empty(&stream->td_list)) {
1910
1911 ehci_to_hcd(ehci)->self.bandwidth_allocated
1912 += stream->bandwidth;
1913 ehci_vdbg (ehci,
1914 "sched devp %s ep%d%s-iso [%d] %dms/%04x\n",
1915 urb->dev->devpath, stream->bEndpointAddress & 0x0f,
1916 (stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out",
1917 (next_uframe >> 3) % ehci->periodic_size,
1918 stream->interval, hc32_to_cpu(ehci, stream->splits));
1919 stream->start = jiffies;
1920 }
1921 ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs++;
1922
1923
1924 for (packet = 0, sitd = NULL;
1925 packet < urb->number_of_packets;
1926 packet++) {
1927
1928
1929 BUG_ON (list_empty (&sched->td_list));
1930
1931
1932
1933 sitd = list_entry (sched->td_list.next,
1934 struct ehci_sitd, sitd_list);
1935 list_move_tail (&sitd->sitd_list, &stream->td_list);
1936 sitd->stream = iso_stream_get (stream);
1937 sitd->urb = usb_get_urb (urb);
1938
1939 sitd_patch(ehci, stream, sitd, sched, packet);
1940 sitd_link (ehci, (next_uframe >> 3) % ehci->periodic_size,
1941 sitd);
1942
1943 next_uframe += stream->interval << 3;
1944 stream->depth += stream->interval << 3;
1945 }
1946 stream->next_uframe = next_uframe % mod;
1947
1948
1949 iso_sched_free (stream, sched);
1950 urb->hcpriv = NULL;
1951
1952 timer_action (ehci, TIMER_IO_WATCHDOG);
1953 return enable_periodic(ehci);
1954}
1955
1956
1957
1958#define SITD_ERRS (SITD_STS_ERR | SITD_STS_DBE | SITD_STS_BABBLE \
1959 | SITD_STS_XACT | SITD_STS_MMF)
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971static unsigned
1972sitd_complete (
1973 struct ehci_hcd *ehci,
1974 struct ehci_sitd *sitd
1975) {
1976 struct urb *urb = sitd->urb;
1977 struct usb_iso_packet_descriptor *desc;
1978 u32 t;
1979 int urb_index = -1;
1980 struct ehci_iso_stream *stream = sitd->stream;
1981 struct usb_device *dev;
1982 unsigned retval = false;
1983
1984 urb_index = sitd->index;
1985 desc = &urb->iso_frame_desc [urb_index];
1986 t = hc32_to_cpup(ehci, &sitd->hw_results);
1987
1988
1989 if (t & SITD_ERRS) {
1990 urb->error_count++;
1991 if (t & SITD_STS_DBE)
1992 desc->status = usb_pipein (urb->pipe)
1993 ? -ENOSR
1994 : -ECOMM;
1995 else if (t & SITD_STS_BABBLE)
1996 desc->status = -EOVERFLOW;
1997 else
1998 desc->status = -EPROTO;
1999 } else {
2000 desc->status = 0;
2001 desc->actual_length = desc->length - SITD_LENGTH (t);
2002 }
2003 stream->depth -= stream->interval << 3;
2004
2005
2006 if ((urb_index + 1) != urb->number_of_packets)
2007 goto done;
2008
2009
2010
2011
2012
2013
2014
2015 dev = urb->dev;
2016 ehci_urb_done(ehci, urb, 0);
2017 retval = true;
2018 urb = NULL;
2019 (void) disable_periodic(ehci);
2020 ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs--;
2021
2022 if (list_empty (&stream->td_list)) {
2023 ehci_to_hcd(ehci)->self.bandwidth_allocated
2024 -= stream->bandwidth;
2025 ehci_vdbg (ehci,
2026 "deschedule devp %s ep%d%s-iso\n",
2027 dev->devpath, stream->bEndpointAddress & 0x0f,
2028 (stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out");
2029 }
2030 iso_stream_put (ehci, stream);
2031
2032done:
2033 usb_put_urb(urb);
2034 sitd->urb = NULL;
2035 sitd->stream = NULL;
2036 list_move(&sitd->sitd_list, &stream->free_list);
2037 iso_stream_put(ehci, stream);
2038
2039 return retval;
2040}
2041
2042
2043static int sitd_submit (struct ehci_hcd *ehci, struct urb *urb,
2044 gfp_t mem_flags)
2045{
2046 int status = -EINVAL;
2047 unsigned long flags;
2048 struct ehci_iso_stream *stream;
2049
2050
2051 stream = iso_stream_find (ehci, urb);
2052 if (stream == NULL) {
2053 ehci_dbg (ehci, "can't get iso stream\n");
2054 return -ENOMEM;
2055 }
2056 if (urb->interval != stream->interval) {
2057 ehci_dbg (ehci, "can't change iso interval %d --> %d\n",
2058 stream->interval, urb->interval);
2059 goto done;
2060 }
2061
2062#ifdef EHCI_URB_TRACE
2063 ehci_dbg (ehci,
2064 "submit %p dev%s ep%d%s-iso len %d\n",
2065 urb, urb->dev->devpath,
2066 usb_pipeendpoint (urb->pipe),
2067 usb_pipein (urb->pipe) ? "in" : "out",
2068 urb->transfer_buffer_length);
2069#endif
2070
2071
2072 status = sitd_urb_transaction (stream, ehci, urb, mem_flags);
2073 if (status < 0) {
2074 ehci_dbg (ehci, "can't init sitds\n");
2075 goto done;
2076 }
2077
2078
2079 spin_lock_irqsave (&ehci->lock, flags);
2080 if (unlikely(!test_bit(HCD_FLAG_HW_ACCESSIBLE,
2081 &ehci_to_hcd(ehci)->flags))) {
2082 status = -ESHUTDOWN;
2083 goto done_not_linked;
2084 }
2085 status = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb);
2086 if (unlikely(status))
2087 goto done_not_linked;
2088 status = iso_stream_schedule(ehci, urb, stream);
2089 if (status == 0)
2090 sitd_link_urb (ehci, urb, ehci->periodic_size << 3, stream);
2091 else
2092 usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb);
2093done_not_linked:
2094 spin_unlock_irqrestore (&ehci->lock, flags);
2095
2096done:
2097 if (status < 0)
2098 iso_stream_put (ehci, stream);
2099 return status;
2100}
2101
2102
2103
2104static void
2105scan_periodic (struct ehci_hcd *ehci)
2106{
2107 unsigned now_uframe, frame, clock, clock_frame, mod;
2108 unsigned modified;
2109
2110 mod = ehci->periodic_size << 3;
2111
2112
2113
2114
2115
2116
2117 now_uframe = ehci->next_uframe;
2118 if (HC_IS_RUNNING (ehci_to_hcd(ehci)->state))
2119 clock = ehci_readl(ehci, &ehci->regs->frame_index);
2120 else
2121 clock = now_uframe + mod - 1;
2122 clock %= mod;
2123 clock_frame = clock >> 3;
2124
2125 for (;;) {
2126 union ehci_shadow q, *q_p;
2127 __hc32 type, *hw_p;
2128 unsigned incomplete = false;
2129
2130 frame = now_uframe >> 3;
2131
2132restart:
2133
2134 q_p = &ehci->pshadow [frame];
2135 hw_p = &ehci->periodic [frame];
2136 q.ptr = q_p->ptr;
2137 type = Q_NEXT_TYPE(ehci, *hw_p);
2138 modified = 0;
2139
2140 while (q.ptr != NULL) {
2141 unsigned uf;
2142 union ehci_shadow temp;
2143 int live;
2144
2145 live = HC_IS_RUNNING (ehci_to_hcd(ehci)->state);
2146 switch (hc32_to_cpu(ehci, type)) {
2147 case Q_TYPE_QH:
2148
2149 temp.qh = qh_get (q.qh);
2150 type = Q_NEXT_TYPE(ehci, q.qh->hw_next);
2151 q = q.qh->qh_next;
2152 modified = qh_completions (ehci, temp.qh);
2153 if (unlikely (list_empty (&temp.qh->qtd_list)))
2154 intr_deschedule (ehci, temp.qh);
2155 qh_put (temp.qh);
2156 break;
2157 case Q_TYPE_FSTN:
2158
2159
2160
2161 if (q.fstn->hw_prev != EHCI_LIST_END(ehci)) {
2162 dbg ("ignoring completions from FSTNs");
2163 }
2164 type = Q_NEXT_TYPE(ehci, q.fstn->hw_next);
2165 q = q.fstn->fstn_next;
2166 break;
2167 case Q_TYPE_ITD:
2168
2169
2170
2171
2172
2173 if (frame == clock_frame && live) {
2174 rmb();
2175 for (uf = 0; uf < 8; uf++) {
2176 if (q.itd->hw_transaction[uf] &
2177 ITD_ACTIVE(ehci))
2178 break;
2179 }
2180 if (uf < 8) {
2181 incomplete = true;
2182 q_p = &q.itd->itd_next;
2183 hw_p = &q.itd->hw_next;
2184 type = Q_NEXT_TYPE(ehci,
2185 q.itd->hw_next);
2186 q = *q_p;
2187 break;
2188 }
2189 }
2190
2191
2192
2193
2194
2195
2196 *q_p = q.itd->itd_next;
2197 *hw_p = q.itd->hw_next;
2198 type = Q_NEXT_TYPE(ehci, q.itd->hw_next);
2199 wmb();
2200 modified = itd_complete (ehci, q.itd);
2201 q = *q_p;
2202 break;
2203 case Q_TYPE_SITD:
2204
2205
2206
2207
2208
2209 if (frame == clock_frame && live &&
2210 (q.sitd->hw_results &
2211 SITD_ACTIVE(ehci))) {
2212 incomplete = true;
2213 q_p = &q.sitd->sitd_next;
2214 hw_p = &q.sitd->hw_next;
2215 type = Q_NEXT_TYPE(ehci,
2216 q.sitd->hw_next);
2217 q = *q_p;
2218 break;
2219 }
2220
2221
2222
2223
2224
2225 *q_p = q.sitd->sitd_next;
2226 *hw_p = q.sitd->hw_next;
2227 type = Q_NEXT_TYPE(ehci, q.sitd->hw_next);
2228 wmb();
2229 modified = sitd_complete (ehci, q.sitd);
2230 q = *q_p;
2231 break;
2232 default:
2233 dbg ("corrupt type %d frame %d shadow %p",
2234 type, frame, q.ptr);
2235
2236 q.ptr = NULL;
2237 }
2238
2239
2240 if (unlikely (modified)) {
2241 if (likely(ehci->periodic_sched > 0))
2242 goto restart;
2243
2244 now_uframe = clock;
2245 break;
2246 }
2247 }
2248
2249
2250
2251
2252
2253 if (incomplete && HC_IS_RUNNING(ehci_to_hcd(ehci)->state)) {
2254 ehci->next_uframe = now_uframe;
2255 break;
2256 }
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266 if (now_uframe == clock) {
2267 unsigned now;
2268
2269 if (!HC_IS_RUNNING (ehci_to_hcd(ehci)->state)
2270 || ehci->periodic_sched == 0)
2271 break;
2272 ehci->next_uframe = now_uframe;
2273 now = ehci_readl(ehci, &ehci->regs->frame_index) % mod;
2274 if (now_uframe == now)
2275 break;
2276
2277
2278 clock = now;
2279 clock_frame = clock >> 3;
2280 } else {
2281 now_uframe++;
2282 now_uframe %= mod;
2283 }
2284 }
2285}