ehci-sched.c 57 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241
  1. /*
  2. * Copyright (c) 2001-2004 by David Brownell
  3. * Copyright (c) 2003 Michal Sojka, for high-speed iso transfers
  4. *
  5. * This program is free software; you can redistribute it and/or modify it
  6. * under the terms of the GNU General Public License as published by the
  7. * Free Software Foundation; either version 2 of the License, or (at your
  8. * option) any later version.
  9. *
  10. * This program is distributed in the hope that it will be useful, but
  11. * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  12. * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
  13. * for more details.
  14. *
  15. * You should have received a copy of the GNU General Public License
  16. * along with this program; if not, write to the Free Software Foundation,
  17. * Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  18. */
  19. /* this file is part of ehci-hcd.c */
  20. /*-------------------------------------------------------------------------*/
  21. /*
  22. * EHCI scheduled transaction support: interrupt, iso, split iso
  23. * These are called "periodic" transactions in the EHCI spec.
  24. *
  25. * Note that for interrupt transfers, the QH/QTD manipulation is shared
  26. * with the "asynchronous" transaction support (control/bulk transfers).
  27. * The only real difference is in how interrupt transfers are scheduled.
  28. *
  29. * For ISO, we make an "iso_stream" head to serve the same role as a QH.
  30. * It keeps track of every ITD (or SITD) that's linked, and holds enough
  31. * pre-calculated schedule data to make appending to the queue be quick.
  32. */
  33. static int ehci_get_frame (struct usb_hcd *hcd);
  34. /*-------------------------------------------------------------------------*/
  35. /*
  36. * periodic_next_shadow - return "next" pointer on shadow list
  37. * @periodic: host pointer to qh/itd/sitd
  38. * @tag: hardware tag for type of this record
  39. */
  40. static union ehci_shadow *
  41. periodic_next_shadow(struct ehci_hcd *ehci, union ehci_shadow *periodic,
  42. __hc32 tag)
  43. {
  44. switch (hc32_to_cpu(ehci, tag)) {
  45. case Q_TYPE_QH:
  46. return &periodic->qh->qh_next;
  47. case Q_TYPE_FSTN:
  48. return &periodic->fstn->fstn_next;
  49. case Q_TYPE_ITD:
  50. return &periodic->itd->itd_next;
  51. // case Q_TYPE_SITD:
  52. default:
  53. return &periodic->sitd->sitd_next;
  54. }
  55. }
  56. /* caller must hold ehci->lock */
  57. static void periodic_unlink (struct ehci_hcd *ehci, unsigned frame, void *ptr)
  58. {
  59. union ehci_shadow *prev_p = &ehci->pshadow[frame];
  60. __hc32 *hw_p = &ehci->periodic[frame];
  61. union ehci_shadow here = *prev_p;
  62. /* find predecessor of "ptr"; hw and shadow lists are in sync */
  63. while (here.ptr && here.ptr != ptr) {
  64. prev_p = periodic_next_shadow(ehci, prev_p,
  65. Q_NEXT_TYPE(ehci, *hw_p));
  66. hw_p = here.hw_next;
  67. here = *prev_p;
  68. }
  69. /* an interrupt entry (at list end) could have been shared */
  70. if (!here.ptr)
  71. return;
  72. /* update shadow and hardware lists ... the old "next" pointers
  73. * from ptr may still be in use, the caller updates them.
  74. */
  75. *prev_p = *periodic_next_shadow(ehci, &here,
  76. Q_NEXT_TYPE(ehci, *hw_p));
  77. *hw_p = *here.hw_next;
  78. }
  79. /* how many of the uframe's 125 usecs are allocated? */
  80. static unsigned short
  81. periodic_usecs (struct ehci_hcd *ehci, unsigned frame, unsigned uframe)
  82. {
  83. __hc32 *hw_p = &ehci->periodic [frame];
  84. union ehci_shadow *q = &ehci->pshadow [frame];
  85. unsigned usecs = 0;
  86. while (q->ptr) {
  87. switch (hc32_to_cpu(ehci, Q_NEXT_TYPE(ehci, *hw_p))) {
  88. case Q_TYPE_QH:
  89. /* is it in the S-mask? */
  90. if (q->qh->hw_info2 & cpu_to_hc32(ehci, 1 << uframe))
  91. usecs += q->qh->usecs;
  92. /* ... or C-mask? */
  93. if (q->qh->hw_info2 & cpu_to_hc32(ehci,
  94. 1 << (8 + uframe)))
  95. usecs += q->qh->c_usecs;
  96. hw_p = &q->qh->hw_next;
  97. q = &q->qh->qh_next;
  98. break;
  99. // case Q_TYPE_FSTN:
  100. default:
  101. /* for "save place" FSTNs, count the relevant INTR
  102. * bandwidth from the previous frame
  103. */
  104. if (q->fstn->hw_prev != EHCI_LIST_END(ehci)) {
  105. ehci_dbg (ehci, "ignoring FSTN cost ...\n");
  106. }
  107. hw_p = &q->fstn->hw_next;
  108. q = &q->fstn->fstn_next;
  109. break;
  110. case Q_TYPE_ITD:
  111. usecs += q->itd->usecs [uframe];
  112. hw_p = &q->itd->hw_next;
  113. q = &q->itd->itd_next;
  114. break;
  115. case Q_TYPE_SITD:
  116. /* is it in the S-mask? (count SPLIT, DATA) */
  117. if (q->sitd->hw_uframe & cpu_to_hc32(ehci,
  118. 1 << uframe)) {
  119. if (q->sitd->hw_fullspeed_ep &
  120. cpu_to_hc32(ehci, 1<<31))
  121. usecs += q->sitd->stream->usecs;
  122. else /* worst case for OUT start-split */
  123. usecs += HS_USECS_ISO (188);
  124. }
  125. /* ... C-mask? (count CSPLIT, DATA) */
  126. if (q->sitd->hw_uframe &
  127. cpu_to_hc32(ehci, 1 << (8 + uframe))) {
  128. /* worst case for IN complete-split */
  129. usecs += q->sitd->stream->c_usecs;
  130. }
  131. hw_p = &q->sitd->hw_next;
  132. q = &q->sitd->sitd_next;
  133. break;
  134. }
  135. }
  136. #ifdef DEBUG
  137. if (usecs > 100)
  138. ehci_err (ehci, "uframe %d sched overrun: %d usecs\n",
  139. frame * 8 + uframe, usecs);
  140. #endif
  141. return usecs;
  142. }
  143. /*-------------------------------------------------------------------------*/
  144. static int same_tt (struct usb_device *dev1, struct usb_device *dev2)
  145. {
  146. if (!dev1->tt || !dev2->tt)
  147. return 0;
  148. if (dev1->tt != dev2->tt)
  149. return 0;
  150. if (dev1->tt->multi)
  151. return dev1->ttport == dev2->ttport;
  152. else
  153. return 1;
  154. }
  155. #ifdef CONFIG_USB_EHCI_TT_NEWSCHED
  156. /* Which uframe does the low/fullspeed transfer start in?
  157. *
  158. * The parameter is the mask of ssplits in "H-frame" terms
  159. * and this returns the transfer start uframe in "B-frame" terms,
  160. * which allows both to match, e.g. a ssplit in "H-frame" uframe 0
  161. * will cause a transfer in "B-frame" uframe 0. "B-frames" lag
  162. * "H-frames" by 1 uframe. See the EHCI spec sec 4.5 and figure 4.7.
  163. */
  164. static inline unsigned char tt_start_uframe(struct ehci_hcd *ehci, __hc32 mask)
  165. {
  166. unsigned char smask = QH_SMASK & hc32_to_cpu(ehci, mask);
  167. if (!smask) {
  168. ehci_err(ehci, "invalid empty smask!\n");
  169. /* uframe 7 can't have bw so this will indicate failure */
  170. return 7;
  171. }
  172. return ffs(smask) - 1;
  173. }
  174. static const unsigned char
  175. max_tt_usecs[] = { 125, 125, 125, 125, 125, 125, 30, 0 };
  176. /* carryover low/fullspeed bandwidth that crosses uframe boundries */
  177. static inline void carryover_tt_bandwidth(unsigned short tt_usecs[8])
  178. {
  179. int i;
  180. for (i=0; i<7; i++) {
  181. if (max_tt_usecs[i] < tt_usecs[i]) {
  182. tt_usecs[i+1] += tt_usecs[i] - max_tt_usecs[i];
  183. tt_usecs[i] = max_tt_usecs[i];
  184. }
  185. }
  186. }
  187. /* How many of the tt's periodic downstream 1000 usecs are allocated?
  188. *
  189. * While this measures the bandwidth in terms of usecs/uframe,
  190. * the low/fullspeed bus has no notion of uframes, so any particular
  191. * low/fullspeed transfer can "carry over" from one uframe to the next,
  192. * since the TT just performs downstream transfers in sequence.
  193. *
  194. * For example two seperate 100 usec transfers can start in the same uframe,
  195. * and the second one would "carry over" 75 usecs into the next uframe.
  196. */
  197. static void
  198. periodic_tt_usecs (
  199. struct ehci_hcd *ehci,
  200. struct usb_device *dev,
  201. unsigned frame,
  202. unsigned short tt_usecs[8]
  203. )
  204. {
  205. __hc32 *hw_p = &ehci->periodic [frame];
  206. union ehci_shadow *q = &ehci->pshadow [frame];
  207. unsigned char uf;
  208. memset(tt_usecs, 0, 16);
  209. while (q->ptr) {
  210. switch (hc32_to_cpu(ehci, Q_NEXT_TYPE(ehci, *hw_p))) {
  211. case Q_TYPE_ITD:
  212. hw_p = &q->itd->hw_next;
  213. q = &q->itd->itd_next;
  214. continue;
  215. case Q_TYPE_QH:
  216. if (same_tt(dev, q->qh->dev)) {
  217. uf = tt_start_uframe(ehci, q->qh->hw_info2);
  218. tt_usecs[uf] += q->qh->tt_usecs;
  219. }
  220. hw_p = &q->qh->hw_next;
  221. q = &q->qh->qh_next;
  222. continue;
  223. case Q_TYPE_SITD:
  224. if (same_tt(dev, q->sitd->urb->dev)) {
  225. uf = tt_start_uframe(ehci, q->sitd->hw_uframe);
  226. tt_usecs[uf] += q->sitd->stream->tt_usecs;
  227. }
  228. hw_p = &q->sitd->hw_next;
  229. q = &q->sitd->sitd_next;
  230. continue;
  231. // case Q_TYPE_FSTN:
  232. default:
  233. ehci_dbg(ehci, "ignoring periodic frame %d FSTN\n",
  234. frame);
  235. hw_p = &q->fstn->hw_next;
  236. q = &q->fstn->fstn_next;
  237. }
  238. }
  239. carryover_tt_bandwidth(tt_usecs);
  240. if (max_tt_usecs[7] < tt_usecs[7])
  241. ehci_err(ehci, "frame %d tt sched overrun: %d usecs\n",
  242. frame, tt_usecs[7] - max_tt_usecs[7]);
  243. }
  244. /*
  245. * Return true if the device's tt's downstream bus is available for a
  246. * periodic transfer of the specified length (usecs), starting at the
  247. * specified frame/uframe. Note that (as summarized in section 11.19
  248. * of the usb 2.0 spec) TTs can buffer multiple transactions for each
  249. * uframe.
  250. *
  251. * The uframe parameter is when the fullspeed/lowspeed transfer
  252. * should be executed in "B-frame" terms, which is the same as the
  253. * highspeed ssplit's uframe (which is in "H-frame" terms). For example
  254. * a ssplit in "H-frame" 0 causes a transfer in "B-frame" 0.
  255. * See the EHCI spec sec 4.5 and fig 4.7.
  256. *
  257. * This checks if the full/lowspeed bus, at the specified starting uframe,
  258. * has the specified bandwidth available, according to rules listed
  259. * in USB 2.0 spec section 11.18.1 fig 11-60.
  260. *
  261. * This does not check if the transfer would exceed the max ssplit
  262. * limit of 16, specified in USB 2.0 spec section 11.18.4 requirement #4,
  263. * since proper scheduling limits ssplits to less than 16 per uframe.
  264. */
  265. static int tt_available (
  266. struct ehci_hcd *ehci,
  267. unsigned period,
  268. struct usb_device *dev,
  269. unsigned frame,
  270. unsigned uframe,
  271. u16 usecs
  272. )
  273. {
  274. if ((period == 0) || (uframe >= 7)) /* error */
  275. return 0;
  276. for (; frame < ehci->periodic_size; frame += period) {
  277. unsigned short tt_usecs[8];
  278. periodic_tt_usecs (ehci, dev, frame, tt_usecs);
  279. ehci_vdbg(ehci, "tt frame %d check %d usecs start uframe %d in"
  280. " schedule %d/%d/%d/%d/%d/%d/%d/%d\n",
  281. frame, usecs, uframe,
  282. tt_usecs[0], tt_usecs[1], tt_usecs[2], tt_usecs[3],
  283. tt_usecs[4], tt_usecs[5], tt_usecs[6], tt_usecs[7]);
  284. if (max_tt_usecs[uframe] <= tt_usecs[uframe]) {
  285. ehci_vdbg(ehci, "frame %d uframe %d fully scheduled\n",
  286. frame, uframe);
  287. return 0;
  288. }
  289. /* special case for isoc transfers larger than 125us:
  290. * the first and each subsequent fully used uframe
  291. * must be empty, so as to not illegally delay
  292. * already scheduled transactions
  293. */
  294. if (125 < usecs) {
  295. int ufs = (usecs / 125) - 1;
  296. int i;
  297. for (i = uframe; i < (uframe + ufs) && i < 8; i++)
  298. if (0 < tt_usecs[i]) {
  299. ehci_vdbg(ehci,
  300. "multi-uframe xfer can't fit "
  301. "in frame %d uframe %d\n",
  302. frame, i);
  303. return 0;
  304. }
  305. }
  306. tt_usecs[uframe] += usecs;
  307. carryover_tt_bandwidth(tt_usecs);
  308. /* fail if the carryover pushed bw past the last uframe's limit */
  309. if (max_tt_usecs[7] < tt_usecs[7]) {
  310. ehci_vdbg(ehci,
  311. "tt unavailable usecs %d frame %d uframe %d\n",
  312. usecs, frame, uframe);
  313. return 0;
  314. }
  315. }
  316. return 1;
  317. }
  318. #else
  319. /* return true iff the device's transaction translator is available
  320. * for a periodic transfer starting at the specified frame, using
  321. * all the uframes in the mask.
  322. */
  323. static int tt_no_collision (
  324. struct ehci_hcd *ehci,
  325. unsigned period,
  326. struct usb_device *dev,
  327. unsigned frame,
  328. u32 uf_mask
  329. )
  330. {
  331. if (period == 0) /* error */
  332. return 0;
  333. /* note bandwidth wastage: split never follows csplit
  334. * (different dev or endpoint) until the next uframe.
  335. * calling convention doesn't make that distinction.
  336. */
  337. for (; frame < ehci->periodic_size; frame += period) {
  338. union ehci_shadow here;
  339. __hc32 type;
  340. here = ehci->pshadow [frame];
  341. type = Q_NEXT_TYPE(ehci, ehci->periodic [frame]);
  342. while (here.ptr) {
  343. switch (hc32_to_cpu(ehci, type)) {
  344. case Q_TYPE_ITD:
  345. type = Q_NEXT_TYPE(ehci, here.itd->hw_next);
  346. here = here.itd->itd_next;
  347. continue;
  348. case Q_TYPE_QH:
  349. if (same_tt (dev, here.qh->dev)) {
  350. u32 mask;
  351. mask = hc32_to_cpu(ehci,
  352. here.qh->hw_info2);
  353. /* "knows" no gap is needed */
  354. mask |= mask >> 8;
  355. if (mask & uf_mask)
  356. break;
  357. }
  358. type = Q_NEXT_TYPE(ehci, here.qh->hw_next);
  359. here = here.qh->qh_next;
  360. continue;
  361. case Q_TYPE_SITD:
  362. if (same_tt (dev, here.sitd->urb->dev)) {
  363. u16 mask;
  364. mask = hc32_to_cpu(ehci, here.sitd
  365. ->hw_uframe);
  366. /* FIXME assumes no gap for IN! */
  367. mask |= mask >> 8;
  368. if (mask & uf_mask)
  369. break;
  370. }
  371. type = Q_NEXT_TYPE(ehci, here.sitd->hw_next);
  372. here = here.sitd->sitd_next;
  373. continue;
  374. // case Q_TYPE_FSTN:
  375. default:
  376. ehci_dbg (ehci,
  377. "periodic frame %d bogus type %d\n",
  378. frame, type);
  379. }
  380. /* collision or error */
  381. return 0;
  382. }
  383. }
  384. /* no collision */
  385. return 1;
  386. }
  387. #endif /* CONFIG_USB_EHCI_TT_NEWSCHED */
  388. /*-------------------------------------------------------------------------*/
  389. static int enable_periodic (struct ehci_hcd *ehci)
  390. {
  391. u32 cmd;
  392. int status;
  393. /* did clearing PSE did take effect yet?
  394. * takes effect only at frame boundaries...
  395. */
  396. status = handshake(ehci, &ehci->regs->status, STS_PSS, 0, 9 * 125);
  397. if (status != 0) {
  398. ehci_to_hcd(ehci)->state = HC_STATE_HALT;
  399. return status;
  400. }
  401. cmd = ehci_readl(ehci, &ehci->regs->command) | CMD_PSE;
  402. ehci_writel(ehci, cmd, &ehci->regs->command);
  403. /* posted write ... PSS happens later */
  404. ehci_to_hcd(ehci)->state = HC_STATE_RUNNING;
  405. /* make sure ehci_work scans these */
  406. ehci->next_uframe = ehci_readl(ehci, &ehci->regs->frame_index)
  407. % (ehci->periodic_size << 3);
  408. return 0;
  409. }
  410. static int disable_periodic (struct ehci_hcd *ehci)
  411. {
  412. u32 cmd;
  413. int status;
  414. /* did setting PSE not take effect yet?
  415. * takes effect only at frame boundaries...
  416. */
  417. status = handshake(ehci, &ehci->regs->status, STS_PSS, STS_PSS, 9 * 125);
  418. if (status != 0) {
  419. ehci_to_hcd(ehci)->state = HC_STATE_HALT;
  420. return status;
  421. }
  422. cmd = ehci_readl(ehci, &ehci->regs->command) & ~CMD_PSE;
  423. ehci_writel(ehci, cmd, &ehci->regs->command);
  424. /* posted write ... */
  425. ehci->next_uframe = -1;
  426. return 0;
  427. }
  428. /*-------------------------------------------------------------------------*/
  429. /* periodic schedule slots have iso tds (normal or split) first, then a
  430. * sparse tree for active interrupt transfers.
  431. *
  432. * this just links in a qh; caller guarantees uframe masks are set right.
  433. * no FSTN support (yet; ehci 0.96+)
  434. */
  435. static int qh_link_periodic (struct ehci_hcd *ehci, struct ehci_qh *qh)
  436. {
  437. unsigned i;
  438. unsigned period = qh->period;
  439. dev_dbg (&qh->dev->dev,
  440. "link qh%d-%04x/%p start %d [%d/%d us]\n",
  441. period, hc32_to_cpup(ehci, &qh->hw_info2) & (QH_CMASK | QH_SMASK),
  442. qh, qh->start, qh->usecs, qh->c_usecs);
  443. /* high bandwidth, or otherwise every microframe */
  444. if (period == 0)
  445. period = 1;
  446. for (i = qh->start; i < ehci->periodic_size; i += period) {
  447. union ehci_shadow *prev = &ehci->pshadow[i];
  448. __hc32 *hw_p = &ehci->periodic[i];
  449. union ehci_shadow here = *prev;
  450. __hc32 type = 0;
  451. /* skip the iso nodes at list head */
  452. while (here.ptr) {
  453. type = Q_NEXT_TYPE(ehci, *hw_p);
  454. if (type == cpu_to_hc32(ehci, Q_TYPE_QH))
  455. break;
  456. prev = periodic_next_shadow(ehci, prev, type);
  457. hw_p = &here.qh->hw_next;
  458. here = *prev;
  459. }
  460. /* sorting each branch by period (slow-->fast)
  461. * enables sharing interior tree nodes
  462. */
  463. while (here.ptr && qh != here.qh) {
  464. if (qh->period > here.qh->period)
  465. break;
  466. prev = &here.qh->qh_next;
  467. hw_p = &here.qh->hw_next;
  468. here = *prev;
  469. }
  470. /* link in this qh, unless some earlier pass did that */
  471. if (qh != here.qh) {
  472. qh->qh_next = here;
  473. if (here.qh)
  474. qh->hw_next = *hw_p;
  475. wmb ();
  476. prev->qh = qh;
  477. *hw_p = QH_NEXT (ehci, qh->qh_dma);
  478. }
  479. }
  480. qh->qh_state = QH_STATE_LINKED;
  481. qh_get (qh);
  482. /* update per-qh bandwidth for usbfs */
  483. ehci_to_hcd(ehci)->self.bandwidth_allocated += qh->period
  484. ? ((qh->usecs + qh->c_usecs) / qh->period)
  485. : (qh->usecs * 8);
  486. /* maybe enable periodic schedule processing */
  487. if (!ehci->periodic_sched++)
  488. return enable_periodic (ehci);
  489. return 0;
  490. }
  491. static void qh_unlink_periodic (struct ehci_hcd *ehci, struct ehci_qh *qh)
  492. {
  493. unsigned i;
  494. unsigned period;
  495. // FIXME:
  496. // IF this isn't high speed
  497. // and this qh is active in the current uframe
  498. // (and overlay token SplitXstate is false?)
  499. // THEN
  500. // qh->hw_info1 |= __constant_cpu_to_hc32(1 << 7 /* "ignore" */);
  501. /* high bandwidth, or otherwise part of every microframe */
  502. if ((period = qh->period) == 0)
  503. period = 1;
  504. for (i = qh->start; i < ehci->periodic_size; i += period)
  505. periodic_unlink (ehci, i, qh);
  506. /* update per-qh bandwidth for usbfs */
  507. ehci_to_hcd(ehci)->self.bandwidth_allocated -= qh->period
  508. ? ((qh->usecs + qh->c_usecs) / qh->period)
  509. : (qh->usecs * 8);
  510. dev_dbg (&qh->dev->dev,
  511. "unlink qh%d-%04x/%p start %d [%d/%d us]\n",
  512. qh->period,
  513. hc32_to_cpup(ehci, &qh->hw_info2) & (QH_CMASK | QH_SMASK),
  514. qh, qh->start, qh->usecs, qh->c_usecs);
  515. /* qh->qh_next still "live" to HC */
  516. qh->qh_state = QH_STATE_UNLINK;
  517. qh->qh_next.ptr = NULL;
  518. qh_put (qh);
  519. /* maybe turn off periodic schedule */
  520. ehci->periodic_sched--;
  521. if (!ehci->periodic_sched)
  522. (void) disable_periodic (ehci);
  523. }
  524. static void intr_deschedule (struct ehci_hcd *ehci, struct ehci_qh *qh)
  525. {
  526. unsigned wait;
  527. qh_unlink_periodic (ehci, qh);
  528. /* simple/paranoid: always delay, expecting the HC needs to read
  529. * qh->hw_next or finish a writeback after SPLIT/CSPLIT ... and
  530. * expect khubd to clean up after any CSPLITs we won't issue.
  531. * active high speed queues may need bigger delays...
  532. */
  533. if (list_empty (&qh->qtd_list)
  534. || (cpu_to_hc32(ehci, QH_CMASK)
  535. & qh->hw_info2) != 0)
  536. wait = 2;
  537. else
  538. wait = 55; /* worst case: 3 * 1024 */
  539. udelay (wait);
  540. qh->qh_state = QH_STATE_IDLE;
  541. qh->hw_next = EHCI_LIST_END(ehci);
  542. wmb ();
  543. }
  544. /*-------------------------------------------------------------------------*/
  545. static int check_period (
  546. struct ehci_hcd *ehci,
  547. unsigned frame,
  548. unsigned uframe,
  549. unsigned period,
  550. unsigned usecs
  551. ) {
  552. int claimed;
  553. /* complete split running into next frame?
  554. * given FSTN support, we could sometimes check...
  555. */
  556. if (uframe >= 8)
  557. return 0;
  558. /*
  559. * 80% periodic == 100 usec/uframe available
  560. * convert "usecs we need" to "max already claimed"
  561. */
  562. usecs = 100 - usecs;
  563. /* we "know" 2 and 4 uframe intervals were rejected; so
  564. * for period 0, check _every_ microframe in the schedule.
  565. */
  566. if (unlikely (period == 0)) {
  567. do {
  568. for (uframe = 0; uframe < 7; uframe++) {
  569. claimed = periodic_usecs (ehci, frame, uframe);
  570. if (claimed > usecs)
  571. return 0;
  572. }
  573. } while ((frame += 1) < ehci->periodic_size);
  574. /* just check the specified uframe, at that period */
  575. } else {
  576. do {
  577. claimed = periodic_usecs (ehci, frame, uframe);
  578. if (claimed > usecs)
  579. return 0;
  580. } while ((frame += period) < ehci->periodic_size);
  581. }
  582. // success!
  583. return 1;
  584. }
  585. static int check_intr_schedule (
  586. struct ehci_hcd *ehci,
  587. unsigned frame,
  588. unsigned uframe,
  589. const struct ehci_qh *qh,
  590. __hc32 *c_maskp
  591. )
  592. {
  593. int retval = -ENOSPC;
  594. u8 mask = 0;
  595. if (qh->c_usecs && uframe >= 6) /* FSTN territory? */
  596. goto done;
  597. if (!check_period (ehci, frame, uframe, qh->period, qh->usecs))
  598. goto done;
  599. if (!qh->c_usecs) {
  600. retval = 0;
  601. *c_maskp = 0;
  602. goto done;
  603. }
  604. #ifdef CONFIG_USB_EHCI_TT_NEWSCHED
  605. if (tt_available (ehci, qh->period, qh->dev, frame, uframe,
  606. qh->tt_usecs)) {
  607. unsigned i;
  608. /* TODO : this may need FSTN for SSPLIT in uframe 5. */
  609. for (i=uframe+1; i<8 && i<uframe+4; i++)
  610. if (!check_period (ehci, frame, i,
  611. qh->period, qh->c_usecs))
  612. goto done;
  613. else
  614. mask |= 1 << i;
  615. retval = 0;
  616. *c_maskp = cpu_to_hc32(ehci, mask << 8);
  617. }
  618. #else
  619. /* Make sure this tt's buffer is also available for CSPLITs.
  620. * We pessimize a bit; probably the typical full speed case
  621. * doesn't need the second CSPLIT.
  622. *
  623. * NOTE: both SPLIT and CSPLIT could be checked in just
  624. * one smart pass...
  625. */
  626. mask = 0x03 << (uframe + qh->gap_uf);
  627. *c_maskp = cpu_to_hc32(ehci, mask << 8);
  628. mask |= 1 << uframe;
  629. if (tt_no_collision (ehci, qh->period, qh->dev, frame, mask)) {
  630. if (!check_period (ehci, frame, uframe + qh->gap_uf + 1,
  631. qh->period, qh->c_usecs))
  632. goto done;
  633. if (!check_period (ehci, frame, uframe + qh->gap_uf,
  634. qh->period, qh->c_usecs))
  635. goto done;
  636. retval = 0;
  637. }
  638. #endif
  639. done:
  640. return retval;
  641. }
  642. /* "first fit" scheduling policy used the first time through,
  643. * or when the previous schedule slot can't be re-used.
  644. */
  645. static int qh_schedule(struct ehci_hcd *ehci, struct ehci_qh *qh)
  646. {
  647. int status;
  648. unsigned uframe;
  649. __hc32 c_mask;
  650. unsigned frame; /* 0..(qh->period - 1), or NO_FRAME */
  651. qh_refresh(ehci, qh);
  652. qh->hw_next = EHCI_LIST_END(ehci);
  653. frame = qh->start;
  654. /* reuse the previous schedule slots, if we can */
  655. if (frame < qh->period) {
  656. uframe = ffs(hc32_to_cpup(ehci, &qh->hw_info2) & QH_SMASK);
  657. status = check_intr_schedule (ehci, frame, --uframe,
  658. qh, &c_mask);
  659. } else {
  660. uframe = 0;
  661. c_mask = 0;
  662. status = -ENOSPC;
  663. }
  664. /* else scan the schedule to find a group of slots such that all
  665. * uframes have enough periodic bandwidth available.
  666. */
  667. if (status) {
  668. /* "normal" case, uframing flexible except with splits */
  669. if (qh->period) {
  670. frame = qh->period - 1;
  671. do {
  672. for (uframe = 0; uframe < 8; uframe++) {
  673. status = check_intr_schedule (ehci,
  674. frame, uframe, qh,
  675. &c_mask);
  676. if (status == 0)
  677. break;
  678. }
  679. } while (status && frame--);
  680. /* qh->period == 0 means every uframe */
  681. } else {
  682. frame = 0;
  683. status = check_intr_schedule (ehci, 0, 0, qh, &c_mask);
  684. }
  685. if (status)
  686. goto done;
  687. qh->start = frame;
  688. /* reset S-frame and (maybe) C-frame masks */
  689. qh->hw_info2 &= cpu_to_hc32(ehci, ~(QH_CMASK | QH_SMASK));
  690. qh->hw_info2 |= qh->period
  691. ? cpu_to_hc32(ehci, 1 << uframe)
  692. : cpu_to_hc32(ehci, QH_SMASK);
  693. qh->hw_info2 |= c_mask;
  694. } else
  695. ehci_dbg (ehci, "reused qh %p schedule\n", qh);
  696. /* stuff into the periodic schedule */
  697. status = qh_link_periodic (ehci, qh);
  698. done:
  699. return status;
  700. }
  701. static int intr_submit (
  702. struct ehci_hcd *ehci,
  703. struct usb_host_endpoint *ep,
  704. struct urb *urb,
  705. struct list_head *qtd_list,
  706. gfp_t mem_flags
  707. ) {
  708. unsigned epnum;
  709. unsigned long flags;
  710. struct ehci_qh *qh;
  711. int status = 0;
  712. struct list_head empty;
  713. /* get endpoint and transfer/schedule data */
  714. epnum = ep->desc.bEndpointAddress;
  715. spin_lock_irqsave (&ehci->lock, flags);
  716. if (unlikely(!test_bit(HCD_FLAG_HW_ACCESSIBLE,
  717. &ehci_to_hcd(ehci)->flags))) {
  718. status = -ESHUTDOWN;
  719. goto done;
  720. }
  721. /* get qh and force any scheduling errors */
  722. INIT_LIST_HEAD (&empty);
  723. qh = qh_append_tds (ehci, urb, &empty, epnum, &ep->hcpriv);
  724. if (qh == NULL) {
  725. status = -ENOMEM;
  726. goto done;
  727. }
  728. if (qh->qh_state == QH_STATE_IDLE) {
  729. if ((status = qh_schedule (ehci, qh)) != 0)
  730. goto done;
  731. }
  732. /* then queue the urb's tds to the qh */
  733. qh = qh_append_tds (ehci, urb, qtd_list, epnum, &ep->hcpriv);
  734. BUG_ON (qh == NULL);
  735. /* ... update usbfs periodic stats */
  736. ehci_to_hcd(ehci)->self.bandwidth_int_reqs++;
  737. done:
  738. spin_unlock_irqrestore (&ehci->lock, flags);
  739. if (status)
  740. qtd_list_free (ehci, urb, qtd_list);
  741. return status;
  742. }
  743. /*-------------------------------------------------------------------------*/
  744. /* ehci_iso_stream ops work with both ITD and SITD */
  745. static struct ehci_iso_stream *
  746. iso_stream_alloc (gfp_t mem_flags)
  747. {
  748. struct ehci_iso_stream *stream;
  749. stream = kzalloc(sizeof *stream, mem_flags);
  750. if (likely (stream != NULL)) {
  751. INIT_LIST_HEAD(&stream->td_list);
  752. INIT_LIST_HEAD(&stream->free_list);
  753. stream->next_uframe = -1;
  754. stream->refcount = 1;
  755. }
  756. return stream;
  757. }
  758. static void
  759. iso_stream_init (
  760. struct ehci_hcd *ehci,
  761. struct ehci_iso_stream *stream,
  762. struct usb_device *dev,
  763. int pipe,
  764. unsigned interval
  765. )
  766. {
  767. static const u8 smask_out [] = { 0x01, 0x03, 0x07, 0x0f, 0x1f, 0x3f };
  768. u32 buf1;
  769. unsigned epnum, maxp;
  770. int is_input;
  771. long bandwidth;
  772. /*
  773. * this might be a "high bandwidth" highspeed endpoint,
  774. * as encoded in the ep descriptor's wMaxPacket field
  775. */
  776. epnum = usb_pipeendpoint (pipe);
  777. is_input = usb_pipein (pipe) ? USB_DIR_IN : 0;
  778. maxp = usb_maxpacket(dev, pipe, !is_input);
  779. if (is_input) {
  780. buf1 = (1 << 11);
  781. } else {
  782. buf1 = 0;
  783. }
  784. /* knows about ITD vs SITD */
  785. if (dev->speed == USB_SPEED_HIGH) {
  786. unsigned multi = hb_mult(maxp);
  787. stream->highspeed = 1;
  788. maxp = max_packet(maxp);
  789. buf1 |= maxp;
  790. maxp *= multi;
  791. stream->buf0 = cpu_to_hc32(ehci, (epnum << 8) | dev->devnum);
  792. stream->buf1 = cpu_to_hc32(ehci, buf1);
  793. stream->buf2 = cpu_to_hc32(ehci, multi);
  794. /* usbfs wants to report the average usecs per frame tied up
  795. * when transfers on this endpoint are scheduled ...
  796. */
  797. stream->usecs = HS_USECS_ISO (maxp);
  798. bandwidth = stream->usecs * 8;
  799. bandwidth /= 1 << (interval - 1);
  800. } else {
  801. u32 addr;
  802. int think_time;
  803. int hs_transfers;
  804. addr = dev->ttport << 24;
  805. if (!ehci_is_TDI(ehci)
  806. || (dev->tt->hub !=
  807. ehci_to_hcd(ehci)->self.root_hub))
  808. addr |= dev->tt->hub->devnum << 16;
  809. addr |= epnum << 8;
  810. addr |= dev->devnum;
  811. stream->usecs = HS_USECS_ISO (maxp);
  812. think_time = dev->tt ? dev->tt->think_time : 0;
  813. stream->tt_usecs = NS_TO_US (think_time + usb_calc_bus_time (
  814. dev->speed, is_input, 1, maxp));
  815. hs_transfers = max (1u, (maxp + 187) / 188);
  816. if (is_input) {
  817. u32 tmp;
  818. addr |= 1 << 31;
  819. stream->c_usecs = stream->usecs;
  820. stream->usecs = HS_USECS_ISO (1);
  821. stream->raw_mask = 1;
  822. /* c-mask as specified in USB 2.0 11.18.4 3.c */
  823. tmp = (1 << (hs_transfers + 2)) - 1;
  824. stream->raw_mask |= tmp << (8 + 2);
  825. } else
  826. stream->raw_mask = smask_out [hs_transfers - 1];
  827. bandwidth = stream->usecs + stream->c_usecs;
  828. bandwidth /= 1 << (interval + 2);
  829. /* stream->splits gets created from raw_mask later */
  830. stream->address = cpu_to_hc32(ehci, addr);
  831. }
  832. stream->bandwidth = bandwidth;
  833. stream->udev = dev;
  834. stream->bEndpointAddress = is_input | epnum;
  835. stream->interval = interval;
  836. stream->maxp = maxp;
  837. }
  838. static void
  839. iso_stream_put(struct ehci_hcd *ehci, struct ehci_iso_stream *stream)
  840. {
  841. stream->refcount--;
  842. /* free whenever just a dev->ep reference remains.
  843. * not like a QH -- no persistent state (toggle, halt)
  844. */
  845. if (stream->refcount == 1) {
  846. int is_in;
  847. // BUG_ON (!list_empty(&stream->td_list));
  848. while (!list_empty (&stream->free_list)) {
  849. struct list_head *entry;
  850. entry = stream->free_list.next;
  851. list_del (entry);
  852. /* knows about ITD vs SITD */
  853. if (stream->highspeed) {
  854. struct ehci_itd *itd;
  855. itd = list_entry (entry, struct ehci_itd,
  856. itd_list);
  857. dma_pool_free (ehci->itd_pool, itd,
  858. itd->itd_dma);
  859. } else {
  860. struct ehci_sitd *sitd;
  861. sitd = list_entry (entry, struct ehci_sitd,
  862. sitd_list);
  863. dma_pool_free (ehci->sitd_pool, sitd,
  864. sitd->sitd_dma);
  865. }
  866. }
  867. is_in = (stream->bEndpointAddress & USB_DIR_IN) ? 0x10 : 0;
  868. stream->bEndpointAddress &= 0x0f;
  869. stream->ep->hcpriv = NULL;
  870. if (stream->rescheduled) {
  871. ehci_info (ehci, "ep%d%s-iso rescheduled "
  872. "%lu times in %lu seconds\n",
  873. stream->bEndpointAddress, is_in ? "in" : "out",
  874. stream->rescheduled,
  875. ((jiffies - stream->start)/HZ)
  876. );
  877. }
  878. kfree(stream);
  879. }
  880. }
  881. static inline struct ehci_iso_stream *
  882. iso_stream_get (struct ehci_iso_stream *stream)
  883. {
  884. if (likely (stream != NULL))
  885. stream->refcount++;
  886. return stream;
  887. }
  888. static struct ehci_iso_stream *
  889. iso_stream_find (struct ehci_hcd *ehci, struct urb *urb)
  890. {
  891. unsigned epnum;
  892. struct ehci_iso_stream *stream;
  893. struct usb_host_endpoint *ep;
  894. unsigned long flags;
  895. epnum = usb_pipeendpoint (urb->pipe);
  896. if (usb_pipein(urb->pipe))
  897. ep = urb->dev->ep_in[epnum];
  898. else
  899. ep = urb->dev->ep_out[epnum];
  900. spin_lock_irqsave (&ehci->lock, flags);
  901. stream = ep->hcpriv;
  902. if (unlikely (stream == NULL)) {
  903. stream = iso_stream_alloc(GFP_ATOMIC);
  904. if (likely (stream != NULL)) {
  905. /* dev->ep owns the initial refcount */
  906. ep->hcpriv = stream;
  907. stream->ep = ep;
  908. iso_stream_init(ehci, stream, urb->dev, urb->pipe,
  909. urb->interval);
  910. }
  911. /* if dev->ep [epnum] is a QH, info1.maxpacket is nonzero */
  912. } else if (unlikely (stream->hw_info1 != 0)) {
  913. ehci_dbg (ehci, "dev %s ep%d%s, not iso??\n",
  914. urb->dev->devpath, epnum,
  915. usb_pipein(urb->pipe) ? "in" : "out");
  916. stream = NULL;
  917. }
  918. /* caller guarantees an eventual matching iso_stream_put */
  919. stream = iso_stream_get (stream);
  920. spin_unlock_irqrestore (&ehci->lock, flags);
  921. return stream;
  922. }
  923. /*-------------------------------------------------------------------------*/
  924. /* ehci_iso_sched ops can be ITD-only or SITD-only */
  925. static struct ehci_iso_sched *
  926. iso_sched_alloc (unsigned packets, gfp_t mem_flags)
  927. {
  928. struct ehci_iso_sched *iso_sched;
  929. int size = sizeof *iso_sched;
  930. size += packets * sizeof (struct ehci_iso_packet);
  931. iso_sched = kzalloc(size, mem_flags);
  932. if (likely (iso_sched != NULL)) {
  933. INIT_LIST_HEAD (&iso_sched->td_list);
  934. }
  935. return iso_sched;
  936. }
  937. static inline void
  938. itd_sched_init(
  939. struct ehci_hcd *ehci,
  940. struct ehci_iso_sched *iso_sched,
  941. struct ehci_iso_stream *stream,
  942. struct urb *urb
  943. )
  944. {
  945. unsigned i;
  946. dma_addr_t dma = urb->transfer_dma;
  947. /* how many uframes are needed for these transfers */
  948. iso_sched->span = urb->number_of_packets * stream->interval;
  949. /* figure out per-uframe itd fields that we'll need later
  950. * when we fit new itds into the schedule.
  951. */
  952. for (i = 0; i < urb->number_of_packets; i++) {
  953. struct ehci_iso_packet *uframe = &iso_sched->packet [i];
  954. unsigned length;
  955. dma_addr_t buf;
  956. u32 trans;
  957. length = urb->iso_frame_desc [i].length;
  958. buf = dma + urb->iso_frame_desc [i].offset;
  959. trans = EHCI_ISOC_ACTIVE;
  960. trans |= buf & 0x0fff;
  961. if (unlikely (((i + 1) == urb->number_of_packets))
  962. && !(urb->transfer_flags & URB_NO_INTERRUPT))
  963. trans |= EHCI_ITD_IOC;
  964. trans |= length << 16;
  965. uframe->transaction = cpu_to_hc32(ehci, trans);
  966. /* might need to cross a buffer page within a uframe */
  967. uframe->bufp = (buf & ~(u64)0x0fff);
  968. buf += length;
  969. if (unlikely ((uframe->bufp != (buf & ~(u64)0x0fff))))
  970. uframe->cross = 1;
  971. }
  972. }
  973. static void
  974. iso_sched_free (
  975. struct ehci_iso_stream *stream,
  976. struct ehci_iso_sched *iso_sched
  977. )
  978. {
  979. if (!iso_sched)
  980. return;
  981. // caller must hold ehci->lock!
  982. list_splice (&iso_sched->td_list, &stream->free_list);
  983. kfree (iso_sched);
  984. }
  985. static int
  986. itd_urb_transaction (
  987. struct ehci_iso_stream *stream,
  988. struct ehci_hcd *ehci,
  989. struct urb *urb,
  990. gfp_t mem_flags
  991. )
  992. {
  993. struct ehci_itd *itd;
  994. dma_addr_t itd_dma;
  995. int i;
  996. unsigned num_itds;
  997. struct ehci_iso_sched *sched;
  998. unsigned long flags;
  999. sched = iso_sched_alloc (urb->number_of_packets, mem_flags);
  1000. if (unlikely (sched == NULL))
  1001. return -ENOMEM;
  1002. itd_sched_init(ehci, sched, stream, urb);
  1003. if (urb->interval < 8)
  1004. num_itds = 1 + (sched->span + 7) / 8;
  1005. else
  1006. num_itds = urb->number_of_packets;
  1007. /* allocate/init ITDs */
  1008. spin_lock_irqsave (&ehci->lock, flags);
  1009. for (i = 0; i < num_itds; i++) {
  1010. /* free_list.next might be cache-hot ... but maybe
  1011. * the HC caches it too. avoid that issue for now.
  1012. */
  1013. /* prefer previously-allocated itds */
  1014. if (likely (!list_empty(&stream->free_list))) {
  1015. itd = list_entry (stream->free_list.prev,
  1016. struct ehci_itd, itd_list);
  1017. list_del (&itd->itd_list);
  1018. itd_dma = itd->itd_dma;
  1019. } else
  1020. itd = NULL;
  1021. if (!itd) {
  1022. spin_unlock_irqrestore (&ehci->lock, flags);
  1023. itd = dma_pool_alloc (ehci->itd_pool, mem_flags,
  1024. &itd_dma);
  1025. spin_lock_irqsave (&ehci->lock, flags);
  1026. }
  1027. if (unlikely (NULL == itd)) {
  1028. iso_sched_free (stream, sched);
  1029. spin_unlock_irqrestore (&ehci->lock, flags);
  1030. return -ENOMEM;
  1031. }
  1032. memset (itd, 0, sizeof *itd);
  1033. itd->itd_dma = itd_dma;
  1034. list_add (&itd->itd_list, &sched->td_list);
  1035. }
  1036. spin_unlock_irqrestore (&ehci->lock, flags);
  1037. /* temporarily store schedule info in hcpriv */
  1038. urb->hcpriv = sched;
  1039. urb->error_count = 0;
  1040. return 0;
  1041. }
  1042. /*-------------------------------------------------------------------------*/
  1043. static inline int
  1044. itd_slot_ok (
  1045. struct ehci_hcd *ehci,
  1046. u32 mod,
  1047. u32 uframe,
  1048. u8 usecs,
  1049. u32 period
  1050. )
  1051. {
  1052. uframe %= period;
  1053. do {
  1054. /* can't commit more than 80% periodic == 100 usec */
  1055. if (periodic_usecs (ehci, uframe >> 3, uframe & 0x7)
  1056. > (100 - usecs))
  1057. return 0;
  1058. /* we know urb->interval is 2^N uframes */
  1059. uframe += period;
  1060. } while (uframe < mod);
  1061. return 1;
  1062. }
  1063. static inline int
  1064. sitd_slot_ok (
  1065. struct ehci_hcd *ehci,
  1066. u32 mod,
  1067. struct ehci_iso_stream *stream,
  1068. u32 uframe,
  1069. struct ehci_iso_sched *sched,
  1070. u32 period_uframes
  1071. )
  1072. {
  1073. u32 mask, tmp;
  1074. u32 frame, uf;
  1075. mask = stream->raw_mask << (uframe & 7);
  1076. /* for IN, don't wrap CSPLIT into the next frame */
  1077. if (mask & ~0xffff)
  1078. return 0;
  1079. /* this multi-pass logic is simple, but performance may
  1080. * suffer when the schedule data isn't cached.
  1081. */
  1082. /* check bandwidth */
  1083. uframe %= period_uframes;
  1084. do {
  1085. u32 max_used;
  1086. frame = uframe >> 3;
  1087. uf = uframe & 7;
  1088. #ifdef CONFIG_USB_EHCI_TT_NEWSCHED
  1089. /* The tt's fullspeed bus bandwidth must be available.
  1090. * tt_available scheduling guarantees 10+% for control/bulk.
  1091. */
  1092. if (!tt_available (ehci, period_uframes << 3,
  1093. stream->udev, frame, uf, stream->tt_usecs))
  1094. return 0;
  1095. #else
  1096. /* tt must be idle for start(s), any gap, and csplit.
  1097. * assume scheduling slop leaves 10+% for control/bulk.
  1098. */
  1099. if (!tt_no_collision (ehci, period_uframes << 3,
  1100. stream->udev, frame, mask))
  1101. return 0;
  1102. #endif
  1103. /* check starts (OUT uses more than one) */
  1104. max_used = 100 - stream->usecs;
  1105. for (tmp = stream->raw_mask & 0xff; tmp; tmp >>= 1, uf++) {
  1106. if (periodic_usecs (ehci, frame, uf) > max_used)
  1107. return 0;
  1108. }
  1109. /* for IN, check CSPLIT */
  1110. if (stream->c_usecs) {
  1111. uf = uframe & 7;
  1112. max_used = 100 - stream->c_usecs;
  1113. do {
  1114. tmp = 1 << uf;
  1115. tmp <<= 8;
  1116. if ((stream->raw_mask & tmp) == 0)
  1117. continue;
  1118. if (periodic_usecs (ehci, frame, uf)
  1119. > max_used)
  1120. return 0;
  1121. } while (++uf < 8);
  1122. }
  1123. /* we know urb->interval is 2^N uframes */
  1124. uframe += period_uframes;
  1125. } while (uframe < mod);
  1126. stream->splits = cpu_to_hc32(ehci, stream->raw_mask << (uframe & 7));
  1127. return 1;
  1128. }
  1129. /*
  1130. * This scheduler plans almost as far into the future as it has actual
  1131. * periodic schedule slots. (Affected by TUNE_FLS, which defaults to
  1132. * "as small as possible" to be cache-friendlier.) That limits the size
  1133. * transfers you can stream reliably; avoid more than 64 msec per urb.
  1134. * Also avoid queue depths of less than ehci's worst irq latency (affected
  1135. * by the per-urb URB_NO_INTERRUPT hint, the log2_irq_thresh module parameter,
  1136. * and other factors); or more than about 230 msec total (for portability,
  1137. * given EHCI_TUNE_FLS and the slop). Or, write a smarter scheduler!
  1138. */
  1139. #define SCHEDULE_SLOP 10 /* frames */
  1140. static int
  1141. iso_stream_schedule (
  1142. struct ehci_hcd *ehci,
  1143. struct urb *urb,
  1144. struct ehci_iso_stream *stream
  1145. )
  1146. {
  1147. u32 now, start, max, period;
  1148. int status;
  1149. unsigned mod = ehci->periodic_size << 3;
  1150. struct ehci_iso_sched *sched = urb->hcpriv;
  1151. if (sched->span > (mod - 8 * SCHEDULE_SLOP)) {
  1152. ehci_dbg (ehci, "iso request %p too long\n", urb);
  1153. status = -EFBIG;
  1154. goto fail;
  1155. }
  1156. if ((stream->depth + sched->span) > mod) {
  1157. ehci_dbg (ehci, "request %p would overflow (%d+%d>%d)\n",
  1158. urb, stream->depth, sched->span, mod);
  1159. status = -EFBIG;
  1160. goto fail;
  1161. }
  1162. now = ehci_readl(ehci, &ehci->regs->frame_index) % mod;
  1163. /* when's the last uframe this urb could start? */
  1164. max = now + mod;
  1165. /* typical case: reuse current schedule. stream is still active,
  1166. * and no gaps from host falling behind (irq delays etc)
  1167. */
  1168. if (likely (!list_empty (&stream->td_list))) {
  1169. start = stream->next_uframe;
  1170. if (start < now)
  1171. start += mod;
  1172. if (likely ((start + sched->span) < max))
  1173. goto ready;
  1174. /* else fell behind; someday, try to reschedule */
  1175. status = -EL2NSYNC;
  1176. goto fail;
  1177. }
  1178. /* need to schedule; when's the next (u)frame we could start?
  1179. * this is bigger than ehci->i_thresh allows; scheduling itself
  1180. * isn't free, the slop should handle reasonably slow cpus. it
  1181. * can also help high bandwidth if the dma and irq loads don't
  1182. * jump until after the queue is primed.
  1183. */
  1184. start = SCHEDULE_SLOP * 8 + (now & ~0x07);
  1185. start %= mod;
  1186. stream->next_uframe = start;
  1187. /* NOTE: assumes URB_ISO_ASAP, to limit complexity/bugs */
  1188. period = urb->interval;
  1189. if (!stream->highspeed)
  1190. period <<= 3;
  1191. /* find a uframe slot with enough bandwidth */
  1192. for (; start < (stream->next_uframe + period); start++) {
  1193. int enough_space;
  1194. /* check schedule: enough space? */
  1195. if (stream->highspeed)
  1196. enough_space = itd_slot_ok (ehci, mod, start,
  1197. stream->usecs, period);
  1198. else {
  1199. if ((start % 8) >= 6)
  1200. continue;
  1201. enough_space = sitd_slot_ok (ehci, mod, stream,
  1202. start, sched, period);
  1203. }
  1204. /* schedule it here if there's enough bandwidth */
  1205. if (enough_space) {
  1206. stream->next_uframe = start % mod;
  1207. goto ready;
  1208. }
  1209. }
  1210. /* no room in the schedule */
  1211. ehci_dbg (ehci, "iso %ssched full %p (now %d max %d)\n",
  1212. list_empty (&stream->td_list) ? "" : "re",
  1213. urb, now, max);
  1214. status = -ENOSPC;
  1215. fail:
  1216. iso_sched_free (stream, sched);
  1217. urb->hcpriv = NULL;
  1218. return status;
  1219. ready:
  1220. /* report high speed start in uframes; full speed, in frames */
  1221. urb->start_frame = stream->next_uframe;
  1222. if (!stream->highspeed)
  1223. urb->start_frame >>= 3;
  1224. return 0;
  1225. }
  1226. /*-------------------------------------------------------------------------*/
  1227. static inline void
  1228. itd_init(struct ehci_hcd *ehci, struct ehci_iso_stream *stream,
  1229. struct ehci_itd *itd)
  1230. {
  1231. int i;
  1232. /* it's been recently zeroed */
  1233. itd->hw_next = EHCI_LIST_END(ehci);
  1234. itd->hw_bufp [0] = stream->buf0;
  1235. itd->hw_bufp [1] = stream->buf1;
  1236. itd->hw_bufp [2] = stream->buf2;
  1237. for (i = 0; i < 8; i++)
  1238. itd->index[i] = -1;
  1239. /* All other fields are filled when scheduling */
  1240. }
  1241. static inline void
  1242. itd_patch(
  1243. struct ehci_hcd *ehci,
  1244. struct ehci_itd *itd,
  1245. struct ehci_iso_sched *iso_sched,
  1246. unsigned index,
  1247. u16 uframe
  1248. )
  1249. {
  1250. struct ehci_iso_packet *uf = &iso_sched->packet [index];
  1251. unsigned pg = itd->pg;
  1252. // BUG_ON (pg == 6 && uf->cross);
  1253. uframe &= 0x07;
  1254. itd->index [uframe] = index;
  1255. itd->hw_transaction[uframe] = uf->transaction;
  1256. itd->hw_transaction[uframe] |= cpu_to_hc32(ehci, pg << 12);
  1257. itd->hw_bufp[pg] |= cpu_to_hc32(ehci, uf->bufp & ~(u32)0);
  1258. itd->hw_bufp_hi[pg] |= cpu_to_hc32(ehci, (u32)(uf->bufp >> 32));
  1259. /* iso_frame_desc[].offset must be strictly increasing */
  1260. if (unlikely (uf->cross)) {
  1261. u64 bufp = uf->bufp + 4096;
  1262. itd->pg = ++pg;
  1263. itd->hw_bufp[pg] |= cpu_to_hc32(ehci, bufp & ~(u32)0);
  1264. itd->hw_bufp_hi[pg] |= cpu_to_hc32(ehci, (u32)(bufp >> 32));
  1265. }
  1266. }
  1267. static inline void
  1268. itd_link (struct ehci_hcd *ehci, unsigned frame, struct ehci_itd *itd)
  1269. {
  1270. /* always prepend ITD/SITD ... only QH tree is order-sensitive */
  1271. itd->itd_next = ehci->pshadow [frame];
  1272. itd->hw_next = ehci->periodic [frame];
  1273. ehci->pshadow [frame].itd = itd;
  1274. itd->frame = frame;
  1275. wmb ();
  1276. ehci->periodic[frame] = cpu_to_hc32(ehci, itd->itd_dma | Q_TYPE_ITD);
  1277. }
  1278. /* fit urb's itds into the selected schedule slot; activate as needed */
  1279. static int
  1280. itd_link_urb (
  1281. struct ehci_hcd *ehci,
  1282. struct urb *urb,
  1283. unsigned mod,
  1284. struct ehci_iso_stream *stream
  1285. )
  1286. {
  1287. int packet;
  1288. unsigned next_uframe, uframe, frame;
  1289. struct ehci_iso_sched *iso_sched = urb->hcpriv;
  1290. struct ehci_itd *itd;
  1291. next_uframe = stream->next_uframe % mod;
  1292. if (unlikely (list_empty(&stream->td_list))) {
  1293. ehci_to_hcd(ehci)->self.bandwidth_allocated
  1294. += stream->bandwidth;
  1295. ehci_vdbg (ehci,
  1296. "schedule devp %s ep%d%s-iso period %d start %d.%d\n",
  1297. urb->dev->devpath, stream->bEndpointAddress & 0x0f,
  1298. (stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out",
  1299. urb->interval,
  1300. next_uframe >> 3, next_uframe & 0x7);
  1301. stream->start = jiffies;
  1302. }
  1303. ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs++;
  1304. /* fill iTDs uframe by uframe */
  1305. for (packet = 0, itd = NULL; packet < urb->number_of_packets; ) {
  1306. if (itd == NULL) {
  1307. /* ASSERT: we have all necessary itds */
  1308. // BUG_ON (list_empty (&iso_sched->td_list));
  1309. /* ASSERT: no itds for this endpoint in this uframe */
  1310. itd = list_entry (iso_sched->td_list.next,
  1311. struct ehci_itd, itd_list);
  1312. list_move_tail (&itd->itd_list, &stream->td_list);
  1313. itd->stream = iso_stream_get (stream);
  1314. itd->urb = usb_get_urb (urb);
  1315. itd_init (ehci, stream, itd);
  1316. }
  1317. uframe = next_uframe & 0x07;
  1318. frame = next_uframe >> 3;
  1319. itd->usecs [uframe] = stream->usecs;
  1320. itd_patch(ehci, itd, iso_sched, packet, uframe);
  1321. next_uframe += stream->interval;
  1322. stream->depth += stream->interval;
  1323. next_uframe %= mod;
  1324. packet++;
  1325. /* link completed itds into the schedule */
  1326. if (((next_uframe >> 3) != frame)
  1327. || packet == urb->number_of_packets) {
  1328. itd_link (ehci, frame % ehci->periodic_size, itd);
  1329. itd = NULL;
  1330. }
  1331. }
  1332. stream->next_uframe = next_uframe;
  1333. /* don't need that schedule data any more */
  1334. iso_sched_free (stream, iso_sched);
  1335. urb->hcpriv = NULL;
  1336. timer_action (ehci, TIMER_IO_WATCHDOG);
  1337. if (unlikely (!ehci->periodic_sched++))
  1338. return enable_periodic (ehci);
  1339. return 0;
  1340. }
  1341. #define ISO_ERRS (EHCI_ISOC_BUF_ERR | EHCI_ISOC_BABBLE | EHCI_ISOC_XACTERR)
  1342. static unsigned
  1343. itd_complete (
  1344. struct ehci_hcd *ehci,
  1345. struct ehci_itd *itd
  1346. ) {
  1347. struct urb *urb = itd->urb;
  1348. struct usb_iso_packet_descriptor *desc;
  1349. u32 t;
  1350. unsigned uframe;
  1351. int urb_index = -1;
  1352. struct ehci_iso_stream *stream = itd->stream;
  1353. struct usb_device *dev;
  1354. /* for each uframe with a packet */
  1355. for (uframe = 0; uframe < 8; uframe++) {
  1356. if (likely (itd->index[uframe] == -1))
  1357. continue;
  1358. urb_index = itd->index[uframe];
  1359. desc = &urb->iso_frame_desc [urb_index];
  1360. t = hc32_to_cpup(ehci, &itd->hw_transaction [uframe]);
  1361. itd->hw_transaction [uframe] = 0;
  1362. stream->depth -= stream->interval;
  1363. /* report transfer status */
  1364. if (unlikely (t & ISO_ERRS)) {
  1365. urb->error_count++;
  1366. if (t & EHCI_ISOC_BUF_ERR)
  1367. desc->status = usb_pipein (urb->pipe)
  1368. ? -ENOSR /* hc couldn't read */
  1369. : -ECOMM; /* hc couldn't write */
  1370. else if (t & EHCI_ISOC_BABBLE)
  1371. desc->status = -EOVERFLOW;
  1372. else /* (t & EHCI_ISOC_XACTERR) */
  1373. desc->status = -EPROTO;
  1374. /* HC need not update length with this error */
  1375. if (!(t & EHCI_ISOC_BABBLE))
  1376. desc->actual_length = EHCI_ITD_LENGTH (t);
  1377. } else if (likely ((t & EHCI_ISOC_ACTIVE) == 0)) {
  1378. desc->status = 0;
  1379. desc->actual_length = EHCI_ITD_LENGTH (t);
  1380. }
  1381. }
  1382. usb_put_urb (urb);
  1383. itd->urb = NULL;
  1384. itd->stream = NULL;
  1385. list_move (&itd->itd_list, &stream->free_list);
  1386. iso_stream_put (ehci, stream);
  1387. /* handle completion now? */
  1388. if (likely ((urb_index + 1) != urb->number_of_packets))
  1389. return 0;
  1390. /* ASSERT: it's really the last itd for this urb
  1391. list_for_each_entry (itd, &stream->td_list, itd_list)
  1392. BUG_ON (itd->urb == urb);
  1393. */
  1394. /* give urb back to the driver ... can be out-of-order */
  1395. dev = urb->dev;
  1396. ehci_urb_done (ehci, urb);
  1397. urb = NULL;
  1398. /* defer stopping schedule; completion can submit */
  1399. ehci->periodic_sched--;
  1400. if (unlikely (!ehci->periodic_sched))
  1401. (void) disable_periodic (ehci);
  1402. ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs--;
  1403. if (unlikely (list_empty (&stream->td_list))) {
  1404. ehci_to_hcd(ehci)->self.bandwidth_allocated
  1405. -= stream->bandwidth;
  1406. ehci_vdbg (ehci,
  1407. "deschedule devp %s ep%d%s-iso\n",
  1408. dev->devpath, stream->bEndpointAddress & 0x0f,
  1409. (stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out");
  1410. }
  1411. iso_stream_put (ehci, stream);
  1412. return 1;
  1413. }
  1414. /*-------------------------------------------------------------------------*/
  1415. static int itd_submit (struct ehci_hcd *ehci, struct urb *urb,
  1416. gfp_t mem_flags)
  1417. {
  1418. int status = -EINVAL;
  1419. unsigned long flags;
  1420. struct ehci_iso_stream *stream;
  1421. /* Get iso_stream head */
  1422. stream = iso_stream_find (ehci, urb);
  1423. if (unlikely (stream == NULL)) {
  1424. ehci_dbg (ehci, "can't get iso stream\n");
  1425. return -ENOMEM;
  1426. }
  1427. if (unlikely (urb->interval != stream->interval)) {
  1428. ehci_dbg (ehci, "can't change iso interval %d --> %d\n",
  1429. stream->interval, urb->interval);
  1430. goto done;
  1431. }
  1432. #ifdef EHCI_URB_TRACE
  1433. ehci_dbg (ehci,
  1434. "%s %s urb %p ep%d%s len %d, %d pkts %d uframes [%p]\n",
  1435. __FUNCTION__, urb->dev->devpath, urb,
  1436. usb_pipeendpoint (urb->pipe),
  1437. usb_pipein (urb->pipe) ? "in" : "out",
  1438. urb->transfer_buffer_length,
  1439. urb->number_of_packets, urb->interval,
  1440. stream);
  1441. #endif
  1442. /* allocate ITDs w/o locking anything */
  1443. status = itd_urb_transaction (stream, ehci, urb, mem_flags);
  1444. if (unlikely (status < 0)) {
  1445. ehci_dbg (ehci, "can't init itds\n");
  1446. goto done;
  1447. }
  1448. /* schedule ... need to lock */
  1449. spin_lock_irqsave (&ehci->lock, flags);
  1450. if (unlikely(!test_bit(HCD_FLAG_HW_ACCESSIBLE,
  1451. &ehci_to_hcd(ehci)->flags)))
  1452. status = -ESHUTDOWN;
  1453. else
  1454. status = iso_stream_schedule (ehci, urb, stream);
  1455. if (likely (status == 0))
  1456. itd_link_urb (ehci, urb, ehci->periodic_size << 3, stream);
  1457. spin_unlock_irqrestore (&ehci->lock, flags);
  1458. done:
  1459. if (unlikely (status < 0))
  1460. iso_stream_put (ehci, stream);
  1461. return status;
  1462. }
  1463. #ifdef CONFIG_USB_EHCI_SPLIT_ISO
  1464. /*-------------------------------------------------------------------------*/
  1465. /*
  1466. * "Split ISO TDs" ... used for USB 1.1 devices going through the
  1467. * TTs in USB 2.0 hubs. These need microframe scheduling.
  1468. */
  1469. static inline void
  1470. sitd_sched_init(
  1471. struct ehci_hcd *ehci,
  1472. struct ehci_iso_sched *iso_sched,
  1473. struct ehci_iso_stream *stream,
  1474. struct urb *urb
  1475. )
  1476. {
  1477. unsigned i;
  1478. dma_addr_t dma = urb->transfer_dma;
  1479. /* how many frames are needed for these transfers */
  1480. iso_sched->span = urb->number_of_packets * stream->interval;
  1481. /* figure out per-frame sitd fields that we'll need later
  1482. * when we fit new sitds into the schedule.
  1483. */
  1484. for (i = 0; i < urb->number_of_packets; i++) {
  1485. struct ehci_iso_packet *packet = &iso_sched->packet [i];
  1486. unsigned length;
  1487. dma_addr_t buf;
  1488. u32 trans;
  1489. length = urb->iso_frame_desc [i].length & 0x03ff;
  1490. buf = dma + urb->iso_frame_desc [i].offset;
  1491. trans = SITD_STS_ACTIVE;
  1492. if (((i + 1) == urb->number_of_packets)
  1493. && !(urb->transfer_flags & URB_NO_INTERRUPT))
  1494. trans |= SITD_IOC;
  1495. trans |= length << 16;
  1496. packet->transaction = cpu_to_hc32(ehci, trans);
  1497. /* might need to cross a buffer page within a td */
  1498. packet->bufp = buf;
  1499. packet->buf1 = (buf + length) & ~0x0fff;
  1500. if (packet->buf1 != (buf & ~(u64)0x0fff))
  1501. packet->cross = 1;
  1502. /* OUT uses multiple start-splits */
  1503. if (stream->bEndpointAddress & USB_DIR_IN)
  1504. continue;
  1505. length = (length + 187) / 188;
  1506. if (length > 1) /* BEGIN vs ALL */
  1507. length |= 1 << 3;
  1508. packet->buf1 |= length;
  1509. }
  1510. }
  1511. static int
  1512. sitd_urb_transaction (
  1513. struct ehci_iso_stream *stream,
  1514. struct ehci_hcd *ehci,
  1515. struct urb *urb,
  1516. gfp_t mem_flags
  1517. )
  1518. {
  1519. struct ehci_sitd *sitd;
  1520. dma_addr_t sitd_dma;
  1521. int i;
  1522. struct ehci_iso_sched *iso_sched;
  1523. unsigned long flags;
  1524. iso_sched = iso_sched_alloc (urb->number_of_packets, mem_flags);
  1525. if (iso_sched == NULL)
  1526. return -ENOMEM;
  1527. sitd_sched_init(ehci, iso_sched, stream, urb);
  1528. /* allocate/init sITDs */
  1529. spin_lock_irqsave (&ehci->lock, flags);
  1530. for (i = 0; i < urb->number_of_packets; i++) {
  1531. /* NOTE: for now, we don't try to handle wraparound cases
  1532. * for IN (using sitd->hw_backpointer, like a FSTN), which
  1533. * means we never need two sitds for full speed packets.
  1534. */
  1535. /* free_list.next might be cache-hot ... but maybe
  1536. * the HC caches it too. avoid that issue for now.
  1537. */
  1538. /* prefer previously-allocated sitds */
  1539. if (!list_empty(&stream->free_list)) {
  1540. sitd = list_entry (stream->free_list.prev,
  1541. struct ehci_sitd, sitd_list);
  1542. list_del (&sitd->sitd_list);
  1543. sitd_dma = sitd->sitd_dma;
  1544. } else
  1545. sitd = NULL;
  1546. if (!sitd) {
  1547. spin_unlock_irqrestore (&ehci->lock, flags);
  1548. sitd = dma_pool_alloc (ehci->sitd_pool, mem_flags,
  1549. &sitd_dma);
  1550. spin_lock_irqsave (&ehci->lock, flags);
  1551. }
  1552. if (!sitd) {
  1553. iso_sched_free (stream, iso_sched);
  1554. spin_unlock_irqrestore (&ehci->lock, flags);
  1555. return -ENOMEM;
  1556. }
  1557. memset (sitd, 0, sizeof *sitd);
  1558. sitd->sitd_dma = sitd_dma;
  1559. list_add (&sitd->sitd_list, &iso_sched->td_list);
  1560. }
  1561. /* temporarily store schedule info in hcpriv */
  1562. urb->hcpriv = iso_sched;
  1563. urb->error_count = 0;
  1564. spin_unlock_irqrestore (&ehci->lock, flags);
  1565. return 0;
  1566. }
  1567. /*-------------------------------------------------------------------------*/
  1568. static inline void
  1569. sitd_patch(
  1570. struct ehci_hcd *ehci,
  1571. struct ehci_iso_stream *stream,
  1572. struct ehci_sitd *sitd,
  1573. struct ehci_iso_sched *iso_sched,
  1574. unsigned index
  1575. )
  1576. {
  1577. struct ehci_iso_packet *uf = &iso_sched->packet [index];
  1578. u64 bufp = uf->bufp;
  1579. sitd->hw_next = EHCI_LIST_END(ehci);
  1580. sitd->hw_fullspeed_ep = stream->address;
  1581. sitd->hw_uframe = stream->splits;
  1582. sitd->hw_results = uf->transaction;
  1583. sitd->hw_backpointer = EHCI_LIST_END(ehci);
  1584. bufp = uf->bufp;
  1585. sitd->hw_buf[0] = cpu_to_hc32(ehci, bufp);
  1586. sitd->hw_buf_hi[0] = cpu_to_hc32(ehci, bufp >> 32);
  1587. sitd->hw_buf[1] = cpu_to_hc32(ehci, uf->buf1);
  1588. if (uf->cross)
  1589. bufp += 4096;
  1590. sitd->hw_buf_hi[1] = cpu_to_hc32(ehci, bufp >> 32);
  1591. sitd->index = index;
  1592. }
  1593. static inline void
  1594. sitd_link (struct ehci_hcd *ehci, unsigned frame, struct ehci_sitd *sitd)
  1595. {
  1596. /* note: sitd ordering could matter (CSPLIT then SSPLIT) */
  1597. sitd->sitd_next = ehci->pshadow [frame];
  1598. sitd->hw_next = ehci->periodic [frame];
  1599. ehci->pshadow [frame].sitd = sitd;
  1600. sitd->frame = frame;
  1601. wmb ();
  1602. ehci->periodic[frame] = cpu_to_hc32(ehci, sitd->sitd_dma | Q_TYPE_SITD);
  1603. }
  1604. /* fit urb's sitds into the selected schedule slot; activate as needed */
  1605. static int
  1606. sitd_link_urb (
  1607. struct ehci_hcd *ehci,
  1608. struct urb *urb,
  1609. unsigned mod,
  1610. struct ehci_iso_stream *stream
  1611. )
  1612. {
  1613. int packet;
  1614. unsigned next_uframe;
  1615. struct ehci_iso_sched *sched = urb->hcpriv;
  1616. struct ehci_sitd *sitd;
  1617. next_uframe = stream->next_uframe;
  1618. if (list_empty(&stream->td_list)) {
  1619. /* usbfs ignores TT bandwidth */
  1620. ehci_to_hcd(ehci)->self.bandwidth_allocated
  1621. += stream->bandwidth;
  1622. ehci_vdbg (ehci,
  1623. "sched devp %s ep%d%s-iso [%d] %dms/%04x\n",
  1624. urb->dev->devpath, stream->bEndpointAddress & 0x0f,
  1625. (stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out",
  1626. (next_uframe >> 3) % ehci->periodic_size,
  1627. stream->interval, hc32_to_cpu(ehci, stream->splits));
  1628. stream->start = jiffies;
  1629. }
  1630. ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs++;
  1631. /* fill sITDs frame by frame */
  1632. for (packet = 0, sitd = NULL;
  1633. packet < urb->number_of_packets;
  1634. packet++) {
  1635. /* ASSERT: we have all necessary sitds */
  1636. BUG_ON (list_empty (&sched->td_list));
  1637. /* ASSERT: no itds for this endpoint in this frame */
  1638. sitd = list_entry (sched->td_list.next,
  1639. struct ehci_sitd, sitd_list);
  1640. list_move_tail (&sitd->sitd_list, &stream->td_list);
  1641. sitd->stream = iso_stream_get (stream);
  1642. sitd->urb = usb_get_urb (urb);
  1643. sitd_patch(ehci, stream, sitd, sched, packet);
  1644. sitd_link (ehci, (next_uframe >> 3) % ehci->periodic_size,
  1645. sitd);
  1646. next_uframe += stream->interval << 3;
  1647. stream->depth += stream->interval << 3;
  1648. }
  1649. stream->next_uframe = next_uframe % mod;
  1650. /* don't need that schedule data any more */
  1651. iso_sched_free (stream, sched);
  1652. urb->hcpriv = NULL;
  1653. timer_action (ehci, TIMER_IO_WATCHDOG);
  1654. if (!ehci->periodic_sched++)
  1655. return enable_periodic (ehci);
  1656. return 0;
  1657. }
  1658. /*-------------------------------------------------------------------------*/
  1659. #define SITD_ERRS (SITD_STS_ERR | SITD_STS_DBE | SITD_STS_BABBLE \
  1660. | SITD_STS_XACT | SITD_STS_MMF)
  1661. static unsigned
  1662. sitd_complete (
  1663. struct ehci_hcd *ehci,
  1664. struct ehci_sitd *sitd
  1665. ) {
  1666. struct urb *urb = sitd->urb;
  1667. struct usb_iso_packet_descriptor *desc;
  1668. u32 t;
  1669. int urb_index = -1;
  1670. struct ehci_iso_stream *stream = sitd->stream;
  1671. struct usb_device *dev;
  1672. urb_index = sitd->index;
  1673. desc = &urb->iso_frame_desc [urb_index];
  1674. t = hc32_to_cpup(ehci, &sitd->hw_results);
  1675. /* report transfer status */
  1676. if (t & SITD_ERRS) {
  1677. urb->error_count++;
  1678. if (t & SITD_STS_DBE)
  1679. desc->status = usb_pipein (urb->pipe)
  1680. ? -ENOSR /* hc couldn't read */
  1681. : -ECOMM; /* hc couldn't write */
  1682. else if (t & SITD_STS_BABBLE)
  1683. desc->status = -EOVERFLOW;
  1684. else /* XACT, MMF, etc */
  1685. desc->status = -EPROTO;
  1686. } else {
  1687. desc->status = 0;
  1688. desc->actual_length = desc->length - SITD_LENGTH (t);
  1689. }
  1690. usb_put_urb (urb);
  1691. sitd->urb = NULL;
  1692. sitd->stream = NULL;
  1693. list_move (&sitd->sitd_list, &stream->free_list);
  1694. stream->depth -= stream->interval << 3;
  1695. iso_stream_put (ehci, stream);
  1696. /* handle completion now? */
  1697. if ((urb_index + 1) != urb->number_of_packets)
  1698. return 0;
  1699. /* ASSERT: it's really the last sitd for this urb
  1700. list_for_each_entry (sitd, &stream->td_list, sitd_list)
  1701. BUG_ON (sitd->urb == urb);
  1702. */
  1703. /* give urb back to the driver */
  1704. dev = urb->dev;
  1705. ehci_urb_done (ehci, urb);
  1706. urb = NULL;
  1707. /* defer stopping schedule; completion can submit */
  1708. ehci->periodic_sched--;
  1709. if (!ehci->periodic_sched)
  1710. (void) disable_periodic (ehci);
  1711. ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs--;
  1712. if (list_empty (&stream->td_list)) {
  1713. ehci_to_hcd(ehci)->self.bandwidth_allocated
  1714. -= stream->bandwidth;
  1715. ehci_vdbg (ehci,
  1716. "deschedule devp %s ep%d%s-iso\n",
  1717. dev->devpath, stream->bEndpointAddress & 0x0f,
  1718. (stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out");
  1719. }
  1720. iso_stream_put (ehci, stream);
  1721. return 1;
  1722. }
  1723. static int sitd_submit (struct ehci_hcd *ehci, struct urb *urb,
  1724. gfp_t mem_flags)
  1725. {
  1726. int status = -EINVAL;
  1727. unsigned long flags;
  1728. struct ehci_iso_stream *stream;
  1729. /* Get iso_stream head */
  1730. stream = iso_stream_find (ehci, urb);
  1731. if (stream == NULL) {
  1732. ehci_dbg (ehci, "can't get iso stream\n");
  1733. return -ENOMEM;
  1734. }
  1735. if (urb->interval != stream->interval) {
  1736. ehci_dbg (ehci, "can't change iso interval %d --> %d\n",
  1737. stream->interval, urb->interval);
  1738. goto done;
  1739. }
  1740. #ifdef EHCI_URB_TRACE
  1741. ehci_dbg (ehci,
  1742. "submit %p dev%s ep%d%s-iso len %d\n",
  1743. urb, urb->dev->devpath,
  1744. usb_pipeendpoint (urb->pipe),
  1745. usb_pipein (urb->pipe) ? "in" : "out",
  1746. urb->transfer_buffer_length);
  1747. #endif
  1748. /* allocate SITDs */
  1749. status = sitd_urb_transaction (stream, ehci, urb, mem_flags);
  1750. if (status < 0) {
  1751. ehci_dbg (ehci, "can't init sitds\n");
  1752. goto done;
  1753. }
  1754. /* schedule ... need to lock */
  1755. spin_lock_irqsave (&ehci->lock, flags);
  1756. if (unlikely(!test_bit(HCD_FLAG_HW_ACCESSIBLE,
  1757. &ehci_to_hcd(ehci)->flags)))
  1758. status = -ESHUTDOWN;
  1759. else
  1760. status = iso_stream_schedule (ehci, urb, stream);
  1761. if (status == 0)
  1762. sitd_link_urb (ehci, urb, ehci->periodic_size << 3, stream);
  1763. spin_unlock_irqrestore (&ehci->lock, flags);
  1764. done:
  1765. if (status < 0)
  1766. iso_stream_put (ehci, stream);
  1767. return status;
  1768. }
  1769. #else
  1770. static inline int
  1771. sitd_submit (struct ehci_hcd *ehci, struct urb *urb, gfp_t mem_flags)
  1772. {
  1773. ehci_dbg (ehci, "split iso support is disabled\n");
  1774. return -ENOSYS;
  1775. }
  1776. static inline unsigned
  1777. sitd_complete (
  1778. struct ehci_hcd *ehci,
  1779. struct ehci_sitd *sitd
  1780. ) {
  1781. ehci_err (ehci, "sitd_complete %p?\n", sitd);
  1782. return 0;
  1783. }
  1784. #endif /* USB_EHCI_SPLIT_ISO */
  1785. /*-------------------------------------------------------------------------*/
  1786. static void
  1787. scan_periodic (struct ehci_hcd *ehci)
  1788. {
  1789. unsigned frame, clock, now_uframe, mod;
  1790. unsigned modified;
  1791. mod = ehci->periodic_size << 3;
  1792. /*
  1793. * When running, scan from last scan point up to "now"
  1794. * else clean up by scanning everything that's left.
  1795. * Touches as few pages as possible: cache-friendly.
  1796. */
  1797. now_uframe = ehci->next_uframe;
  1798. if (HC_IS_RUNNING (ehci_to_hcd(ehci)->state))
  1799. clock = ehci_readl(ehci, &ehci->regs->frame_index);
  1800. else
  1801. clock = now_uframe + mod - 1;
  1802. clock %= mod;
  1803. for (;;) {
  1804. union ehci_shadow q, *q_p;
  1805. __hc32 type, *hw_p;
  1806. unsigned uframes;
  1807. /* don't scan past the live uframe */
  1808. frame = now_uframe >> 3;
  1809. if (frame == (clock >> 3))
  1810. uframes = now_uframe & 0x07;
  1811. else {
  1812. /* safe to scan the whole frame at once */
  1813. now_uframe |= 0x07;
  1814. uframes = 8;
  1815. }
  1816. restart:
  1817. /* scan each element in frame's queue for completions */
  1818. q_p = &ehci->pshadow [frame];
  1819. hw_p = &ehci->periodic [frame];
  1820. q.ptr = q_p->ptr;
  1821. type = Q_NEXT_TYPE(ehci, *hw_p);
  1822. modified = 0;
  1823. while (q.ptr != NULL) {
  1824. unsigned uf;
  1825. union ehci_shadow temp;
  1826. int live;
  1827. live = HC_IS_RUNNING (ehci_to_hcd(ehci)->state);
  1828. switch (hc32_to_cpu(ehci, type)) {
  1829. case Q_TYPE_QH:
  1830. /* handle any completions */
  1831. temp.qh = qh_get (q.qh);
  1832. type = Q_NEXT_TYPE(ehci, q.qh->hw_next);
  1833. q = q.qh->qh_next;
  1834. modified = qh_completions (ehci, temp.qh);
  1835. if (unlikely (list_empty (&temp.qh->qtd_list)))
  1836. intr_deschedule (ehci, temp.qh);
  1837. qh_put (temp.qh);
  1838. break;
  1839. case Q_TYPE_FSTN:
  1840. /* for "save place" FSTNs, look at QH entries
  1841. * in the previous frame for completions.
  1842. */
  1843. if (q.fstn->hw_prev != EHCI_LIST_END(ehci)) {
  1844. dbg ("ignoring completions from FSTNs");
  1845. }
  1846. type = Q_NEXT_TYPE(ehci, q.fstn->hw_next);
  1847. q = q.fstn->fstn_next;
  1848. break;
  1849. case Q_TYPE_ITD:
  1850. /* skip itds for later in the frame */
  1851. rmb ();
  1852. for (uf = live ? uframes : 8; uf < 8; uf++) {
  1853. if (0 == (q.itd->hw_transaction [uf]
  1854. & ITD_ACTIVE(ehci)))
  1855. continue;
  1856. q_p = &q.itd->itd_next;
  1857. hw_p = &q.itd->hw_next;
  1858. type = Q_NEXT_TYPE(ehci,
  1859. q.itd->hw_next);
  1860. q = *q_p;
  1861. break;
  1862. }
  1863. if (uf != 8)
  1864. break;
  1865. /* this one's ready ... HC won't cache the
  1866. * pointer for much longer, if at all.
  1867. */
  1868. *q_p = q.itd->itd_next;
  1869. *hw_p = q.itd->hw_next;
  1870. type = Q_NEXT_TYPE(ehci, q.itd->hw_next);
  1871. wmb();
  1872. modified = itd_complete (ehci, q.itd);
  1873. q = *q_p;
  1874. break;
  1875. case Q_TYPE_SITD:
  1876. if ((q.sitd->hw_results & SITD_ACTIVE(ehci))
  1877. && live) {
  1878. q_p = &q.sitd->sitd_next;
  1879. hw_p = &q.sitd->hw_next;
  1880. type = Q_NEXT_TYPE(ehci,
  1881. q.sitd->hw_next);
  1882. q = *q_p;
  1883. break;
  1884. }
  1885. *q_p = q.sitd->sitd_next;
  1886. *hw_p = q.sitd->hw_next;
  1887. type = Q_NEXT_TYPE(ehci, q.sitd->hw_next);
  1888. wmb();
  1889. modified = sitd_complete (ehci, q.sitd);
  1890. q = *q_p;
  1891. break;
  1892. default:
  1893. dbg ("corrupt type %d frame %d shadow %p",
  1894. type, frame, q.ptr);
  1895. // BUG ();
  1896. q.ptr = NULL;
  1897. }
  1898. /* assume completion callbacks modify the queue */
  1899. if (unlikely (modified))
  1900. goto restart;
  1901. }
  1902. /* stop when we catch up to the HC */
  1903. // FIXME: this assumes we won't get lapped when
  1904. // latencies climb; that should be rare, but...
  1905. // detect it, and just go all the way around.
  1906. // FLR might help detect this case, so long as latencies
  1907. // don't exceed periodic_size msec (default 1.024 sec).
  1908. // FIXME: likewise assumes HC doesn't halt mid-scan
  1909. if (now_uframe == clock) {
  1910. unsigned now;
  1911. if (!HC_IS_RUNNING (ehci_to_hcd(ehci)->state))
  1912. break;
  1913. ehci->next_uframe = now_uframe;
  1914. now = ehci_readl(ehci, &ehci->regs->frame_index) % mod;
  1915. if (now_uframe == now)
  1916. break;
  1917. /* rescan the rest of this frame, then ... */
  1918. clock = now;
  1919. } else {
  1920. now_uframe++;
  1921. now_uframe %= mod;
  1922. }
  1923. }
  1924. }