xhci-mem.c 75 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498
  1. /*
  2. * xHCI host controller driver
  3. *
  4. * Copyright (C) 2008 Intel Corp.
  5. *
  6. * Author: Sarah Sharp
  7. * Some code borrowed from the Linux EHCI driver.
  8. *
  9. * This program is free software; you can redistribute it and/or modify
  10. * it under the terms of the GNU General Public License version 2 as
  11. * published by the Free Software Foundation.
  12. *
  13. * This program is distributed in the hope that it will be useful, but
  14. * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  15. * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
  16. * for more details.
  17. *
  18. * You should have received a copy of the GNU General Public License
  19. * along with this program; if not, write to the Free Software Foundation,
  20. * Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  21. */
  22. #include <linux/usb.h>
  23. #include <linux/pci.h>
  24. #include <linux/slab.h>
  25. #include <linux/dmapool.h>
  26. #include "xhci.h"
  27. /*
  28. * Allocates a generic ring segment from the ring pool, sets the dma address,
  29. * initializes the segment to zero, and sets the private next pointer to NULL.
  30. *
  31. * Section 4.11.1.1:
  32. * "All components of all Command and Transfer TRBs shall be initialized to '0'"
  33. */
  34. static struct xhci_segment *xhci_segment_alloc(struct xhci_hcd *xhci,
  35. unsigned int cycle_state, gfp_t flags)
  36. {
  37. struct xhci_segment *seg;
  38. dma_addr_t dma;
  39. int i;
  40. seg = kzalloc(sizeof *seg, flags);
  41. if (!seg)
  42. return NULL;
  43. seg->trbs = dma_pool_alloc(xhci->segment_pool, flags, &dma);
  44. if (!seg->trbs) {
  45. kfree(seg);
  46. return NULL;
  47. }
  48. memset(seg->trbs, 0, SEGMENT_SIZE);
  49. /* If the cycle state is 0, set the cycle bit to 1 for all the TRBs */
  50. if (cycle_state == 0) {
  51. for (i = 0; i < TRBS_PER_SEGMENT; i++)
  52. seg->trbs[i].link.control |= TRB_CYCLE;
  53. }
  54. seg->dma = dma;
  55. seg->next = NULL;
  56. return seg;
  57. }
  58. static void xhci_segment_free(struct xhci_hcd *xhci, struct xhci_segment *seg)
  59. {
  60. if (seg->trbs) {
  61. dma_pool_free(xhci->segment_pool, seg->trbs, seg->dma);
  62. seg->trbs = NULL;
  63. }
  64. kfree(seg);
  65. }
  66. static void xhci_free_segments_for_ring(struct xhci_hcd *xhci,
  67. struct xhci_segment *first)
  68. {
  69. struct xhci_segment *seg;
  70. seg = first->next;
  71. while (seg != first) {
  72. struct xhci_segment *next = seg->next;
  73. xhci_segment_free(xhci, seg);
  74. seg = next;
  75. }
  76. xhci_segment_free(xhci, first);
  77. }
  78. /*
  79. * Make the prev segment point to the next segment.
  80. *
  81. * Change the last TRB in the prev segment to be a Link TRB which points to the
  82. * DMA address of the next segment. The caller needs to set any Link TRB
  83. * related flags, such as End TRB, Toggle Cycle, and no snoop.
  84. */
  85. static void xhci_link_segments(struct xhci_hcd *xhci, struct xhci_segment *prev,
  86. struct xhci_segment *next, enum xhci_ring_type type)
  87. {
  88. u32 val;
  89. if (!prev || !next)
  90. return;
  91. prev->next = next;
  92. if (type != TYPE_EVENT) {
  93. prev->trbs[TRBS_PER_SEGMENT-1].link.segment_ptr =
  94. cpu_to_le64(next->dma);
  95. /* Set the last TRB in the segment to have a TRB type ID of Link TRB */
  96. val = le32_to_cpu(prev->trbs[TRBS_PER_SEGMENT-1].link.control);
  97. val &= ~TRB_TYPE_BITMASK;
  98. val |= TRB_TYPE(TRB_LINK);
  99. /* Always set the chain bit with 0.95 hardware */
  100. /* Set chain bit for isoc rings on AMD 0.96 host */
  101. if (xhci_link_trb_quirk(xhci) ||
  102. (type == TYPE_ISOC &&
  103. (xhci->quirks & XHCI_AMD_0x96_HOST)))
  104. val |= TRB_CHAIN;
  105. prev->trbs[TRBS_PER_SEGMENT-1].link.control = cpu_to_le32(val);
  106. }
  107. }
  108. /*
  109. * Link the ring to the new segments.
  110. * Set Toggle Cycle for the new ring if needed.
  111. */
  112. static void xhci_link_rings(struct xhci_hcd *xhci, struct xhci_ring *ring,
  113. struct xhci_segment *first, struct xhci_segment *last,
  114. unsigned int num_segs)
  115. {
  116. struct xhci_segment *next;
  117. if (!ring || !first || !last)
  118. return;
  119. next = ring->enq_seg->next;
  120. xhci_link_segments(xhci, ring->enq_seg, first, ring->type);
  121. xhci_link_segments(xhci, last, next, ring->type);
  122. ring->num_segs += num_segs;
  123. ring->num_trbs_free += (TRBS_PER_SEGMENT - 1) * num_segs;
  124. if (ring->type != TYPE_EVENT && ring->enq_seg == ring->last_seg) {
  125. ring->last_seg->trbs[TRBS_PER_SEGMENT-1].link.control
  126. &= ~cpu_to_le32(LINK_TOGGLE);
  127. last->trbs[TRBS_PER_SEGMENT-1].link.control
  128. |= cpu_to_le32(LINK_TOGGLE);
  129. ring->last_seg = last;
  130. }
  131. }
  132. /* XXX: Do we need the hcd structure in all these functions? */
  133. void xhci_ring_free(struct xhci_hcd *xhci, struct xhci_ring *ring)
  134. {
  135. if (!ring)
  136. return;
  137. if (ring->first_seg)
  138. xhci_free_segments_for_ring(xhci, ring->first_seg);
  139. kfree(ring);
  140. }
  141. static void xhci_initialize_ring_info(struct xhci_ring *ring,
  142. unsigned int cycle_state)
  143. {
  144. /* The ring is empty, so the enqueue pointer == dequeue pointer */
  145. ring->enqueue = ring->first_seg->trbs;
  146. ring->enq_seg = ring->first_seg;
  147. ring->dequeue = ring->enqueue;
  148. ring->deq_seg = ring->first_seg;
  149. /* The ring is initialized to 0. The producer must write 1 to the cycle
  150. * bit to handover ownership of the TRB, so PCS = 1. The consumer must
  151. * compare CCS to the cycle bit to check ownership, so CCS = 1.
  152. *
  153. * New rings are initialized with cycle state equal to 1; if we are
  154. * handling ring expansion, set the cycle state equal to the old ring.
  155. */
  156. ring->cycle_state = cycle_state;
  157. /* Not necessary for new rings, but needed for re-initialized rings */
  158. ring->enq_updates = 0;
  159. ring->deq_updates = 0;
  160. /*
  161. * Each segment has a link TRB, and leave an extra TRB for SW
  162. * accounting purpose
  163. */
  164. ring->num_trbs_free = ring->num_segs * (TRBS_PER_SEGMENT - 1) - 1;
  165. }
  166. /* Allocate segments and link them for a ring */
  167. static int xhci_alloc_segments_for_ring(struct xhci_hcd *xhci,
  168. struct xhci_segment **first, struct xhci_segment **last,
  169. unsigned int num_segs, unsigned int cycle_state,
  170. enum xhci_ring_type type, gfp_t flags)
  171. {
  172. struct xhci_segment *prev;
  173. prev = xhci_segment_alloc(xhci, cycle_state, flags);
  174. if (!prev)
  175. return -ENOMEM;
  176. num_segs--;
  177. *first = prev;
  178. while (num_segs > 0) {
  179. struct xhci_segment *next;
  180. next = xhci_segment_alloc(xhci, cycle_state, flags);
  181. if (!next) {
  182. xhci_free_segments_for_ring(xhci, *first);
  183. return -ENOMEM;
  184. }
  185. xhci_link_segments(xhci, prev, next, type);
  186. prev = next;
  187. num_segs--;
  188. }
  189. xhci_link_segments(xhci, prev, *first, type);
  190. *last = prev;
  191. return 0;
  192. }
  193. /**
  194. * Create a new ring with zero or more segments.
  195. *
  196. * Link each segment together into a ring.
  197. * Set the end flag and the cycle toggle bit on the last segment.
  198. * See section 4.9.1 and figures 15 and 16.
  199. */
  200. static struct xhci_ring *xhci_ring_alloc(struct xhci_hcd *xhci,
  201. unsigned int num_segs, unsigned int cycle_state,
  202. enum xhci_ring_type type, gfp_t flags)
  203. {
  204. struct xhci_ring *ring;
  205. int ret;
  206. ring = kzalloc(sizeof *(ring), flags);
  207. if (!ring)
  208. return NULL;
  209. ring->num_segs = num_segs;
  210. INIT_LIST_HEAD(&ring->td_list);
  211. ring->type = type;
  212. if (num_segs == 0)
  213. return ring;
  214. ret = xhci_alloc_segments_for_ring(xhci, &ring->first_seg,
  215. &ring->last_seg, num_segs, cycle_state, type, flags);
  216. if (ret)
  217. goto fail;
  218. /* Only event ring does not use link TRB */
  219. if (type != TYPE_EVENT) {
  220. /* See section 4.9.2.1 and 6.4.4.1 */
  221. ring->last_seg->trbs[TRBS_PER_SEGMENT - 1].link.control |=
  222. cpu_to_le32(LINK_TOGGLE);
  223. }
  224. xhci_initialize_ring_info(ring, cycle_state);
  225. return ring;
  226. fail:
  227. xhci_ring_free(xhci, ring);
  228. return NULL;
  229. }
  230. void xhci_free_or_cache_endpoint_ring(struct xhci_hcd *xhci,
  231. struct xhci_virt_device *virt_dev,
  232. unsigned int ep_index)
  233. {
  234. int rings_cached;
  235. rings_cached = virt_dev->num_rings_cached;
  236. if (rings_cached < XHCI_MAX_RINGS_CACHED) {
  237. virt_dev->ring_cache[rings_cached] =
  238. virt_dev->eps[ep_index].ring;
  239. virt_dev->num_rings_cached++;
  240. xhci_dbg(xhci, "Cached old ring, "
  241. "%d ring%s cached\n",
  242. virt_dev->num_rings_cached,
  243. (virt_dev->num_rings_cached > 1) ? "s" : "");
  244. } else {
  245. xhci_ring_free(xhci, virt_dev->eps[ep_index].ring);
  246. xhci_dbg(xhci, "Ring cache full (%d rings), "
  247. "freeing ring\n",
  248. virt_dev->num_rings_cached);
  249. }
  250. virt_dev->eps[ep_index].ring = NULL;
  251. }
  252. /* Zero an endpoint ring (except for link TRBs) and move the enqueue and dequeue
  253. * pointers to the beginning of the ring.
  254. */
  255. static void xhci_reinit_cached_ring(struct xhci_hcd *xhci,
  256. struct xhci_ring *ring, unsigned int cycle_state,
  257. enum xhci_ring_type type)
  258. {
  259. struct xhci_segment *seg = ring->first_seg;
  260. int i;
  261. do {
  262. memset(seg->trbs, 0,
  263. sizeof(union xhci_trb)*TRBS_PER_SEGMENT);
  264. if (cycle_state == 0) {
  265. for (i = 0; i < TRBS_PER_SEGMENT; i++)
  266. seg->trbs[i].link.control |= TRB_CYCLE;
  267. }
  268. /* All endpoint rings have link TRBs */
  269. xhci_link_segments(xhci, seg, seg->next, type);
  270. seg = seg->next;
  271. } while (seg != ring->first_seg);
  272. ring->type = type;
  273. xhci_initialize_ring_info(ring, cycle_state);
  274. /* td list should be empty since all URBs have been cancelled,
  275. * but just in case...
  276. */
  277. INIT_LIST_HEAD(&ring->td_list);
  278. }
  279. /*
  280. * Expand an existing ring.
  281. * Look for a cached ring or allocate a new ring which has same segment numbers
  282. * and link the two rings.
  283. */
  284. int xhci_ring_expansion(struct xhci_hcd *xhci, struct xhci_ring *ring,
  285. unsigned int num_trbs, gfp_t flags)
  286. {
  287. struct xhci_segment *first;
  288. struct xhci_segment *last;
  289. unsigned int num_segs;
  290. unsigned int num_segs_needed;
  291. int ret;
  292. num_segs_needed = (num_trbs + (TRBS_PER_SEGMENT - 1) - 1) /
  293. (TRBS_PER_SEGMENT - 1);
  294. /* Allocate number of segments we needed, or double the ring size */
  295. num_segs = ring->num_segs > num_segs_needed ?
  296. ring->num_segs : num_segs_needed;
  297. ret = xhci_alloc_segments_for_ring(xhci, &first, &last,
  298. num_segs, ring->cycle_state, ring->type, flags);
  299. if (ret)
  300. return -ENOMEM;
  301. xhci_link_rings(xhci, ring, first, last, num_segs);
  302. xhci_dbg(xhci, "ring expansion succeed, now has %d segments\n",
  303. ring->num_segs);
  304. return 0;
  305. }
  306. #define CTX_SIZE(_hcc) (HCC_64BYTE_CONTEXT(_hcc) ? 64 : 32)
  307. static struct xhci_container_ctx *xhci_alloc_container_ctx(struct xhci_hcd *xhci,
  308. int type, gfp_t flags)
  309. {
  310. struct xhci_container_ctx *ctx = kzalloc(sizeof(*ctx), flags);
  311. if (!ctx)
  312. return NULL;
  313. BUG_ON((type != XHCI_CTX_TYPE_DEVICE) && (type != XHCI_CTX_TYPE_INPUT));
  314. ctx->type = type;
  315. ctx->size = HCC_64BYTE_CONTEXT(xhci->hcc_params) ? 2048 : 1024;
  316. if (type == XHCI_CTX_TYPE_INPUT)
  317. ctx->size += CTX_SIZE(xhci->hcc_params);
  318. ctx->bytes = dma_pool_alloc(xhci->device_pool, flags, &ctx->dma);
  319. memset(ctx->bytes, 0, ctx->size);
  320. return ctx;
  321. }
  322. static void xhci_free_container_ctx(struct xhci_hcd *xhci,
  323. struct xhci_container_ctx *ctx)
  324. {
  325. if (!ctx)
  326. return;
  327. dma_pool_free(xhci->device_pool, ctx->bytes, ctx->dma);
  328. kfree(ctx);
  329. }
  330. struct xhci_input_control_ctx *xhci_get_input_control_ctx(struct xhci_hcd *xhci,
  331. struct xhci_container_ctx *ctx)
  332. {
  333. BUG_ON(ctx->type != XHCI_CTX_TYPE_INPUT);
  334. return (struct xhci_input_control_ctx *)ctx->bytes;
  335. }
  336. struct xhci_slot_ctx *xhci_get_slot_ctx(struct xhci_hcd *xhci,
  337. struct xhci_container_ctx *ctx)
  338. {
  339. if (ctx->type == XHCI_CTX_TYPE_DEVICE)
  340. return (struct xhci_slot_ctx *)ctx->bytes;
  341. return (struct xhci_slot_ctx *)
  342. (ctx->bytes + CTX_SIZE(xhci->hcc_params));
  343. }
  344. struct xhci_ep_ctx *xhci_get_ep_ctx(struct xhci_hcd *xhci,
  345. struct xhci_container_ctx *ctx,
  346. unsigned int ep_index)
  347. {
  348. /* increment ep index by offset of start of ep ctx array */
  349. ep_index++;
  350. if (ctx->type == XHCI_CTX_TYPE_INPUT)
  351. ep_index++;
  352. return (struct xhci_ep_ctx *)
  353. (ctx->bytes + (ep_index * CTX_SIZE(xhci->hcc_params)));
  354. }
  355. /***************** Streams structures manipulation *************************/
  356. static void xhci_free_stream_ctx(struct xhci_hcd *xhci,
  357. unsigned int num_stream_ctxs,
  358. struct xhci_stream_ctx *stream_ctx, dma_addr_t dma)
  359. {
  360. struct pci_dev *pdev = to_pci_dev(xhci_to_hcd(xhci)->self.controller);
  361. if (num_stream_ctxs > MEDIUM_STREAM_ARRAY_SIZE)
  362. dma_free_coherent(&pdev->dev,
  363. sizeof(struct xhci_stream_ctx)*num_stream_ctxs,
  364. stream_ctx, dma);
  365. else if (num_stream_ctxs <= SMALL_STREAM_ARRAY_SIZE)
  366. return dma_pool_free(xhci->small_streams_pool,
  367. stream_ctx, dma);
  368. else
  369. return dma_pool_free(xhci->medium_streams_pool,
  370. stream_ctx, dma);
  371. }
  372. /*
  373. * The stream context array for each endpoint with bulk streams enabled can
  374. * vary in size, based on:
  375. * - how many streams the endpoint supports,
  376. * - the maximum primary stream array size the host controller supports,
  377. * - and how many streams the device driver asks for.
  378. *
  379. * The stream context array must be a power of 2, and can be as small as
  380. * 64 bytes or as large as 1MB.
  381. */
  382. static struct xhci_stream_ctx *xhci_alloc_stream_ctx(struct xhci_hcd *xhci,
  383. unsigned int num_stream_ctxs, dma_addr_t *dma,
  384. gfp_t mem_flags)
  385. {
  386. struct pci_dev *pdev = to_pci_dev(xhci_to_hcd(xhci)->self.controller);
  387. if (num_stream_ctxs > MEDIUM_STREAM_ARRAY_SIZE)
  388. return dma_alloc_coherent(&pdev->dev,
  389. sizeof(struct xhci_stream_ctx)*num_stream_ctxs,
  390. dma, mem_flags);
  391. else if (num_stream_ctxs <= SMALL_STREAM_ARRAY_SIZE)
  392. return dma_pool_alloc(xhci->small_streams_pool,
  393. mem_flags, dma);
  394. else
  395. return dma_pool_alloc(xhci->medium_streams_pool,
  396. mem_flags, dma);
  397. }
  398. struct xhci_ring *xhci_dma_to_transfer_ring(
  399. struct xhci_virt_ep *ep,
  400. u64 address)
  401. {
  402. if (ep->ep_state & EP_HAS_STREAMS)
  403. return radix_tree_lookup(&ep->stream_info->trb_address_map,
  404. address >> SEGMENT_SHIFT);
  405. return ep->ring;
  406. }
  407. /* Only use this when you know stream_info is valid */
  408. #ifdef CONFIG_USB_XHCI_HCD_DEBUGGING
  409. static struct xhci_ring *dma_to_stream_ring(
  410. struct xhci_stream_info *stream_info,
  411. u64 address)
  412. {
  413. return radix_tree_lookup(&stream_info->trb_address_map,
  414. address >> SEGMENT_SHIFT);
  415. }
  416. #endif /* CONFIG_USB_XHCI_HCD_DEBUGGING */
  417. struct xhci_ring *xhci_stream_id_to_ring(
  418. struct xhci_virt_device *dev,
  419. unsigned int ep_index,
  420. unsigned int stream_id)
  421. {
  422. struct xhci_virt_ep *ep = &dev->eps[ep_index];
  423. if (stream_id == 0)
  424. return ep->ring;
  425. if (!ep->stream_info)
  426. return NULL;
  427. if (stream_id > ep->stream_info->num_streams)
  428. return NULL;
  429. return ep->stream_info->stream_rings[stream_id];
  430. }
  431. #ifdef CONFIG_USB_XHCI_HCD_DEBUGGING
  432. static int xhci_test_radix_tree(struct xhci_hcd *xhci,
  433. unsigned int num_streams,
  434. struct xhci_stream_info *stream_info)
  435. {
  436. u32 cur_stream;
  437. struct xhci_ring *cur_ring;
  438. u64 addr;
  439. for (cur_stream = 1; cur_stream < num_streams; cur_stream++) {
  440. struct xhci_ring *mapped_ring;
  441. int trb_size = sizeof(union xhci_trb);
  442. cur_ring = stream_info->stream_rings[cur_stream];
  443. for (addr = cur_ring->first_seg->dma;
  444. addr < cur_ring->first_seg->dma + SEGMENT_SIZE;
  445. addr += trb_size) {
  446. mapped_ring = dma_to_stream_ring(stream_info, addr);
  447. if (cur_ring != mapped_ring) {
  448. xhci_warn(xhci, "WARN: DMA address 0x%08llx "
  449. "didn't map to stream ID %u; "
  450. "mapped to ring %p\n",
  451. (unsigned long long) addr,
  452. cur_stream,
  453. mapped_ring);
  454. return -EINVAL;
  455. }
  456. }
  457. /* One TRB after the end of the ring segment shouldn't return a
  458. * pointer to the current ring (although it may be a part of a
  459. * different ring).
  460. */
  461. mapped_ring = dma_to_stream_ring(stream_info, addr);
  462. if (mapped_ring != cur_ring) {
  463. /* One TRB before should also fail */
  464. addr = cur_ring->first_seg->dma - trb_size;
  465. mapped_ring = dma_to_stream_ring(stream_info, addr);
  466. }
  467. if (mapped_ring == cur_ring) {
  468. xhci_warn(xhci, "WARN: Bad DMA address 0x%08llx "
  469. "mapped to valid stream ID %u; "
  470. "mapped ring = %p\n",
  471. (unsigned long long) addr,
  472. cur_stream,
  473. mapped_ring);
  474. return -EINVAL;
  475. }
  476. }
  477. return 0;
  478. }
  479. #endif /* CONFIG_USB_XHCI_HCD_DEBUGGING */
  480. /*
  481. * Change an endpoint's internal structure so it supports stream IDs. The
  482. * number of requested streams includes stream 0, which cannot be used by device
  483. * drivers.
  484. *
  485. * The number of stream contexts in the stream context array may be bigger than
  486. * the number of streams the driver wants to use. This is because the number of
  487. * stream context array entries must be a power of two.
  488. *
  489. * We need a radix tree for mapping physical addresses of TRBs to which stream
  490. * ID they belong to. We need to do this because the host controller won't tell
  491. * us which stream ring the TRB came from. We could store the stream ID in an
  492. * event data TRB, but that doesn't help us for the cancellation case, since the
  493. * endpoint may stop before it reaches that event data TRB.
  494. *
  495. * The radix tree maps the upper portion of the TRB DMA address to a ring
  496. * segment that has the same upper portion of DMA addresses. For example, say I
  497. * have segments of size 1KB, that are always 64-byte aligned. A segment may
  498. * start at 0x10c91000 and end at 0x10c913f0. If I use the upper 10 bits, the
  499. * key to the stream ID is 0x43244. I can use the DMA address of the TRB to
  500. * pass the radix tree a key to get the right stream ID:
  501. *
  502. * 0x10c90fff >> 10 = 0x43243
  503. * 0x10c912c0 >> 10 = 0x43244
  504. * 0x10c91400 >> 10 = 0x43245
  505. *
  506. * Obviously, only those TRBs with DMA addresses that are within the segment
  507. * will make the radix tree return the stream ID for that ring.
  508. *
  509. * Caveats for the radix tree:
  510. *
  511. * The radix tree uses an unsigned long as a key pair. On 32-bit systems, an
  512. * unsigned long will be 32-bits; on a 64-bit system an unsigned long will be
  513. * 64-bits. Since we only request 32-bit DMA addresses, we can use that as the
  514. * key on 32-bit or 64-bit systems (it would also be fine if we asked for 64-bit
  515. * PCI DMA addresses on a 64-bit system). There might be a problem on 32-bit
  516. * extended systems (where the DMA address can be bigger than 32-bits),
  517. * if we allow the PCI dma mask to be bigger than 32-bits. So don't do that.
  518. */
  519. struct xhci_stream_info *xhci_alloc_stream_info(struct xhci_hcd *xhci,
  520. unsigned int num_stream_ctxs,
  521. unsigned int num_streams, gfp_t mem_flags)
  522. {
  523. struct xhci_stream_info *stream_info;
  524. u32 cur_stream;
  525. struct xhci_ring *cur_ring;
  526. unsigned long key;
  527. u64 addr;
  528. int ret;
  529. xhci_dbg(xhci, "Allocating %u streams and %u "
  530. "stream context array entries.\n",
  531. num_streams, num_stream_ctxs);
  532. if (xhci->cmd_ring_reserved_trbs == MAX_RSVD_CMD_TRBS) {
  533. xhci_dbg(xhci, "Command ring has no reserved TRBs available\n");
  534. return NULL;
  535. }
  536. xhci->cmd_ring_reserved_trbs++;
  537. stream_info = kzalloc(sizeof(struct xhci_stream_info), mem_flags);
  538. if (!stream_info)
  539. goto cleanup_trbs;
  540. stream_info->num_streams = num_streams;
  541. stream_info->num_stream_ctxs = num_stream_ctxs;
  542. /* Initialize the array of virtual pointers to stream rings. */
  543. stream_info->stream_rings = kzalloc(
  544. sizeof(struct xhci_ring *)*num_streams,
  545. mem_flags);
  546. if (!stream_info->stream_rings)
  547. goto cleanup_info;
  548. /* Initialize the array of DMA addresses for stream rings for the HW. */
  549. stream_info->stream_ctx_array = xhci_alloc_stream_ctx(xhci,
  550. num_stream_ctxs, &stream_info->ctx_array_dma,
  551. mem_flags);
  552. if (!stream_info->stream_ctx_array)
  553. goto cleanup_ctx;
  554. memset(stream_info->stream_ctx_array, 0,
  555. sizeof(struct xhci_stream_ctx)*num_stream_ctxs);
  556. /* Allocate everything needed to free the stream rings later */
  557. stream_info->free_streams_command =
  558. xhci_alloc_command(xhci, true, true, mem_flags);
  559. if (!stream_info->free_streams_command)
  560. goto cleanup_ctx;
  561. INIT_RADIX_TREE(&stream_info->trb_address_map, GFP_ATOMIC);
  562. /* Allocate rings for all the streams that the driver will use,
  563. * and add their segment DMA addresses to the radix tree.
  564. * Stream 0 is reserved.
  565. */
  566. for (cur_stream = 1; cur_stream < num_streams; cur_stream++) {
  567. stream_info->stream_rings[cur_stream] =
  568. xhci_ring_alloc(xhci, 2, 1, TYPE_STREAM, mem_flags);
  569. cur_ring = stream_info->stream_rings[cur_stream];
  570. if (!cur_ring)
  571. goto cleanup_rings;
  572. cur_ring->stream_id = cur_stream;
  573. /* Set deq ptr, cycle bit, and stream context type */
  574. addr = cur_ring->first_seg->dma |
  575. SCT_FOR_CTX(SCT_PRI_TR) |
  576. cur_ring->cycle_state;
  577. stream_info->stream_ctx_array[cur_stream].stream_ring =
  578. cpu_to_le64(addr);
  579. xhci_dbg(xhci, "Setting stream %d ring ptr to 0x%08llx\n",
  580. cur_stream, (unsigned long long) addr);
  581. key = (unsigned long)
  582. (cur_ring->first_seg->dma >> SEGMENT_SHIFT);
  583. ret = radix_tree_insert(&stream_info->trb_address_map,
  584. key, cur_ring);
  585. if (ret) {
  586. xhci_ring_free(xhci, cur_ring);
  587. stream_info->stream_rings[cur_stream] = NULL;
  588. goto cleanup_rings;
  589. }
  590. }
  591. /* Leave the other unused stream ring pointers in the stream context
  592. * array initialized to zero. This will cause the xHC to give us an
  593. * error if the device asks for a stream ID we don't have setup (if it
  594. * was any other way, the host controller would assume the ring is
  595. * "empty" and wait forever for data to be queued to that stream ID).
  596. */
  597. #if XHCI_DEBUG
  598. /* Do a little test on the radix tree to make sure it returns the
  599. * correct values.
  600. */
  601. if (xhci_test_radix_tree(xhci, num_streams, stream_info))
  602. goto cleanup_rings;
  603. #endif
  604. return stream_info;
  605. cleanup_rings:
  606. for (cur_stream = 1; cur_stream < num_streams; cur_stream++) {
  607. cur_ring = stream_info->stream_rings[cur_stream];
  608. if (cur_ring) {
  609. addr = cur_ring->first_seg->dma;
  610. radix_tree_delete(&stream_info->trb_address_map,
  611. addr >> SEGMENT_SHIFT);
  612. xhci_ring_free(xhci, cur_ring);
  613. stream_info->stream_rings[cur_stream] = NULL;
  614. }
  615. }
  616. xhci_free_command(xhci, stream_info->free_streams_command);
  617. cleanup_ctx:
  618. kfree(stream_info->stream_rings);
  619. cleanup_info:
  620. kfree(stream_info);
  621. cleanup_trbs:
  622. xhci->cmd_ring_reserved_trbs--;
  623. return NULL;
  624. }
  625. /*
  626. * Sets the MaxPStreams field and the Linear Stream Array field.
  627. * Sets the dequeue pointer to the stream context array.
  628. */
  629. void xhci_setup_streams_ep_input_ctx(struct xhci_hcd *xhci,
  630. struct xhci_ep_ctx *ep_ctx,
  631. struct xhci_stream_info *stream_info)
  632. {
  633. u32 max_primary_streams;
  634. /* MaxPStreams is the number of stream context array entries, not the
  635. * number we're actually using. Must be in 2^(MaxPstreams + 1) format.
  636. * fls(0) = 0, fls(0x1) = 1, fls(0x10) = 2, fls(0x100) = 3, etc.
  637. */
  638. max_primary_streams = fls(stream_info->num_stream_ctxs) - 2;
  639. xhci_dbg(xhci, "Setting number of stream ctx array entries to %u\n",
  640. 1 << (max_primary_streams + 1));
  641. ep_ctx->ep_info &= cpu_to_le32(~EP_MAXPSTREAMS_MASK);
  642. ep_ctx->ep_info |= cpu_to_le32(EP_MAXPSTREAMS(max_primary_streams)
  643. | EP_HAS_LSA);
  644. ep_ctx->deq = cpu_to_le64(stream_info->ctx_array_dma);
  645. }
  646. /*
  647. * Sets the MaxPStreams field and the Linear Stream Array field to 0.
  648. * Reinstalls the "normal" endpoint ring (at its previous dequeue mark,
  649. * not at the beginning of the ring).
  650. */
  651. void xhci_setup_no_streams_ep_input_ctx(struct xhci_hcd *xhci,
  652. struct xhci_ep_ctx *ep_ctx,
  653. struct xhci_virt_ep *ep)
  654. {
  655. dma_addr_t addr;
  656. ep_ctx->ep_info &= cpu_to_le32(~(EP_MAXPSTREAMS_MASK | EP_HAS_LSA));
  657. addr = xhci_trb_virt_to_dma(ep->ring->deq_seg, ep->ring->dequeue);
  658. ep_ctx->deq = cpu_to_le64(addr | ep->ring->cycle_state);
  659. }
  660. /* Frees all stream contexts associated with the endpoint,
  661. *
  662. * Caller should fix the endpoint context streams fields.
  663. */
  664. void xhci_free_stream_info(struct xhci_hcd *xhci,
  665. struct xhci_stream_info *stream_info)
  666. {
  667. int cur_stream;
  668. struct xhci_ring *cur_ring;
  669. dma_addr_t addr;
  670. if (!stream_info)
  671. return;
  672. for (cur_stream = 1; cur_stream < stream_info->num_streams;
  673. cur_stream++) {
  674. cur_ring = stream_info->stream_rings[cur_stream];
  675. if (cur_ring) {
  676. addr = cur_ring->first_seg->dma;
  677. radix_tree_delete(&stream_info->trb_address_map,
  678. addr >> SEGMENT_SHIFT);
  679. xhci_ring_free(xhci, cur_ring);
  680. stream_info->stream_rings[cur_stream] = NULL;
  681. }
  682. }
  683. xhci_free_command(xhci, stream_info->free_streams_command);
  684. xhci->cmd_ring_reserved_trbs--;
  685. if (stream_info->stream_ctx_array)
  686. xhci_free_stream_ctx(xhci,
  687. stream_info->num_stream_ctxs,
  688. stream_info->stream_ctx_array,
  689. stream_info->ctx_array_dma);
  690. if (stream_info)
  691. kfree(stream_info->stream_rings);
  692. kfree(stream_info);
  693. }
  694. /***************** Device context manipulation *************************/
  695. static void xhci_init_endpoint_timer(struct xhci_hcd *xhci,
  696. struct xhci_virt_ep *ep)
  697. {
  698. init_timer(&ep->stop_cmd_timer);
  699. ep->stop_cmd_timer.data = (unsigned long) ep;
  700. ep->stop_cmd_timer.function = xhci_stop_endpoint_command_watchdog;
  701. ep->xhci = xhci;
  702. }
  703. static void xhci_free_tt_info(struct xhci_hcd *xhci,
  704. struct xhci_virt_device *virt_dev,
  705. int slot_id)
  706. {
  707. struct list_head *tt;
  708. struct list_head *tt_list_head;
  709. struct list_head *tt_next;
  710. struct xhci_tt_bw_info *tt_info;
  711. /* If the device never made it past the Set Address stage,
  712. * it may not have the real_port set correctly.
  713. */
  714. if (virt_dev->real_port == 0 ||
  715. virt_dev->real_port > HCS_MAX_PORTS(xhci->hcs_params1)) {
  716. xhci_dbg(xhci, "Bad real port.\n");
  717. return;
  718. }
  719. tt_list_head = &(xhci->rh_bw[virt_dev->real_port - 1].tts);
  720. if (list_empty(tt_list_head))
  721. return;
  722. list_for_each(tt, tt_list_head) {
  723. tt_info = list_entry(tt, struct xhci_tt_bw_info, tt_list);
  724. if (tt_info->slot_id == slot_id)
  725. break;
  726. }
  727. /* Cautionary measure in case the hub was disconnected before we
  728. * stored the TT information.
  729. */
  730. if (tt_info->slot_id != slot_id)
  731. return;
  732. tt_next = tt->next;
  733. tt_info = list_entry(tt, struct xhci_tt_bw_info,
  734. tt_list);
  735. /* Multi-TT hubs will have more than one entry */
  736. do {
  737. list_del(tt);
  738. kfree(tt_info);
  739. tt = tt_next;
  740. if (list_empty(tt_list_head))
  741. break;
  742. tt_next = tt->next;
  743. tt_info = list_entry(tt, struct xhci_tt_bw_info,
  744. tt_list);
  745. } while (tt_info->slot_id == slot_id);
  746. }
  747. int xhci_alloc_tt_info(struct xhci_hcd *xhci,
  748. struct xhci_virt_device *virt_dev,
  749. struct usb_device *hdev,
  750. struct usb_tt *tt, gfp_t mem_flags)
  751. {
  752. struct xhci_tt_bw_info *tt_info;
  753. unsigned int num_ports;
  754. int i, j;
  755. if (!tt->multi)
  756. num_ports = 1;
  757. else
  758. num_ports = hdev->maxchild;
  759. for (i = 0; i < num_ports; i++, tt_info++) {
  760. struct xhci_interval_bw_table *bw_table;
  761. tt_info = kzalloc(sizeof(*tt_info), mem_flags);
  762. if (!tt_info)
  763. goto free_tts;
  764. INIT_LIST_HEAD(&tt_info->tt_list);
  765. list_add(&tt_info->tt_list,
  766. &xhci->rh_bw[virt_dev->real_port - 1].tts);
  767. tt_info->slot_id = virt_dev->udev->slot_id;
  768. if (tt->multi)
  769. tt_info->ttport = i+1;
  770. bw_table = &tt_info->bw_table;
  771. for (j = 0; j < XHCI_MAX_INTERVAL; j++)
  772. INIT_LIST_HEAD(&bw_table->interval_bw[j].endpoints);
  773. }
  774. return 0;
  775. free_tts:
  776. xhci_free_tt_info(xhci, virt_dev, virt_dev->udev->slot_id);
  777. return -ENOMEM;
  778. }
  779. /* All the xhci_tds in the ring's TD list should be freed at this point.
  780. * Should be called with xhci->lock held if there is any chance the TT lists
  781. * will be manipulated by the configure endpoint, allocate device, or update
  782. * hub functions while this function is removing the TT entries from the list.
  783. */
  784. void xhci_free_virt_device(struct xhci_hcd *xhci, int slot_id)
  785. {
  786. struct xhci_virt_device *dev;
  787. int i;
  788. int old_active_eps = 0;
  789. /* Slot ID 0 is reserved */
  790. if (slot_id == 0 || !xhci->devs[slot_id])
  791. return;
  792. dev = xhci->devs[slot_id];
  793. xhci->dcbaa->dev_context_ptrs[slot_id] = 0;
  794. if (!dev)
  795. return;
  796. if (dev->tt_info)
  797. old_active_eps = dev->tt_info->active_eps;
  798. for (i = 0; i < 31; ++i) {
  799. if (dev->eps[i].ring)
  800. xhci_ring_free(xhci, dev->eps[i].ring);
  801. if (dev->eps[i].stream_info)
  802. xhci_free_stream_info(xhci,
  803. dev->eps[i].stream_info);
  804. /* Endpoints on the TT/root port lists should have been removed
  805. * when usb_disable_device() was called for the device.
  806. * We can't drop them anyway, because the udev might have gone
  807. * away by this point, and we can't tell what speed it was.
  808. */
  809. if (!list_empty(&dev->eps[i].bw_endpoint_list))
  810. xhci_warn(xhci, "Slot %u endpoint %u "
  811. "not removed from BW list!\n",
  812. slot_id, i);
  813. }
  814. /* If this is a hub, free the TT(s) from the TT list */
  815. xhci_free_tt_info(xhci, dev, slot_id);
  816. /* If necessary, update the number of active TTs on this root port */
  817. xhci_update_tt_active_eps(xhci, dev, old_active_eps);
  818. if (dev->ring_cache) {
  819. for (i = 0; i < dev->num_rings_cached; i++)
  820. xhci_ring_free(xhci, dev->ring_cache[i]);
  821. kfree(dev->ring_cache);
  822. }
  823. if (dev->in_ctx)
  824. xhci_free_container_ctx(xhci, dev->in_ctx);
  825. if (dev->out_ctx)
  826. xhci_free_container_ctx(xhci, dev->out_ctx);
  827. kfree(xhci->devs[slot_id]);
  828. xhci->devs[slot_id] = NULL;
  829. }
  830. int xhci_alloc_virt_device(struct xhci_hcd *xhci, int slot_id,
  831. struct usb_device *udev, gfp_t flags)
  832. {
  833. struct xhci_virt_device *dev;
  834. int i;
  835. /* Slot ID 0 is reserved */
  836. if (slot_id == 0 || xhci->devs[slot_id]) {
  837. xhci_warn(xhci, "Bad Slot ID %d\n", slot_id);
  838. return 0;
  839. }
  840. xhci->devs[slot_id] = kzalloc(sizeof(*xhci->devs[slot_id]), flags);
  841. if (!xhci->devs[slot_id])
  842. return 0;
  843. dev = xhci->devs[slot_id];
  844. /* Allocate the (output) device context that will be used in the HC. */
  845. dev->out_ctx = xhci_alloc_container_ctx(xhci, XHCI_CTX_TYPE_DEVICE, flags);
  846. if (!dev->out_ctx)
  847. goto fail;
  848. xhci_dbg(xhci, "Slot %d output ctx = 0x%llx (dma)\n", slot_id,
  849. (unsigned long long)dev->out_ctx->dma);
  850. /* Allocate the (input) device context for address device command */
  851. dev->in_ctx = xhci_alloc_container_ctx(xhci, XHCI_CTX_TYPE_INPUT, flags);
  852. if (!dev->in_ctx)
  853. goto fail;
  854. xhci_dbg(xhci, "Slot %d input ctx = 0x%llx (dma)\n", slot_id,
  855. (unsigned long long)dev->in_ctx->dma);
  856. /* Initialize the cancellation list and watchdog timers for each ep */
  857. for (i = 0; i < 31; i++) {
  858. xhci_init_endpoint_timer(xhci, &dev->eps[i]);
  859. INIT_LIST_HEAD(&dev->eps[i].cancelled_td_list);
  860. INIT_LIST_HEAD(&dev->eps[i].bw_endpoint_list);
  861. }
  862. /* Allocate endpoint 0 ring */
  863. dev->eps[0].ring = xhci_ring_alloc(xhci, 2, 1, TYPE_CTRL, flags);
  864. if (!dev->eps[0].ring)
  865. goto fail;
  866. /* Allocate pointers to the ring cache */
  867. dev->ring_cache = kzalloc(
  868. sizeof(struct xhci_ring *)*XHCI_MAX_RINGS_CACHED,
  869. flags);
  870. if (!dev->ring_cache)
  871. goto fail;
  872. dev->num_rings_cached = 0;
  873. init_completion(&dev->cmd_completion);
  874. INIT_LIST_HEAD(&dev->cmd_list);
  875. dev->udev = udev;
  876. /* Point to output device context in dcbaa. */
  877. xhci->dcbaa->dev_context_ptrs[slot_id] = cpu_to_le64(dev->out_ctx->dma);
  878. xhci_dbg(xhci, "Set slot id %d dcbaa entry %p to 0x%llx\n",
  879. slot_id,
  880. &xhci->dcbaa->dev_context_ptrs[slot_id],
  881. le64_to_cpu(xhci->dcbaa->dev_context_ptrs[slot_id]));
  882. return 1;
  883. fail:
  884. xhci_free_virt_device(xhci, slot_id);
  885. return 0;
  886. }
  887. void xhci_copy_ep0_dequeue_into_input_ctx(struct xhci_hcd *xhci,
  888. struct usb_device *udev)
  889. {
  890. struct xhci_virt_device *virt_dev;
  891. struct xhci_ep_ctx *ep0_ctx;
  892. struct xhci_ring *ep_ring;
  893. virt_dev = xhci->devs[udev->slot_id];
  894. ep0_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, 0);
  895. ep_ring = virt_dev->eps[0].ring;
  896. /*
  897. * FIXME we don't keep track of the dequeue pointer very well after a
  898. * Set TR dequeue pointer, so we're setting the dequeue pointer of the
  899. * host to our enqueue pointer. This should only be called after a
  900. * configured device has reset, so all control transfers should have
  901. * been completed or cancelled before the reset.
  902. */
  903. ep0_ctx->deq = cpu_to_le64(xhci_trb_virt_to_dma(ep_ring->enq_seg,
  904. ep_ring->enqueue)
  905. | ep_ring->cycle_state);
  906. }
  907. /*
  908. * The xHCI roothub may have ports of differing speeds in any order in the port
  909. * status registers. xhci->port_array provides an array of the port speed for
  910. * each offset into the port status registers.
  911. *
  912. * The xHCI hardware wants to know the roothub port number that the USB device
  913. * is attached to (or the roothub port its ancestor hub is attached to). All we
  914. * know is the index of that port under either the USB 2.0 or the USB 3.0
  915. * roothub, but that doesn't give us the real index into the HW port status
  916. * registers. Scan through the xHCI roothub port array, looking for the Nth
  917. * entry of the correct port speed. Return the port number of that entry.
  918. */
  919. static u32 xhci_find_real_port_number(struct xhci_hcd *xhci,
  920. struct usb_device *udev)
  921. {
  922. struct usb_device *top_dev;
  923. unsigned int num_similar_speed_ports;
  924. unsigned int faked_port_num;
  925. int i;
  926. for (top_dev = udev; top_dev->parent && top_dev->parent->parent;
  927. top_dev = top_dev->parent)
  928. /* Found device below root hub */;
  929. faked_port_num = top_dev->portnum;
  930. for (i = 0, num_similar_speed_ports = 0;
  931. i < HCS_MAX_PORTS(xhci->hcs_params1); i++) {
  932. u8 port_speed = xhci->port_array[i];
  933. /*
  934. * Skip ports that don't have known speeds, or have duplicate
  935. * Extended Capabilities port speed entries.
  936. */
  937. if (port_speed == 0 || port_speed == DUPLICATE_ENTRY)
  938. continue;
  939. /*
  940. * USB 3.0 ports are always under a USB 3.0 hub. USB 2.0 and
  941. * 1.1 ports are under the USB 2.0 hub. If the port speed
  942. * matches the device speed, it's a similar speed port.
  943. */
  944. if ((port_speed == 0x03) == (udev->speed == USB_SPEED_SUPER))
  945. num_similar_speed_ports++;
  946. if (num_similar_speed_ports == faked_port_num)
  947. /* Roothub ports are numbered from 1 to N */
  948. return i+1;
  949. }
  950. return 0;
  951. }
  952. /* Setup an xHCI virtual device for a Set Address command */
  953. int xhci_setup_addressable_virt_dev(struct xhci_hcd *xhci, struct usb_device *udev)
  954. {
  955. struct xhci_virt_device *dev;
  956. struct xhci_ep_ctx *ep0_ctx;
  957. struct xhci_slot_ctx *slot_ctx;
  958. u32 port_num;
  959. struct usb_device *top_dev;
  960. dev = xhci->devs[udev->slot_id];
  961. /* Slot ID 0 is reserved */
  962. if (udev->slot_id == 0 || !dev) {
  963. xhci_warn(xhci, "Slot ID %d is not assigned to this device\n",
  964. udev->slot_id);
  965. return -EINVAL;
  966. }
  967. ep0_ctx = xhci_get_ep_ctx(xhci, dev->in_ctx, 0);
  968. slot_ctx = xhci_get_slot_ctx(xhci, dev->in_ctx);
  969. /* 3) Only the control endpoint is valid - one endpoint context */
  970. slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(1) | udev->route);
  971. switch (udev->speed) {
  972. case USB_SPEED_SUPER:
  973. slot_ctx->dev_info |= cpu_to_le32(SLOT_SPEED_SS);
  974. break;
  975. case USB_SPEED_HIGH:
  976. slot_ctx->dev_info |= cpu_to_le32(SLOT_SPEED_HS);
  977. break;
  978. case USB_SPEED_FULL:
  979. slot_ctx->dev_info |= cpu_to_le32(SLOT_SPEED_FS);
  980. break;
  981. case USB_SPEED_LOW:
  982. slot_ctx->dev_info |= cpu_to_le32(SLOT_SPEED_LS);
  983. break;
  984. case USB_SPEED_WIRELESS:
  985. xhci_dbg(xhci, "FIXME xHCI doesn't support wireless speeds\n");
  986. return -EINVAL;
  987. break;
  988. default:
  989. /* Speed was set earlier, this shouldn't happen. */
  990. BUG();
  991. }
  992. /* Find the root hub port this device is under */
  993. port_num = xhci_find_real_port_number(xhci, udev);
  994. if (!port_num)
  995. return -EINVAL;
  996. slot_ctx->dev_info2 |= cpu_to_le32(ROOT_HUB_PORT(port_num));
  997. /* Set the port number in the virtual_device to the faked port number */
  998. for (top_dev = udev; top_dev->parent && top_dev->parent->parent;
  999. top_dev = top_dev->parent)
  1000. /* Found device below root hub */;
  1001. dev->fake_port = top_dev->portnum;
  1002. dev->real_port = port_num;
  1003. xhci_dbg(xhci, "Set root hub portnum to %d\n", port_num);
  1004. xhci_dbg(xhci, "Set fake root hub portnum to %d\n", dev->fake_port);
  1005. /* Find the right bandwidth table that this device will be a part of.
  1006. * If this is a full speed device attached directly to a root port (or a
  1007. * decendent of one), it counts as a primary bandwidth domain, not a
  1008. * secondary bandwidth domain under a TT. An xhci_tt_info structure
  1009. * will never be created for the HS root hub.
  1010. */
  1011. if (!udev->tt || !udev->tt->hub->parent) {
  1012. dev->bw_table = &xhci->rh_bw[port_num - 1].bw_table;
  1013. } else {
  1014. struct xhci_root_port_bw_info *rh_bw;
  1015. struct xhci_tt_bw_info *tt_bw;
  1016. rh_bw = &xhci->rh_bw[port_num - 1];
  1017. /* Find the right TT. */
  1018. list_for_each_entry(tt_bw, &rh_bw->tts, tt_list) {
  1019. if (tt_bw->slot_id != udev->tt->hub->slot_id)
  1020. continue;
  1021. if (!dev->udev->tt->multi ||
  1022. (udev->tt->multi &&
  1023. tt_bw->ttport == dev->udev->ttport)) {
  1024. dev->bw_table = &tt_bw->bw_table;
  1025. dev->tt_info = tt_bw;
  1026. break;
  1027. }
  1028. }
  1029. if (!dev->tt_info)
  1030. xhci_warn(xhci, "WARN: Didn't find a matching TT\n");
  1031. }
  1032. /* Is this a LS/FS device under an external HS hub? */
  1033. if (udev->tt && udev->tt->hub->parent) {
  1034. slot_ctx->tt_info = cpu_to_le32(udev->tt->hub->slot_id |
  1035. (udev->ttport << 8));
  1036. if (udev->tt->multi)
  1037. slot_ctx->dev_info |= cpu_to_le32(DEV_MTT);
  1038. }
  1039. xhci_dbg(xhci, "udev->tt = %p\n", udev->tt);
  1040. xhci_dbg(xhci, "udev->ttport = 0x%x\n", udev->ttport);
  1041. /* Step 4 - ring already allocated */
  1042. /* Step 5 */
  1043. ep0_ctx->ep_info2 = cpu_to_le32(EP_TYPE(CTRL_EP));
  1044. /*
  1045. * XXX: Not sure about wireless USB devices.
  1046. */
  1047. switch (udev->speed) {
  1048. case USB_SPEED_SUPER:
  1049. ep0_ctx->ep_info2 |= cpu_to_le32(MAX_PACKET(512));
  1050. break;
  1051. case USB_SPEED_HIGH:
  1052. /* USB core guesses at a 64-byte max packet first for FS devices */
  1053. case USB_SPEED_FULL:
  1054. ep0_ctx->ep_info2 |= cpu_to_le32(MAX_PACKET(64));
  1055. break;
  1056. case USB_SPEED_LOW:
  1057. ep0_ctx->ep_info2 |= cpu_to_le32(MAX_PACKET(8));
  1058. break;
  1059. case USB_SPEED_WIRELESS:
  1060. xhci_dbg(xhci, "FIXME xHCI doesn't support wireless speeds\n");
  1061. return -EINVAL;
  1062. break;
  1063. default:
  1064. /* New speed? */
  1065. BUG();
  1066. }
  1067. /* EP 0 can handle "burst" sizes of 1, so Max Burst Size field is 0 */
  1068. ep0_ctx->ep_info2 |= cpu_to_le32(MAX_BURST(0) | ERROR_COUNT(3));
  1069. ep0_ctx->deq = cpu_to_le64(dev->eps[0].ring->first_seg->dma |
  1070. dev->eps[0].ring->cycle_state);
  1071. /* Steps 7 and 8 were done in xhci_alloc_virt_device() */
  1072. return 0;
  1073. }
  1074. /*
  1075. * Convert interval expressed as 2^(bInterval - 1) == interval into
  1076. * straight exponent value 2^n == interval.
  1077. *
  1078. */
  1079. static unsigned int xhci_parse_exponent_interval(struct usb_device *udev,
  1080. struct usb_host_endpoint *ep)
  1081. {
  1082. unsigned int interval;
  1083. interval = clamp_val(ep->desc.bInterval, 1, 16) - 1;
  1084. if (interval != ep->desc.bInterval - 1)
  1085. dev_warn(&udev->dev,
  1086. "ep %#x - rounding interval to %d %sframes\n",
  1087. ep->desc.bEndpointAddress,
  1088. 1 << interval,
  1089. udev->speed == USB_SPEED_FULL ? "" : "micro");
  1090. if (udev->speed == USB_SPEED_FULL) {
  1091. /*
  1092. * Full speed isoc endpoints specify interval in frames,
  1093. * not microframes. We are using microframes everywhere,
  1094. * so adjust accordingly.
  1095. */
  1096. interval += 3; /* 1 frame = 2^3 uframes */
  1097. }
  1098. return interval;
  1099. }
  1100. /*
  1101. * Convert bInterval expressed in microframes (in 1-255 range) to exponent of
  1102. * microframes, rounded down to nearest power of 2.
  1103. */
  1104. static unsigned int xhci_microframes_to_exponent(struct usb_device *udev,
  1105. struct usb_host_endpoint *ep, unsigned int desc_interval,
  1106. unsigned int min_exponent, unsigned int max_exponent)
  1107. {
  1108. unsigned int interval;
  1109. interval = fls(desc_interval) - 1;
  1110. interval = clamp_val(interval, min_exponent, max_exponent);
  1111. if ((1 << interval) != desc_interval)
  1112. dev_warn(&udev->dev,
  1113. "ep %#x - rounding interval to %d microframes, ep desc says %d microframes\n",
  1114. ep->desc.bEndpointAddress,
  1115. 1 << interval,
  1116. desc_interval);
  1117. return interval;
  1118. }
  1119. static unsigned int xhci_parse_microframe_interval(struct usb_device *udev,
  1120. struct usb_host_endpoint *ep)
  1121. {
  1122. return xhci_microframes_to_exponent(udev, ep,
  1123. ep->desc.bInterval, 0, 15);
  1124. }
  1125. static unsigned int xhci_parse_frame_interval(struct usb_device *udev,
  1126. struct usb_host_endpoint *ep)
  1127. {
  1128. return xhci_microframes_to_exponent(udev, ep,
  1129. ep->desc.bInterval * 8, 3, 10);
  1130. }
  1131. /* Return the polling or NAK interval.
  1132. *
  1133. * The polling interval is expressed in "microframes". If xHCI's Interval field
  1134. * is set to N, it will service the endpoint every 2^(Interval)*125us.
  1135. *
  1136. * The NAK interval is one NAK per 1 to 255 microframes, or no NAKs if interval
  1137. * is set to 0.
  1138. */
  1139. static unsigned int xhci_get_endpoint_interval(struct usb_device *udev,
  1140. struct usb_host_endpoint *ep)
  1141. {
  1142. unsigned int interval = 0;
  1143. switch (udev->speed) {
  1144. case USB_SPEED_HIGH:
  1145. /* Max NAK rate */
  1146. if (usb_endpoint_xfer_control(&ep->desc) ||
  1147. usb_endpoint_xfer_bulk(&ep->desc)) {
  1148. interval = xhci_parse_microframe_interval(udev, ep);
  1149. break;
  1150. }
  1151. /* Fall through - SS and HS isoc/int have same decoding */
  1152. case USB_SPEED_SUPER:
  1153. if (usb_endpoint_xfer_int(&ep->desc) ||
  1154. usb_endpoint_xfer_isoc(&ep->desc)) {
  1155. interval = xhci_parse_exponent_interval(udev, ep);
  1156. }
  1157. break;
  1158. case USB_SPEED_FULL:
  1159. if (usb_endpoint_xfer_isoc(&ep->desc)) {
  1160. interval = xhci_parse_exponent_interval(udev, ep);
  1161. break;
  1162. }
  1163. /*
  1164. * Fall through for interrupt endpoint interval decoding
  1165. * since it uses the same rules as low speed interrupt
  1166. * endpoints.
  1167. */
  1168. case USB_SPEED_LOW:
  1169. if (usb_endpoint_xfer_int(&ep->desc) ||
  1170. usb_endpoint_xfer_isoc(&ep->desc)) {
  1171. interval = xhci_parse_frame_interval(udev, ep);
  1172. }
  1173. break;
  1174. default:
  1175. BUG();
  1176. }
  1177. return EP_INTERVAL(interval);
  1178. }
  1179. /* The "Mult" field in the endpoint context is only set for SuperSpeed isoc eps.
  1180. * High speed endpoint descriptors can define "the number of additional
  1181. * transaction opportunities per microframe", but that goes in the Max Burst
  1182. * endpoint context field.
  1183. */
  1184. static u32 xhci_get_endpoint_mult(struct usb_device *udev,
  1185. struct usb_host_endpoint *ep)
  1186. {
  1187. if (udev->speed != USB_SPEED_SUPER ||
  1188. !usb_endpoint_xfer_isoc(&ep->desc))
  1189. return 0;
  1190. return ep->ss_ep_comp.bmAttributes;
  1191. }
  1192. static u32 xhci_get_endpoint_type(struct usb_device *udev,
  1193. struct usb_host_endpoint *ep)
  1194. {
  1195. int in;
  1196. u32 type;
  1197. in = usb_endpoint_dir_in(&ep->desc);
  1198. if (usb_endpoint_xfer_control(&ep->desc)) {
  1199. type = EP_TYPE(CTRL_EP);
  1200. } else if (usb_endpoint_xfer_bulk(&ep->desc)) {
  1201. if (in)
  1202. type = EP_TYPE(BULK_IN_EP);
  1203. else
  1204. type = EP_TYPE(BULK_OUT_EP);
  1205. } else if (usb_endpoint_xfer_isoc(&ep->desc)) {
  1206. if (in)
  1207. type = EP_TYPE(ISOC_IN_EP);
  1208. else
  1209. type = EP_TYPE(ISOC_OUT_EP);
  1210. } else if (usb_endpoint_xfer_int(&ep->desc)) {
  1211. if (in)
  1212. type = EP_TYPE(INT_IN_EP);
  1213. else
  1214. type = EP_TYPE(INT_OUT_EP);
  1215. } else {
  1216. BUG();
  1217. }
  1218. return type;
  1219. }
  1220. /* Return the maximum endpoint service interval time (ESIT) payload.
  1221. * Basically, this is the maxpacket size, multiplied by the burst size
  1222. * and mult size.
  1223. */
  1224. static u32 xhci_get_max_esit_payload(struct xhci_hcd *xhci,
  1225. struct usb_device *udev,
  1226. struct usb_host_endpoint *ep)
  1227. {
  1228. int max_burst;
  1229. int max_packet;
  1230. /* Only applies for interrupt or isochronous endpoints */
  1231. if (usb_endpoint_xfer_control(&ep->desc) ||
  1232. usb_endpoint_xfer_bulk(&ep->desc))
  1233. return 0;
  1234. if (udev->speed == USB_SPEED_SUPER)
  1235. return le16_to_cpu(ep->ss_ep_comp.wBytesPerInterval);
  1236. max_packet = GET_MAX_PACKET(usb_endpoint_maxp(&ep->desc));
  1237. max_burst = (usb_endpoint_maxp(&ep->desc) & 0x1800) >> 11;
  1238. /* A 0 in max burst means 1 transfer per ESIT */
  1239. return max_packet * (max_burst + 1);
  1240. }
  1241. /* Set up an endpoint with one ring segment. Do not allocate stream rings.
  1242. * Drivers will have to call usb_alloc_streams() to do that.
  1243. */
  1244. int xhci_endpoint_init(struct xhci_hcd *xhci,
  1245. struct xhci_virt_device *virt_dev,
  1246. struct usb_device *udev,
  1247. struct usb_host_endpoint *ep,
  1248. gfp_t mem_flags)
  1249. {
  1250. unsigned int ep_index;
  1251. struct xhci_ep_ctx *ep_ctx;
  1252. struct xhci_ring *ep_ring;
  1253. unsigned int max_packet;
  1254. unsigned int max_burst;
  1255. enum xhci_ring_type type;
  1256. u32 max_esit_payload;
  1257. ep_index = xhci_get_endpoint_index(&ep->desc);
  1258. ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, ep_index);
  1259. type = usb_endpoint_type(&ep->desc);
  1260. /* Set up the endpoint ring */
  1261. virt_dev->eps[ep_index].new_ring =
  1262. xhci_ring_alloc(xhci, 2, 1, type, mem_flags);
  1263. if (!virt_dev->eps[ep_index].new_ring) {
  1264. /* Attempt to use the ring cache */
  1265. if (virt_dev->num_rings_cached == 0)
  1266. return -ENOMEM;
  1267. virt_dev->eps[ep_index].new_ring =
  1268. virt_dev->ring_cache[virt_dev->num_rings_cached];
  1269. virt_dev->ring_cache[virt_dev->num_rings_cached] = NULL;
  1270. virt_dev->num_rings_cached--;
  1271. xhci_reinit_cached_ring(xhci, virt_dev->eps[ep_index].new_ring,
  1272. 1, type);
  1273. }
  1274. virt_dev->eps[ep_index].skip = false;
  1275. ep_ring = virt_dev->eps[ep_index].new_ring;
  1276. ep_ctx->deq = cpu_to_le64(ep_ring->first_seg->dma | ep_ring->cycle_state);
  1277. ep_ctx->ep_info = cpu_to_le32(xhci_get_endpoint_interval(udev, ep)
  1278. | EP_MULT(xhci_get_endpoint_mult(udev, ep)));
  1279. /* FIXME dig Mult and streams info out of ep companion desc */
  1280. /* Allow 3 retries for everything but isoc;
  1281. * CErr shall be set to 0 for Isoch endpoints.
  1282. */
  1283. if (!usb_endpoint_xfer_isoc(&ep->desc))
  1284. ep_ctx->ep_info2 = cpu_to_le32(ERROR_COUNT(3));
  1285. else
  1286. ep_ctx->ep_info2 = cpu_to_le32(ERROR_COUNT(0));
  1287. ep_ctx->ep_info2 |= cpu_to_le32(xhci_get_endpoint_type(udev, ep));
  1288. /* Set the max packet size and max burst */
  1289. switch (udev->speed) {
  1290. case USB_SPEED_SUPER:
  1291. max_packet = usb_endpoint_maxp(&ep->desc);
  1292. ep_ctx->ep_info2 |= cpu_to_le32(MAX_PACKET(max_packet));
  1293. /* dig out max burst from ep companion desc */
  1294. max_packet = ep->ss_ep_comp.bMaxBurst;
  1295. ep_ctx->ep_info2 |= cpu_to_le32(MAX_BURST(max_packet));
  1296. break;
  1297. case USB_SPEED_HIGH:
  1298. /* bits 11:12 specify the number of additional transaction
  1299. * opportunities per microframe (USB 2.0, section 9.6.6)
  1300. */
  1301. if (usb_endpoint_xfer_isoc(&ep->desc) ||
  1302. usb_endpoint_xfer_int(&ep->desc)) {
  1303. max_burst = (usb_endpoint_maxp(&ep->desc)
  1304. & 0x1800) >> 11;
  1305. ep_ctx->ep_info2 |= cpu_to_le32(MAX_BURST(max_burst));
  1306. }
  1307. /* Fall through */
  1308. case USB_SPEED_FULL:
  1309. case USB_SPEED_LOW:
  1310. max_packet = GET_MAX_PACKET(usb_endpoint_maxp(&ep->desc));
  1311. ep_ctx->ep_info2 |= cpu_to_le32(MAX_PACKET(max_packet));
  1312. break;
  1313. default:
  1314. BUG();
  1315. }
  1316. max_esit_payload = xhci_get_max_esit_payload(xhci, udev, ep);
  1317. ep_ctx->tx_info = cpu_to_le32(MAX_ESIT_PAYLOAD_FOR_EP(max_esit_payload));
  1318. /*
  1319. * XXX no idea how to calculate the average TRB buffer length for bulk
  1320. * endpoints, as the driver gives us no clue how big each scatter gather
  1321. * list entry (or buffer) is going to be.
  1322. *
  1323. * For isochronous and interrupt endpoints, we set it to the max
  1324. * available, until we have new API in the USB core to allow drivers to
  1325. * declare how much bandwidth they actually need.
  1326. *
  1327. * Normally, it would be calculated by taking the total of the buffer
  1328. * lengths in the TD and then dividing by the number of TRBs in a TD,
  1329. * including link TRBs, No-op TRBs, and Event data TRBs. Since we don't
  1330. * use Event Data TRBs, and we don't chain in a link TRB on short
  1331. * transfers, we're basically dividing by 1.
  1332. *
  1333. * xHCI 1.0 specification indicates that the Average TRB Length should
  1334. * be set to 8 for control endpoints.
  1335. */
  1336. if (usb_endpoint_xfer_control(&ep->desc) && xhci->hci_version == 0x100)
  1337. ep_ctx->tx_info |= cpu_to_le32(AVG_TRB_LENGTH_FOR_EP(8));
  1338. else
  1339. ep_ctx->tx_info |=
  1340. cpu_to_le32(AVG_TRB_LENGTH_FOR_EP(max_esit_payload));
  1341. /* FIXME Debug endpoint context */
  1342. return 0;
  1343. }
  1344. void xhci_endpoint_zero(struct xhci_hcd *xhci,
  1345. struct xhci_virt_device *virt_dev,
  1346. struct usb_host_endpoint *ep)
  1347. {
  1348. unsigned int ep_index;
  1349. struct xhci_ep_ctx *ep_ctx;
  1350. ep_index = xhci_get_endpoint_index(&ep->desc);
  1351. ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, ep_index);
  1352. ep_ctx->ep_info = 0;
  1353. ep_ctx->ep_info2 = 0;
  1354. ep_ctx->deq = 0;
  1355. ep_ctx->tx_info = 0;
  1356. /* Don't free the endpoint ring until the set interface or configuration
  1357. * request succeeds.
  1358. */
  1359. }
  1360. void xhci_clear_endpoint_bw_info(struct xhci_bw_info *bw_info)
  1361. {
  1362. bw_info->ep_interval = 0;
  1363. bw_info->mult = 0;
  1364. bw_info->num_packets = 0;
  1365. bw_info->max_packet_size = 0;
  1366. bw_info->type = 0;
  1367. bw_info->max_esit_payload = 0;
  1368. }
  1369. void xhci_update_bw_info(struct xhci_hcd *xhci,
  1370. struct xhci_container_ctx *in_ctx,
  1371. struct xhci_input_control_ctx *ctrl_ctx,
  1372. struct xhci_virt_device *virt_dev)
  1373. {
  1374. struct xhci_bw_info *bw_info;
  1375. struct xhci_ep_ctx *ep_ctx;
  1376. unsigned int ep_type;
  1377. int i;
  1378. for (i = 1; i < 31; ++i) {
  1379. bw_info = &virt_dev->eps[i].bw_info;
  1380. /* We can't tell what endpoint type is being dropped, but
  1381. * unconditionally clearing the bandwidth info for non-periodic
  1382. * endpoints should be harmless because the info will never be
  1383. * set in the first place.
  1384. */
  1385. if (!EP_IS_ADDED(ctrl_ctx, i) && EP_IS_DROPPED(ctrl_ctx, i)) {
  1386. /* Dropped endpoint */
  1387. xhci_clear_endpoint_bw_info(bw_info);
  1388. continue;
  1389. }
  1390. if (EP_IS_ADDED(ctrl_ctx, i)) {
  1391. ep_ctx = xhci_get_ep_ctx(xhci, in_ctx, i);
  1392. ep_type = CTX_TO_EP_TYPE(le32_to_cpu(ep_ctx->ep_info2));
  1393. /* Ignore non-periodic endpoints */
  1394. if (ep_type != ISOC_OUT_EP && ep_type != INT_OUT_EP &&
  1395. ep_type != ISOC_IN_EP &&
  1396. ep_type != INT_IN_EP)
  1397. continue;
  1398. /* Added or changed endpoint */
  1399. bw_info->ep_interval = CTX_TO_EP_INTERVAL(
  1400. le32_to_cpu(ep_ctx->ep_info));
  1401. /* Number of packets and mult are zero-based in the
  1402. * input context, but we want one-based for the
  1403. * interval table.
  1404. */
  1405. bw_info->mult = CTX_TO_EP_MULT(
  1406. le32_to_cpu(ep_ctx->ep_info)) + 1;
  1407. bw_info->num_packets = CTX_TO_MAX_BURST(
  1408. le32_to_cpu(ep_ctx->ep_info2)) + 1;
  1409. bw_info->max_packet_size = MAX_PACKET_DECODED(
  1410. le32_to_cpu(ep_ctx->ep_info2));
  1411. bw_info->type = ep_type;
  1412. bw_info->max_esit_payload = CTX_TO_MAX_ESIT_PAYLOAD(
  1413. le32_to_cpu(ep_ctx->tx_info));
  1414. }
  1415. }
  1416. }
  1417. /* Copy output xhci_ep_ctx to the input xhci_ep_ctx copy.
  1418. * Useful when you want to change one particular aspect of the endpoint and then
  1419. * issue a configure endpoint command.
  1420. */
  1421. void xhci_endpoint_copy(struct xhci_hcd *xhci,
  1422. struct xhci_container_ctx *in_ctx,
  1423. struct xhci_container_ctx *out_ctx,
  1424. unsigned int ep_index)
  1425. {
  1426. struct xhci_ep_ctx *out_ep_ctx;
  1427. struct xhci_ep_ctx *in_ep_ctx;
  1428. out_ep_ctx = xhci_get_ep_ctx(xhci, out_ctx, ep_index);
  1429. in_ep_ctx = xhci_get_ep_ctx(xhci, in_ctx, ep_index);
  1430. in_ep_ctx->ep_info = out_ep_ctx->ep_info;
  1431. in_ep_ctx->ep_info2 = out_ep_ctx->ep_info2;
  1432. in_ep_ctx->deq = out_ep_ctx->deq;
  1433. in_ep_ctx->tx_info = out_ep_ctx->tx_info;
  1434. }
  1435. /* Copy output xhci_slot_ctx to the input xhci_slot_ctx.
  1436. * Useful when you want to change one particular aspect of the endpoint and then
  1437. * issue a configure endpoint command. Only the context entries field matters,
  1438. * but we'll copy the whole thing anyway.
  1439. */
  1440. void xhci_slot_copy(struct xhci_hcd *xhci,
  1441. struct xhci_container_ctx *in_ctx,
  1442. struct xhci_container_ctx *out_ctx)
  1443. {
  1444. struct xhci_slot_ctx *in_slot_ctx;
  1445. struct xhci_slot_ctx *out_slot_ctx;
  1446. in_slot_ctx = xhci_get_slot_ctx(xhci, in_ctx);
  1447. out_slot_ctx = xhci_get_slot_ctx(xhci, out_ctx);
  1448. in_slot_ctx->dev_info = out_slot_ctx->dev_info;
  1449. in_slot_ctx->dev_info2 = out_slot_ctx->dev_info2;
  1450. in_slot_ctx->tt_info = out_slot_ctx->tt_info;
  1451. in_slot_ctx->dev_state = out_slot_ctx->dev_state;
  1452. }
  1453. /* Set up the scratchpad buffer array and scratchpad buffers, if needed. */
  1454. static int scratchpad_alloc(struct xhci_hcd *xhci, gfp_t flags)
  1455. {
  1456. int i;
  1457. struct device *dev = xhci_to_hcd(xhci)->self.controller;
  1458. int num_sp = HCS_MAX_SCRATCHPAD(xhci->hcs_params2);
  1459. xhci_dbg(xhci, "Allocating %d scratchpad buffers\n", num_sp);
  1460. if (!num_sp)
  1461. return 0;
  1462. xhci->scratchpad = kzalloc(sizeof(*xhci->scratchpad), flags);
  1463. if (!xhci->scratchpad)
  1464. goto fail_sp;
  1465. xhci->scratchpad->sp_array = dma_alloc_coherent(dev,
  1466. num_sp * sizeof(u64),
  1467. &xhci->scratchpad->sp_dma, flags);
  1468. if (!xhci->scratchpad->sp_array)
  1469. goto fail_sp2;
  1470. xhci->scratchpad->sp_buffers = kzalloc(sizeof(void *) * num_sp, flags);
  1471. if (!xhci->scratchpad->sp_buffers)
  1472. goto fail_sp3;
  1473. xhci->scratchpad->sp_dma_buffers =
  1474. kzalloc(sizeof(dma_addr_t) * num_sp, flags);
  1475. if (!xhci->scratchpad->sp_dma_buffers)
  1476. goto fail_sp4;
  1477. xhci->dcbaa->dev_context_ptrs[0] = cpu_to_le64(xhci->scratchpad->sp_dma);
  1478. for (i = 0; i < num_sp; i++) {
  1479. dma_addr_t dma;
  1480. void *buf = dma_alloc_coherent(dev, xhci->page_size, &dma,
  1481. flags);
  1482. if (!buf)
  1483. goto fail_sp5;
  1484. xhci->scratchpad->sp_array[i] = dma;
  1485. xhci->scratchpad->sp_buffers[i] = buf;
  1486. xhci->scratchpad->sp_dma_buffers[i] = dma;
  1487. }
  1488. return 0;
  1489. fail_sp5:
  1490. for (i = i - 1; i >= 0; i--) {
  1491. dma_free_coherent(dev, xhci->page_size,
  1492. xhci->scratchpad->sp_buffers[i],
  1493. xhci->scratchpad->sp_dma_buffers[i]);
  1494. }
  1495. kfree(xhci->scratchpad->sp_dma_buffers);
  1496. fail_sp4:
  1497. kfree(xhci->scratchpad->sp_buffers);
  1498. fail_sp3:
  1499. dma_free_coherent(dev, num_sp * sizeof(u64),
  1500. xhci->scratchpad->sp_array,
  1501. xhci->scratchpad->sp_dma);
  1502. fail_sp2:
  1503. kfree(xhci->scratchpad);
  1504. xhci->scratchpad = NULL;
  1505. fail_sp:
  1506. return -ENOMEM;
  1507. }
  1508. static void scratchpad_free(struct xhci_hcd *xhci)
  1509. {
  1510. int num_sp;
  1511. int i;
  1512. struct pci_dev *pdev = to_pci_dev(xhci_to_hcd(xhci)->self.controller);
  1513. if (!xhci->scratchpad)
  1514. return;
  1515. num_sp = HCS_MAX_SCRATCHPAD(xhci->hcs_params2);
  1516. for (i = 0; i < num_sp; i++) {
  1517. dma_free_coherent(&pdev->dev, xhci->page_size,
  1518. xhci->scratchpad->sp_buffers[i],
  1519. xhci->scratchpad->sp_dma_buffers[i]);
  1520. }
  1521. kfree(xhci->scratchpad->sp_dma_buffers);
  1522. kfree(xhci->scratchpad->sp_buffers);
  1523. dma_free_coherent(&pdev->dev, num_sp * sizeof(u64),
  1524. xhci->scratchpad->sp_array,
  1525. xhci->scratchpad->sp_dma);
  1526. kfree(xhci->scratchpad);
  1527. xhci->scratchpad = NULL;
  1528. }
  1529. struct xhci_command *xhci_alloc_command(struct xhci_hcd *xhci,
  1530. bool allocate_in_ctx, bool allocate_completion,
  1531. gfp_t mem_flags)
  1532. {
  1533. struct xhci_command *command;
  1534. command = kzalloc(sizeof(*command), mem_flags);
  1535. if (!command)
  1536. return NULL;
  1537. if (allocate_in_ctx) {
  1538. command->in_ctx =
  1539. xhci_alloc_container_ctx(xhci, XHCI_CTX_TYPE_INPUT,
  1540. mem_flags);
  1541. if (!command->in_ctx) {
  1542. kfree(command);
  1543. return NULL;
  1544. }
  1545. }
  1546. if (allocate_completion) {
  1547. command->completion =
  1548. kzalloc(sizeof(struct completion), mem_flags);
  1549. if (!command->completion) {
  1550. xhci_free_container_ctx(xhci, command->in_ctx);
  1551. kfree(command);
  1552. return NULL;
  1553. }
  1554. init_completion(command->completion);
  1555. }
  1556. command->status = 0;
  1557. INIT_LIST_HEAD(&command->cmd_list);
  1558. return command;
  1559. }
  1560. void xhci_urb_free_priv(struct xhci_hcd *xhci, struct urb_priv *urb_priv)
  1561. {
  1562. if (urb_priv) {
  1563. kfree(urb_priv->td[0]);
  1564. kfree(urb_priv);
  1565. }
  1566. }
  1567. void xhci_free_command(struct xhci_hcd *xhci,
  1568. struct xhci_command *command)
  1569. {
  1570. xhci_free_container_ctx(xhci,
  1571. command->in_ctx);
  1572. kfree(command->completion);
  1573. kfree(command);
  1574. }
  1575. void xhci_mem_cleanup(struct xhci_hcd *xhci)
  1576. {
  1577. struct pci_dev *pdev = to_pci_dev(xhci_to_hcd(xhci)->self.controller);
  1578. struct dev_info *dev_info, *next;
  1579. struct list_head *tt_list_head;
  1580. struct list_head *tt;
  1581. struct list_head *endpoints;
  1582. struct list_head *ep, *q;
  1583. struct xhci_tt_bw_info *tt_info;
  1584. struct xhci_interval_bw_table *bwt;
  1585. struct xhci_virt_ep *virt_ep;
  1586. unsigned long flags;
  1587. int size;
  1588. int i;
  1589. /* Free the Event Ring Segment Table and the actual Event Ring */
  1590. size = sizeof(struct xhci_erst_entry)*(xhci->erst.num_entries);
  1591. if (xhci->erst.entries)
  1592. dma_free_coherent(&pdev->dev, size,
  1593. xhci->erst.entries, xhci->erst.erst_dma_addr);
  1594. xhci->erst.entries = NULL;
  1595. xhci_dbg(xhci, "Freed ERST\n");
  1596. if (xhci->event_ring)
  1597. xhci_ring_free(xhci, xhci->event_ring);
  1598. xhci->event_ring = NULL;
  1599. xhci_dbg(xhci, "Freed event ring\n");
  1600. if (xhci->lpm_command)
  1601. xhci_free_command(xhci, xhci->lpm_command);
  1602. xhci->cmd_ring_reserved_trbs = 0;
  1603. if (xhci->cmd_ring)
  1604. xhci_ring_free(xhci, xhci->cmd_ring);
  1605. xhci->cmd_ring = NULL;
  1606. xhci_dbg(xhci, "Freed command ring\n");
  1607. for (i = 1; i < MAX_HC_SLOTS; ++i)
  1608. xhci_free_virt_device(xhci, i);
  1609. if (xhci->segment_pool)
  1610. dma_pool_destroy(xhci->segment_pool);
  1611. xhci->segment_pool = NULL;
  1612. xhci_dbg(xhci, "Freed segment pool\n");
  1613. if (xhci->device_pool)
  1614. dma_pool_destroy(xhci->device_pool);
  1615. xhci->device_pool = NULL;
  1616. xhci_dbg(xhci, "Freed device context pool\n");
  1617. if (xhci->small_streams_pool)
  1618. dma_pool_destroy(xhci->small_streams_pool);
  1619. xhci->small_streams_pool = NULL;
  1620. xhci_dbg(xhci, "Freed small stream array pool\n");
  1621. if (xhci->medium_streams_pool)
  1622. dma_pool_destroy(xhci->medium_streams_pool);
  1623. xhci->medium_streams_pool = NULL;
  1624. xhci_dbg(xhci, "Freed medium stream array pool\n");
  1625. if (xhci->dcbaa)
  1626. dma_free_coherent(&pdev->dev, sizeof(*xhci->dcbaa),
  1627. xhci->dcbaa, xhci->dcbaa->dma);
  1628. xhci->dcbaa = NULL;
  1629. scratchpad_free(xhci);
  1630. spin_lock_irqsave(&xhci->lock, flags);
  1631. list_for_each_entry_safe(dev_info, next, &xhci->lpm_failed_devs, list) {
  1632. list_del(&dev_info->list);
  1633. kfree(dev_info);
  1634. }
  1635. spin_unlock_irqrestore(&xhci->lock, flags);
  1636. bwt = &xhci->rh_bw->bw_table;
  1637. for (i = 0; i < XHCI_MAX_INTERVAL; i++) {
  1638. endpoints = &bwt->interval_bw[i].endpoints;
  1639. list_for_each_safe(ep, q, endpoints) {
  1640. virt_ep = list_entry(ep, struct xhci_virt_ep, bw_endpoint_list);
  1641. list_del(&virt_ep->bw_endpoint_list);
  1642. kfree(virt_ep);
  1643. }
  1644. }
  1645. tt_list_head = &xhci->rh_bw->tts;
  1646. list_for_each_safe(tt, q, tt_list_head) {
  1647. tt_info = list_entry(tt, struct xhci_tt_bw_info, tt_list);
  1648. list_del(tt);
  1649. kfree(tt_info);
  1650. }
  1651. xhci->num_usb2_ports = 0;
  1652. xhci->num_usb3_ports = 0;
  1653. xhci->num_active_eps = 0;
  1654. kfree(xhci->usb2_ports);
  1655. kfree(xhci->usb3_ports);
  1656. kfree(xhci->port_array);
  1657. kfree(xhci->rh_bw);
  1658. xhci->page_size = 0;
  1659. xhci->page_shift = 0;
  1660. xhci->bus_state[0].bus_suspended = 0;
  1661. xhci->bus_state[1].bus_suspended = 0;
  1662. }
  1663. static int xhci_test_trb_in_td(struct xhci_hcd *xhci,
  1664. struct xhci_segment *input_seg,
  1665. union xhci_trb *start_trb,
  1666. union xhci_trb *end_trb,
  1667. dma_addr_t input_dma,
  1668. struct xhci_segment *result_seg,
  1669. char *test_name, int test_number)
  1670. {
  1671. unsigned long long start_dma;
  1672. unsigned long long end_dma;
  1673. struct xhci_segment *seg;
  1674. start_dma = xhci_trb_virt_to_dma(input_seg, start_trb);
  1675. end_dma = xhci_trb_virt_to_dma(input_seg, end_trb);
  1676. seg = trb_in_td(input_seg, start_trb, end_trb, input_dma);
  1677. if (seg != result_seg) {
  1678. xhci_warn(xhci, "WARN: %s TRB math test %d failed!\n",
  1679. test_name, test_number);
  1680. xhci_warn(xhci, "Tested TRB math w/ seg %p and "
  1681. "input DMA 0x%llx\n",
  1682. input_seg,
  1683. (unsigned long long) input_dma);
  1684. xhci_warn(xhci, "starting TRB %p (0x%llx DMA), "
  1685. "ending TRB %p (0x%llx DMA)\n",
  1686. start_trb, start_dma,
  1687. end_trb, end_dma);
  1688. xhci_warn(xhci, "Expected seg %p, got seg %p\n",
  1689. result_seg, seg);
  1690. return -1;
  1691. }
  1692. return 0;
  1693. }
  1694. /* TRB math checks for xhci_trb_in_td(), using the command and event rings. */
  1695. static int xhci_check_trb_in_td_math(struct xhci_hcd *xhci, gfp_t mem_flags)
  1696. {
  1697. struct {
  1698. dma_addr_t input_dma;
  1699. struct xhci_segment *result_seg;
  1700. } simple_test_vector [] = {
  1701. /* A zeroed DMA field should fail */
  1702. { 0, NULL },
  1703. /* One TRB before the ring start should fail */
  1704. { xhci->event_ring->first_seg->dma - 16, NULL },
  1705. /* One byte before the ring start should fail */
  1706. { xhci->event_ring->first_seg->dma - 1, NULL },
  1707. /* Starting TRB should succeed */
  1708. { xhci->event_ring->first_seg->dma, xhci->event_ring->first_seg },
  1709. /* Ending TRB should succeed */
  1710. { xhci->event_ring->first_seg->dma + (TRBS_PER_SEGMENT - 1)*16,
  1711. xhci->event_ring->first_seg },
  1712. /* One byte after the ring end should fail */
  1713. { xhci->event_ring->first_seg->dma + (TRBS_PER_SEGMENT - 1)*16 + 1, NULL },
  1714. /* One TRB after the ring end should fail */
  1715. { xhci->event_ring->first_seg->dma + (TRBS_PER_SEGMENT)*16, NULL },
  1716. /* An address of all ones should fail */
  1717. { (dma_addr_t) (~0), NULL },
  1718. };
  1719. struct {
  1720. struct xhci_segment *input_seg;
  1721. union xhci_trb *start_trb;
  1722. union xhci_trb *end_trb;
  1723. dma_addr_t input_dma;
  1724. struct xhci_segment *result_seg;
  1725. } complex_test_vector [] = {
  1726. /* Test feeding a valid DMA address from a different ring */
  1727. { .input_seg = xhci->event_ring->first_seg,
  1728. .start_trb = xhci->event_ring->first_seg->trbs,
  1729. .end_trb = &xhci->event_ring->first_seg->trbs[TRBS_PER_SEGMENT - 1],
  1730. .input_dma = xhci->cmd_ring->first_seg->dma,
  1731. .result_seg = NULL,
  1732. },
  1733. /* Test feeding a valid end TRB from a different ring */
  1734. { .input_seg = xhci->event_ring->first_seg,
  1735. .start_trb = xhci->event_ring->first_seg->trbs,
  1736. .end_trb = &xhci->cmd_ring->first_seg->trbs[TRBS_PER_SEGMENT - 1],
  1737. .input_dma = xhci->cmd_ring->first_seg->dma,
  1738. .result_seg = NULL,
  1739. },
  1740. /* Test feeding a valid start and end TRB from a different ring */
  1741. { .input_seg = xhci->event_ring->first_seg,
  1742. .start_trb = xhci->cmd_ring->first_seg->trbs,
  1743. .end_trb = &xhci->cmd_ring->first_seg->trbs[TRBS_PER_SEGMENT - 1],
  1744. .input_dma = xhci->cmd_ring->first_seg->dma,
  1745. .result_seg = NULL,
  1746. },
  1747. /* TRB in this ring, but after this TD */
  1748. { .input_seg = xhci->event_ring->first_seg,
  1749. .start_trb = &xhci->event_ring->first_seg->trbs[0],
  1750. .end_trb = &xhci->event_ring->first_seg->trbs[3],
  1751. .input_dma = xhci->event_ring->first_seg->dma + 4*16,
  1752. .result_seg = NULL,
  1753. },
  1754. /* TRB in this ring, but before this TD */
  1755. { .input_seg = xhci->event_ring->first_seg,
  1756. .start_trb = &xhci->event_ring->first_seg->trbs[3],
  1757. .end_trb = &xhci->event_ring->first_seg->trbs[6],
  1758. .input_dma = xhci->event_ring->first_seg->dma + 2*16,
  1759. .result_seg = NULL,
  1760. },
  1761. /* TRB in this ring, but after this wrapped TD */
  1762. { .input_seg = xhci->event_ring->first_seg,
  1763. .start_trb = &xhci->event_ring->first_seg->trbs[TRBS_PER_SEGMENT - 3],
  1764. .end_trb = &xhci->event_ring->first_seg->trbs[1],
  1765. .input_dma = xhci->event_ring->first_seg->dma + 2*16,
  1766. .result_seg = NULL,
  1767. },
  1768. /* TRB in this ring, but before this wrapped TD */
  1769. { .input_seg = xhci->event_ring->first_seg,
  1770. .start_trb = &xhci->event_ring->first_seg->trbs[TRBS_PER_SEGMENT - 3],
  1771. .end_trb = &xhci->event_ring->first_seg->trbs[1],
  1772. .input_dma = xhci->event_ring->first_seg->dma + (TRBS_PER_SEGMENT - 4)*16,
  1773. .result_seg = NULL,
  1774. },
  1775. /* TRB not in this ring, and we have a wrapped TD */
  1776. { .input_seg = xhci->event_ring->first_seg,
  1777. .start_trb = &xhci->event_ring->first_seg->trbs[TRBS_PER_SEGMENT - 3],
  1778. .end_trb = &xhci->event_ring->first_seg->trbs[1],
  1779. .input_dma = xhci->cmd_ring->first_seg->dma + 2*16,
  1780. .result_seg = NULL,
  1781. },
  1782. };
  1783. unsigned int num_tests;
  1784. int i, ret;
  1785. num_tests = ARRAY_SIZE(simple_test_vector);
  1786. for (i = 0; i < num_tests; i++) {
  1787. ret = xhci_test_trb_in_td(xhci,
  1788. xhci->event_ring->first_seg,
  1789. xhci->event_ring->first_seg->trbs,
  1790. &xhci->event_ring->first_seg->trbs[TRBS_PER_SEGMENT - 1],
  1791. simple_test_vector[i].input_dma,
  1792. simple_test_vector[i].result_seg,
  1793. "Simple", i);
  1794. if (ret < 0)
  1795. return ret;
  1796. }
  1797. num_tests = ARRAY_SIZE(complex_test_vector);
  1798. for (i = 0; i < num_tests; i++) {
  1799. ret = xhci_test_trb_in_td(xhci,
  1800. complex_test_vector[i].input_seg,
  1801. complex_test_vector[i].start_trb,
  1802. complex_test_vector[i].end_trb,
  1803. complex_test_vector[i].input_dma,
  1804. complex_test_vector[i].result_seg,
  1805. "Complex", i);
  1806. if (ret < 0)
  1807. return ret;
  1808. }
  1809. xhci_dbg(xhci, "TRB math tests passed.\n");
  1810. return 0;
  1811. }
  1812. static void xhci_set_hc_event_deq(struct xhci_hcd *xhci)
  1813. {
  1814. u64 temp;
  1815. dma_addr_t deq;
  1816. deq = xhci_trb_virt_to_dma(xhci->event_ring->deq_seg,
  1817. xhci->event_ring->dequeue);
  1818. if (deq == 0 && !in_interrupt())
  1819. xhci_warn(xhci, "WARN something wrong with SW event ring "
  1820. "dequeue ptr.\n");
  1821. /* Update HC event ring dequeue pointer */
  1822. temp = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue);
  1823. temp &= ERST_PTR_MASK;
  1824. /* Don't clear the EHB bit (which is RW1C) because
  1825. * there might be more events to service.
  1826. */
  1827. temp &= ~ERST_EHB;
  1828. xhci_dbg(xhci, "// Write event ring dequeue pointer, "
  1829. "preserving EHB bit\n");
  1830. xhci_write_64(xhci, ((u64) deq & (u64) ~ERST_PTR_MASK) | temp,
  1831. &xhci->ir_set->erst_dequeue);
  1832. }
  1833. static void xhci_add_in_port(struct xhci_hcd *xhci, unsigned int num_ports,
  1834. __le32 __iomem *addr, u8 major_revision)
  1835. {
  1836. u32 temp, port_offset, port_count;
  1837. int i;
  1838. if (major_revision > 0x03) {
  1839. xhci_warn(xhci, "Ignoring unknown port speed, "
  1840. "Ext Cap %p, revision = 0x%x\n",
  1841. addr, major_revision);
  1842. /* Ignoring port protocol we can't understand. FIXME */
  1843. return;
  1844. }
  1845. /* Port offset and count in the third dword, see section 7.2 */
  1846. temp = xhci_readl(xhci, addr + 2);
  1847. port_offset = XHCI_EXT_PORT_OFF(temp);
  1848. port_count = XHCI_EXT_PORT_COUNT(temp);
  1849. xhci_dbg(xhci, "Ext Cap %p, port offset = %u, "
  1850. "count = %u, revision = 0x%x\n",
  1851. addr, port_offset, port_count, major_revision);
  1852. /* Port count includes the current port offset */
  1853. if (port_offset == 0 || (port_offset + port_count - 1) > num_ports)
  1854. /* WTF? "Valid values are ‘1’ to MaxPorts" */
  1855. return;
  1856. /* Check the host's USB2 LPM capability */
  1857. if ((xhci->hci_version == 0x96) && (major_revision != 0x03) &&
  1858. (temp & XHCI_L1C)) {
  1859. xhci_dbg(xhci, "xHCI 0.96: support USB2 software lpm\n");
  1860. xhci->sw_lpm_support = 1;
  1861. }
  1862. if ((xhci->hci_version >= 0x100) && (major_revision != 0x03)) {
  1863. xhci_dbg(xhci, "xHCI 1.0: support USB2 software lpm\n");
  1864. xhci->sw_lpm_support = 1;
  1865. if (temp & XHCI_HLC) {
  1866. xhci_dbg(xhci, "xHCI 1.0: support USB2 hardware lpm\n");
  1867. xhci->hw_lpm_support = 1;
  1868. }
  1869. }
  1870. port_offset--;
  1871. for (i = port_offset; i < (port_offset + port_count); i++) {
  1872. /* Duplicate entry. Ignore the port if the revisions differ. */
  1873. if (xhci->port_array[i] != 0) {
  1874. xhci_warn(xhci, "Duplicate port entry, Ext Cap %p,"
  1875. " port %u\n", addr, i);
  1876. xhci_warn(xhci, "Port was marked as USB %u, "
  1877. "duplicated as USB %u\n",
  1878. xhci->port_array[i], major_revision);
  1879. /* Only adjust the roothub port counts if we haven't
  1880. * found a similar duplicate.
  1881. */
  1882. if (xhci->port_array[i] != major_revision &&
  1883. xhci->port_array[i] != DUPLICATE_ENTRY) {
  1884. if (xhci->port_array[i] == 0x03)
  1885. xhci->num_usb3_ports--;
  1886. else
  1887. xhci->num_usb2_ports--;
  1888. xhci->port_array[i] = DUPLICATE_ENTRY;
  1889. }
  1890. /* FIXME: Should we disable the port? */
  1891. continue;
  1892. }
  1893. xhci->port_array[i] = major_revision;
  1894. if (major_revision == 0x03)
  1895. xhci->num_usb3_ports++;
  1896. else
  1897. xhci->num_usb2_ports++;
  1898. }
  1899. /* FIXME: Should we disable ports not in the Extended Capabilities? */
  1900. }
  1901. /*
  1902. * Scan the Extended Capabilities for the "Supported Protocol Capabilities" that
  1903. * specify what speeds each port is supposed to be. We can't count on the port
  1904. * speed bits in the PORTSC register being correct until a device is connected,
  1905. * but we need to set up the two fake roothubs with the correct number of USB
  1906. * 3.0 and USB 2.0 ports at host controller initialization time.
  1907. */
  1908. static int xhci_setup_port_arrays(struct xhci_hcd *xhci, gfp_t flags)
  1909. {
  1910. __le32 __iomem *addr;
  1911. u32 offset;
  1912. unsigned int num_ports;
  1913. int i, j, port_index;
  1914. addr = &xhci->cap_regs->hcc_params;
  1915. offset = XHCI_HCC_EXT_CAPS(xhci_readl(xhci, addr));
  1916. if (offset == 0) {
  1917. xhci_err(xhci, "No Extended Capability registers, "
  1918. "unable to set up roothub.\n");
  1919. return -ENODEV;
  1920. }
  1921. num_ports = HCS_MAX_PORTS(xhci->hcs_params1);
  1922. xhci->port_array = kzalloc(sizeof(*xhci->port_array)*num_ports, flags);
  1923. if (!xhci->port_array)
  1924. return -ENOMEM;
  1925. xhci->rh_bw = kzalloc(sizeof(*xhci->rh_bw)*num_ports, flags);
  1926. if (!xhci->rh_bw)
  1927. return -ENOMEM;
  1928. for (i = 0; i < num_ports; i++) {
  1929. struct xhci_interval_bw_table *bw_table;
  1930. INIT_LIST_HEAD(&xhci->rh_bw[i].tts);
  1931. bw_table = &xhci->rh_bw[i].bw_table;
  1932. for (j = 0; j < XHCI_MAX_INTERVAL; j++)
  1933. INIT_LIST_HEAD(&bw_table->interval_bw[j].endpoints);
  1934. }
  1935. /*
  1936. * For whatever reason, the first capability offset is from the
  1937. * capability register base, not from the HCCPARAMS register.
  1938. * See section 5.3.6 for offset calculation.
  1939. */
  1940. addr = &xhci->cap_regs->hc_capbase + offset;
  1941. while (1) {
  1942. u32 cap_id;
  1943. cap_id = xhci_readl(xhci, addr);
  1944. if (XHCI_EXT_CAPS_ID(cap_id) == XHCI_EXT_CAPS_PROTOCOL)
  1945. xhci_add_in_port(xhci, num_ports, addr,
  1946. (u8) XHCI_EXT_PORT_MAJOR(cap_id));
  1947. offset = XHCI_EXT_CAPS_NEXT(cap_id);
  1948. if (!offset || (xhci->num_usb2_ports + xhci->num_usb3_ports)
  1949. == num_ports)
  1950. break;
  1951. /*
  1952. * Once you're into the Extended Capabilities, the offset is
  1953. * always relative to the register holding the offset.
  1954. */
  1955. addr += offset;
  1956. }
  1957. if (xhci->num_usb2_ports == 0 && xhci->num_usb3_ports == 0) {
  1958. xhci_warn(xhci, "No ports on the roothubs?\n");
  1959. return -ENODEV;
  1960. }
  1961. xhci_dbg(xhci, "Found %u USB 2.0 ports and %u USB 3.0 ports.\n",
  1962. xhci->num_usb2_ports, xhci->num_usb3_ports);
  1963. /* Place limits on the number of roothub ports so that the hub
  1964. * descriptors aren't longer than the USB core will allocate.
  1965. */
  1966. if (xhci->num_usb3_ports > 15) {
  1967. xhci_dbg(xhci, "Limiting USB 3.0 roothub ports to 15.\n");
  1968. xhci->num_usb3_ports = 15;
  1969. }
  1970. if (xhci->num_usb2_ports > USB_MAXCHILDREN) {
  1971. xhci_dbg(xhci, "Limiting USB 2.0 roothub ports to %u.\n",
  1972. USB_MAXCHILDREN);
  1973. xhci->num_usb2_ports = USB_MAXCHILDREN;
  1974. }
  1975. /*
  1976. * Note we could have all USB 3.0 ports, or all USB 2.0 ports.
  1977. * Not sure how the USB core will handle a hub with no ports...
  1978. */
  1979. if (xhci->num_usb2_ports) {
  1980. xhci->usb2_ports = kmalloc(sizeof(*xhci->usb2_ports)*
  1981. xhci->num_usb2_ports, flags);
  1982. if (!xhci->usb2_ports)
  1983. return -ENOMEM;
  1984. port_index = 0;
  1985. for (i = 0; i < num_ports; i++) {
  1986. if (xhci->port_array[i] == 0x03 ||
  1987. xhci->port_array[i] == 0 ||
  1988. xhci->port_array[i] == DUPLICATE_ENTRY)
  1989. continue;
  1990. xhci->usb2_ports[port_index] =
  1991. &xhci->op_regs->port_status_base +
  1992. NUM_PORT_REGS*i;
  1993. xhci_dbg(xhci, "USB 2.0 port at index %u, "
  1994. "addr = %p\n", i,
  1995. xhci->usb2_ports[port_index]);
  1996. port_index++;
  1997. if (port_index == xhci->num_usb2_ports)
  1998. break;
  1999. }
  2000. }
  2001. if (xhci->num_usb3_ports) {
  2002. xhci->usb3_ports = kmalloc(sizeof(*xhci->usb3_ports)*
  2003. xhci->num_usb3_ports, flags);
  2004. if (!xhci->usb3_ports)
  2005. return -ENOMEM;
  2006. port_index = 0;
  2007. for (i = 0; i < num_ports; i++)
  2008. if (xhci->port_array[i] == 0x03) {
  2009. xhci->usb3_ports[port_index] =
  2010. &xhci->op_regs->port_status_base +
  2011. NUM_PORT_REGS*i;
  2012. xhci_dbg(xhci, "USB 3.0 port at index %u, "
  2013. "addr = %p\n", i,
  2014. xhci->usb3_ports[port_index]);
  2015. port_index++;
  2016. if (port_index == xhci->num_usb3_ports)
  2017. break;
  2018. }
  2019. }
  2020. return 0;
  2021. }
  2022. int xhci_mem_init(struct xhci_hcd *xhci, gfp_t flags)
  2023. {
  2024. dma_addr_t dma;
  2025. struct device *dev = xhci_to_hcd(xhci)->self.controller;
  2026. unsigned int val, val2;
  2027. u64 val_64;
  2028. struct xhci_segment *seg;
  2029. u32 page_size, temp;
  2030. int i;
  2031. page_size = xhci_readl(xhci, &xhci->op_regs->page_size);
  2032. xhci_dbg(xhci, "Supported page size register = 0x%x\n", page_size);
  2033. for (i = 0; i < 16; i++) {
  2034. if ((0x1 & page_size) != 0)
  2035. break;
  2036. page_size = page_size >> 1;
  2037. }
  2038. if (i < 16)
  2039. xhci_dbg(xhci, "Supported page size of %iK\n", (1 << (i+12)) / 1024);
  2040. else
  2041. xhci_warn(xhci, "WARN: no supported page size\n");
  2042. /* Use 4K pages, since that's common and the minimum the HC supports */
  2043. xhci->page_shift = 12;
  2044. xhci->page_size = 1 << xhci->page_shift;
  2045. xhci_dbg(xhci, "HCD page size set to %iK\n", xhci->page_size / 1024);
  2046. /*
  2047. * Program the Number of Device Slots Enabled field in the CONFIG
  2048. * register with the max value of slots the HC can handle.
  2049. */
  2050. val = HCS_MAX_SLOTS(xhci_readl(xhci, &xhci->cap_regs->hcs_params1));
  2051. xhci_dbg(xhci, "// xHC can handle at most %d device slots.\n",
  2052. (unsigned int) val);
  2053. val2 = xhci_readl(xhci, &xhci->op_regs->config_reg);
  2054. val |= (val2 & ~HCS_SLOTS_MASK);
  2055. xhci_dbg(xhci, "// Setting Max device slots reg = 0x%x.\n",
  2056. (unsigned int) val);
  2057. xhci_writel(xhci, val, &xhci->op_regs->config_reg);
  2058. /*
  2059. * Section 5.4.8 - doorbell array must be
  2060. * "physically contiguous and 64-byte (cache line) aligned".
  2061. */
  2062. xhci->dcbaa = dma_alloc_coherent(dev, sizeof(*xhci->dcbaa), &dma,
  2063. GFP_KERNEL);
  2064. if (!xhci->dcbaa)
  2065. goto fail;
  2066. memset(xhci->dcbaa, 0, sizeof *(xhci->dcbaa));
  2067. xhci->dcbaa->dma = dma;
  2068. xhci_dbg(xhci, "// Device context base array address = 0x%llx (DMA), %p (virt)\n",
  2069. (unsigned long long)xhci->dcbaa->dma, xhci->dcbaa);
  2070. xhci_write_64(xhci, dma, &xhci->op_regs->dcbaa_ptr);
  2071. /*
  2072. * Initialize the ring segment pool. The ring must be a contiguous
  2073. * structure comprised of TRBs. The TRBs must be 16 byte aligned,
  2074. * however, the command ring segment needs 64-byte aligned segments,
  2075. * so we pick the greater alignment need.
  2076. */
  2077. xhci->segment_pool = dma_pool_create("xHCI ring segments", dev,
  2078. SEGMENT_SIZE, 64, xhci->page_size);
  2079. /* See Table 46 and Note on Figure 55 */
  2080. xhci->device_pool = dma_pool_create("xHCI input/output contexts", dev,
  2081. 2112, 64, xhci->page_size);
  2082. if (!xhci->segment_pool || !xhci->device_pool)
  2083. goto fail;
  2084. /* Linear stream context arrays don't have any boundary restrictions,
  2085. * and only need to be 16-byte aligned.
  2086. */
  2087. xhci->small_streams_pool =
  2088. dma_pool_create("xHCI 256 byte stream ctx arrays",
  2089. dev, SMALL_STREAM_ARRAY_SIZE, 16, 0);
  2090. xhci->medium_streams_pool =
  2091. dma_pool_create("xHCI 1KB stream ctx arrays",
  2092. dev, MEDIUM_STREAM_ARRAY_SIZE, 16, 0);
  2093. /* Any stream context array bigger than MEDIUM_STREAM_ARRAY_SIZE
  2094. * will be allocated with dma_alloc_coherent()
  2095. */
  2096. if (!xhci->small_streams_pool || !xhci->medium_streams_pool)
  2097. goto fail;
  2098. /* Set up the command ring to have one segments for now. */
  2099. xhci->cmd_ring = xhci_ring_alloc(xhci, 1, 1, TYPE_COMMAND, flags);
  2100. if (!xhci->cmd_ring)
  2101. goto fail;
  2102. xhci_dbg(xhci, "Allocated command ring at %p\n", xhci->cmd_ring);
  2103. xhci_dbg(xhci, "First segment DMA is 0x%llx\n",
  2104. (unsigned long long)xhci->cmd_ring->first_seg->dma);
  2105. /* Set the address in the Command Ring Control register */
  2106. val_64 = xhci_read_64(xhci, &xhci->op_regs->cmd_ring);
  2107. val_64 = (val_64 & (u64) CMD_RING_RSVD_BITS) |
  2108. (xhci->cmd_ring->first_seg->dma & (u64) ~CMD_RING_RSVD_BITS) |
  2109. xhci->cmd_ring->cycle_state;
  2110. xhci_dbg(xhci, "// Setting command ring address to 0x%x\n", val);
  2111. xhci_write_64(xhci, val_64, &xhci->op_regs->cmd_ring);
  2112. xhci_dbg_cmd_ptrs(xhci);
  2113. xhci->lpm_command = xhci_alloc_command(xhci, true, true, flags);
  2114. if (!xhci->lpm_command)
  2115. goto fail;
  2116. /* Reserve one command ring TRB for disabling LPM.
  2117. * Since the USB core grabs the shared usb_bus bandwidth mutex before
  2118. * disabling LPM, we only need to reserve one TRB for all devices.
  2119. */
  2120. xhci->cmd_ring_reserved_trbs++;
  2121. val = xhci_readl(xhci, &xhci->cap_regs->db_off);
  2122. val &= DBOFF_MASK;
  2123. xhci_dbg(xhci, "// Doorbell array is located at offset 0x%x"
  2124. " from cap regs base addr\n", val);
  2125. xhci->dba = (void __iomem *) xhci->cap_regs + val;
  2126. xhci_dbg_regs(xhci);
  2127. xhci_print_run_regs(xhci);
  2128. /* Set ir_set to interrupt register set 0 */
  2129. xhci->ir_set = &xhci->run_regs->ir_set[0];
  2130. /*
  2131. * Event ring setup: Allocate a normal ring, but also setup
  2132. * the event ring segment table (ERST). Section 4.9.3.
  2133. */
  2134. xhci_dbg(xhci, "// Allocating event ring\n");
  2135. xhci->event_ring = xhci_ring_alloc(xhci, ERST_NUM_SEGS, 1, TYPE_EVENT,
  2136. flags);
  2137. if (!xhci->event_ring)
  2138. goto fail;
  2139. if (xhci_check_trb_in_td_math(xhci, flags) < 0)
  2140. goto fail;
  2141. xhci->erst.entries = dma_alloc_coherent(dev,
  2142. sizeof(struct xhci_erst_entry) * ERST_NUM_SEGS, &dma,
  2143. GFP_KERNEL);
  2144. if (!xhci->erst.entries)
  2145. goto fail;
  2146. xhci_dbg(xhci, "// Allocated event ring segment table at 0x%llx\n",
  2147. (unsigned long long)dma);
  2148. memset(xhci->erst.entries, 0, sizeof(struct xhci_erst_entry)*ERST_NUM_SEGS);
  2149. xhci->erst.num_entries = ERST_NUM_SEGS;
  2150. xhci->erst.erst_dma_addr = dma;
  2151. xhci_dbg(xhci, "Set ERST to 0; private num segs = %i, virt addr = %p, dma addr = 0x%llx\n",
  2152. xhci->erst.num_entries,
  2153. xhci->erst.entries,
  2154. (unsigned long long)xhci->erst.erst_dma_addr);
  2155. /* set ring base address and size for each segment table entry */
  2156. for (val = 0, seg = xhci->event_ring->first_seg; val < ERST_NUM_SEGS; val++) {
  2157. struct xhci_erst_entry *entry = &xhci->erst.entries[val];
  2158. entry->seg_addr = cpu_to_le64(seg->dma);
  2159. entry->seg_size = cpu_to_le32(TRBS_PER_SEGMENT);
  2160. entry->rsvd = 0;
  2161. seg = seg->next;
  2162. }
  2163. /* set ERST count with the number of entries in the segment table */
  2164. val = xhci_readl(xhci, &xhci->ir_set->erst_size);
  2165. val &= ERST_SIZE_MASK;
  2166. val |= ERST_NUM_SEGS;
  2167. xhci_dbg(xhci, "// Write ERST size = %i to ir_set 0 (some bits preserved)\n",
  2168. val);
  2169. xhci_writel(xhci, val, &xhci->ir_set->erst_size);
  2170. xhci_dbg(xhci, "// Set ERST entries to point to event ring.\n");
  2171. /* set the segment table base address */
  2172. xhci_dbg(xhci, "// Set ERST base address for ir_set 0 = 0x%llx\n",
  2173. (unsigned long long)xhci->erst.erst_dma_addr);
  2174. val_64 = xhci_read_64(xhci, &xhci->ir_set->erst_base);
  2175. val_64 &= ERST_PTR_MASK;
  2176. val_64 |= (xhci->erst.erst_dma_addr & (u64) ~ERST_PTR_MASK);
  2177. xhci_write_64(xhci, val_64, &xhci->ir_set->erst_base);
  2178. /* Set the event ring dequeue address */
  2179. xhci_set_hc_event_deq(xhci);
  2180. xhci_dbg(xhci, "Wrote ERST address to ir_set 0.\n");
  2181. xhci_print_ir_set(xhci, 0);
  2182. /*
  2183. * XXX: Might need to set the Interrupter Moderation Register to
  2184. * something other than the default (~1ms minimum between interrupts).
  2185. * See section 5.5.1.2.
  2186. */
  2187. init_completion(&xhci->addr_dev);
  2188. for (i = 0; i < MAX_HC_SLOTS; ++i)
  2189. xhci->devs[i] = NULL;
  2190. for (i = 0; i < USB_MAXCHILDREN; ++i) {
  2191. xhci->bus_state[0].resume_done[i] = 0;
  2192. xhci->bus_state[1].resume_done[i] = 0;
  2193. }
  2194. if (scratchpad_alloc(xhci, flags))
  2195. goto fail;
  2196. if (xhci_setup_port_arrays(xhci, flags))
  2197. goto fail;
  2198. INIT_LIST_HEAD(&xhci->lpm_failed_devs);
  2199. /* Enable USB 3.0 device notifications for function remote wake, which
  2200. * is necessary for allowing USB 3.0 devices to do remote wakeup from
  2201. * U3 (device suspend).
  2202. */
  2203. temp = xhci_readl(xhci, &xhci->op_regs->dev_notification);
  2204. temp &= ~DEV_NOTE_MASK;
  2205. temp |= DEV_NOTE_FWAKE;
  2206. xhci_writel(xhci, temp, &xhci->op_regs->dev_notification);
  2207. return 0;
  2208. fail:
  2209. xhci_warn(xhci, "Couldn't initialize memory\n");
  2210. xhci_halt(xhci);
  2211. xhci_reset(xhci);
  2212. xhci_mem_cleanup(xhci);
  2213. return -ENOMEM;
  2214. }