cppi_dma.c 44 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553
  1. /*
  2. * Copyright (C) 2005-2006 by Texas Instruments
  3. *
  4. * This file implements a DMA interface using TI's CPPI DMA.
  5. * For now it's DaVinci-only, but CPPI isn't specific to DaVinci or USB.
  6. * The TUSB6020, using VLYNQ, has CPPI that looks much like DaVinci.
  7. */
  8. #include <linux/module.h>
  9. #include <linux/platform_device.h>
  10. #include <linux/slab.h>
  11. #include <linux/usb.h>
  12. #include "musb_core.h"
  13. #include "musb_debug.h"
  14. #include "cppi_dma.h"
  15. /* CPPI DMA status 7-mar-2006:
  16. *
  17. * - See musb_{host,gadget}.c for more info
  18. *
  19. * - Correct RX DMA generally forces the engine into irq-per-packet mode,
  20. * which can easily saturate the CPU under non-mass-storage loads.
  21. *
  22. * NOTES 24-aug-2006 (2.6.18-rc4):
  23. *
  24. * - peripheral RXDMA wedged in a test with packets of length 512/512/1.
  25. * evidently after the 1 byte packet was received and acked, the queue
  26. * of BDs got garbaged so it wouldn't empty the fifo. (rxcsr 0x2003,
  27. * and RX DMA0: 4 left, 80000000 8feff880, 8feff860 8feff860; 8f321401
  28. * 004001ff 00000001 .. 8feff860) Host was just getting NAKed on tx
  29. * of its next (512 byte) packet. IRQ issues?
  30. *
  31. * REVISIT: the "transfer DMA" glue between CPPI and USB fifos will
  32. * evidently also directly update the RX and TX CSRs ... so audit all
  33. * host and peripheral side DMA code to avoid CSR access after DMA has
  34. * been started.
  35. */
  36. /* REVISIT now we can avoid preallocating these descriptors; or
  37. * more simply, switch to a global freelist not per-channel ones.
  38. * Note: at full speed, 64 descriptors == 4K bulk data.
  39. */
  40. #define NUM_TXCHAN_BD 64
  41. #define NUM_RXCHAN_BD 64
  42. static inline void cpu_drain_writebuffer(void)
  43. {
  44. wmb();
  45. #ifdef CONFIG_CPU_ARM926T
  46. /* REVISIT this "should not be needed",
  47. * but lack of it sure seemed to hurt ...
  48. */
  49. asm("mcr p15, 0, r0, c7, c10, 4 @ drain write buffer\n");
  50. #endif
  51. }
  52. static inline struct cppi_descriptor *cppi_bd_alloc(struct cppi_channel *c)
  53. {
  54. struct cppi_descriptor *bd = c->freelist;
  55. if (bd)
  56. c->freelist = bd->next;
  57. return bd;
  58. }
  59. static inline void
  60. cppi_bd_free(struct cppi_channel *c, struct cppi_descriptor *bd)
  61. {
  62. if (!bd)
  63. return;
  64. bd->next = c->freelist;
  65. c->freelist = bd;
  66. }
  67. /*
  68. * Start DMA controller
  69. *
  70. * Initialize the DMA controller as necessary.
  71. */
  72. /* zero out entire rx state RAM entry for the channel */
  73. static void cppi_reset_rx(struct cppi_rx_stateram __iomem *rx)
  74. {
  75. musb_writel(&rx->rx_skipbytes, 0, 0);
  76. musb_writel(&rx->rx_head, 0, 0);
  77. musb_writel(&rx->rx_sop, 0, 0);
  78. musb_writel(&rx->rx_current, 0, 0);
  79. musb_writel(&rx->rx_buf_current, 0, 0);
  80. musb_writel(&rx->rx_len_len, 0, 0);
  81. musb_writel(&rx->rx_cnt_cnt, 0, 0);
  82. }
  83. /* zero out entire tx state RAM entry for the channel */
  84. static void cppi_reset_tx(struct cppi_tx_stateram __iomem *tx, u32 ptr)
  85. {
  86. musb_writel(&tx->tx_head, 0, 0);
  87. musb_writel(&tx->tx_buf, 0, 0);
  88. musb_writel(&tx->tx_current, 0, 0);
  89. musb_writel(&tx->tx_buf_current, 0, 0);
  90. musb_writel(&tx->tx_info, 0, 0);
  91. musb_writel(&tx->tx_rem_len, 0, 0);
  92. /* musb_writel(&tx->tx_dummy, 0, 0); */
  93. musb_writel(&tx->tx_complete, 0, ptr);
  94. }
  95. static void cppi_pool_init(struct cppi *cppi, struct cppi_channel *c)
  96. {
  97. int j;
  98. /* initialize channel fields */
  99. c->head = NULL;
  100. c->tail = NULL;
  101. c->last_processed = NULL;
  102. c->channel.status = MUSB_DMA_STATUS_UNKNOWN;
  103. c->controller = cppi;
  104. c->is_rndis = 0;
  105. c->freelist = NULL;
  106. /* build the BD Free list for the channel */
  107. for (j = 0; j < NUM_TXCHAN_BD + 1; j++) {
  108. struct cppi_descriptor *bd;
  109. dma_addr_t dma;
  110. bd = dma_pool_alloc(cppi->pool, GFP_KERNEL, &dma);
  111. bd->dma = dma;
  112. cppi_bd_free(c, bd);
  113. }
  114. }
  115. static int cppi_channel_abort(struct dma_channel *);
  116. static void cppi_pool_free(struct cppi_channel *c)
  117. {
  118. struct cppi *cppi = c->controller;
  119. struct cppi_descriptor *bd;
  120. (void) cppi_channel_abort(&c->channel);
  121. c->channel.status = MUSB_DMA_STATUS_UNKNOWN;
  122. c->controller = NULL;
  123. /* free all its bds */
  124. bd = c->last_processed;
  125. do {
  126. if (bd)
  127. dma_pool_free(cppi->pool, bd, bd->dma);
  128. bd = cppi_bd_alloc(c);
  129. } while (bd);
  130. c->last_processed = NULL;
  131. }
  132. static int cppi_controller_start(struct dma_controller *c)
  133. {
  134. struct cppi *controller;
  135. void __iomem *tibase;
  136. int i;
  137. controller = container_of(c, struct cppi, controller);
  138. /* do whatever is necessary to start controller */
  139. for (i = 0; i < ARRAY_SIZE(controller->tx); i++) {
  140. controller->tx[i].transmit = true;
  141. controller->tx[i].index = i;
  142. }
  143. for (i = 0; i < ARRAY_SIZE(controller->rx); i++) {
  144. controller->rx[i].transmit = false;
  145. controller->rx[i].index = i;
  146. }
  147. /* setup BD list on a per channel basis */
  148. for (i = 0; i < ARRAY_SIZE(controller->tx); i++)
  149. cppi_pool_init(controller, controller->tx + i);
  150. for (i = 0; i < ARRAY_SIZE(controller->rx); i++)
  151. cppi_pool_init(controller, controller->rx + i);
  152. tibase = controller->tibase;
  153. INIT_LIST_HEAD(&controller->tx_complete);
  154. /* initialise tx/rx channel head pointers to zero */
  155. for (i = 0; i < ARRAY_SIZE(controller->tx); i++) {
  156. struct cppi_channel *tx_ch = controller->tx + i;
  157. struct cppi_tx_stateram __iomem *tx;
  158. INIT_LIST_HEAD(&tx_ch->tx_complete);
  159. tx = tibase + DAVINCI_TXCPPI_STATERAM_OFFSET(i);
  160. tx_ch->state_ram = tx;
  161. cppi_reset_tx(tx, 0);
  162. }
  163. for (i = 0; i < ARRAY_SIZE(controller->rx); i++) {
  164. struct cppi_channel *rx_ch = controller->rx + i;
  165. struct cppi_rx_stateram __iomem *rx;
  166. INIT_LIST_HEAD(&rx_ch->tx_complete);
  167. rx = tibase + DAVINCI_RXCPPI_STATERAM_OFFSET(i);
  168. rx_ch->state_ram = rx;
  169. cppi_reset_rx(rx);
  170. }
  171. /* enable individual cppi channels */
  172. musb_writel(tibase, DAVINCI_TXCPPI_INTENAB_REG,
  173. DAVINCI_DMA_ALL_CHANNELS_ENABLE);
  174. musb_writel(tibase, DAVINCI_RXCPPI_INTENAB_REG,
  175. DAVINCI_DMA_ALL_CHANNELS_ENABLE);
  176. /* enable tx/rx CPPI control */
  177. musb_writel(tibase, DAVINCI_TXCPPI_CTRL_REG, DAVINCI_DMA_CTRL_ENABLE);
  178. musb_writel(tibase, DAVINCI_RXCPPI_CTRL_REG, DAVINCI_DMA_CTRL_ENABLE);
  179. /* disable RNDIS mode, also host rx RNDIS autorequest */
  180. musb_writel(tibase, DAVINCI_RNDIS_REG, 0);
  181. musb_writel(tibase, DAVINCI_AUTOREQ_REG, 0);
  182. return 0;
  183. }
  184. /*
  185. * Stop DMA controller
  186. *
  187. * De-Init the DMA controller as necessary.
  188. */
  189. static int cppi_controller_stop(struct dma_controller *c)
  190. {
  191. struct cppi *controller;
  192. void __iomem *tibase;
  193. int i;
  194. struct musb *musb;
  195. controller = container_of(c, struct cppi, controller);
  196. musb = controller->musb;
  197. tibase = controller->tibase;
  198. /* DISABLE INDIVIDUAL CHANNEL Interrupts */
  199. musb_writel(tibase, DAVINCI_TXCPPI_INTCLR_REG,
  200. DAVINCI_DMA_ALL_CHANNELS_ENABLE);
  201. musb_writel(tibase, DAVINCI_RXCPPI_INTCLR_REG,
  202. DAVINCI_DMA_ALL_CHANNELS_ENABLE);
  203. dev_dbg(musb->controller, "Tearing down RX and TX Channels\n");
  204. for (i = 0; i < ARRAY_SIZE(controller->tx); i++) {
  205. /* FIXME restructure of txdma to use bds like rxdma */
  206. controller->tx[i].last_processed = NULL;
  207. cppi_pool_free(controller->tx + i);
  208. }
  209. for (i = 0; i < ARRAY_SIZE(controller->rx); i++)
  210. cppi_pool_free(controller->rx + i);
  211. /* in Tx Case proper teardown is supported. We resort to disabling
  212. * Tx/Rx CPPI after cleanup of Tx channels. Before TX teardown is
  213. * complete TX CPPI cannot be disabled.
  214. */
  215. /*disable tx/rx cppi */
  216. musb_writel(tibase, DAVINCI_TXCPPI_CTRL_REG, DAVINCI_DMA_CTRL_DISABLE);
  217. musb_writel(tibase, DAVINCI_RXCPPI_CTRL_REG, DAVINCI_DMA_CTRL_DISABLE);
  218. return 0;
  219. }
  220. /* While dma channel is allocated, we only want the core irqs active
  221. * for fault reports, otherwise we'd get irqs that we don't care about.
  222. * Except for TX irqs, where dma done != fifo empty and reusable ...
  223. *
  224. * NOTE: docs don't say either way, but irq masking **enables** irqs.
  225. *
  226. * REVISIT same issue applies to pure PIO usage too, and non-cppi dma...
  227. */
  228. static inline void core_rxirq_disable(void __iomem *tibase, unsigned epnum)
  229. {
  230. musb_writel(tibase, DAVINCI_USB_INT_MASK_CLR_REG, 1 << (epnum + 8));
  231. }
  232. static inline void core_rxirq_enable(void __iomem *tibase, unsigned epnum)
  233. {
  234. musb_writel(tibase, DAVINCI_USB_INT_MASK_SET_REG, 1 << (epnum + 8));
  235. }
  236. /*
  237. * Allocate a CPPI Channel for DMA. With CPPI, channels are bound to
  238. * each transfer direction of a non-control endpoint, so allocating
  239. * (and deallocating) is mostly a way to notice bad housekeeping on
  240. * the software side. We assume the irqs are always active.
  241. */
  242. static struct dma_channel *
  243. cppi_channel_allocate(struct dma_controller *c,
  244. struct musb_hw_ep *ep, u8 transmit)
  245. {
  246. struct cppi *controller;
  247. u8 index;
  248. struct cppi_channel *cppi_ch;
  249. void __iomem *tibase;
  250. struct musb *musb;
  251. controller = container_of(c, struct cppi, controller);
  252. tibase = controller->tibase;
  253. musb = controller->musb;
  254. /* ep0 doesn't use DMA; remember cppi indices are 0..N-1 */
  255. index = ep->epnum - 1;
  256. /* return the corresponding CPPI Channel Handle, and
  257. * probably disable the non-CPPI irq until we need it.
  258. */
  259. if (transmit) {
  260. if (index >= ARRAY_SIZE(controller->tx)) {
  261. dev_dbg(musb->controller, "no %cX%d CPPI channel\n", 'T', index);
  262. return NULL;
  263. }
  264. cppi_ch = controller->tx + index;
  265. } else {
  266. if (index >= ARRAY_SIZE(controller->rx)) {
  267. dev_dbg(musb->controller, "no %cX%d CPPI channel\n", 'R', index);
  268. return NULL;
  269. }
  270. cppi_ch = controller->rx + index;
  271. core_rxirq_disable(tibase, ep->epnum);
  272. }
  273. /* REVISIT make this an error later once the same driver code works
  274. * with the other DMA engine too
  275. */
  276. if (cppi_ch->hw_ep)
  277. dev_dbg(musb->controller, "re-allocating DMA%d %cX channel %p\n",
  278. index, transmit ? 'T' : 'R', cppi_ch);
  279. cppi_ch->hw_ep = ep;
  280. cppi_ch->channel.status = MUSB_DMA_STATUS_FREE;
  281. cppi_ch->channel.max_len = 0x7fffffff;
  282. dev_dbg(musb->controller, "Allocate CPPI%d %cX\n", index, transmit ? 'T' : 'R');
  283. return &cppi_ch->channel;
  284. }
  285. /* Release a CPPI Channel. */
  286. static void cppi_channel_release(struct dma_channel *channel)
  287. {
  288. struct cppi_channel *c;
  289. void __iomem *tibase;
  290. /* REVISIT: for paranoia, check state and abort if needed... */
  291. c = container_of(channel, struct cppi_channel, channel);
  292. tibase = c->controller->tibase;
  293. if (!c->hw_ep)
  294. dev_dbg(c->controller->musb->controller,
  295. "releasing idle DMA channel %p\n", c);
  296. else if (!c->transmit)
  297. core_rxirq_enable(tibase, c->index + 1);
  298. /* for now, leave its cppi IRQ enabled (we won't trigger it) */
  299. c->hw_ep = NULL;
  300. channel->status = MUSB_DMA_STATUS_UNKNOWN;
  301. }
  302. /* Context: controller irqlocked */
  303. static void
  304. cppi_dump_rx(int level, struct cppi_channel *c, const char *tag)
  305. {
  306. void __iomem *base = c->controller->mregs;
  307. struct cppi_rx_stateram __iomem *rx = c->state_ram;
  308. musb_ep_select(base, c->index + 1);
  309. dev_dbg(c->controller->musb->controller,
  310. "RX DMA%d%s: %d left, csr %04x, "
  311. "%08x H%08x S%08x C%08x, "
  312. "B%08x L%08x %08x .. %08x"
  313. "\n",
  314. c->index, tag,
  315. musb_readl(c->controller->tibase,
  316. DAVINCI_RXCPPI_BUFCNT0_REG + 4 * c->index),
  317. musb_readw(c->hw_ep->regs, MUSB_RXCSR),
  318. musb_readl(&rx->rx_skipbytes, 0),
  319. musb_readl(&rx->rx_head, 0),
  320. musb_readl(&rx->rx_sop, 0),
  321. musb_readl(&rx->rx_current, 0),
  322. musb_readl(&rx->rx_buf_current, 0),
  323. musb_readl(&rx->rx_len_len, 0),
  324. musb_readl(&rx->rx_cnt_cnt, 0),
  325. musb_readl(&rx->rx_complete, 0)
  326. );
  327. }
  328. /* Context: controller irqlocked */
  329. static void
  330. cppi_dump_tx(int level, struct cppi_channel *c, const char *tag)
  331. {
  332. void __iomem *base = c->controller->mregs;
  333. struct cppi_tx_stateram __iomem *tx = c->state_ram;
  334. musb_ep_select(base, c->index + 1);
  335. dev_dbg(c->controller->musb->controller,
  336. "TX DMA%d%s: csr %04x, "
  337. "H%08x S%08x C%08x %08x, "
  338. "F%08x L%08x .. %08x"
  339. "\n",
  340. c->index, tag,
  341. musb_readw(c->hw_ep->regs, MUSB_TXCSR),
  342. musb_readl(&tx->tx_head, 0),
  343. musb_readl(&tx->tx_buf, 0),
  344. musb_readl(&tx->tx_current, 0),
  345. musb_readl(&tx->tx_buf_current, 0),
  346. musb_readl(&tx->tx_info, 0),
  347. musb_readl(&tx->tx_rem_len, 0),
  348. /* dummy/unused word 6 */
  349. musb_readl(&tx->tx_complete, 0)
  350. );
  351. }
  352. /* Context: controller irqlocked */
  353. static inline void
  354. cppi_rndis_update(struct cppi_channel *c, int is_rx,
  355. void __iomem *tibase, int is_rndis)
  356. {
  357. /* we may need to change the rndis flag for this cppi channel */
  358. if (c->is_rndis != is_rndis) {
  359. u32 value = musb_readl(tibase, DAVINCI_RNDIS_REG);
  360. u32 temp = 1 << (c->index);
  361. if (is_rx)
  362. temp <<= 16;
  363. if (is_rndis)
  364. value |= temp;
  365. else
  366. value &= ~temp;
  367. musb_writel(tibase, DAVINCI_RNDIS_REG, value);
  368. c->is_rndis = is_rndis;
  369. }
  370. }
  371. static void cppi_dump_rxbd(const char *tag, struct cppi_descriptor *bd)
  372. {
  373. pr_debug("RXBD/%s %08x: "
  374. "nxt %08x buf %08x off.blen %08x opt.plen %08x\n",
  375. tag, bd->dma,
  376. bd->hw_next, bd->hw_bufp, bd->hw_off_len,
  377. bd->hw_options);
  378. }
  379. static void cppi_dump_rxq(int level, const char *tag, struct cppi_channel *rx)
  380. {
  381. struct cppi_descriptor *bd;
  382. cppi_dump_rx(level, rx, tag);
  383. if (rx->last_processed)
  384. cppi_dump_rxbd("last", rx->last_processed);
  385. for (bd = rx->head; bd; bd = bd->next)
  386. cppi_dump_rxbd("active", bd);
  387. }
  388. /* NOTE: DaVinci autoreq is ignored except for host side "RNDIS" mode RX;
  389. * so we won't ever use it (see "CPPI RX Woes" below).
  390. */
  391. static inline int cppi_autoreq_update(struct cppi_channel *rx,
  392. void __iomem *tibase, int onepacket, unsigned n_bds)
  393. {
  394. u32 val;
  395. #ifdef RNDIS_RX_IS_USABLE
  396. u32 tmp;
  397. /* assert(is_host_active(musb)) */
  398. /* start from "AutoReq never" */
  399. tmp = musb_readl(tibase, DAVINCI_AUTOREQ_REG);
  400. val = tmp & ~((0x3) << (rx->index * 2));
  401. /* HCD arranged reqpkt for packet #1. we arrange int
  402. * for all but the last one, maybe in two segments.
  403. */
  404. if (!onepacket) {
  405. #if 0
  406. /* use two segments, autoreq "all" then the last "never" */
  407. val |= ((0x3) << (rx->index * 2));
  408. n_bds--;
  409. #else
  410. /* one segment, autoreq "all-but-last" */
  411. val |= ((0x1) << (rx->index * 2));
  412. #endif
  413. }
  414. if (val != tmp) {
  415. int n = 100;
  416. /* make sure that autoreq is updated before continuing */
  417. musb_writel(tibase, DAVINCI_AUTOREQ_REG, val);
  418. do {
  419. tmp = musb_readl(tibase, DAVINCI_AUTOREQ_REG);
  420. if (tmp == val)
  421. break;
  422. cpu_relax();
  423. } while (n-- > 0);
  424. }
  425. #endif
  426. /* REQPKT is turned off after each segment */
  427. if (n_bds && rx->channel.actual_len) {
  428. void __iomem *regs = rx->hw_ep->regs;
  429. val = musb_readw(regs, MUSB_RXCSR);
  430. if (!(val & MUSB_RXCSR_H_REQPKT)) {
  431. val |= MUSB_RXCSR_H_REQPKT | MUSB_RXCSR_H_WZC_BITS;
  432. musb_writew(regs, MUSB_RXCSR, val);
  433. /* flush writebuffer */
  434. val = musb_readw(regs, MUSB_RXCSR);
  435. }
  436. }
  437. return n_bds;
  438. }
  439. /* Buffer enqueuing Logic:
  440. *
  441. * - RX builds new queues each time, to help handle routine "early
  442. * termination" cases (faults, including errors and short reads)
  443. * more correctly.
  444. *
  445. * - for now, TX reuses the same queue of BDs every time
  446. *
  447. * REVISIT long term, we want a normal dynamic model.
  448. * ... the goal will be to append to the
  449. * existing queue, processing completed "dma buffers" (segments) on the fly.
  450. *
  451. * Otherwise we force an IRQ latency between requests, which slows us a lot
  452. * (especially in "transparent" dma). Unfortunately that model seems to be
  453. * inherent in the DMA model from the Mentor code, except in the rare case
  454. * of transfers big enough (~128+ KB) that we could append "middle" segments
  455. * in the TX paths. (RX can't do this, see below.)
  456. *
  457. * That's true even in the CPPI- friendly iso case, where most urbs have
  458. * several small segments provided in a group and where the "packet at a time"
  459. * "transparent" DMA model is always correct, even on the RX side.
  460. */
  461. /*
  462. * CPPI TX:
  463. * ========
  464. * TX is a lot more reasonable than RX; it doesn't need to run in
  465. * irq-per-packet mode very often. RNDIS mode seems to behave too
  466. * (except how it handles the exactly-N-packets case). Building a
  467. * txdma queue with multiple requests (urb or usb_request) looks
  468. * like it would work ... but fault handling would need much testing.
  469. *
  470. * The main issue with TX mode RNDIS relates to transfer lengths that
  471. * are an exact multiple of the packet length. It appears that there's
  472. * a hiccup in that case (maybe the DMA completes before the ZLP gets
  473. * written?) boiling down to not being able to rely on CPPI writing any
  474. * terminating zero length packet before the next transfer is written.
  475. * So that's punted to PIO; better yet, gadget drivers can avoid it.
  476. *
  477. * Plus, there's allegedly an undocumented constraint that rndis transfer
  478. * length be a multiple of 64 bytes ... but the chip doesn't act that
  479. * way, and we really don't _want_ that behavior anyway.
  480. *
  481. * On TX, "transparent" mode works ... although experiments have shown
  482. * problems trying to use the SOP/EOP bits in different USB packets.
  483. *
  484. * REVISIT try to handle terminating zero length packets using CPPI
  485. * instead of doing it by PIO after an IRQ. (Meanwhile, make Ethernet
  486. * links avoid that issue by forcing them to avoid zlps.)
  487. */
  488. static void
  489. cppi_next_tx_segment(struct musb *musb, struct cppi_channel *tx)
  490. {
  491. unsigned maxpacket = tx->maxpacket;
  492. dma_addr_t addr = tx->buf_dma + tx->offset;
  493. size_t length = tx->buf_len - tx->offset;
  494. struct cppi_descriptor *bd;
  495. unsigned n_bds;
  496. unsigned i;
  497. struct cppi_tx_stateram __iomem *tx_ram = tx->state_ram;
  498. int rndis;
  499. /* TX can use the CPPI "rndis" mode, where we can probably fit this
  500. * transfer in one BD and one IRQ. The only time we would NOT want
  501. * to use it is when hardware constraints prevent it, or if we'd
  502. * trigger the "send a ZLP?" confusion.
  503. */
  504. rndis = (maxpacket & 0x3f) == 0
  505. && length > maxpacket
  506. && length < 0xffff
  507. && (length % maxpacket) != 0;
  508. if (rndis) {
  509. maxpacket = length;
  510. n_bds = 1;
  511. } else {
  512. n_bds = length / maxpacket;
  513. if (!length || (length % maxpacket))
  514. n_bds++;
  515. n_bds = min(n_bds, (unsigned) NUM_TXCHAN_BD);
  516. length = min(n_bds * maxpacket, length);
  517. }
  518. dev_dbg(musb->controller, "TX DMA%d, pktSz %d %s bds %d dma 0x%llx len %u\n",
  519. tx->index,
  520. maxpacket,
  521. rndis ? "rndis" : "transparent",
  522. n_bds,
  523. (unsigned long long)addr, length);
  524. cppi_rndis_update(tx, 0, musb->ctrl_base, rndis);
  525. /* assuming here that channel_program is called during
  526. * transfer initiation ... current code maintains state
  527. * for one outstanding request only (no queues, not even
  528. * the implicit ones of an iso urb).
  529. */
  530. bd = tx->freelist;
  531. tx->head = bd;
  532. tx->last_processed = NULL;
  533. /* FIXME use BD pool like RX side does, and just queue
  534. * the minimum number for this request.
  535. */
  536. /* Prepare queue of BDs first, then hand it to hardware.
  537. * All BDs except maybe the last should be of full packet
  538. * size; for RNDIS there _is_ only that last packet.
  539. */
  540. for (i = 0; i < n_bds; ) {
  541. if (++i < n_bds && bd->next)
  542. bd->hw_next = bd->next->dma;
  543. else
  544. bd->hw_next = 0;
  545. bd->hw_bufp = tx->buf_dma + tx->offset;
  546. /* FIXME set EOP only on the last packet,
  547. * SOP only on the first ... avoid IRQs
  548. */
  549. if ((tx->offset + maxpacket) <= tx->buf_len) {
  550. tx->offset += maxpacket;
  551. bd->hw_off_len = maxpacket;
  552. bd->hw_options = CPPI_SOP_SET | CPPI_EOP_SET
  553. | CPPI_OWN_SET | maxpacket;
  554. } else {
  555. /* only this one may be a partial USB Packet */
  556. u32 partial_len;
  557. partial_len = tx->buf_len - tx->offset;
  558. tx->offset = tx->buf_len;
  559. bd->hw_off_len = partial_len;
  560. bd->hw_options = CPPI_SOP_SET | CPPI_EOP_SET
  561. | CPPI_OWN_SET | partial_len;
  562. if (partial_len == 0)
  563. bd->hw_options |= CPPI_ZERO_SET;
  564. }
  565. dev_dbg(musb->controller, "TXBD %p: nxt %08x buf %08x len %04x opt %08x\n",
  566. bd, bd->hw_next, bd->hw_bufp,
  567. bd->hw_off_len, bd->hw_options);
  568. /* update the last BD enqueued to the list */
  569. tx->tail = bd;
  570. bd = bd->next;
  571. }
  572. /* BDs live in DMA-coherent memory, but writes might be pending */
  573. cpu_drain_writebuffer();
  574. /* Write to the HeadPtr in state RAM to trigger */
  575. musb_writel(&tx_ram->tx_head, 0, (u32)tx->freelist->dma);
  576. cppi_dump_tx(5, tx, "/S");
  577. }
  578. /*
  579. * CPPI RX Woes:
  580. * =============
  581. * Consider a 1KB bulk RX buffer in two scenarios: (a) it's fed two 300 byte
  582. * packets back-to-back, and (b) it's fed two 512 byte packets back-to-back.
  583. * (Full speed transfers have similar scenarios.)
  584. *
  585. * The correct behavior for Linux is that (a) fills the buffer with 300 bytes,
  586. * and the next packet goes into a buffer that's queued later; while (b) fills
  587. * the buffer with 1024 bytes. How to do that with CPPI?
  588. *
  589. * - RX queues in "rndis" mode -- one single BD -- handle (a) correctly, but
  590. * (b) loses **BADLY** because nothing (!) happens when that second packet
  591. * fills the buffer, much less when a third one arrives. (Which makes this
  592. * not a "true" RNDIS mode. In the RNDIS protocol short-packet termination
  593. * is optional, and it's fine if peripherals -- not hosts! -- pad messages
  594. * out to end-of-buffer. Standard PCI host controller DMA descriptors
  595. * implement that mode by default ... which is no accident.)
  596. *
  597. * - RX queues in "transparent" mode -- two BDs with 512 bytes each -- have
  598. * converse problems: (b) is handled right, but (a) loses badly. CPPI RX
  599. * ignores SOP/EOP markings and processes both of those BDs; so both packets
  600. * are loaded into the buffer (with a 212 byte gap between them), and the next
  601. * buffer queued will NOT get its 300 bytes of data. (It seems like SOP/EOP
  602. * are intended as outputs for RX queues, not inputs...)
  603. *
  604. * - A variant of "transparent" mode -- one BD at a time -- is the only way to
  605. * reliably make both cases work, with software handling both cases correctly
  606. * and at the significant penalty of needing an IRQ per packet. (The lack of
  607. * I/O overlap can be slightly ameliorated by enabling double buffering.)
  608. *
  609. * So how to get rid of IRQ-per-packet? The transparent multi-BD case could
  610. * be used in special cases like mass storage, which sets URB_SHORT_NOT_OK
  611. * (or maybe its peripheral side counterpart) to flag (a) scenarios as errors
  612. * with guaranteed driver level fault recovery and scrubbing out what's left
  613. * of that garbaged datastream.
  614. *
  615. * But there seems to be no way to identify the cases where CPPI RNDIS mode
  616. * is appropriate -- which do NOT include RNDIS host drivers, but do include
  617. * the CDC Ethernet driver! -- and the documentation is incomplete/wrong.
  618. * So we can't _ever_ use RX RNDIS mode ... except by using a heuristic
  619. * that applies best on the peripheral side (and which could fail rudely).
  620. *
  621. * Leaving only "transparent" mode; we avoid multi-bd modes in almost all
  622. * cases other than mass storage class. Otherwise we're correct but slow,
  623. * since CPPI penalizes our need for a "true RNDIS" default mode.
  624. */
  625. /* Heuristic, intended to kick in for ethernet/rndis peripheral ONLY
  626. *
  627. * IFF
  628. * (a) peripheral mode ... since rndis peripherals could pad their
  629. * writes to hosts, causing i/o failure; or we'd have to cope with
  630. * a largely unknowable variety of host side protocol variants
  631. * (b) and short reads are NOT errors ... since full reads would
  632. * cause those same i/o failures
  633. * (c) and read length is
  634. * - less than 64KB (max per cppi descriptor)
  635. * - not a multiple of 4096 (g_zero default, full reads typical)
  636. * - N (>1) packets long, ditto (full reads not EXPECTED)
  637. * THEN
  638. * try rx rndis mode
  639. *
  640. * Cost of heuristic failing: RXDMA wedges at the end of transfers that
  641. * fill out the whole buffer. Buggy host side usb network drivers could
  642. * trigger that, but "in the field" such bugs seem to be all but unknown.
  643. *
  644. * So this module parameter lets the heuristic be disabled. When using
  645. * gadgetfs, the heuristic will probably need to be disabled.
  646. */
  647. static bool cppi_rx_rndis = 1;
  648. module_param(cppi_rx_rndis, bool, 0);
  649. MODULE_PARM_DESC(cppi_rx_rndis, "enable/disable RX RNDIS heuristic");
  650. /**
  651. * cppi_next_rx_segment - dma read for the next chunk of a buffer
  652. * @musb: the controller
  653. * @rx: dma channel
  654. * @onepacket: true unless caller treats short reads as errors, and
  655. * performs fault recovery above usbcore.
  656. * Context: controller irqlocked
  657. *
  658. * See above notes about why we can't use multi-BD RX queues except in
  659. * rare cases (mass storage class), and can never use the hardware "rndis"
  660. * mode (since it's not a "true" RNDIS mode) with complete safety..
  661. *
  662. * It's ESSENTIAL that callers specify "onepacket" mode unless they kick in
  663. * code to recover from corrupted datastreams after each short transfer.
  664. */
  665. static void
  666. cppi_next_rx_segment(struct musb *musb, struct cppi_channel *rx, int onepacket)
  667. {
  668. unsigned maxpacket = rx->maxpacket;
  669. dma_addr_t addr = rx->buf_dma + rx->offset;
  670. size_t length = rx->buf_len - rx->offset;
  671. struct cppi_descriptor *bd, *tail;
  672. unsigned n_bds;
  673. unsigned i;
  674. void __iomem *tibase = musb->ctrl_base;
  675. int is_rndis = 0;
  676. struct cppi_rx_stateram __iomem *rx_ram = rx->state_ram;
  677. struct cppi_descriptor *d;
  678. if (onepacket) {
  679. /* almost every USB driver, host or peripheral side */
  680. n_bds = 1;
  681. /* maybe apply the heuristic above */
  682. if (cppi_rx_rndis
  683. && is_peripheral_active(musb)
  684. && length > maxpacket
  685. && (length & ~0xffff) == 0
  686. && (length & 0x0fff) != 0
  687. && (length & (maxpacket - 1)) == 0) {
  688. maxpacket = length;
  689. is_rndis = 1;
  690. }
  691. } else {
  692. /* virtually nothing except mass storage class */
  693. if (length > 0xffff) {
  694. n_bds = 0xffff / maxpacket;
  695. length = n_bds * maxpacket;
  696. } else {
  697. n_bds = length / maxpacket;
  698. if (length % maxpacket)
  699. n_bds++;
  700. }
  701. if (n_bds == 1)
  702. onepacket = 1;
  703. else
  704. n_bds = min(n_bds, (unsigned) NUM_RXCHAN_BD);
  705. }
  706. /* In host mode, autorequest logic can generate some IN tokens; it's
  707. * tricky since we can't leave REQPKT set in RXCSR after the transfer
  708. * finishes. So: multipacket transfers involve two or more segments.
  709. * And always at least two IRQs ... RNDIS mode is not an option.
  710. */
  711. if (is_host_active(musb))
  712. n_bds = cppi_autoreq_update(rx, tibase, onepacket, n_bds);
  713. cppi_rndis_update(rx, 1, musb->ctrl_base, is_rndis);
  714. length = min(n_bds * maxpacket, length);
  715. dev_dbg(musb->controller, "RX DMA%d seg, maxp %d %s bds %d (cnt %d) "
  716. "dma 0x%llx len %u %u/%u\n",
  717. rx->index, maxpacket,
  718. onepacket
  719. ? (is_rndis ? "rndis" : "onepacket")
  720. : "multipacket",
  721. n_bds,
  722. musb_readl(tibase,
  723. DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4))
  724. & 0xffff,
  725. (unsigned long long)addr, length,
  726. rx->channel.actual_len, rx->buf_len);
  727. /* only queue one segment at a time, since the hardware prevents
  728. * correct queue shutdown after unexpected short packets
  729. */
  730. bd = cppi_bd_alloc(rx);
  731. rx->head = bd;
  732. /* Build BDs for all packets in this segment */
  733. for (i = 0, tail = NULL; bd && i < n_bds; i++, tail = bd) {
  734. u32 bd_len;
  735. if (i) {
  736. bd = cppi_bd_alloc(rx);
  737. if (!bd)
  738. break;
  739. tail->next = bd;
  740. tail->hw_next = bd->dma;
  741. }
  742. bd->hw_next = 0;
  743. /* all but the last packet will be maxpacket size */
  744. if (maxpacket < length)
  745. bd_len = maxpacket;
  746. else
  747. bd_len = length;
  748. bd->hw_bufp = addr;
  749. addr += bd_len;
  750. rx->offset += bd_len;
  751. bd->hw_off_len = (0 /*offset*/ << 16) + bd_len;
  752. bd->buflen = bd_len;
  753. bd->hw_options = CPPI_OWN_SET | (i == 0 ? length : 0);
  754. length -= bd_len;
  755. }
  756. /* we always expect at least one reusable BD! */
  757. if (!tail) {
  758. WARNING("rx dma%d -- no BDs? need %d\n", rx->index, n_bds);
  759. return;
  760. } else if (i < n_bds)
  761. WARNING("rx dma%d -- only %d of %d BDs\n", rx->index, i, n_bds);
  762. tail->next = NULL;
  763. tail->hw_next = 0;
  764. bd = rx->head;
  765. rx->tail = tail;
  766. /* short reads and other faults should terminate this entire
  767. * dma segment. we want one "dma packet" per dma segment, not
  768. * one per USB packet, terminating the whole queue at once...
  769. * NOTE that current hardware seems to ignore SOP and EOP.
  770. */
  771. bd->hw_options |= CPPI_SOP_SET;
  772. tail->hw_options |= CPPI_EOP_SET;
  773. for (d = rx->head; d; d = d->next)
  774. cppi_dump_rxbd("S", d);
  775. /* in case the preceding transfer left some state... */
  776. tail = rx->last_processed;
  777. if (tail) {
  778. tail->next = bd;
  779. tail->hw_next = bd->dma;
  780. }
  781. core_rxirq_enable(tibase, rx->index + 1);
  782. /* BDs live in DMA-coherent memory, but writes might be pending */
  783. cpu_drain_writebuffer();
  784. /* REVISIT specs say to write this AFTER the BUFCNT register
  785. * below ... but that loses badly.
  786. */
  787. musb_writel(&rx_ram->rx_head, 0, bd->dma);
  788. /* bufferCount must be at least 3, and zeroes on completion
  789. * unless it underflows below zero, or stops at two, or keeps
  790. * growing ... grr.
  791. */
  792. i = musb_readl(tibase,
  793. DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4))
  794. & 0xffff;
  795. if (!i)
  796. musb_writel(tibase,
  797. DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4),
  798. n_bds + 2);
  799. else if (n_bds > (i - 3))
  800. musb_writel(tibase,
  801. DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4),
  802. n_bds - (i - 3));
  803. i = musb_readl(tibase,
  804. DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4))
  805. & 0xffff;
  806. if (i < (2 + n_bds)) {
  807. dev_dbg(musb->controller, "bufcnt%d underrun - %d (for %d)\n",
  808. rx->index, i, n_bds);
  809. musb_writel(tibase,
  810. DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4),
  811. n_bds + 2);
  812. }
  813. cppi_dump_rx(4, rx, "/S");
  814. }
  815. /**
  816. * cppi_channel_program - program channel for data transfer
  817. * @ch: the channel
  818. * @maxpacket: max packet size
  819. * @mode: For RX, 1 unless the usb protocol driver promised to treat
  820. * all short reads as errors and kick in high level fault recovery.
  821. * For TX, ignored because of RNDIS mode races/glitches.
  822. * @dma_addr: dma address of buffer
  823. * @len: length of buffer
  824. * Context: controller irqlocked
  825. */
  826. static int cppi_channel_program(struct dma_channel *ch,
  827. u16 maxpacket, u8 mode,
  828. dma_addr_t dma_addr, u32 len)
  829. {
  830. struct cppi_channel *cppi_ch;
  831. struct cppi *controller;
  832. struct musb *musb;
  833. cppi_ch = container_of(ch, struct cppi_channel, channel);
  834. controller = cppi_ch->controller;
  835. musb = controller->musb;
  836. switch (ch->status) {
  837. case MUSB_DMA_STATUS_BUS_ABORT:
  838. case MUSB_DMA_STATUS_CORE_ABORT:
  839. /* fault irq handler should have handled cleanup */
  840. WARNING("%cX DMA%d not cleaned up after abort!\n",
  841. cppi_ch->transmit ? 'T' : 'R',
  842. cppi_ch->index);
  843. /* WARN_ON(1); */
  844. break;
  845. case MUSB_DMA_STATUS_BUSY:
  846. WARNING("program active channel? %cX DMA%d\n",
  847. cppi_ch->transmit ? 'T' : 'R',
  848. cppi_ch->index);
  849. /* WARN_ON(1); */
  850. break;
  851. case MUSB_DMA_STATUS_UNKNOWN:
  852. dev_dbg(musb->controller, "%cX DMA%d not allocated!\n",
  853. cppi_ch->transmit ? 'T' : 'R',
  854. cppi_ch->index);
  855. /* FALLTHROUGH */
  856. case MUSB_DMA_STATUS_FREE:
  857. break;
  858. }
  859. ch->status = MUSB_DMA_STATUS_BUSY;
  860. /* set transfer parameters, then queue up its first segment */
  861. cppi_ch->buf_dma = dma_addr;
  862. cppi_ch->offset = 0;
  863. cppi_ch->maxpacket = maxpacket;
  864. cppi_ch->buf_len = len;
  865. cppi_ch->channel.actual_len = 0;
  866. /* TX channel? or RX? */
  867. if (cppi_ch->transmit)
  868. cppi_next_tx_segment(musb, cppi_ch);
  869. else
  870. cppi_next_rx_segment(musb, cppi_ch, mode);
  871. return true;
  872. }
  873. static bool cppi_rx_scan(struct cppi *cppi, unsigned ch)
  874. {
  875. struct cppi_channel *rx = &cppi->rx[ch];
  876. struct cppi_rx_stateram __iomem *state = rx->state_ram;
  877. struct cppi_descriptor *bd;
  878. struct cppi_descriptor *last = rx->last_processed;
  879. bool completed = false;
  880. bool acked = false;
  881. int i;
  882. dma_addr_t safe2ack;
  883. void __iomem *regs = rx->hw_ep->regs;
  884. struct musb *musb = cppi->musb;
  885. cppi_dump_rx(6, rx, "/K");
  886. bd = last ? last->next : rx->head;
  887. if (!bd)
  888. return false;
  889. /* run through all completed BDs */
  890. for (i = 0, safe2ack = musb_readl(&state->rx_complete, 0);
  891. (safe2ack || completed) && bd && i < NUM_RXCHAN_BD;
  892. i++, bd = bd->next) {
  893. u16 len;
  894. /* catch latest BD writes from CPPI */
  895. rmb();
  896. if (!completed && (bd->hw_options & CPPI_OWN_SET))
  897. break;
  898. dev_dbg(musb->controller, "C/RXBD %llx: nxt %08x buf %08x "
  899. "off.len %08x opt.len %08x (%d)\n",
  900. (unsigned long long)bd->dma, bd->hw_next, bd->hw_bufp,
  901. bd->hw_off_len, bd->hw_options,
  902. rx->channel.actual_len);
  903. /* actual packet received length */
  904. if ((bd->hw_options & CPPI_SOP_SET) && !completed)
  905. len = bd->hw_off_len & CPPI_RECV_PKTLEN_MASK;
  906. else
  907. len = 0;
  908. if (bd->hw_options & CPPI_EOQ_MASK)
  909. completed = true;
  910. if (!completed && len < bd->buflen) {
  911. /* NOTE: when we get a short packet, RXCSR_H_REQPKT
  912. * must have been cleared, and no more DMA packets may
  913. * active be in the queue... TI docs didn't say, but
  914. * CPPI ignores those BDs even though OWN is still set.
  915. */
  916. completed = true;
  917. dev_dbg(musb->controller, "rx short %d/%d (%d)\n",
  918. len, bd->buflen,
  919. rx->channel.actual_len);
  920. }
  921. /* If we got here, we expect to ack at least one BD; meanwhile
  922. * CPPI may completing other BDs while we scan this list...
  923. *
  924. * RACE: we can notice OWN cleared before CPPI raises the
  925. * matching irq by writing that BD as the completion pointer.
  926. * In such cases, stop scanning and wait for the irq, avoiding
  927. * lost acks and states where BD ownership is unclear.
  928. */
  929. if (bd->dma == safe2ack) {
  930. musb_writel(&state->rx_complete, 0, safe2ack);
  931. safe2ack = musb_readl(&state->rx_complete, 0);
  932. acked = true;
  933. if (bd->dma == safe2ack)
  934. safe2ack = 0;
  935. }
  936. rx->channel.actual_len += len;
  937. cppi_bd_free(rx, last);
  938. last = bd;
  939. /* stop scanning on end-of-segment */
  940. if (bd->hw_next == 0)
  941. completed = true;
  942. }
  943. rx->last_processed = last;
  944. /* dma abort, lost ack, or ... */
  945. if (!acked && last) {
  946. int csr;
  947. if (safe2ack == 0 || safe2ack == rx->last_processed->dma)
  948. musb_writel(&state->rx_complete, 0, safe2ack);
  949. if (safe2ack == 0) {
  950. cppi_bd_free(rx, last);
  951. rx->last_processed = NULL;
  952. /* if we land here on the host side, H_REQPKT will
  953. * be clear and we need to restart the queue...
  954. */
  955. WARN_ON(rx->head);
  956. }
  957. musb_ep_select(cppi->mregs, rx->index + 1);
  958. csr = musb_readw(regs, MUSB_RXCSR);
  959. if (csr & MUSB_RXCSR_DMAENAB) {
  960. dev_dbg(musb->controller, "list%d %p/%p, last %llx%s, csr %04x\n",
  961. rx->index,
  962. rx->head, rx->tail,
  963. rx->last_processed
  964. ? (unsigned long long)
  965. rx->last_processed->dma
  966. : 0,
  967. completed ? ", completed" : "",
  968. csr);
  969. cppi_dump_rxq(4, "/what?", rx);
  970. }
  971. }
  972. if (!completed) {
  973. int csr;
  974. rx->head = bd;
  975. /* REVISIT seems like "autoreq all but EOP" doesn't...
  976. * setting it here "should" be racey, but seems to work
  977. */
  978. csr = musb_readw(rx->hw_ep->regs, MUSB_RXCSR);
  979. if (is_host_active(cppi->musb)
  980. && bd
  981. && !(csr & MUSB_RXCSR_H_REQPKT)) {
  982. csr |= MUSB_RXCSR_H_REQPKT;
  983. musb_writew(regs, MUSB_RXCSR,
  984. MUSB_RXCSR_H_WZC_BITS | csr);
  985. csr = musb_readw(rx->hw_ep->regs, MUSB_RXCSR);
  986. }
  987. } else {
  988. rx->head = NULL;
  989. rx->tail = NULL;
  990. }
  991. cppi_dump_rx(6, rx, completed ? "/completed" : "/cleaned");
  992. return completed;
  993. }
  994. irqreturn_t cppi_interrupt(int irq, void *dev_id)
  995. {
  996. struct musb *musb = dev_id;
  997. struct cppi *cppi;
  998. void __iomem *tibase;
  999. struct musb_hw_ep *hw_ep = NULL;
  1000. u32 rx, tx;
  1001. int i, index;
  1002. unsigned long uninitialized_var(flags);
  1003. cppi = container_of(musb->dma_controller, struct cppi, controller);
  1004. if (cppi->irq)
  1005. spin_lock_irqsave(&musb->lock, flags);
  1006. tibase = musb->ctrl_base;
  1007. tx = musb_readl(tibase, DAVINCI_TXCPPI_MASKED_REG);
  1008. rx = musb_readl(tibase, DAVINCI_RXCPPI_MASKED_REG);
  1009. if (!tx && !rx) {
  1010. if (cppi->irq)
  1011. spin_unlock_irqrestore(&musb->lock, flags);
  1012. return IRQ_NONE;
  1013. }
  1014. dev_dbg(musb->controller, "CPPI IRQ Tx%x Rx%x\n", tx, rx);
  1015. /* process TX channels */
  1016. for (index = 0; tx; tx = tx >> 1, index++) {
  1017. struct cppi_channel *tx_ch;
  1018. struct cppi_tx_stateram __iomem *tx_ram;
  1019. bool completed = false;
  1020. struct cppi_descriptor *bd;
  1021. if (!(tx & 1))
  1022. continue;
  1023. tx_ch = cppi->tx + index;
  1024. tx_ram = tx_ch->state_ram;
  1025. /* FIXME need a cppi_tx_scan() routine, which
  1026. * can also be called from abort code
  1027. */
  1028. cppi_dump_tx(5, tx_ch, "/E");
  1029. bd = tx_ch->head;
  1030. /*
  1031. * If Head is null then this could mean that a abort interrupt
  1032. * that needs to be acknowledged.
  1033. */
  1034. if (NULL == bd) {
  1035. dev_dbg(musb->controller, "null BD\n");
  1036. musb_writel(&tx_ram->tx_complete, 0, 0);
  1037. continue;
  1038. }
  1039. /* run through all completed BDs */
  1040. for (i = 0; !completed && bd && i < NUM_TXCHAN_BD;
  1041. i++, bd = bd->next) {
  1042. u16 len;
  1043. /* catch latest BD writes from CPPI */
  1044. rmb();
  1045. if (bd->hw_options & CPPI_OWN_SET)
  1046. break;
  1047. dev_dbg(musb->controller, "C/TXBD %p n %x b %x off %x opt %x\n",
  1048. bd, bd->hw_next, bd->hw_bufp,
  1049. bd->hw_off_len, bd->hw_options);
  1050. len = bd->hw_off_len & CPPI_BUFFER_LEN_MASK;
  1051. tx_ch->channel.actual_len += len;
  1052. tx_ch->last_processed = bd;
  1053. /* write completion register to acknowledge
  1054. * processing of completed BDs, and possibly
  1055. * release the IRQ; EOQ might not be set ...
  1056. *
  1057. * REVISIT use the same ack strategy as rx
  1058. *
  1059. * REVISIT have observed bit 18 set; huh??
  1060. */
  1061. /* if ((bd->hw_options & CPPI_EOQ_MASK)) */
  1062. musb_writel(&tx_ram->tx_complete, 0, bd->dma);
  1063. /* stop scanning on end-of-segment */
  1064. if (bd->hw_next == 0)
  1065. completed = true;
  1066. }
  1067. /* on end of segment, maybe go to next one */
  1068. if (completed) {
  1069. /* cppi_dump_tx(4, tx_ch, "/complete"); */
  1070. /* transfer more, or report completion */
  1071. if (tx_ch->offset >= tx_ch->buf_len) {
  1072. tx_ch->head = NULL;
  1073. tx_ch->tail = NULL;
  1074. tx_ch->channel.status = MUSB_DMA_STATUS_FREE;
  1075. hw_ep = tx_ch->hw_ep;
  1076. musb_dma_completion(musb, index + 1, 1);
  1077. } else {
  1078. /* Bigger transfer than we could fit in
  1079. * that first batch of descriptors...
  1080. */
  1081. cppi_next_tx_segment(musb, tx_ch);
  1082. }
  1083. } else
  1084. tx_ch->head = bd;
  1085. }
  1086. /* Start processing the RX block */
  1087. for (index = 0; rx; rx = rx >> 1, index++) {
  1088. if (rx & 1) {
  1089. struct cppi_channel *rx_ch;
  1090. rx_ch = cppi->rx + index;
  1091. /* let incomplete dma segments finish */
  1092. if (!cppi_rx_scan(cppi, index))
  1093. continue;
  1094. /* start another dma segment if needed */
  1095. if (rx_ch->channel.actual_len != rx_ch->buf_len
  1096. && rx_ch->channel.actual_len
  1097. == rx_ch->offset) {
  1098. cppi_next_rx_segment(musb, rx_ch, 1);
  1099. continue;
  1100. }
  1101. /* all segments completed! */
  1102. rx_ch->channel.status = MUSB_DMA_STATUS_FREE;
  1103. hw_ep = rx_ch->hw_ep;
  1104. core_rxirq_disable(tibase, index + 1);
  1105. musb_dma_completion(musb, index + 1, 0);
  1106. }
  1107. }
  1108. /* write to CPPI EOI register to re-enable interrupts */
  1109. musb_writel(tibase, DAVINCI_CPPI_EOI_REG, 0);
  1110. if (cppi->irq)
  1111. spin_unlock_irqrestore(&musb->lock, flags);
  1112. return IRQ_HANDLED;
  1113. }
  1114. EXPORT_SYMBOL_GPL(cppi_interrupt);
  1115. /* Instantiate a software object representing a DMA controller. */
  1116. struct dma_controller *dma_controller_create(struct musb *musb, void __iomem *mregs)
  1117. {
  1118. struct cppi *controller;
  1119. struct device *dev = musb->controller;
  1120. struct platform_device *pdev = to_platform_device(dev);
  1121. int irq = platform_get_irq_byname(pdev, "dma");
  1122. controller = kzalloc(sizeof *controller, GFP_KERNEL);
  1123. if (!controller)
  1124. return NULL;
  1125. controller->mregs = mregs;
  1126. controller->tibase = mregs - DAVINCI_BASE_OFFSET;
  1127. controller->musb = musb;
  1128. controller->controller.start = cppi_controller_start;
  1129. controller->controller.stop = cppi_controller_stop;
  1130. controller->controller.channel_alloc = cppi_channel_allocate;
  1131. controller->controller.channel_release = cppi_channel_release;
  1132. controller->controller.channel_program = cppi_channel_program;
  1133. controller->controller.channel_abort = cppi_channel_abort;
  1134. /* NOTE: allocating from on-chip SRAM would give the least
  1135. * contention for memory access, if that ever matters here.
  1136. */
  1137. /* setup BufferPool */
  1138. controller->pool = dma_pool_create("cppi",
  1139. controller->musb->controller,
  1140. sizeof(struct cppi_descriptor),
  1141. CPPI_DESCRIPTOR_ALIGN, 0);
  1142. if (!controller->pool) {
  1143. kfree(controller);
  1144. return NULL;
  1145. }
  1146. if (irq > 0) {
  1147. if (request_irq(irq, cppi_interrupt, 0, "cppi-dma", musb)) {
  1148. dev_err(dev, "request_irq %d failed!\n", irq);
  1149. dma_controller_destroy(&controller->controller);
  1150. return NULL;
  1151. }
  1152. controller->irq = irq;
  1153. }
  1154. return &controller->controller;
  1155. }
  1156. /*
  1157. * Destroy a previously-instantiated DMA controller.
  1158. */
  1159. void dma_controller_destroy(struct dma_controller *c)
  1160. {
  1161. struct cppi *cppi;
  1162. cppi = container_of(c, struct cppi, controller);
  1163. if (cppi->irq)
  1164. free_irq(cppi->irq, cppi->musb);
  1165. /* assert: caller stopped the controller first */
  1166. dma_pool_destroy(cppi->pool);
  1167. kfree(cppi);
  1168. }
  1169. /*
  1170. * Context: controller irqlocked, endpoint selected
  1171. */
  1172. static int cppi_channel_abort(struct dma_channel *channel)
  1173. {
  1174. struct cppi_channel *cppi_ch;
  1175. struct cppi *controller;
  1176. void __iomem *mbase;
  1177. void __iomem *tibase;
  1178. void __iomem *regs;
  1179. u32 value;
  1180. struct cppi_descriptor *queue;
  1181. cppi_ch = container_of(channel, struct cppi_channel, channel);
  1182. controller = cppi_ch->controller;
  1183. switch (channel->status) {
  1184. case MUSB_DMA_STATUS_BUS_ABORT:
  1185. case MUSB_DMA_STATUS_CORE_ABORT:
  1186. /* from RX or TX fault irq handler */
  1187. case MUSB_DMA_STATUS_BUSY:
  1188. /* the hardware needs shutting down */
  1189. regs = cppi_ch->hw_ep->regs;
  1190. break;
  1191. case MUSB_DMA_STATUS_UNKNOWN:
  1192. case MUSB_DMA_STATUS_FREE:
  1193. return 0;
  1194. default:
  1195. return -EINVAL;
  1196. }
  1197. if (!cppi_ch->transmit && cppi_ch->head)
  1198. cppi_dump_rxq(3, "/abort", cppi_ch);
  1199. mbase = controller->mregs;
  1200. tibase = controller->tibase;
  1201. queue = cppi_ch->head;
  1202. cppi_ch->head = NULL;
  1203. cppi_ch->tail = NULL;
  1204. /* REVISIT should rely on caller having done this,
  1205. * and caller should rely on us not changing it.
  1206. * peripheral code is safe ... check host too.
  1207. */
  1208. musb_ep_select(mbase, cppi_ch->index + 1);
  1209. if (cppi_ch->transmit) {
  1210. struct cppi_tx_stateram __iomem *tx_ram;
  1211. /* REVISIT put timeouts on these controller handshakes */
  1212. cppi_dump_tx(6, cppi_ch, " (teardown)");
  1213. /* teardown DMA engine then usb core */
  1214. do {
  1215. value = musb_readl(tibase, DAVINCI_TXCPPI_TEAR_REG);
  1216. } while (!(value & CPPI_TEAR_READY));
  1217. musb_writel(tibase, DAVINCI_TXCPPI_TEAR_REG, cppi_ch->index);
  1218. tx_ram = cppi_ch->state_ram;
  1219. do {
  1220. value = musb_readl(&tx_ram->tx_complete, 0);
  1221. } while (0xFFFFFFFC != value);
  1222. /* FIXME clean up the transfer state ... here?
  1223. * the completion routine should get called with
  1224. * an appropriate status code.
  1225. */
  1226. value = musb_readw(regs, MUSB_TXCSR);
  1227. value &= ~MUSB_TXCSR_DMAENAB;
  1228. value |= MUSB_TXCSR_FLUSHFIFO;
  1229. musb_writew(regs, MUSB_TXCSR, value);
  1230. musb_writew(regs, MUSB_TXCSR, value);
  1231. /*
  1232. * 1. Write to completion Ptr value 0x1(bit 0 set)
  1233. * (write back mode)
  1234. * 2. Wait for abort interrupt and then put the channel in
  1235. * compare mode by writing 1 to the tx_complete register.
  1236. */
  1237. cppi_reset_tx(tx_ram, 1);
  1238. cppi_ch->head = NULL;
  1239. musb_writel(&tx_ram->tx_complete, 0, 1);
  1240. cppi_dump_tx(5, cppi_ch, " (done teardown)");
  1241. /* REVISIT tx side _should_ clean up the same way
  1242. * as the RX side ... this does no cleanup at all!
  1243. */
  1244. } else /* RX */ {
  1245. u16 csr;
  1246. /* NOTE: docs don't guarantee any of this works ... we
  1247. * expect that if the usb core stops telling the cppi core
  1248. * to pull more data from it, then it'll be safe to flush
  1249. * current RX DMA state iff any pending fifo transfer is done.
  1250. */
  1251. core_rxirq_disable(tibase, cppi_ch->index + 1);
  1252. /* for host, ensure ReqPkt is never set again */
  1253. if (is_host_active(cppi_ch->controller->musb)) {
  1254. value = musb_readl(tibase, DAVINCI_AUTOREQ_REG);
  1255. value &= ~((0x3) << (cppi_ch->index * 2));
  1256. musb_writel(tibase, DAVINCI_AUTOREQ_REG, value);
  1257. }
  1258. csr = musb_readw(regs, MUSB_RXCSR);
  1259. /* for host, clear (just) ReqPkt at end of current packet(s) */
  1260. if (is_host_active(cppi_ch->controller->musb)) {
  1261. csr |= MUSB_RXCSR_H_WZC_BITS;
  1262. csr &= ~MUSB_RXCSR_H_REQPKT;
  1263. } else
  1264. csr |= MUSB_RXCSR_P_WZC_BITS;
  1265. /* clear dma enable */
  1266. csr &= ~(MUSB_RXCSR_DMAENAB);
  1267. musb_writew(regs, MUSB_RXCSR, csr);
  1268. csr = musb_readw(regs, MUSB_RXCSR);
  1269. /* Quiesce: wait for current dma to finish (if not cleanup).
  1270. * We can't use bit zero of stateram->rx_sop, since that
  1271. * refers to an entire "DMA packet" not just emptying the
  1272. * current fifo. Most segments need multiple usb packets.
  1273. */
  1274. if (channel->status == MUSB_DMA_STATUS_BUSY)
  1275. udelay(50);
  1276. /* scan the current list, reporting any data that was
  1277. * transferred and acking any IRQ
  1278. */
  1279. cppi_rx_scan(controller, cppi_ch->index);
  1280. /* clobber the existing state once it's idle
  1281. *
  1282. * NOTE: arguably, we should also wait for all the other
  1283. * RX channels to quiesce (how??) and then temporarily
  1284. * disable RXCPPI_CTRL_REG ... but it seems that we can
  1285. * rely on the controller restarting from state ram, with
  1286. * only RXCPPI_BUFCNT state being bogus. BUFCNT will
  1287. * correct itself after the next DMA transfer though.
  1288. *
  1289. * REVISIT does using rndis mode change that?
  1290. */
  1291. cppi_reset_rx(cppi_ch->state_ram);
  1292. /* next DMA request _should_ load cppi head ptr */
  1293. /* ... we don't "free" that list, only mutate it in place. */
  1294. cppi_dump_rx(5, cppi_ch, " (done abort)");
  1295. /* clean up previously pending bds */
  1296. cppi_bd_free(cppi_ch, cppi_ch->last_processed);
  1297. cppi_ch->last_processed = NULL;
  1298. while (queue) {
  1299. struct cppi_descriptor *tmp = queue->next;
  1300. cppi_bd_free(cppi_ch, queue);
  1301. queue = tmp;
  1302. }
  1303. }
  1304. channel->status = MUSB_DMA_STATUS_FREE;
  1305. cppi_ch->buf_dma = 0;
  1306. cppi_ch->offset = 0;
  1307. cppi_ch->buf_len = 0;
  1308. cppi_ch->maxpacket = 0;
  1309. return 0;
  1310. }
  1311. /* TBD Queries:
  1312. *
  1313. * Power Management ... probably turn off cppi during suspend, restart;
  1314. * check state ram? Clocking is presumably shared with usb core.
  1315. */