dw_dmac.c 42 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634
  1. /*
  2. * Driver for the Synopsys DesignWare DMA Controller (aka DMACA on
  3. * AVR32 systems.)
  4. *
  5. * Copyright (C) 2007-2008 Atmel Corporation
  6. * Copyright (C) 2010-2011 ST Microelectronics
  7. *
  8. * This program is free software; you can redistribute it and/or modify
  9. * it under the terms of the GNU General Public License version 2 as
  10. * published by the Free Software Foundation.
  11. */
  12. #include <linux/bitops.h>
  13. #include <linux/clk.h>
  14. #include <linux/delay.h>
  15. #include <linux/dmaengine.h>
  16. #include <linux/dma-mapping.h>
  17. #include <linux/init.h>
  18. #include <linux/interrupt.h>
  19. #include <linux/io.h>
  20. #include <linux/of.h>
  21. #include <linux/mm.h>
  22. #include <linux/module.h>
  23. #include <linux/platform_device.h>
  24. #include <linux/slab.h>
  25. #include "dw_dmac_regs.h"
  26. #include "dmaengine.h"
  27. /*
  28. * This supports the Synopsys "DesignWare AHB Central DMA Controller",
  29. * (DW_ahb_dmac) which is used with various AMBA 2.0 systems (not all
  30. * of which use ARM any more). See the "Databook" from Synopsys for
  31. * information beyond what licensees probably provide.
  32. *
  33. * The driver has currently been tested only with the Atmel AT32AP7000,
  34. * which does not support descriptor writeback.
  35. */
  36. #define DWC_DEFAULT_CTLLO(_chan) ({ \
  37. struct dw_dma_slave *__slave = (_chan->private); \
  38. struct dw_dma_chan *_dwc = to_dw_dma_chan(_chan); \
  39. struct dma_slave_config *_sconfig = &_dwc->dma_sconfig; \
  40. int _dms = __slave ? __slave->dst_master : 0; \
  41. int _sms = __slave ? __slave->src_master : 1; \
  42. u8 _smsize = __slave ? _sconfig->src_maxburst : \
  43. DW_DMA_MSIZE_16; \
  44. u8 _dmsize = __slave ? _sconfig->dst_maxburst : \
  45. DW_DMA_MSIZE_16; \
  46. \
  47. (DWC_CTLL_DST_MSIZE(_dmsize) \
  48. | DWC_CTLL_SRC_MSIZE(_smsize) \
  49. | DWC_CTLL_LLP_D_EN \
  50. | DWC_CTLL_LLP_S_EN \
  51. | DWC_CTLL_DMS(_dms) \
  52. | DWC_CTLL_SMS(_sms)); \
  53. })
  54. /*
  55. * This is configuration-dependent and usually a funny size like 4095.
  56. *
  57. * Note that this is a transfer count, i.e. if we transfer 32-bit
  58. * words, we can do 16380 bytes per descriptor.
  59. *
  60. * This parameter is also system-specific.
  61. */
  62. #define DWC_MAX_COUNT 4095U
  63. /*
  64. * Number of descriptors to allocate for each channel. This should be
  65. * made configurable somehow; preferably, the clients (at least the
  66. * ones using slave transfers) should be able to give us a hint.
  67. */
  68. #define NR_DESCS_PER_CHANNEL 64
  69. /*----------------------------------------------------------------------*/
  70. /*
  71. * Because we're not relying on writeback from the controller (it may not
  72. * even be configured into the core!) we don't need to use dma_pool. These
  73. * descriptors -- and associated data -- are cacheable. We do need to make
  74. * sure their dcache entries are written back before handing them off to
  75. * the controller, though.
  76. */
  77. static struct device *chan2dev(struct dma_chan *chan)
  78. {
  79. return &chan->dev->device;
  80. }
  81. static struct device *chan2parent(struct dma_chan *chan)
  82. {
  83. return chan->dev->device.parent;
  84. }
  85. static struct dw_desc *dwc_first_active(struct dw_dma_chan *dwc)
  86. {
  87. return list_entry(dwc->active_list.next, struct dw_desc, desc_node);
  88. }
  89. static struct dw_desc *dwc_desc_get(struct dw_dma_chan *dwc)
  90. {
  91. struct dw_desc *desc, *_desc;
  92. struct dw_desc *ret = NULL;
  93. unsigned int i = 0;
  94. unsigned long flags;
  95. spin_lock_irqsave(&dwc->lock, flags);
  96. list_for_each_entry_safe(desc, _desc, &dwc->free_list, desc_node) {
  97. if (async_tx_test_ack(&desc->txd)) {
  98. list_del(&desc->desc_node);
  99. ret = desc;
  100. break;
  101. }
  102. dev_dbg(chan2dev(&dwc->chan), "desc %p not ACKed\n", desc);
  103. i++;
  104. }
  105. spin_unlock_irqrestore(&dwc->lock, flags);
  106. dev_vdbg(chan2dev(&dwc->chan), "scanned %u descriptors on freelist\n", i);
  107. return ret;
  108. }
  109. static void dwc_sync_desc_for_cpu(struct dw_dma_chan *dwc, struct dw_desc *desc)
  110. {
  111. struct dw_desc *child;
  112. list_for_each_entry(child, &desc->tx_list, desc_node)
  113. dma_sync_single_for_cpu(chan2parent(&dwc->chan),
  114. child->txd.phys, sizeof(child->lli),
  115. DMA_TO_DEVICE);
  116. dma_sync_single_for_cpu(chan2parent(&dwc->chan),
  117. desc->txd.phys, sizeof(desc->lli),
  118. DMA_TO_DEVICE);
  119. }
  120. /*
  121. * Move a descriptor, including any children, to the free list.
  122. * `desc' must not be on any lists.
  123. */
  124. static void dwc_desc_put(struct dw_dma_chan *dwc, struct dw_desc *desc)
  125. {
  126. unsigned long flags;
  127. if (desc) {
  128. struct dw_desc *child;
  129. dwc_sync_desc_for_cpu(dwc, desc);
  130. spin_lock_irqsave(&dwc->lock, flags);
  131. list_for_each_entry(child, &desc->tx_list, desc_node)
  132. dev_vdbg(chan2dev(&dwc->chan),
  133. "moving child desc %p to freelist\n",
  134. child);
  135. list_splice_init(&desc->tx_list, &dwc->free_list);
  136. dev_vdbg(chan2dev(&dwc->chan), "moving desc %p to freelist\n", desc);
  137. list_add(&desc->desc_node, &dwc->free_list);
  138. spin_unlock_irqrestore(&dwc->lock, flags);
  139. }
  140. }
  141. static void dwc_initialize(struct dw_dma_chan *dwc)
  142. {
  143. struct dw_dma *dw = to_dw_dma(dwc->chan.device);
  144. struct dw_dma_slave *dws = dwc->chan.private;
  145. u32 cfghi = DWC_CFGH_FIFO_MODE;
  146. u32 cfglo = DWC_CFGL_CH_PRIOR(dwc->priority);
  147. if (dwc->initialized == true)
  148. return;
  149. if (dws) {
  150. /*
  151. * We need controller-specific data to set up slave
  152. * transfers.
  153. */
  154. BUG_ON(!dws->dma_dev || dws->dma_dev != dw->dma.dev);
  155. cfghi = dws->cfg_hi;
  156. cfglo |= dws->cfg_lo & ~DWC_CFGL_CH_PRIOR_MASK;
  157. }
  158. channel_writel(dwc, CFG_LO, cfglo);
  159. channel_writel(dwc, CFG_HI, cfghi);
  160. /* Enable interrupts */
  161. channel_set_bit(dw, MASK.XFER, dwc->mask);
  162. channel_set_bit(dw, MASK.ERROR, dwc->mask);
  163. dwc->initialized = true;
  164. }
  165. /*----------------------------------------------------------------------*/
  166. /* Called with dwc->lock held and bh disabled */
  167. static void dwc_dostart(struct dw_dma_chan *dwc, struct dw_desc *first)
  168. {
  169. struct dw_dma *dw = to_dw_dma(dwc->chan.device);
  170. /* ASSERT: channel is idle */
  171. if (dma_readl(dw, CH_EN) & dwc->mask) {
  172. dev_err(chan2dev(&dwc->chan),
  173. "BUG: Attempted to start non-idle channel\n");
  174. dev_err(chan2dev(&dwc->chan),
  175. " SAR: 0x%x DAR: 0x%x LLP: 0x%x CTL: 0x%x:%08x\n",
  176. channel_readl(dwc, SAR),
  177. channel_readl(dwc, DAR),
  178. channel_readl(dwc, LLP),
  179. channel_readl(dwc, CTL_HI),
  180. channel_readl(dwc, CTL_LO));
  181. /* The tasklet will hopefully advance the queue... */
  182. return;
  183. }
  184. dwc_initialize(dwc);
  185. channel_writel(dwc, LLP, first->txd.phys);
  186. channel_writel(dwc, CTL_LO,
  187. DWC_CTLL_LLP_D_EN | DWC_CTLL_LLP_S_EN);
  188. channel_writel(dwc, CTL_HI, 0);
  189. channel_set_bit(dw, CH_EN, dwc->mask);
  190. }
  191. /*----------------------------------------------------------------------*/
  192. static void
  193. dwc_descriptor_complete(struct dw_dma_chan *dwc, struct dw_desc *desc,
  194. bool callback_required)
  195. {
  196. dma_async_tx_callback callback = NULL;
  197. void *param = NULL;
  198. struct dma_async_tx_descriptor *txd = &desc->txd;
  199. struct dw_desc *child;
  200. unsigned long flags;
  201. dev_vdbg(chan2dev(&dwc->chan), "descriptor %u complete\n", txd->cookie);
  202. spin_lock_irqsave(&dwc->lock, flags);
  203. dma_cookie_complete(txd);
  204. if (callback_required) {
  205. callback = txd->callback;
  206. param = txd->callback_param;
  207. }
  208. dwc_sync_desc_for_cpu(dwc, desc);
  209. /* async_tx_ack */
  210. list_for_each_entry(child, &desc->tx_list, desc_node)
  211. async_tx_ack(&child->txd);
  212. async_tx_ack(&desc->txd);
  213. list_splice_init(&desc->tx_list, &dwc->free_list);
  214. list_move(&desc->desc_node, &dwc->free_list);
  215. if (!dwc->chan.private) {
  216. struct device *parent = chan2parent(&dwc->chan);
  217. if (!(txd->flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
  218. if (txd->flags & DMA_COMPL_DEST_UNMAP_SINGLE)
  219. dma_unmap_single(parent, desc->lli.dar,
  220. desc->len, DMA_FROM_DEVICE);
  221. else
  222. dma_unmap_page(parent, desc->lli.dar,
  223. desc->len, DMA_FROM_DEVICE);
  224. }
  225. if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
  226. if (txd->flags & DMA_COMPL_SRC_UNMAP_SINGLE)
  227. dma_unmap_single(parent, desc->lli.sar,
  228. desc->len, DMA_TO_DEVICE);
  229. else
  230. dma_unmap_page(parent, desc->lli.sar,
  231. desc->len, DMA_TO_DEVICE);
  232. }
  233. }
  234. spin_unlock_irqrestore(&dwc->lock, flags);
  235. if (callback_required && callback)
  236. callback(param);
  237. }
  238. static void dwc_complete_all(struct dw_dma *dw, struct dw_dma_chan *dwc)
  239. {
  240. struct dw_desc *desc, *_desc;
  241. LIST_HEAD(list);
  242. unsigned long flags;
  243. spin_lock_irqsave(&dwc->lock, flags);
  244. if (dma_readl(dw, CH_EN) & dwc->mask) {
  245. dev_err(chan2dev(&dwc->chan),
  246. "BUG: XFER bit set, but channel not idle!\n");
  247. /* Try to continue after resetting the channel... */
  248. channel_clear_bit(dw, CH_EN, dwc->mask);
  249. while (dma_readl(dw, CH_EN) & dwc->mask)
  250. cpu_relax();
  251. }
  252. /*
  253. * Submit queued descriptors ASAP, i.e. before we go through
  254. * the completed ones.
  255. */
  256. list_splice_init(&dwc->active_list, &list);
  257. if (!list_empty(&dwc->queue)) {
  258. list_move(dwc->queue.next, &dwc->active_list);
  259. dwc_dostart(dwc, dwc_first_active(dwc));
  260. }
  261. spin_unlock_irqrestore(&dwc->lock, flags);
  262. list_for_each_entry_safe(desc, _desc, &list, desc_node)
  263. dwc_descriptor_complete(dwc, desc, true);
  264. }
  265. static void dwc_scan_descriptors(struct dw_dma *dw, struct dw_dma_chan *dwc)
  266. {
  267. dma_addr_t llp;
  268. struct dw_desc *desc, *_desc;
  269. struct dw_desc *child;
  270. u32 status_xfer;
  271. unsigned long flags;
  272. spin_lock_irqsave(&dwc->lock, flags);
  273. llp = channel_readl(dwc, LLP);
  274. status_xfer = dma_readl(dw, RAW.XFER);
  275. if (status_xfer & dwc->mask) {
  276. /* Everything we've submitted is done */
  277. dma_writel(dw, CLEAR.XFER, dwc->mask);
  278. spin_unlock_irqrestore(&dwc->lock, flags);
  279. dwc_complete_all(dw, dwc);
  280. return;
  281. }
  282. if (list_empty(&dwc->active_list)) {
  283. spin_unlock_irqrestore(&dwc->lock, flags);
  284. return;
  285. }
  286. dev_vdbg(chan2dev(&dwc->chan), "scan_descriptors: llp=0x%llx\n",
  287. (unsigned long long)llp);
  288. list_for_each_entry_safe(desc, _desc, &dwc->active_list, desc_node) {
  289. /* check first descriptors addr */
  290. if (desc->txd.phys == llp) {
  291. spin_unlock_irqrestore(&dwc->lock, flags);
  292. return;
  293. }
  294. /* check first descriptors llp */
  295. if (desc->lli.llp == llp) {
  296. /* This one is currently in progress */
  297. spin_unlock_irqrestore(&dwc->lock, flags);
  298. return;
  299. }
  300. list_for_each_entry(child, &desc->tx_list, desc_node)
  301. if (child->lli.llp == llp) {
  302. /* Currently in progress */
  303. spin_unlock_irqrestore(&dwc->lock, flags);
  304. return;
  305. }
  306. /*
  307. * No descriptors so far seem to be in progress, i.e.
  308. * this one must be done.
  309. */
  310. spin_unlock_irqrestore(&dwc->lock, flags);
  311. dwc_descriptor_complete(dwc, desc, true);
  312. spin_lock_irqsave(&dwc->lock, flags);
  313. }
  314. dev_err(chan2dev(&dwc->chan),
  315. "BUG: All descriptors done, but channel not idle!\n");
  316. /* Try to continue after resetting the channel... */
  317. channel_clear_bit(dw, CH_EN, dwc->mask);
  318. while (dma_readl(dw, CH_EN) & dwc->mask)
  319. cpu_relax();
  320. if (!list_empty(&dwc->queue)) {
  321. list_move(dwc->queue.next, &dwc->active_list);
  322. dwc_dostart(dwc, dwc_first_active(dwc));
  323. }
  324. spin_unlock_irqrestore(&dwc->lock, flags);
  325. }
  326. static void dwc_dump_lli(struct dw_dma_chan *dwc, struct dw_lli *lli)
  327. {
  328. dev_printk(KERN_CRIT, chan2dev(&dwc->chan),
  329. " desc: s0x%llx d0x%llx l0x%llx c0x%x:%x\n",
  330. (unsigned long long)lli->sar,
  331. (unsigned long long)lli->dar,
  332. (unsigned long long)lli->llp,
  333. lli->ctlhi, lli->ctllo);
  334. }
  335. static void dwc_handle_error(struct dw_dma *dw, struct dw_dma_chan *dwc)
  336. {
  337. struct dw_desc *bad_desc;
  338. struct dw_desc *child;
  339. unsigned long flags;
  340. dwc_scan_descriptors(dw, dwc);
  341. spin_lock_irqsave(&dwc->lock, flags);
  342. /*
  343. * The descriptor currently at the head of the active list is
  344. * borked. Since we don't have any way to report errors, we'll
  345. * just have to scream loudly and try to carry on.
  346. */
  347. bad_desc = dwc_first_active(dwc);
  348. list_del_init(&bad_desc->desc_node);
  349. list_move(dwc->queue.next, dwc->active_list.prev);
  350. /* Clear the error flag and try to restart the controller */
  351. dma_writel(dw, CLEAR.ERROR, dwc->mask);
  352. if (!list_empty(&dwc->active_list))
  353. dwc_dostart(dwc, dwc_first_active(dwc));
  354. /*
  355. * KERN_CRITICAL may seem harsh, but since this only happens
  356. * when someone submits a bad physical address in a
  357. * descriptor, we should consider ourselves lucky that the
  358. * controller flagged an error instead of scribbling over
  359. * random memory locations.
  360. */
  361. dev_printk(KERN_CRIT, chan2dev(&dwc->chan),
  362. "Bad descriptor submitted for DMA!\n");
  363. dev_printk(KERN_CRIT, chan2dev(&dwc->chan),
  364. " cookie: %d\n", bad_desc->txd.cookie);
  365. dwc_dump_lli(dwc, &bad_desc->lli);
  366. list_for_each_entry(child, &bad_desc->tx_list, desc_node)
  367. dwc_dump_lli(dwc, &child->lli);
  368. spin_unlock_irqrestore(&dwc->lock, flags);
  369. /* Pretend the descriptor completed successfully */
  370. dwc_descriptor_complete(dwc, bad_desc, true);
  371. }
  372. /* --------------------- Cyclic DMA API extensions -------------------- */
  373. inline dma_addr_t dw_dma_get_src_addr(struct dma_chan *chan)
  374. {
  375. struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
  376. return channel_readl(dwc, SAR);
  377. }
  378. EXPORT_SYMBOL(dw_dma_get_src_addr);
  379. inline dma_addr_t dw_dma_get_dst_addr(struct dma_chan *chan)
  380. {
  381. struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
  382. return channel_readl(dwc, DAR);
  383. }
  384. EXPORT_SYMBOL(dw_dma_get_dst_addr);
  385. /* called with dwc->lock held and all DMAC interrupts disabled */
  386. static void dwc_handle_cyclic(struct dw_dma *dw, struct dw_dma_chan *dwc,
  387. u32 status_err, u32 status_xfer)
  388. {
  389. unsigned long flags;
  390. if (dwc->mask) {
  391. void (*callback)(void *param);
  392. void *callback_param;
  393. dev_vdbg(chan2dev(&dwc->chan), "new cyclic period llp 0x%08x\n",
  394. channel_readl(dwc, LLP));
  395. callback = dwc->cdesc->period_callback;
  396. callback_param = dwc->cdesc->period_callback_param;
  397. if (callback)
  398. callback(callback_param);
  399. }
  400. /*
  401. * Error and transfer complete are highly unlikely, and will most
  402. * likely be due to a configuration error by the user.
  403. */
  404. if (unlikely(status_err & dwc->mask) ||
  405. unlikely(status_xfer & dwc->mask)) {
  406. int i;
  407. dev_err(chan2dev(&dwc->chan), "cyclic DMA unexpected %s "
  408. "interrupt, stopping DMA transfer\n",
  409. status_xfer ? "xfer" : "error");
  410. spin_lock_irqsave(&dwc->lock, flags);
  411. dev_err(chan2dev(&dwc->chan),
  412. " SAR: 0x%x DAR: 0x%x LLP: 0x%x CTL: 0x%x:%08x\n",
  413. channel_readl(dwc, SAR),
  414. channel_readl(dwc, DAR),
  415. channel_readl(dwc, LLP),
  416. channel_readl(dwc, CTL_HI),
  417. channel_readl(dwc, CTL_LO));
  418. channel_clear_bit(dw, CH_EN, dwc->mask);
  419. while (dma_readl(dw, CH_EN) & dwc->mask)
  420. cpu_relax();
  421. /* make sure DMA does not restart by loading a new list */
  422. channel_writel(dwc, LLP, 0);
  423. channel_writel(dwc, CTL_LO, 0);
  424. channel_writel(dwc, CTL_HI, 0);
  425. dma_writel(dw, CLEAR.ERROR, dwc->mask);
  426. dma_writel(dw, CLEAR.XFER, dwc->mask);
  427. for (i = 0; i < dwc->cdesc->periods; i++)
  428. dwc_dump_lli(dwc, &dwc->cdesc->desc[i]->lli);
  429. spin_unlock_irqrestore(&dwc->lock, flags);
  430. }
  431. }
  432. /* ------------------------------------------------------------------------- */
  433. static void dw_dma_tasklet(unsigned long data)
  434. {
  435. struct dw_dma *dw = (struct dw_dma *)data;
  436. struct dw_dma_chan *dwc;
  437. u32 status_xfer;
  438. u32 status_err;
  439. int i;
  440. status_xfer = dma_readl(dw, RAW.XFER);
  441. status_err = dma_readl(dw, RAW.ERROR);
  442. dev_vdbg(dw->dma.dev, "tasklet: status_err=%x\n", status_err);
  443. for (i = 0; i < dw->dma.chancnt; i++) {
  444. dwc = &dw->chan[i];
  445. if (test_bit(DW_DMA_IS_CYCLIC, &dwc->flags))
  446. dwc_handle_cyclic(dw, dwc, status_err, status_xfer);
  447. else if (status_err & (1 << i))
  448. dwc_handle_error(dw, dwc);
  449. else if (status_xfer & (1 << i))
  450. dwc_scan_descriptors(dw, dwc);
  451. }
  452. /*
  453. * Re-enable interrupts.
  454. */
  455. channel_set_bit(dw, MASK.XFER, dw->all_chan_mask);
  456. channel_set_bit(dw, MASK.ERROR, dw->all_chan_mask);
  457. }
  458. static irqreturn_t dw_dma_interrupt(int irq, void *dev_id)
  459. {
  460. struct dw_dma *dw = dev_id;
  461. u32 status;
  462. dev_vdbg(dw->dma.dev, "interrupt: status=0x%x\n",
  463. dma_readl(dw, STATUS_INT));
  464. /*
  465. * Just disable the interrupts. We'll turn them back on in the
  466. * softirq handler.
  467. */
  468. channel_clear_bit(dw, MASK.XFER, dw->all_chan_mask);
  469. channel_clear_bit(dw, MASK.ERROR, dw->all_chan_mask);
  470. status = dma_readl(dw, STATUS_INT);
  471. if (status) {
  472. dev_err(dw->dma.dev,
  473. "BUG: Unexpected interrupts pending: 0x%x\n",
  474. status);
  475. /* Try to recover */
  476. channel_clear_bit(dw, MASK.XFER, (1 << 8) - 1);
  477. channel_clear_bit(dw, MASK.SRC_TRAN, (1 << 8) - 1);
  478. channel_clear_bit(dw, MASK.DST_TRAN, (1 << 8) - 1);
  479. channel_clear_bit(dw, MASK.ERROR, (1 << 8) - 1);
  480. }
  481. tasklet_schedule(&dw->tasklet);
  482. return IRQ_HANDLED;
  483. }
  484. /*----------------------------------------------------------------------*/
  485. static dma_cookie_t dwc_tx_submit(struct dma_async_tx_descriptor *tx)
  486. {
  487. struct dw_desc *desc = txd_to_dw_desc(tx);
  488. struct dw_dma_chan *dwc = to_dw_dma_chan(tx->chan);
  489. dma_cookie_t cookie;
  490. unsigned long flags;
  491. spin_lock_irqsave(&dwc->lock, flags);
  492. cookie = dma_cookie_assign(tx);
  493. /*
  494. * REVISIT: We should attempt to chain as many descriptors as
  495. * possible, perhaps even appending to those already submitted
  496. * for DMA. But this is hard to do in a race-free manner.
  497. */
  498. if (list_empty(&dwc->active_list)) {
  499. dev_vdbg(chan2dev(tx->chan), "tx_submit: started %u\n",
  500. desc->txd.cookie);
  501. list_add_tail(&desc->desc_node, &dwc->active_list);
  502. dwc_dostart(dwc, dwc_first_active(dwc));
  503. } else {
  504. dev_vdbg(chan2dev(tx->chan), "tx_submit: queued %u\n",
  505. desc->txd.cookie);
  506. list_add_tail(&desc->desc_node, &dwc->queue);
  507. }
  508. spin_unlock_irqrestore(&dwc->lock, flags);
  509. return cookie;
  510. }
  511. static struct dma_async_tx_descriptor *
  512. dwc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
  513. size_t len, unsigned long flags)
  514. {
  515. struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
  516. struct dw_desc *desc;
  517. struct dw_desc *first;
  518. struct dw_desc *prev;
  519. size_t xfer_count;
  520. size_t offset;
  521. unsigned int src_width;
  522. unsigned int dst_width;
  523. u32 ctllo;
  524. dev_vdbg(chan2dev(chan),
  525. "prep_dma_memcpy d0x%llx s0x%llx l0x%zx f0x%lx\n",
  526. (unsigned long long)dest, (unsigned long long)src,
  527. len, flags);
  528. if (unlikely(!len)) {
  529. dev_dbg(chan2dev(chan), "prep_dma_memcpy: length is zero!\n");
  530. return NULL;
  531. }
  532. /*
  533. * We can be a lot more clever here, but this should take care
  534. * of the most common optimization.
  535. */
  536. if (!((src | dest | len) & 7))
  537. src_width = dst_width = 3;
  538. else if (!((src | dest | len) & 3))
  539. src_width = dst_width = 2;
  540. else if (!((src | dest | len) & 1))
  541. src_width = dst_width = 1;
  542. else
  543. src_width = dst_width = 0;
  544. ctllo = DWC_DEFAULT_CTLLO(chan)
  545. | DWC_CTLL_DST_WIDTH(dst_width)
  546. | DWC_CTLL_SRC_WIDTH(src_width)
  547. | DWC_CTLL_DST_INC
  548. | DWC_CTLL_SRC_INC
  549. | DWC_CTLL_FC_M2M;
  550. prev = first = NULL;
  551. for (offset = 0; offset < len; offset += xfer_count << src_width) {
  552. xfer_count = min_t(size_t, (len - offset) >> src_width,
  553. DWC_MAX_COUNT);
  554. desc = dwc_desc_get(dwc);
  555. if (!desc)
  556. goto err_desc_get;
  557. desc->lli.sar = src + offset;
  558. desc->lli.dar = dest + offset;
  559. desc->lli.ctllo = ctllo;
  560. desc->lli.ctlhi = xfer_count;
  561. if (!first) {
  562. first = desc;
  563. } else {
  564. prev->lli.llp = desc->txd.phys;
  565. dma_sync_single_for_device(chan2parent(chan),
  566. prev->txd.phys, sizeof(prev->lli),
  567. DMA_TO_DEVICE);
  568. list_add_tail(&desc->desc_node,
  569. &first->tx_list);
  570. }
  571. prev = desc;
  572. }
  573. if (flags & DMA_PREP_INTERRUPT)
  574. /* Trigger interrupt after last block */
  575. prev->lli.ctllo |= DWC_CTLL_INT_EN;
  576. prev->lli.llp = 0;
  577. dma_sync_single_for_device(chan2parent(chan),
  578. prev->txd.phys, sizeof(prev->lli),
  579. DMA_TO_DEVICE);
  580. first->txd.flags = flags;
  581. first->len = len;
  582. return &first->txd;
  583. err_desc_get:
  584. dwc_desc_put(dwc, first);
  585. return NULL;
  586. }
  587. static struct dma_async_tx_descriptor *
  588. dwc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
  589. unsigned int sg_len, enum dma_transfer_direction direction,
  590. unsigned long flags, void *context)
  591. {
  592. struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
  593. struct dw_dma_slave *dws = chan->private;
  594. struct dma_slave_config *sconfig = &dwc->dma_sconfig;
  595. struct dw_desc *prev;
  596. struct dw_desc *first;
  597. u32 ctllo;
  598. dma_addr_t reg;
  599. unsigned int reg_width;
  600. unsigned int mem_width;
  601. unsigned int i;
  602. struct scatterlist *sg;
  603. size_t total_len = 0;
  604. dev_vdbg(chan2dev(chan), "prep_dma_slave\n");
  605. if (unlikely(!dws || !sg_len))
  606. return NULL;
  607. prev = first = NULL;
  608. switch (direction) {
  609. case DMA_MEM_TO_DEV:
  610. reg_width = __fls(sconfig->dst_addr_width);
  611. reg = sconfig->dst_addr;
  612. ctllo = (DWC_DEFAULT_CTLLO(chan)
  613. | DWC_CTLL_DST_WIDTH(reg_width)
  614. | DWC_CTLL_DST_FIX
  615. | DWC_CTLL_SRC_INC);
  616. ctllo |= sconfig->device_fc ? DWC_CTLL_FC(DW_DMA_FC_P_M2P) :
  617. DWC_CTLL_FC(DW_DMA_FC_D_M2P);
  618. for_each_sg(sgl, sg, sg_len, i) {
  619. struct dw_desc *desc;
  620. u32 len, dlen, mem;
  621. mem = sg_dma_address(sg);
  622. len = sg_dma_len(sg);
  623. if (!((mem | len) & 7))
  624. mem_width = 3;
  625. else if (!((mem | len) & 3))
  626. mem_width = 2;
  627. else if (!((mem | len) & 1))
  628. mem_width = 1;
  629. else
  630. mem_width = 0;
  631. slave_sg_todev_fill_desc:
  632. desc = dwc_desc_get(dwc);
  633. if (!desc) {
  634. dev_err(chan2dev(chan),
  635. "not enough descriptors available\n");
  636. goto err_desc_get;
  637. }
  638. desc->lli.sar = mem;
  639. desc->lli.dar = reg;
  640. desc->lli.ctllo = ctllo | DWC_CTLL_SRC_WIDTH(mem_width);
  641. if ((len >> mem_width) > DWC_MAX_COUNT) {
  642. dlen = DWC_MAX_COUNT << mem_width;
  643. mem += dlen;
  644. len -= dlen;
  645. } else {
  646. dlen = len;
  647. len = 0;
  648. }
  649. desc->lli.ctlhi = dlen >> mem_width;
  650. if (!first) {
  651. first = desc;
  652. } else {
  653. prev->lli.llp = desc->txd.phys;
  654. dma_sync_single_for_device(chan2parent(chan),
  655. prev->txd.phys,
  656. sizeof(prev->lli),
  657. DMA_TO_DEVICE);
  658. list_add_tail(&desc->desc_node,
  659. &first->tx_list);
  660. }
  661. prev = desc;
  662. total_len += dlen;
  663. if (len)
  664. goto slave_sg_todev_fill_desc;
  665. }
  666. break;
  667. case DMA_DEV_TO_MEM:
  668. reg_width = __fls(sconfig->src_addr_width);
  669. reg = sconfig->src_addr;
  670. ctllo = (DWC_DEFAULT_CTLLO(chan)
  671. | DWC_CTLL_SRC_WIDTH(reg_width)
  672. | DWC_CTLL_DST_INC
  673. | DWC_CTLL_SRC_FIX);
  674. ctllo |= sconfig->device_fc ? DWC_CTLL_FC(DW_DMA_FC_P_P2M) :
  675. DWC_CTLL_FC(DW_DMA_FC_D_P2M);
  676. for_each_sg(sgl, sg, sg_len, i) {
  677. struct dw_desc *desc;
  678. u32 len, dlen, mem;
  679. mem = sg_dma_address(sg);
  680. len = sg_dma_len(sg);
  681. if (!((mem | len) & 7))
  682. mem_width = 3;
  683. else if (!((mem | len) & 3))
  684. mem_width = 2;
  685. else if (!((mem | len) & 1))
  686. mem_width = 1;
  687. else
  688. mem_width = 0;
  689. slave_sg_fromdev_fill_desc:
  690. desc = dwc_desc_get(dwc);
  691. if (!desc) {
  692. dev_err(chan2dev(chan),
  693. "not enough descriptors available\n");
  694. goto err_desc_get;
  695. }
  696. desc->lli.sar = reg;
  697. desc->lli.dar = mem;
  698. desc->lli.ctllo = ctllo | DWC_CTLL_DST_WIDTH(mem_width);
  699. if ((len >> reg_width) > DWC_MAX_COUNT) {
  700. dlen = DWC_MAX_COUNT << reg_width;
  701. mem += dlen;
  702. len -= dlen;
  703. } else {
  704. dlen = len;
  705. len = 0;
  706. }
  707. desc->lli.ctlhi = dlen >> reg_width;
  708. if (!first) {
  709. first = desc;
  710. } else {
  711. prev->lli.llp = desc->txd.phys;
  712. dma_sync_single_for_device(chan2parent(chan),
  713. prev->txd.phys,
  714. sizeof(prev->lli),
  715. DMA_TO_DEVICE);
  716. list_add_tail(&desc->desc_node,
  717. &first->tx_list);
  718. }
  719. prev = desc;
  720. total_len += dlen;
  721. if (len)
  722. goto slave_sg_fromdev_fill_desc;
  723. }
  724. break;
  725. default:
  726. return NULL;
  727. }
  728. if (flags & DMA_PREP_INTERRUPT)
  729. /* Trigger interrupt after last block */
  730. prev->lli.ctllo |= DWC_CTLL_INT_EN;
  731. prev->lli.llp = 0;
  732. dma_sync_single_for_device(chan2parent(chan),
  733. prev->txd.phys, sizeof(prev->lli),
  734. DMA_TO_DEVICE);
  735. first->len = total_len;
  736. return &first->txd;
  737. err_desc_get:
  738. dwc_desc_put(dwc, first);
  739. return NULL;
  740. }
  741. /*
  742. * Fix sconfig's burst size according to dw_dmac. We need to convert them as:
  743. * 1 -> 0, 4 -> 1, 8 -> 2, 16 -> 3.
  744. *
  745. * NOTE: burst size 2 is not supported by controller.
  746. *
  747. * This can be done by finding least significant bit set: n & (n - 1)
  748. */
  749. static inline void convert_burst(u32 *maxburst)
  750. {
  751. if (*maxburst > 1)
  752. *maxburst = fls(*maxburst) - 2;
  753. else
  754. *maxburst = 0;
  755. }
  756. static int
  757. set_runtime_config(struct dma_chan *chan, struct dma_slave_config *sconfig)
  758. {
  759. struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
  760. /* Check if it is chan is configured for slave transfers */
  761. if (!chan->private)
  762. return -EINVAL;
  763. memcpy(&dwc->dma_sconfig, sconfig, sizeof(*sconfig));
  764. convert_burst(&dwc->dma_sconfig.src_maxburst);
  765. convert_burst(&dwc->dma_sconfig.dst_maxburst);
  766. return 0;
  767. }
  768. static int dwc_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
  769. unsigned long arg)
  770. {
  771. struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
  772. struct dw_dma *dw = to_dw_dma(chan->device);
  773. struct dw_desc *desc, *_desc;
  774. unsigned long flags;
  775. u32 cfglo;
  776. LIST_HEAD(list);
  777. if (cmd == DMA_PAUSE) {
  778. spin_lock_irqsave(&dwc->lock, flags);
  779. cfglo = channel_readl(dwc, CFG_LO);
  780. channel_writel(dwc, CFG_LO, cfglo | DWC_CFGL_CH_SUSP);
  781. while (!(channel_readl(dwc, CFG_LO) & DWC_CFGL_FIFO_EMPTY))
  782. cpu_relax();
  783. dwc->paused = true;
  784. spin_unlock_irqrestore(&dwc->lock, flags);
  785. } else if (cmd == DMA_RESUME) {
  786. if (!dwc->paused)
  787. return 0;
  788. spin_lock_irqsave(&dwc->lock, flags);
  789. cfglo = channel_readl(dwc, CFG_LO);
  790. channel_writel(dwc, CFG_LO, cfglo & ~DWC_CFGL_CH_SUSP);
  791. dwc->paused = false;
  792. spin_unlock_irqrestore(&dwc->lock, flags);
  793. } else if (cmd == DMA_TERMINATE_ALL) {
  794. spin_lock_irqsave(&dwc->lock, flags);
  795. channel_clear_bit(dw, CH_EN, dwc->mask);
  796. while (dma_readl(dw, CH_EN) & dwc->mask)
  797. cpu_relax();
  798. dwc->paused = false;
  799. /* active_list entries will end up before queued entries */
  800. list_splice_init(&dwc->queue, &list);
  801. list_splice_init(&dwc->active_list, &list);
  802. spin_unlock_irqrestore(&dwc->lock, flags);
  803. /* Flush all pending and queued descriptors */
  804. list_for_each_entry_safe(desc, _desc, &list, desc_node)
  805. dwc_descriptor_complete(dwc, desc, false);
  806. } else if (cmd == DMA_SLAVE_CONFIG) {
  807. return set_runtime_config(chan, (struct dma_slave_config *)arg);
  808. } else {
  809. return -ENXIO;
  810. }
  811. return 0;
  812. }
  813. static enum dma_status
  814. dwc_tx_status(struct dma_chan *chan,
  815. dma_cookie_t cookie,
  816. struct dma_tx_state *txstate)
  817. {
  818. struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
  819. enum dma_status ret;
  820. ret = dma_cookie_status(chan, cookie, txstate);
  821. if (ret != DMA_SUCCESS) {
  822. dwc_scan_descriptors(to_dw_dma(chan->device), dwc);
  823. ret = dma_cookie_status(chan, cookie, txstate);
  824. }
  825. if (ret != DMA_SUCCESS)
  826. dma_set_residue(txstate, dwc_first_active(dwc)->len);
  827. if (dwc->paused)
  828. return DMA_PAUSED;
  829. return ret;
  830. }
  831. static void dwc_issue_pending(struct dma_chan *chan)
  832. {
  833. struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
  834. if (!list_empty(&dwc->queue))
  835. dwc_scan_descriptors(to_dw_dma(chan->device), dwc);
  836. }
  837. static int dwc_alloc_chan_resources(struct dma_chan *chan)
  838. {
  839. struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
  840. struct dw_dma *dw = to_dw_dma(chan->device);
  841. struct dw_desc *desc;
  842. int i;
  843. unsigned long flags;
  844. dev_vdbg(chan2dev(chan), "alloc_chan_resources\n");
  845. /* ASSERT: channel is idle */
  846. if (dma_readl(dw, CH_EN) & dwc->mask) {
  847. dev_dbg(chan2dev(chan), "DMA channel not idle?\n");
  848. return -EIO;
  849. }
  850. dma_cookie_init(chan);
  851. /*
  852. * NOTE: some controllers may have additional features that we
  853. * need to initialize here, like "scatter-gather" (which
  854. * doesn't mean what you think it means), and status writeback.
  855. */
  856. spin_lock_irqsave(&dwc->lock, flags);
  857. i = dwc->descs_allocated;
  858. while (dwc->descs_allocated < NR_DESCS_PER_CHANNEL) {
  859. spin_unlock_irqrestore(&dwc->lock, flags);
  860. desc = kzalloc(sizeof(struct dw_desc), GFP_KERNEL);
  861. if (!desc) {
  862. dev_info(chan2dev(chan),
  863. "only allocated %d descriptors\n", i);
  864. spin_lock_irqsave(&dwc->lock, flags);
  865. break;
  866. }
  867. INIT_LIST_HEAD(&desc->tx_list);
  868. dma_async_tx_descriptor_init(&desc->txd, chan);
  869. desc->txd.tx_submit = dwc_tx_submit;
  870. desc->txd.flags = DMA_CTRL_ACK;
  871. desc->txd.phys = dma_map_single(chan2parent(chan), &desc->lli,
  872. sizeof(desc->lli), DMA_TO_DEVICE);
  873. dwc_desc_put(dwc, desc);
  874. spin_lock_irqsave(&dwc->lock, flags);
  875. i = ++dwc->descs_allocated;
  876. }
  877. spin_unlock_irqrestore(&dwc->lock, flags);
  878. dev_dbg(chan2dev(chan),
  879. "alloc_chan_resources allocated %d descriptors\n", i);
  880. return i;
  881. }
  882. static void dwc_free_chan_resources(struct dma_chan *chan)
  883. {
  884. struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
  885. struct dw_dma *dw = to_dw_dma(chan->device);
  886. struct dw_desc *desc, *_desc;
  887. unsigned long flags;
  888. LIST_HEAD(list);
  889. dev_dbg(chan2dev(chan), "free_chan_resources (descs allocated=%u)\n",
  890. dwc->descs_allocated);
  891. /* ASSERT: channel is idle */
  892. BUG_ON(!list_empty(&dwc->active_list));
  893. BUG_ON(!list_empty(&dwc->queue));
  894. BUG_ON(dma_readl(to_dw_dma(chan->device), CH_EN) & dwc->mask);
  895. spin_lock_irqsave(&dwc->lock, flags);
  896. list_splice_init(&dwc->free_list, &list);
  897. dwc->descs_allocated = 0;
  898. dwc->initialized = false;
  899. /* Disable interrupts */
  900. channel_clear_bit(dw, MASK.XFER, dwc->mask);
  901. channel_clear_bit(dw, MASK.ERROR, dwc->mask);
  902. spin_unlock_irqrestore(&dwc->lock, flags);
  903. list_for_each_entry_safe(desc, _desc, &list, desc_node) {
  904. dev_vdbg(chan2dev(chan), " freeing descriptor %p\n", desc);
  905. dma_unmap_single(chan2parent(chan), desc->txd.phys,
  906. sizeof(desc->lli), DMA_TO_DEVICE);
  907. kfree(desc);
  908. }
  909. dev_vdbg(chan2dev(chan), "free_chan_resources done\n");
  910. }
  911. /* --------------------- Cyclic DMA API extensions -------------------- */
  912. /**
  913. * dw_dma_cyclic_start - start the cyclic DMA transfer
  914. * @chan: the DMA channel to start
  915. *
  916. * Must be called with soft interrupts disabled. Returns zero on success or
  917. * -errno on failure.
  918. */
  919. int dw_dma_cyclic_start(struct dma_chan *chan)
  920. {
  921. struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
  922. struct dw_dma *dw = to_dw_dma(dwc->chan.device);
  923. unsigned long flags;
  924. if (!test_bit(DW_DMA_IS_CYCLIC, &dwc->flags)) {
  925. dev_err(chan2dev(&dwc->chan), "missing prep for cyclic DMA\n");
  926. return -ENODEV;
  927. }
  928. spin_lock_irqsave(&dwc->lock, flags);
  929. /* assert channel is idle */
  930. if (dma_readl(dw, CH_EN) & dwc->mask) {
  931. dev_err(chan2dev(&dwc->chan),
  932. "BUG: Attempted to start non-idle channel\n");
  933. dev_err(chan2dev(&dwc->chan),
  934. " SAR: 0x%x DAR: 0x%x LLP: 0x%x CTL: 0x%x:%08x\n",
  935. channel_readl(dwc, SAR),
  936. channel_readl(dwc, DAR),
  937. channel_readl(dwc, LLP),
  938. channel_readl(dwc, CTL_HI),
  939. channel_readl(dwc, CTL_LO));
  940. spin_unlock_irqrestore(&dwc->lock, flags);
  941. return -EBUSY;
  942. }
  943. dma_writel(dw, CLEAR.ERROR, dwc->mask);
  944. dma_writel(dw, CLEAR.XFER, dwc->mask);
  945. /* setup DMAC channel registers */
  946. channel_writel(dwc, LLP, dwc->cdesc->desc[0]->txd.phys);
  947. channel_writel(dwc, CTL_LO, DWC_CTLL_LLP_D_EN | DWC_CTLL_LLP_S_EN);
  948. channel_writel(dwc, CTL_HI, 0);
  949. channel_set_bit(dw, CH_EN, dwc->mask);
  950. spin_unlock_irqrestore(&dwc->lock, flags);
  951. return 0;
  952. }
  953. EXPORT_SYMBOL(dw_dma_cyclic_start);
  954. /**
  955. * dw_dma_cyclic_stop - stop the cyclic DMA transfer
  956. * @chan: the DMA channel to stop
  957. *
  958. * Must be called with soft interrupts disabled.
  959. */
  960. void dw_dma_cyclic_stop(struct dma_chan *chan)
  961. {
  962. struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
  963. struct dw_dma *dw = to_dw_dma(dwc->chan.device);
  964. unsigned long flags;
  965. spin_lock_irqsave(&dwc->lock, flags);
  966. channel_clear_bit(dw, CH_EN, dwc->mask);
  967. while (dma_readl(dw, CH_EN) & dwc->mask)
  968. cpu_relax();
  969. spin_unlock_irqrestore(&dwc->lock, flags);
  970. }
  971. EXPORT_SYMBOL(dw_dma_cyclic_stop);
  972. /**
  973. * dw_dma_cyclic_prep - prepare the cyclic DMA transfer
  974. * @chan: the DMA channel to prepare
  975. * @buf_addr: physical DMA address where the buffer starts
  976. * @buf_len: total number of bytes for the entire buffer
  977. * @period_len: number of bytes for each period
  978. * @direction: transfer direction, to or from device
  979. *
  980. * Must be called before trying to start the transfer. Returns a valid struct
  981. * dw_cyclic_desc if successful or an ERR_PTR(-errno) if not successful.
  982. */
  983. struct dw_cyclic_desc *dw_dma_cyclic_prep(struct dma_chan *chan,
  984. dma_addr_t buf_addr, size_t buf_len, size_t period_len,
  985. enum dma_transfer_direction direction)
  986. {
  987. struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
  988. struct dma_slave_config *sconfig = &dwc->dma_sconfig;
  989. struct dw_cyclic_desc *cdesc;
  990. struct dw_cyclic_desc *retval = NULL;
  991. struct dw_desc *desc;
  992. struct dw_desc *last = NULL;
  993. unsigned long was_cyclic;
  994. unsigned int reg_width;
  995. unsigned int periods;
  996. unsigned int i;
  997. unsigned long flags;
  998. spin_lock_irqsave(&dwc->lock, flags);
  999. if (!list_empty(&dwc->queue) || !list_empty(&dwc->active_list)) {
  1000. spin_unlock_irqrestore(&dwc->lock, flags);
  1001. dev_dbg(chan2dev(&dwc->chan),
  1002. "queue and/or active list are not empty\n");
  1003. return ERR_PTR(-EBUSY);
  1004. }
  1005. was_cyclic = test_and_set_bit(DW_DMA_IS_CYCLIC, &dwc->flags);
  1006. spin_unlock_irqrestore(&dwc->lock, flags);
  1007. if (was_cyclic) {
  1008. dev_dbg(chan2dev(&dwc->chan),
  1009. "channel already prepared for cyclic DMA\n");
  1010. return ERR_PTR(-EBUSY);
  1011. }
  1012. retval = ERR_PTR(-EINVAL);
  1013. if (direction == DMA_MEM_TO_DEV)
  1014. reg_width = __ffs(sconfig->dst_addr_width);
  1015. else
  1016. reg_width = __ffs(sconfig->src_addr_width);
  1017. periods = buf_len / period_len;
  1018. /* Check for too big/unaligned periods and unaligned DMA buffer. */
  1019. if (period_len > (DWC_MAX_COUNT << reg_width))
  1020. goto out_err;
  1021. if (unlikely(period_len & ((1 << reg_width) - 1)))
  1022. goto out_err;
  1023. if (unlikely(buf_addr & ((1 << reg_width) - 1)))
  1024. goto out_err;
  1025. if (unlikely(!(direction & (DMA_MEM_TO_DEV | DMA_DEV_TO_MEM))))
  1026. goto out_err;
  1027. retval = ERR_PTR(-ENOMEM);
  1028. if (periods > NR_DESCS_PER_CHANNEL)
  1029. goto out_err;
  1030. cdesc = kzalloc(sizeof(struct dw_cyclic_desc), GFP_KERNEL);
  1031. if (!cdesc)
  1032. goto out_err;
  1033. cdesc->desc = kzalloc(sizeof(struct dw_desc *) * periods, GFP_KERNEL);
  1034. if (!cdesc->desc)
  1035. goto out_err_alloc;
  1036. for (i = 0; i < periods; i++) {
  1037. desc = dwc_desc_get(dwc);
  1038. if (!desc)
  1039. goto out_err_desc_get;
  1040. switch (direction) {
  1041. case DMA_MEM_TO_DEV:
  1042. desc->lli.dar = sconfig->dst_addr;
  1043. desc->lli.sar = buf_addr + (period_len * i);
  1044. desc->lli.ctllo = (DWC_DEFAULT_CTLLO(chan)
  1045. | DWC_CTLL_DST_WIDTH(reg_width)
  1046. | DWC_CTLL_SRC_WIDTH(reg_width)
  1047. | DWC_CTLL_DST_FIX
  1048. | DWC_CTLL_SRC_INC
  1049. | DWC_CTLL_INT_EN);
  1050. desc->lli.ctllo |= sconfig->device_fc ?
  1051. DWC_CTLL_FC(DW_DMA_FC_P_M2P) :
  1052. DWC_CTLL_FC(DW_DMA_FC_D_M2P);
  1053. break;
  1054. case DMA_DEV_TO_MEM:
  1055. desc->lli.dar = buf_addr + (period_len * i);
  1056. desc->lli.sar = sconfig->src_addr;
  1057. desc->lli.ctllo = (DWC_DEFAULT_CTLLO(chan)
  1058. | DWC_CTLL_SRC_WIDTH(reg_width)
  1059. | DWC_CTLL_DST_WIDTH(reg_width)
  1060. | DWC_CTLL_DST_INC
  1061. | DWC_CTLL_SRC_FIX
  1062. | DWC_CTLL_INT_EN);
  1063. desc->lli.ctllo |= sconfig->device_fc ?
  1064. DWC_CTLL_FC(DW_DMA_FC_P_P2M) :
  1065. DWC_CTLL_FC(DW_DMA_FC_D_P2M);
  1066. break;
  1067. default:
  1068. break;
  1069. }
  1070. desc->lli.ctlhi = (period_len >> reg_width);
  1071. cdesc->desc[i] = desc;
  1072. if (last) {
  1073. last->lli.llp = desc->txd.phys;
  1074. dma_sync_single_for_device(chan2parent(chan),
  1075. last->txd.phys, sizeof(last->lli),
  1076. DMA_TO_DEVICE);
  1077. }
  1078. last = desc;
  1079. }
  1080. /* lets make a cyclic list */
  1081. last->lli.llp = cdesc->desc[0]->txd.phys;
  1082. dma_sync_single_for_device(chan2parent(chan), last->txd.phys,
  1083. sizeof(last->lli), DMA_TO_DEVICE);
  1084. dev_dbg(chan2dev(&dwc->chan), "cyclic prepared buf 0x%llx len %zu "
  1085. "period %zu periods %d\n", (unsigned long long)buf_addr,
  1086. buf_len, period_len, periods);
  1087. cdesc->periods = periods;
  1088. dwc->cdesc = cdesc;
  1089. return cdesc;
  1090. out_err_desc_get:
  1091. while (i--)
  1092. dwc_desc_put(dwc, cdesc->desc[i]);
  1093. out_err_alloc:
  1094. kfree(cdesc);
  1095. out_err:
  1096. clear_bit(DW_DMA_IS_CYCLIC, &dwc->flags);
  1097. return (struct dw_cyclic_desc *)retval;
  1098. }
  1099. EXPORT_SYMBOL(dw_dma_cyclic_prep);
  1100. /**
  1101. * dw_dma_cyclic_free - free a prepared cyclic DMA transfer
  1102. * @chan: the DMA channel to free
  1103. */
  1104. void dw_dma_cyclic_free(struct dma_chan *chan)
  1105. {
  1106. struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
  1107. struct dw_dma *dw = to_dw_dma(dwc->chan.device);
  1108. struct dw_cyclic_desc *cdesc = dwc->cdesc;
  1109. int i;
  1110. unsigned long flags;
  1111. dev_dbg(chan2dev(&dwc->chan), "cyclic free\n");
  1112. if (!cdesc)
  1113. return;
  1114. spin_lock_irqsave(&dwc->lock, flags);
  1115. channel_clear_bit(dw, CH_EN, dwc->mask);
  1116. while (dma_readl(dw, CH_EN) & dwc->mask)
  1117. cpu_relax();
  1118. dma_writel(dw, CLEAR.ERROR, dwc->mask);
  1119. dma_writel(dw, CLEAR.XFER, dwc->mask);
  1120. spin_unlock_irqrestore(&dwc->lock, flags);
  1121. for (i = 0; i < cdesc->periods; i++)
  1122. dwc_desc_put(dwc, cdesc->desc[i]);
  1123. kfree(cdesc->desc);
  1124. kfree(cdesc);
  1125. clear_bit(DW_DMA_IS_CYCLIC, &dwc->flags);
  1126. }
  1127. EXPORT_SYMBOL(dw_dma_cyclic_free);
  1128. /*----------------------------------------------------------------------*/
  1129. static void dw_dma_off(struct dw_dma *dw)
  1130. {
  1131. int i;
  1132. dma_writel(dw, CFG, 0);
  1133. channel_clear_bit(dw, MASK.XFER, dw->all_chan_mask);
  1134. channel_clear_bit(dw, MASK.SRC_TRAN, dw->all_chan_mask);
  1135. channel_clear_bit(dw, MASK.DST_TRAN, dw->all_chan_mask);
  1136. channel_clear_bit(dw, MASK.ERROR, dw->all_chan_mask);
  1137. while (dma_readl(dw, CFG) & DW_CFG_DMA_EN)
  1138. cpu_relax();
  1139. for (i = 0; i < dw->dma.chancnt; i++)
  1140. dw->chan[i].initialized = false;
  1141. }
  1142. static int __init dw_probe(struct platform_device *pdev)
  1143. {
  1144. struct dw_dma_platform_data *pdata;
  1145. struct resource *io;
  1146. struct dw_dma *dw;
  1147. size_t size;
  1148. int irq;
  1149. int err;
  1150. int i;
  1151. pdata = dev_get_platdata(&pdev->dev);
  1152. if (!pdata || pdata->nr_channels > DW_DMA_MAX_NR_CHANNELS)
  1153. return -EINVAL;
  1154. io = platform_get_resource(pdev, IORESOURCE_MEM, 0);
  1155. if (!io)
  1156. return -EINVAL;
  1157. irq = platform_get_irq(pdev, 0);
  1158. if (irq < 0)
  1159. return irq;
  1160. size = sizeof(struct dw_dma);
  1161. size += pdata->nr_channels * sizeof(struct dw_dma_chan);
  1162. dw = kzalloc(size, GFP_KERNEL);
  1163. if (!dw)
  1164. return -ENOMEM;
  1165. if (!request_mem_region(io->start, DW_REGLEN, pdev->dev.driver->name)) {
  1166. err = -EBUSY;
  1167. goto err_kfree;
  1168. }
  1169. dw->regs = ioremap(io->start, DW_REGLEN);
  1170. if (!dw->regs) {
  1171. err = -ENOMEM;
  1172. goto err_release_r;
  1173. }
  1174. dw->clk = clk_get(&pdev->dev, "hclk");
  1175. if (IS_ERR(dw->clk)) {
  1176. err = PTR_ERR(dw->clk);
  1177. goto err_clk;
  1178. }
  1179. clk_prepare_enable(dw->clk);
  1180. /* force dma off, just in case */
  1181. dw_dma_off(dw);
  1182. err = request_irq(irq, dw_dma_interrupt, 0, "dw_dmac", dw);
  1183. if (err)
  1184. goto err_irq;
  1185. platform_set_drvdata(pdev, dw);
  1186. tasklet_init(&dw->tasklet, dw_dma_tasklet, (unsigned long)dw);
  1187. dw->all_chan_mask = (1 << pdata->nr_channels) - 1;
  1188. INIT_LIST_HEAD(&dw->dma.channels);
  1189. for (i = 0; i < pdata->nr_channels; i++) {
  1190. struct dw_dma_chan *dwc = &dw->chan[i];
  1191. dwc->chan.device = &dw->dma;
  1192. dma_cookie_init(&dwc->chan);
  1193. if (pdata->chan_allocation_order == CHAN_ALLOCATION_ASCENDING)
  1194. list_add_tail(&dwc->chan.device_node,
  1195. &dw->dma.channels);
  1196. else
  1197. list_add(&dwc->chan.device_node, &dw->dma.channels);
  1198. /* 7 is highest priority & 0 is lowest. */
  1199. if (pdata->chan_priority == CHAN_PRIORITY_ASCENDING)
  1200. dwc->priority = pdata->nr_channels - i - 1;
  1201. else
  1202. dwc->priority = i;
  1203. dwc->ch_regs = &__dw_regs(dw)->CHAN[i];
  1204. spin_lock_init(&dwc->lock);
  1205. dwc->mask = 1 << i;
  1206. INIT_LIST_HEAD(&dwc->active_list);
  1207. INIT_LIST_HEAD(&dwc->queue);
  1208. INIT_LIST_HEAD(&dwc->free_list);
  1209. channel_clear_bit(dw, CH_EN, dwc->mask);
  1210. }
  1211. /* Clear/disable all interrupts on all channels. */
  1212. dma_writel(dw, CLEAR.XFER, dw->all_chan_mask);
  1213. dma_writel(dw, CLEAR.SRC_TRAN, dw->all_chan_mask);
  1214. dma_writel(dw, CLEAR.DST_TRAN, dw->all_chan_mask);
  1215. dma_writel(dw, CLEAR.ERROR, dw->all_chan_mask);
  1216. channel_clear_bit(dw, MASK.XFER, dw->all_chan_mask);
  1217. channel_clear_bit(dw, MASK.SRC_TRAN, dw->all_chan_mask);
  1218. channel_clear_bit(dw, MASK.DST_TRAN, dw->all_chan_mask);
  1219. channel_clear_bit(dw, MASK.ERROR, dw->all_chan_mask);
  1220. dma_cap_set(DMA_MEMCPY, dw->dma.cap_mask);
  1221. dma_cap_set(DMA_SLAVE, dw->dma.cap_mask);
  1222. if (pdata->is_private)
  1223. dma_cap_set(DMA_PRIVATE, dw->dma.cap_mask);
  1224. dw->dma.dev = &pdev->dev;
  1225. dw->dma.device_alloc_chan_resources = dwc_alloc_chan_resources;
  1226. dw->dma.device_free_chan_resources = dwc_free_chan_resources;
  1227. dw->dma.device_prep_dma_memcpy = dwc_prep_dma_memcpy;
  1228. dw->dma.device_prep_slave_sg = dwc_prep_slave_sg;
  1229. dw->dma.device_control = dwc_control;
  1230. dw->dma.device_tx_status = dwc_tx_status;
  1231. dw->dma.device_issue_pending = dwc_issue_pending;
  1232. dma_writel(dw, CFG, DW_CFG_DMA_EN);
  1233. printk(KERN_INFO "%s: DesignWare DMA Controller, %d channels\n",
  1234. dev_name(&pdev->dev), pdata->nr_channels);
  1235. dma_async_device_register(&dw->dma);
  1236. return 0;
  1237. err_irq:
  1238. clk_disable_unprepare(dw->clk);
  1239. clk_put(dw->clk);
  1240. err_clk:
  1241. iounmap(dw->regs);
  1242. dw->regs = NULL;
  1243. err_release_r:
  1244. release_resource(io);
  1245. err_kfree:
  1246. kfree(dw);
  1247. return err;
  1248. }
  1249. static int __exit dw_remove(struct platform_device *pdev)
  1250. {
  1251. struct dw_dma *dw = platform_get_drvdata(pdev);
  1252. struct dw_dma_chan *dwc, *_dwc;
  1253. struct resource *io;
  1254. dw_dma_off(dw);
  1255. dma_async_device_unregister(&dw->dma);
  1256. free_irq(platform_get_irq(pdev, 0), dw);
  1257. tasklet_kill(&dw->tasklet);
  1258. list_for_each_entry_safe(dwc, _dwc, &dw->dma.channels,
  1259. chan.device_node) {
  1260. list_del(&dwc->chan.device_node);
  1261. channel_clear_bit(dw, CH_EN, dwc->mask);
  1262. }
  1263. clk_disable_unprepare(dw->clk);
  1264. clk_put(dw->clk);
  1265. iounmap(dw->regs);
  1266. dw->regs = NULL;
  1267. io = platform_get_resource(pdev, IORESOURCE_MEM, 0);
  1268. release_mem_region(io->start, DW_REGLEN);
  1269. kfree(dw);
  1270. return 0;
  1271. }
  1272. static void dw_shutdown(struct platform_device *pdev)
  1273. {
  1274. struct dw_dma *dw = platform_get_drvdata(pdev);
  1275. dw_dma_off(platform_get_drvdata(pdev));
  1276. clk_disable_unprepare(dw->clk);
  1277. }
  1278. static int dw_suspend_noirq(struct device *dev)
  1279. {
  1280. struct platform_device *pdev = to_platform_device(dev);
  1281. struct dw_dma *dw = platform_get_drvdata(pdev);
  1282. dw_dma_off(platform_get_drvdata(pdev));
  1283. clk_disable_unprepare(dw->clk);
  1284. return 0;
  1285. }
  1286. static int dw_resume_noirq(struct device *dev)
  1287. {
  1288. struct platform_device *pdev = to_platform_device(dev);
  1289. struct dw_dma *dw = platform_get_drvdata(pdev);
  1290. clk_prepare_enable(dw->clk);
  1291. dma_writel(dw, CFG, DW_CFG_DMA_EN);
  1292. return 0;
  1293. }
  1294. static const struct dev_pm_ops dw_dev_pm_ops = {
  1295. .suspend_noirq = dw_suspend_noirq,
  1296. .resume_noirq = dw_resume_noirq,
  1297. .freeze_noirq = dw_suspend_noirq,
  1298. .thaw_noirq = dw_resume_noirq,
  1299. .restore_noirq = dw_resume_noirq,
  1300. .poweroff_noirq = dw_suspend_noirq,
  1301. };
  1302. #ifdef CONFIG_OF
  1303. static const struct of_device_id dw_dma_id_table[] = {
  1304. { .compatible = "snps,dma-spear1340" },
  1305. {}
  1306. };
  1307. MODULE_DEVICE_TABLE(of, dw_dma_id_table);
  1308. #endif
  1309. static struct platform_driver dw_driver = {
  1310. .remove = __exit_p(dw_remove),
  1311. .shutdown = dw_shutdown,
  1312. .driver = {
  1313. .name = "dw_dmac",
  1314. .pm = &dw_dev_pm_ops,
  1315. .of_match_table = of_match_ptr(dw_dma_id_table),
  1316. },
  1317. };
  1318. static int __init dw_init(void)
  1319. {
  1320. return platform_driver_probe(&dw_driver, dw_probe);
  1321. }
  1322. subsys_initcall(dw_init);
  1323. static void __exit dw_exit(void)
  1324. {
  1325. platform_driver_unregister(&dw_driver);
  1326. }
  1327. module_exit(dw_exit);
  1328. MODULE_LICENSE("GPL v2");
  1329. MODULE_DESCRIPTION("Synopsys DesignWare DMA Controller driver");
  1330. MODULE_AUTHOR("Haavard Skinnemoen (Atmel)");
  1331. MODULE_AUTHOR("Viresh Kumar <viresh.kumar@st.com>");