bfin_dma_5xx.c 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522
  1. /*
  2. * bfin_dma_5xx.c - Blackfin DMA implementation
  3. *
  4. * Copyright 2004-2008 Analog Devices Inc.
  5. *
  6. * Licensed under the GPL-2 or later.
  7. */
  8. #include <linux/errno.h>
  9. #include <linux/interrupt.h>
  10. #include <linux/kernel.h>
  11. #include <linux/module.h>
  12. #include <linux/param.h>
  13. #include <linux/proc_fs.h>
  14. #include <linux/sched.h>
  15. #include <linux/seq_file.h>
  16. #include <linux/spinlock.h>
  17. #include <asm/blackfin.h>
  18. #include <asm/cacheflush.h>
  19. #include <asm/dma.h>
  20. #include <asm/uaccess.h>
  21. #include <asm/early_printk.h>
  22. /*
  23. * To make sure we work around 05000119 - we always check DMA_DONE bit,
  24. * never the DMA_RUN bit
  25. */
  26. struct dma_channel dma_ch[MAX_DMA_CHANNELS];
  27. EXPORT_SYMBOL(dma_ch);
  28. static int __init blackfin_dma_init(void)
  29. {
  30. int i;
  31. printk(KERN_INFO "Blackfin DMA Controller\n");
  32. for (i = 0; i < MAX_DMA_CHANNELS; i++) {
  33. dma_ch[i].chan_status = DMA_CHANNEL_FREE;
  34. dma_ch[i].regs = dma_io_base_addr[i];
  35. mutex_init(&(dma_ch[i].dmalock));
  36. }
  37. /* Mark MEMDMA Channel 0 as requested since we're using it internally */
  38. request_dma(CH_MEM_STREAM0_DEST, "Blackfin dma_memcpy");
  39. request_dma(CH_MEM_STREAM0_SRC, "Blackfin dma_memcpy");
  40. #if defined(CONFIG_DEB_DMA_URGENT)
  41. bfin_write_EBIU_DDRQUE(bfin_read_EBIU_DDRQUE()
  42. | DEB1_URGENT | DEB2_URGENT | DEB3_URGENT);
  43. #endif
  44. return 0;
  45. }
  46. arch_initcall(blackfin_dma_init);
  47. #ifdef CONFIG_PROC_FS
  48. static int proc_dma_show(struct seq_file *m, void *v)
  49. {
  50. int i;
  51. for (i = 0; i < MAX_DMA_CHANNELS; ++i)
  52. if (dma_ch[i].chan_status != DMA_CHANNEL_FREE)
  53. seq_printf(m, "%2d: %s\n", i, dma_ch[i].device_id);
  54. return 0;
  55. }
  56. static int proc_dma_open(struct inode *inode, struct file *file)
  57. {
  58. return single_open(file, proc_dma_show, NULL);
  59. }
  60. static const struct file_operations proc_dma_operations = {
  61. .open = proc_dma_open,
  62. .read = seq_read,
  63. .llseek = seq_lseek,
  64. .release = single_release,
  65. };
  66. static int __init proc_dma_init(void)
  67. {
  68. return proc_create("dma", 0, NULL, &proc_dma_operations) != NULL;
  69. }
  70. late_initcall(proc_dma_init);
  71. #endif
  72. /**
  73. * request_dma - request a DMA channel
  74. *
  75. * Request the specific DMA channel from the system if it's available.
  76. */
  77. int request_dma(unsigned int channel, const char *device_id)
  78. {
  79. pr_debug("request_dma() : BEGIN \n");
  80. if (device_id == NULL)
  81. printk(KERN_WARNING "request_dma(%u): no device_id given\n", channel);
  82. #if defined(CONFIG_BF561) && ANOMALY_05000182
  83. if (channel >= CH_IMEM_STREAM0_DEST && channel <= CH_IMEM_STREAM1_DEST) {
  84. if (get_cclk() > 500000000) {
  85. printk(KERN_WARNING
  86. "Request IMDMA failed due to ANOMALY 05000182\n");
  87. return -EFAULT;
  88. }
  89. }
  90. #endif
  91. mutex_lock(&(dma_ch[channel].dmalock));
  92. if ((dma_ch[channel].chan_status == DMA_CHANNEL_REQUESTED)
  93. || (dma_ch[channel].chan_status == DMA_CHANNEL_ENABLED)) {
  94. mutex_unlock(&(dma_ch[channel].dmalock));
  95. pr_debug("DMA CHANNEL IN USE \n");
  96. return -EBUSY;
  97. } else {
  98. dma_ch[channel].chan_status = DMA_CHANNEL_REQUESTED;
  99. pr_debug("DMA CHANNEL IS ALLOCATED \n");
  100. }
  101. mutex_unlock(&(dma_ch[channel].dmalock));
  102. #ifdef CONFIG_BF54x
  103. if (channel >= CH_UART2_RX && channel <= CH_UART3_TX) {
  104. unsigned int per_map;
  105. per_map = dma_ch[channel].regs->peripheral_map & 0xFFF;
  106. if (strncmp(device_id, "BFIN_UART", 9) == 0)
  107. dma_ch[channel].regs->peripheral_map = per_map |
  108. ((channel - CH_UART2_RX + 0xC)<<12);
  109. else
  110. dma_ch[channel].regs->peripheral_map = per_map |
  111. ((channel - CH_UART2_RX + 0x6)<<12);
  112. }
  113. #endif
  114. dma_ch[channel].device_id = device_id;
  115. dma_ch[channel].irq = 0;
  116. /* This is to be enabled by putting a restriction -
  117. * you have to request DMA, before doing any operations on
  118. * descriptor/channel
  119. */
  120. pr_debug("request_dma() : END \n");
  121. return 0;
  122. }
  123. EXPORT_SYMBOL(request_dma);
  124. int set_dma_callback(unsigned int channel, irq_handler_t callback, void *data)
  125. {
  126. BUG_ON(channel >= MAX_DMA_CHANNELS ||
  127. dma_ch[channel].chan_status == DMA_CHANNEL_FREE);
  128. if (callback != NULL) {
  129. int ret;
  130. unsigned int irq = channel2irq(channel);
  131. ret = request_irq(irq, callback, IRQF_DISABLED,
  132. dma_ch[channel].device_id, data);
  133. if (ret)
  134. return ret;
  135. dma_ch[channel].irq = irq;
  136. dma_ch[channel].data = data;
  137. }
  138. return 0;
  139. }
  140. EXPORT_SYMBOL(set_dma_callback);
  141. /**
  142. * clear_dma_buffer - clear DMA fifos for specified channel
  143. *
  144. * Set the Buffer Clear bit in the Configuration register of specific DMA
  145. * channel. This will stop the descriptor based DMA operation.
  146. */
  147. static void clear_dma_buffer(unsigned int channel)
  148. {
  149. dma_ch[channel].regs->cfg |= RESTART;
  150. SSYNC();
  151. dma_ch[channel].regs->cfg &= ~RESTART;
  152. }
  153. void free_dma(unsigned int channel)
  154. {
  155. pr_debug("freedma() : BEGIN \n");
  156. BUG_ON(channel >= MAX_DMA_CHANNELS ||
  157. dma_ch[channel].chan_status == DMA_CHANNEL_FREE);
  158. /* Halt the DMA */
  159. disable_dma(channel);
  160. clear_dma_buffer(channel);
  161. if (dma_ch[channel].irq)
  162. free_irq(dma_ch[channel].irq, dma_ch[channel].data);
  163. /* Clear the DMA Variable in the Channel */
  164. mutex_lock(&(dma_ch[channel].dmalock));
  165. dma_ch[channel].chan_status = DMA_CHANNEL_FREE;
  166. mutex_unlock(&(dma_ch[channel].dmalock));
  167. pr_debug("freedma() : END \n");
  168. }
  169. EXPORT_SYMBOL(free_dma);
  170. #ifdef CONFIG_PM
  171. # ifndef MAX_DMA_SUSPEND_CHANNELS
  172. # define MAX_DMA_SUSPEND_CHANNELS MAX_DMA_CHANNELS
  173. # endif
  174. int blackfin_dma_suspend(void)
  175. {
  176. int i;
  177. for (i = 0; i < MAX_DMA_SUSPEND_CHANNELS; ++i) {
  178. if (dma_ch[i].chan_status == DMA_CHANNEL_ENABLED) {
  179. printk(KERN_ERR "DMA Channel %d failed to suspend\n", i);
  180. return -EBUSY;
  181. }
  182. dma_ch[i].saved_peripheral_map = dma_ch[i].regs->peripheral_map;
  183. }
  184. return 0;
  185. }
  186. void blackfin_dma_resume(void)
  187. {
  188. int i;
  189. for (i = 0; i < MAX_DMA_CHANNELS; ++i) {
  190. dma_ch[i].regs->cfg = 0;
  191. if (i < MAX_DMA_SUSPEND_CHANNELS)
  192. dma_ch[i].regs->peripheral_map = dma_ch[i].saved_peripheral_map;
  193. }
  194. }
  195. #endif
  196. /**
  197. * blackfin_dma_early_init - minimal DMA init
  198. *
  199. * Setup a few DMA registers so we can safely do DMA transfers early on in
  200. * the kernel booting process. Really this just means using dma_memcpy().
  201. */
  202. void __init blackfin_dma_early_init(void)
  203. {
  204. early_shadow_stamp();
  205. bfin_write_MDMA_S0_CONFIG(0);
  206. bfin_write_MDMA_S1_CONFIG(0);
  207. }
  208. void __init early_dma_memcpy(void *pdst, const void *psrc, size_t size)
  209. {
  210. unsigned long dst = (unsigned long)pdst;
  211. unsigned long src = (unsigned long)psrc;
  212. struct dma_register *dst_ch, *src_ch;
  213. early_shadow_stamp();
  214. /* We assume that everything is 4 byte aligned, so include
  215. * a basic sanity check
  216. */
  217. BUG_ON(dst % 4);
  218. BUG_ON(src % 4);
  219. BUG_ON(size % 4);
  220. src_ch = 0;
  221. /* Find an avalible memDMA channel */
  222. while (1) {
  223. if (src_ch == (struct dma_register *)MDMA_S0_NEXT_DESC_PTR) {
  224. dst_ch = (struct dma_register *)MDMA_D1_NEXT_DESC_PTR;
  225. src_ch = (struct dma_register *)MDMA_S1_NEXT_DESC_PTR;
  226. } else {
  227. dst_ch = (struct dma_register *)MDMA_D0_NEXT_DESC_PTR;
  228. src_ch = (struct dma_register *)MDMA_S0_NEXT_DESC_PTR;
  229. }
  230. if (!bfin_read16(&src_ch->cfg))
  231. break;
  232. else if (bfin_read16(&dst_ch->irq_status) & DMA_DONE) {
  233. bfin_write16(&src_ch->cfg, 0);
  234. break;
  235. }
  236. }
  237. /* Force a sync in case a previous config reset on this channel
  238. * occurred. This is needed so subsequent writes to DMA registers
  239. * are not spuriously lost/corrupted.
  240. */
  241. __builtin_bfin_ssync();
  242. /* Destination */
  243. bfin_write32(&dst_ch->start_addr, dst);
  244. bfin_write16(&dst_ch->x_count, size >> 2);
  245. bfin_write16(&dst_ch->x_modify, 1 << 2);
  246. bfin_write16(&dst_ch->irq_status, DMA_DONE | DMA_ERR);
  247. /* Source */
  248. bfin_write32(&src_ch->start_addr, src);
  249. bfin_write16(&src_ch->x_count, size >> 2);
  250. bfin_write16(&src_ch->x_modify, 1 << 2);
  251. bfin_write16(&src_ch->irq_status, DMA_DONE | DMA_ERR);
  252. /* Enable */
  253. bfin_write16(&src_ch->cfg, DMAEN | WDSIZE_32);
  254. bfin_write16(&dst_ch->cfg, WNR | DI_EN | DMAEN | WDSIZE_32);
  255. /* Since we are atomic now, don't use the workaround ssync */
  256. __builtin_bfin_ssync();
  257. }
  258. void __init early_dma_memcpy_done(void)
  259. {
  260. early_shadow_stamp();
  261. while ((bfin_read_MDMA_S0_CONFIG() && !(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE)) ||
  262. (bfin_read_MDMA_S1_CONFIG() && !(bfin_read_MDMA_D1_IRQ_STATUS() & DMA_DONE)))
  263. continue;
  264. bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
  265. bfin_write_MDMA_D1_IRQ_STATUS(DMA_DONE | DMA_ERR);
  266. /*
  267. * Now that DMA is done, we would normally flush cache, but
  268. * i/d cache isn't running this early, so we don't bother,
  269. * and just clear out the DMA channel for next time
  270. */
  271. bfin_write_MDMA_S0_CONFIG(0);
  272. bfin_write_MDMA_S1_CONFIG(0);
  273. bfin_write_MDMA_D0_CONFIG(0);
  274. bfin_write_MDMA_D1_CONFIG(0);
  275. __builtin_bfin_ssync();
  276. }
  277. /**
  278. * __dma_memcpy - program the MDMA registers
  279. *
  280. * Actually program MDMA0 and wait for the transfer to finish. Disable IRQs
  281. * while programming registers so that everything is fully configured. Wait
  282. * for DMA to finish with IRQs enabled. If interrupted, the initial DMA_DONE
  283. * check will make sure we don't clobber any existing transfer.
  284. */
  285. static void __dma_memcpy(u32 daddr, s16 dmod, u32 saddr, s16 smod, size_t cnt, u32 conf)
  286. {
  287. static DEFINE_SPINLOCK(mdma_lock);
  288. unsigned long flags;
  289. spin_lock_irqsave(&mdma_lock, flags);
  290. /* Force a sync in case a previous config reset on this channel
  291. * occurred. This is needed so subsequent writes to DMA registers
  292. * are not spuriously lost/corrupted. Do it under irq lock and
  293. * without the anomaly version (because we are atomic already).
  294. */
  295. __builtin_bfin_ssync();
  296. if (bfin_read_MDMA_S0_CONFIG())
  297. while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE))
  298. continue;
  299. if (conf & DMA2D) {
  300. /* For larger bit sizes, we've already divided down cnt so it
  301. * is no longer a multiple of 64k. So we have to break down
  302. * the limit here so it is a multiple of the incoming size.
  303. * There is no limitation here in terms of total size other
  304. * than the hardware though as the bits lost in the shift are
  305. * made up by MODIFY (== we can hit the whole address space).
  306. * X: (2^(16 - 0)) * 1 == (2^(16 - 1)) * 2 == (2^(16 - 2)) * 4
  307. */
  308. u32 shift = abs(dmod) >> 1;
  309. size_t ycnt = cnt >> (16 - shift);
  310. cnt = 1 << (16 - shift);
  311. bfin_write_MDMA_D0_Y_COUNT(ycnt);
  312. bfin_write_MDMA_S0_Y_COUNT(ycnt);
  313. bfin_write_MDMA_D0_Y_MODIFY(dmod);
  314. bfin_write_MDMA_S0_Y_MODIFY(smod);
  315. }
  316. bfin_write_MDMA_D0_START_ADDR(daddr);
  317. bfin_write_MDMA_D0_X_COUNT(cnt);
  318. bfin_write_MDMA_D0_X_MODIFY(dmod);
  319. bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
  320. bfin_write_MDMA_S0_START_ADDR(saddr);
  321. bfin_write_MDMA_S0_X_COUNT(cnt);
  322. bfin_write_MDMA_S0_X_MODIFY(smod);
  323. bfin_write_MDMA_S0_IRQ_STATUS(DMA_DONE | DMA_ERR);
  324. bfin_write_MDMA_S0_CONFIG(DMAEN | conf);
  325. bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | conf);
  326. spin_unlock_irqrestore(&mdma_lock, flags);
  327. SSYNC();
  328. while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE))
  329. if (bfin_read_MDMA_S0_CONFIG())
  330. continue;
  331. else
  332. return;
  333. bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
  334. bfin_write_MDMA_S0_CONFIG(0);
  335. bfin_write_MDMA_D0_CONFIG(0);
  336. }
  337. /**
  338. * _dma_memcpy - translate C memcpy settings into MDMA settings
  339. *
  340. * Handle all the high level steps before we touch the MDMA registers. So
  341. * handle direction, tweaking of sizes, and formatting of addresses.
  342. */
  343. static void *_dma_memcpy(void *pdst, const void *psrc, size_t size)
  344. {
  345. u32 conf, shift;
  346. s16 mod;
  347. unsigned long dst = (unsigned long)pdst;
  348. unsigned long src = (unsigned long)psrc;
  349. if (size == 0)
  350. return NULL;
  351. if (dst % 4 == 0 && src % 4 == 0 && size % 4 == 0) {
  352. conf = WDSIZE_32;
  353. shift = 2;
  354. } else if (dst % 2 == 0 && src % 2 == 0 && size % 2 == 0) {
  355. conf = WDSIZE_16;
  356. shift = 1;
  357. } else {
  358. conf = WDSIZE_8;
  359. shift = 0;
  360. }
  361. /* If the two memory regions have a chance of overlapping, make
  362. * sure the memcpy still works as expected. Do this by having the
  363. * copy run backwards instead.
  364. */
  365. mod = 1 << shift;
  366. if (src < dst) {
  367. mod *= -1;
  368. dst += size + mod;
  369. src += size + mod;
  370. }
  371. size >>= shift;
  372. if (size > 0x10000)
  373. conf |= DMA2D;
  374. __dma_memcpy(dst, mod, src, mod, size, conf);
  375. return pdst;
  376. }
  377. /**
  378. * dma_memcpy - DMA memcpy under mutex lock
  379. *
  380. * Do not check arguments before starting the DMA memcpy. Break the transfer
  381. * up into two pieces. The first transfer is in multiples of 64k and the
  382. * second transfer is the piece smaller than 64k.
  383. */
  384. void *dma_memcpy(void *pdst, const void *psrc, size_t size)
  385. {
  386. unsigned long dst = (unsigned long)pdst;
  387. unsigned long src = (unsigned long)psrc;
  388. size_t bulk, rest;
  389. if (bfin_addr_dcacheable(src))
  390. blackfin_dcache_flush_range(src, src + size);
  391. if (bfin_addr_dcacheable(dst))
  392. blackfin_dcache_invalidate_range(dst, dst + size);
  393. bulk = size & ~0xffff;
  394. rest = size - bulk;
  395. if (bulk)
  396. _dma_memcpy(pdst, psrc, bulk);
  397. _dma_memcpy(pdst + bulk, psrc + bulk, rest);
  398. return pdst;
  399. }
  400. EXPORT_SYMBOL(dma_memcpy);
  401. /**
  402. * safe_dma_memcpy - DMA memcpy w/argument checking
  403. *
  404. * Verify arguments are safe before heading to dma_memcpy().
  405. */
  406. void *safe_dma_memcpy(void *dst, const void *src, size_t size)
  407. {
  408. if (!access_ok(VERIFY_WRITE, dst, size))
  409. return NULL;
  410. if (!access_ok(VERIFY_READ, src, size))
  411. return NULL;
  412. return dma_memcpy(dst, src, size);
  413. }
  414. EXPORT_SYMBOL(safe_dma_memcpy);
  415. static void _dma_out(unsigned long addr, unsigned long buf, unsigned short len,
  416. u16 size, u16 dma_size)
  417. {
  418. blackfin_dcache_flush_range(buf, buf + len * size);
  419. __dma_memcpy(addr, 0, buf, size, len, dma_size);
  420. }
  421. static void _dma_in(unsigned long addr, unsigned long buf, unsigned short len,
  422. u16 size, u16 dma_size)
  423. {
  424. blackfin_dcache_invalidate_range(buf, buf + len * size);
  425. __dma_memcpy(buf, size, addr, 0, len, dma_size);
  426. }
  427. #define MAKE_DMA_IO(io, bwl, isize, dmasize, cnst) \
  428. void dma_##io##s##bwl(unsigned long addr, cnst void *buf, unsigned short len) \
  429. { \
  430. _dma_##io(addr, (unsigned long)buf, len, isize, WDSIZE_##dmasize); \
  431. } \
  432. EXPORT_SYMBOL(dma_##io##s##bwl)
  433. MAKE_DMA_IO(out, b, 1, 8, const);
  434. MAKE_DMA_IO(in, b, 1, 8, );
  435. MAKE_DMA_IO(out, w, 2, 16, const);
  436. MAKE_DMA_IO(in, w, 2, 16, );
  437. MAKE_DMA_IO(out, l, 4, 32, const);
  438. MAKE_DMA_IO(in, l, 4, 32, );