ep93xx_dma.c 36 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380
  1. /*
  2. * Driver for the Cirrus Logic EP93xx DMA Controller
  3. *
  4. * Copyright (C) 2011 Mika Westerberg
  5. *
  6. * DMA M2P implementation is based on the original
  7. * arch/arm/mach-ep93xx/dma-m2p.c which has following copyrights:
  8. *
  9. * Copyright (C) 2006 Lennert Buytenhek <buytenh@wantstofly.org>
  10. * Copyright (C) 2006 Applied Data Systems
  11. * Copyright (C) 2009 Ryan Mallon <rmallon@gmail.com>
  12. *
  13. * This driver is based on dw_dmac and amba-pl08x drivers.
  14. *
  15. * This program is free software; you can redistribute it and/or modify
  16. * it under the terms of the GNU General Public License as published by
  17. * the Free Software Foundation; either version 2 of the License, or
  18. * (at your option) any later version.
  19. */
  20. #include <linux/clk.h>
  21. #include <linux/init.h>
  22. #include <linux/interrupt.h>
  23. #include <linux/dmaengine.h>
  24. #include <linux/module.h>
  25. #include <linux/platform_device.h>
  26. #include <linux/slab.h>
  27. #include <mach/dma.h>
  28. #include "dmaengine.h"
  29. /* M2P registers */
  30. #define M2P_CONTROL 0x0000
  31. #define M2P_CONTROL_STALLINT BIT(0)
  32. #define M2P_CONTROL_NFBINT BIT(1)
  33. #define M2P_CONTROL_CH_ERROR_INT BIT(3)
  34. #define M2P_CONTROL_ENABLE BIT(4)
  35. #define M2P_CONTROL_ICE BIT(6)
  36. #define M2P_INTERRUPT 0x0004
  37. #define M2P_INTERRUPT_STALL BIT(0)
  38. #define M2P_INTERRUPT_NFB BIT(1)
  39. #define M2P_INTERRUPT_ERROR BIT(3)
  40. #define M2P_PPALLOC 0x0008
  41. #define M2P_STATUS 0x000c
  42. #define M2P_MAXCNT0 0x0020
  43. #define M2P_BASE0 0x0024
  44. #define M2P_MAXCNT1 0x0030
  45. #define M2P_BASE1 0x0034
  46. #define M2P_STATE_IDLE 0
  47. #define M2P_STATE_STALL 1
  48. #define M2P_STATE_ON 2
  49. #define M2P_STATE_NEXT 3
  50. /* M2M registers */
  51. #define M2M_CONTROL 0x0000
  52. #define M2M_CONTROL_DONEINT BIT(2)
  53. #define M2M_CONTROL_ENABLE BIT(3)
  54. #define M2M_CONTROL_START BIT(4)
  55. #define M2M_CONTROL_DAH BIT(11)
  56. #define M2M_CONTROL_SAH BIT(12)
  57. #define M2M_CONTROL_PW_SHIFT 9
  58. #define M2M_CONTROL_PW_8 (0 << M2M_CONTROL_PW_SHIFT)
  59. #define M2M_CONTROL_PW_16 (1 << M2M_CONTROL_PW_SHIFT)
  60. #define M2M_CONTROL_PW_32 (2 << M2M_CONTROL_PW_SHIFT)
  61. #define M2M_CONTROL_PW_MASK (3 << M2M_CONTROL_PW_SHIFT)
  62. #define M2M_CONTROL_TM_SHIFT 13
  63. #define M2M_CONTROL_TM_TX (1 << M2M_CONTROL_TM_SHIFT)
  64. #define M2M_CONTROL_TM_RX (2 << M2M_CONTROL_TM_SHIFT)
  65. #define M2M_CONTROL_RSS_SHIFT 22
  66. #define M2M_CONTROL_RSS_SSPRX (1 << M2M_CONTROL_RSS_SHIFT)
  67. #define M2M_CONTROL_RSS_SSPTX (2 << M2M_CONTROL_RSS_SHIFT)
  68. #define M2M_CONTROL_RSS_IDE (3 << M2M_CONTROL_RSS_SHIFT)
  69. #define M2M_CONTROL_NO_HDSK BIT(24)
  70. #define M2M_CONTROL_PWSC_SHIFT 25
  71. #define M2M_INTERRUPT 0x0004
  72. #define M2M_INTERRUPT_DONEINT BIT(1)
  73. #define M2M_BCR0 0x0010
  74. #define M2M_BCR1 0x0014
  75. #define M2M_SAR_BASE0 0x0018
  76. #define M2M_SAR_BASE1 0x001c
  77. #define M2M_DAR_BASE0 0x002c
  78. #define M2M_DAR_BASE1 0x0030
  79. #define DMA_MAX_CHAN_BYTES 0xffff
  80. #define DMA_MAX_CHAN_DESCRIPTORS 32
  81. struct ep93xx_dma_engine;
  82. /**
  83. * struct ep93xx_dma_desc - EP93xx specific transaction descriptor
  84. * @src_addr: source address of the transaction
  85. * @dst_addr: destination address of the transaction
  86. * @size: size of the transaction (in bytes)
  87. * @complete: this descriptor is completed
  88. * @txd: dmaengine API descriptor
  89. * @tx_list: list of linked descriptors
  90. * @node: link used for putting this into a channel queue
  91. */
  92. struct ep93xx_dma_desc {
  93. u32 src_addr;
  94. u32 dst_addr;
  95. size_t size;
  96. bool complete;
  97. struct dma_async_tx_descriptor txd;
  98. struct list_head tx_list;
  99. struct list_head node;
  100. };
  101. /**
  102. * struct ep93xx_dma_chan - an EP93xx DMA M2P/M2M channel
  103. * @chan: dmaengine API channel
  104. * @edma: pointer to to the engine device
  105. * @regs: memory mapped registers
  106. * @irq: interrupt number of the channel
  107. * @clk: clock used by this channel
  108. * @tasklet: channel specific tasklet used for callbacks
  109. * @lock: lock protecting the fields following
  110. * @flags: flags for the channel
  111. * @buffer: which buffer to use next (0/1)
  112. * @active: flattened chain of descriptors currently being processed
  113. * @queue: pending descriptors which are handled next
  114. * @free_list: list of free descriptors which can be used
  115. * @runtime_addr: physical address currently used as dest/src (M2M only). This
  116. * is set via %DMA_SLAVE_CONFIG before slave operation is
  117. * prepared
  118. * @runtime_ctrl: M2M runtime values for the control register.
  119. *
  120. * As EP93xx DMA controller doesn't support real chained DMA descriptors we
  121. * will have slightly different scheme here: @active points to a head of
  122. * flattened DMA descriptor chain.
  123. *
  124. * @queue holds pending transactions. These are linked through the first
  125. * descriptor in the chain. When a descriptor is moved to the @active queue,
  126. * the first and chained descriptors are flattened into a single list.
  127. *
  128. * @chan.private holds pointer to &struct ep93xx_dma_data which contains
  129. * necessary channel configuration information. For memcpy channels this must
  130. * be %NULL.
  131. */
  132. struct ep93xx_dma_chan {
  133. struct dma_chan chan;
  134. const struct ep93xx_dma_engine *edma;
  135. void __iomem *regs;
  136. int irq;
  137. struct clk *clk;
  138. struct tasklet_struct tasklet;
  139. /* protects the fields following */
  140. spinlock_t lock;
  141. unsigned long flags;
  142. /* Channel is configured for cyclic transfers */
  143. #define EP93XX_DMA_IS_CYCLIC 0
  144. int buffer;
  145. struct list_head active;
  146. struct list_head queue;
  147. struct list_head free_list;
  148. u32 runtime_addr;
  149. u32 runtime_ctrl;
  150. };
  151. /**
  152. * struct ep93xx_dma_engine - the EP93xx DMA engine instance
  153. * @dma_dev: holds the dmaengine device
  154. * @m2m: is this an M2M or M2P device
  155. * @hw_setup: method which sets the channel up for operation
  156. * @hw_shutdown: shuts the channel down and flushes whatever is left
  157. * @hw_submit: pushes active descriptor(s) to the hardware
  158. * @hw_interrupt: handle the interrupt
  159. * @num_channels: number of channels for this instance
  160. * @channels: array of channels
  161. *
  162. * There is one instance of this struct for the M2P channels and one for the
  163. * M2M channels. hw_xxx() methods are used to perform operations which are
  164. * different on M2M and M2P channels. These methods are called with channel
  165. * lock held and interrupts disabled so they cannot sleep.
  166. */
  167. struct ep93xx_dma_engine {
  168. struct dma_device dma_dev;
  169. bool m2m;
  170. int (*hw_setup)(struct ep93xx_dma_chan *);
  171. void (*hw_shutdown)(struct ep93xx_dma_chan *);
  172. void (*hw_submit)(struct ep93xx_dma_chan *);
  173. int (*hw_interrupt)(struct ep93xx_dma_chan *);
  174. #define INTERRUPT_UNKNOWN 0
  175. #define INTERRUPT_DONE 1
  176. #define INTERRUPT_NEXT_BUFFER 2
  177. size_t num_channels;
  178. struct ep93xx_dma_chan channels[];
  179. };
  180. static inline struct device *chan2dev(struct ep93xx_dma_chan *edmac)
  181. {
  182. return &edmac->chan.dev->device;
  183. }
  184. static struct ep93xx_dma_chan *to_ep93xx_dma_chan(struct dma_chan *chan)
  185. {
  186. return container_of(chan, struct ep93xx_dma_chan, chan);
  187. }
  188. /**
  189. * ep93xx_dma_set_active - set new active descriptor chain
  190. * @edmac: channel
  191. * @desc: head of the new active descriptor chain
  192. *
  193. * Sets @desc to be the head of the new active descriptor chain. This is the
  194. * chain which is processed next. The active list must be empty before calling
  195. * this function.
  196. *
  197. * Called with @edmac->lock held and interrupts disabled.
  198. */
  199. static void ep93xx_dma_set_active(struct ep93xx_dma_chan *edmac,
  200. struct ep93xx_dma_desc *desc)
  201. {
  202. BUG_ON(!list_empty(&edmac->active));
  203. list_add_tail(&desc->node, &edmac->active);
  204. /* Flatten the @desc->tx_list chain into @edmac->active list */
  205. while (!list_empty(&desc->tx_list)) {
  206. struct ep93xx_dma_desc *d = list_first_entry(&desc->tx_list,
  207. struct ep93xx_dma_desc, node);
  208. /*
  209. * We copy the callback parameters from the first descriptor
  210. * to all the chained descriptors. This way we can call the
  211. * callback without having to find out the first descriptor in
  212. * the chain. Useful for cyclic transfers.
  213. */
  214. d->txd.callback = desc->txd.callback;
  215. d->txd.callback_param = desc->txd.callback_param;
  216. list_move_tail(&d->node, &edmac->active);
  217. }
  218. }
  219. /* Called with @edmac->lock held and interrupts disabled */
  220. static struct ep93xx_dma_desc *
  221. ep93xx_dma_get_active(struct ep93xx_dma_chan *edmac)
  222. {
  223. if (list_empty(&edmac->active))
  224. return NULL;
  225. return list_first_entry(&edmac->active, struct ep93xx_dma_desc, node);
  226. }
  227. /**
  228. * ep93xx_dma_advance_active - advances to the next active descriptor
  229. * @edmac: channel
  230. *
  231. * Function advances active descriptor to the next in the @edmac->active and
  232. * returns %true if we still have descriptors in the chain to process.
  233. * Otherwise returns %false.
  234. *
  235. * When the channel is in cyclic mode always returns %true.
  236. *
  237. * Called with @edmac->lock held and interrupts disabled.
  238. */
  239. static bool ep93xx_dma_advance_active(struct ep93xx_dma_chan *edmac)
  240. {
  241. struct ep93xx_dma_desc *desc;
  242. list_rotate_left(&edmac->active);
  243. if (test_bit(EP93XX_DMA_IS_CYCLIC, &edmac->flags))
  244. return true;
  245. desc = ep93xx_dma_get_active(edmac);
  246. if (!desc)
  247. return false;
  248. /*
  249. * If txd.cookie is set it means that we are back in the first
  250. * descriptor in the chain and hence done with it.
  251. */
  252. return !desc->txd.cookie;
  253. }
  254. /*
  255. * M2P DMA implementation
  256. */
  257. static void m2p_set_control(struct ep93xx_dma_chan *edmac, u32 control)
  258. {
  259. writel(control, edmac->regs + M2P_CONTROL);
  260. /*
  261. * EP93xx User's Guide states that we must perform a dummy read after
  262. * write to the control register.
  263. */
  264. readl(edmac->regs + M2P_CONTROL);
  265. }
  266. static int m2p_hw_setup(struct ep93xx_dma_chan *edmac)
  267. {
  268. struct ep93xx_dma_data *data = edmac->chan.private;
  269. u32 control;
  270. writel(data->port & 0xf, edmac->regs + M2P_PPALLOC);
  271. control = M2P_CONTROL_CH_ERROR_INT | M2P_CONTROL_ICE
  272. | M2P_CONTROL_ENABLE;
  273. m2p_set_control(edmac, control);
  274. return 0;
  275. }
  276. static inline u32 m2p_channel_state(struct ep93xx_dma_chan *edmac)
  277. {
  278. return (readl(edmac->regs + M2P_STATUS) >> 4) & 0x3;
  279. }
  280. static void m2p_hw_shutdown(struct ep93xx_dma_chan *edmac)
  281. {
  282. u32 control;
  283. control = readl(edmac->regs + M2P_CONTROL);
  284. control &= ~(M2P_CONTROL_STALLINT | M2P_CONTROL_NFBINT);
  285. m2p_set_control(edmac, control);
  286. while (m2p_channel_state(edmac) >= M2P_STATE_ON)
  287. cpu_relax();
  288. m2p_set_control(edmac, 0);
  289. while (m2p_channel_state(edmac) == M2P_STATE_STALL)
  290. cpu_relax();
  291. }
  292. static void m2p_fill_desc(struct ep93xx_dma_chan *edmac)
  293. {
  294. struct ep93xx_dma_desc *desc;
  295. u32 bus_addr;
  296. desc = ep93xx_dma_get_active(edmac);
  297. if (!desc) {
  298. dev_warn(chan2dev(edmac), "M2P: empty descriptor list\n");
  299. return;
  300. }
  301. if (ep93xx_dma_chan_direction(&edmac->chan) == DMA_MEM_TO_DEV)
  302. bus_addr = desc->src_addr;
  303. else
  304. bus_addr = desc->dst_addr;
  305. if (edmac->buffer == 0) {
  306. writel(desc->size, edmac->regs + M2P_MAXCNT0);
  307. writel(bus_addr, edmac->regs + M2P_BASE0);
  308. } else {
  309. writel(desc->size, edmac->regs + M2P_MAXCNT1);
  310. writel(bus_addr, edmac->regs + M2P_BASE1);
  311. }
  312. edmac->buffer ^= 1;
  313. }
  314. static void m2p_hw_submit(struct ep93xx_dma_chan *edmac)
  315. {
  316. u32 control = readl(edmac->regs + M2P_CONTROL);
  317. m2p_fill_desc(edmac);
  318. control |= M2P_CONTROL_STALLINT;
  319. if (ep93xx_dma_advance_active(edmac)) {
  320. m2p_fill_desc(edmac);
  321. control |= M2P_CONTROL_NFBINT;
  322. }
  323. m2p_set_control(edmac, control);
  324. }
  325. static int m2p_hw_interrupt(struct ep93xx_dma_chan *edmac)
  326. {
  327. u32 irq_status = readl(edmac->regs + M2P_INTERRUPT);
  328. u32 control;
  329. if (irq_status & M2P_INTERRUPT_ERROR) {
  330. struct ep93xx_dma_desc *desc = ep93xx_dma_get_active(edmac);
  331. /* Clear the error interrupt */
  332. writel(1, edmac->regs + M2P_INTERRUPT);
  333. /*
  334. * It seems that there is no easy way of reporting errors back
  335. * to client so we just report the error here and continue as
  336. * usual.
  337. *
  338. * Revisit this when there is a mechanism to report back the
  339. * errors.
  340. */
  341. dev_err(chan2dev(edmac),
  342. "DMA transfer failed! Details:\n"
  343. "\tcookie : %d\n"
  344. "\tsrc_addr : 0x%08x\n"
  345. "\tdst_addr : 0x%08x\n"
  346. "\tsize : %zu\n",
  347. desc->txd.cookie, desc->src_addr, desc->dst_addr,
  348. desc->size);
  349. }
  350. switch (irq_status & (M2P_INTERRUPT_STALL | M2P_INTERRUPT_NFB)) {
  351. case M2P_INTERRUPT_STALL:
  352. /* Disable interrupts */
  353. control = readl(edmac->regs + M2P_CONTROL);
  354. control &= ~(M2P_CONTROL_STALLINT | M2P_CONTROL_NFBINT);
  355. m2p_set_control(edmac, control);
  356. return INTERRUPT_DONE;
  357. case M2P_INTERRUPT_NFB:
  358. if (ep93xx_dma_advance_active(edmac))
  359. m2p_fill_desc(edmac);
  360. return INTERRUPT_NEXT_BUFFER;
  361. }
  362. return INTERRUPT_UNKNOWN;
  363. }
  364. /*
  365. * M2M DMA implementation
  366. *
  367. * For the M2M transfers we don't use NFB at all. This is because it simply
  368. * doesn't work well with memcpy transfers. When you submit both buffers it is
  369. * extremely unlikely that you get an NFB interrupt, but it instead reports
  370. * DONE interrupt and both buffers are already transferred which means that we
  371. * weren't able to update the next buffer.
  372. *
  373. * So for now we "simulate" NFB by just submitting buffer after buffer
  374. * without double buffering.
  375. */
  376. static int m2m_hw_setup(struct ep93xx_dma_chan *edmac)
  377. {
  378. const struct ep93xx_dma_data *data = edmac->chan.private;
  379. u32 control = 0;
  380. if (!data) {
  381. /* This is memcpy channel, nothing to configure */
  382. writel(control, edmac->regs + M2M_CONTROL);
  383. return 0;
  384. }
  385. switch (data->port) {
  386. case EP93XX_DMA_SSP:
  387. /*
  388. * This was found via experimenting - anything less than 5
  389. * causes the channel to perform only a partial transfer which
  390. * leads to problems since we don't get DONE interrupt then.
  391. */
  392. control = (5 << M2M_CONTROL_PWSC_SHIFT);
  393. control |= M2M_CONTROL_NO_HDSK;
  394. if (data->direction == DMA_MEM_TO_DEV) {
  395. control |= M2M_CONTROL_DAH;
  396. control |= M2M_CONTROL_TM_TX;
  397. control |= M2M_CONTROL_RSS_SSPTX;
  398. } else {
  399. control |= M2M_CONTROL_SAH;
  400. control |= M2M_CONTROL_TM_RX;
  401. control |= M2M_CONTROL_RSS_SSPRX;
  402. }
  403. break;
  404. case EP93XX_DMA_IDE:
  405. /*
  406. * This IDE part is totally untested. Values below are taken
  407. * from the EP93xx Users's Guide and might not be correct.
  408. */
  409. if (data->direction == DMA_MEM_TO_DEV) {
  410. /* Worst case from the UG */
  411. control = (3 << M2M_CONTROL_PWSC_SHIFT);
  412. control |= M2M_CONTROL_DAH;
  413. control |= M2M_CONTROL_TM_TX;
  414. } else {
  415. control = (2 << M2M_CONTROL_PWSC_SHIFT);
  416. control |= M2M_CONTROL_SAH;
  417. control |= M2M_CONTROL_TM_RX;
  418. }
  419. control |= M2M_CONTROL_NO_HDSK;
  420. control |= M2M_CONTROL_RSS_IDE;
  421. control |= M2M_CONTROL_PW_16;
  422. break;
  423. default:
  424. return -EINVAL;
  425. }
  426. writel(control, edmac->regs + M2M_CONTROL);
  427. return 0;
  428. }
  429. static void m2m_hw_shutdown(struct ep93xx_dma_chan *edmac)
  430. {
  431. /* Just disable the channel */
  432. writel(0, edmac->regs + M2M_CONTROL);
  433. }
  434. static void m2m_fill_desc(struct ep93xx_dma_chan *edmac)
  435. {
  436. struct ep93xx_dma_desc *desc;
  437. desc = ep93xx_dma_get_active(edmac);
  438. if (!desc) {
  439. dev_warn(chan2dev(edmac), "M2M: empty descriptor list\n");
  440. return;
  441. }
  442. if (edmac->buffer == 0) {
  443. writel(desc->src_addr, edmac->regs + M2M_SAR_BASE0);
  444. writel(desc->dst_addr, edmac->regs + M2M_DAR_BASE0);
  445. writel(desc->size, edmac->regs + M2M_BCR0);
  446. } else {
  447. writel(desc->src_addr, edmac->regs + M2M_SAR_BASE1);
  448. writel(desc->dst_addr, edmac->regs + M2M_DAR_BASE1);
  449. writel(desc->size, edmac->regs + M2M_BCR1);
  450. }
  451. edmac->buffer ^= 1;
  452. }
  453. static void m2m_hw_submit(struct ep93xx_dma_chan *edmac)
  454. {
  455. struct ep93xx_dma_data *data = edmac->chan.private;
  456. u32 control = readl(edmac->regs + M2M_CONTROL);
  457. /*
  458. * Since we allow clients to configure PW (peripheral width) we always
  459. * clear PW bits here and then set them according what is given in
  460. * the runtime configuration.
  461. */
  462. control &= ~M2M_CONTROL_PW_MASK;
  463. control |= edmac->runtime_ctrl;
  464. m2m_fill_desc(edmac);
  465. control |= M2M_CONTROL_DONEINT;
  466. /*
  467. * Now we can finally enable the channel. For M2M channel this must be
  468. * done _after_ the BCRx registers are programmed.
  469. */
  470. control |= M2M_CONTROL_ENABLE;
  471. writel(control, edmac->regs + M2M_CONTROL);
  472. if (!data) {
  473. /*
  474. * For memcpy channels the software trigger must be asserted
  475. * in order to start the memcpy operation.
  476. */
  477. control |= M2M_CONTROL_START;
  478. writel(control, edmac->regs + M2M_CONTROL);
  479. }
  480. }
  481. static int m2m_hw_interrupt(struct ep93xx_dma_chan *edmac)
  482. {
  483. u32 control;
  484. if (!(readl(edmac->regs + M2M_INTERRUPT) & M2M_INTERRUPT_DONEINT))
  485. return INTERRUPT_UNKNOWN;
  486. /* Clear the DONE bit */
  487. writel(0, edmac->regs + M2M_INTERRUPT);
  488. /* Disable interrupts and the channel */
  489. control = readl(edmac->regs + M2M_CONTROL);
  490. control &= ~(M2M_CONTROL_DONEINT | M2M_CONTROL_ENABLE);
  491. writel(control, edmac->regs + M2M_CONTROL);
  492. /*
  493. * Since we only get DONE interrupt we have to find out ourselves
  494. * whether there still is something to process. So we try to advance
  495. * the chain an see whether it succeeds.
  496. */
  497. if (ep93xx_dma_advance_active(edmac)) {
  498. edmac->edma->hw_submit(edmac);
  499. return INTERRUPT_NEXT_BUFFER;
  500. }
  501. return INTERRUPT_DONE;
  502. }
  503. /*
  504. * DMA engine API implementation
  505. */
  506. static struct ep93xx_dma_desc *
  507. ep93xx_dma_desc_get(struct ep93xx_dma_chan *edmac)
  508. {
  509. struct ep93xx_dma_desc *desc, *_desc;
  510. struct ep93xx_dma_desc *ret = NULL;
  511. unsigned long flags;
  512. spin_lock_irqsave(&edmac->lock, flags);
  513. list_for_each_entry_safe(desc, _desc, &edmac->free_list, node) {
  514. if (async_tx_test_ack(&desc->txd)) {
  515. list_del_init(&desc->node);
  516. /* Re-initialize the descriptor */
  517. desc->src_addr = 0;
  518. desc->dst_addr = 0;
  519. desc->size = 0;
  520. desc->complete = false;
  521. desc->txd.cookie = 0;
  522. desc->txd.callback = NULL;
  523. desc->txd.callback_param = NULL;
  524. ret = desc;
  525. break;
  526. }
  527. }
  528. spin_unlock_irqrestore(&edmac->lock, flags);
  529. return ret;
  530. }
  531. static void ep93xx_dma_desc_put(struct ep93xx_dma_chan *edmac,
  532. struct ep93xx_dma_desc *desc)
  533. {
  534. if (desc) {
  535. unsigned long flags;
  536. spin_lock_irqsave(&edmac->lock, flags);
  537. list_splice_init(&desc->tx_list, &edmac->free_list);
  538. list_add(&desc->node, &edmac->free_list);
  539. spin_unlock_irqrestore(&edmac->lock, flags);
  540. }
  541. }
  542. /**
  543. * ep93xx_dma_advance_work - start processing the next pending transaction
  544. * @edmac: channel
  545. *
  546. * If we have pending transactions queued and we are currently idling, this
  547. * function takes the next queued transaction from the @edmac->queue and
  548. * pushes it to the hardware for execution.
  549. */
  550. static void ep93xx_dma_advance_work(struct ep93xx_dma_chan *edmac)
  551. {
  552. struct ep93xx_dma_desc *new;
  553. unsigned long flags;
  554. spin_lock_irqsave(&edmac->lock, flags);
  555. if (!list_empty(&edmac->active) || list_empty(&edmac->queue)) {
  556. spin_unlock_irqrestore(&edmac->lock, flags);
  557. return;
  558. }
  559. /* Take the next descriptor from the pending queue */
  560. new = list_first_entry(&edmac->queue, struct ep93xx_dma_desc, node);
  561. list_del_init(&new->node);
  562. ep93xx_dma_set_active(edmac, new);
  563. /* Push it to the hardware */
  564. edmac->edma->hw_submit(edmac);
  565. spin_unlock_irqrestore(&edmac->lock, flags);
  566. }
  567. static void ep93xx_dma_unmap_buffers(struct ep93xx_dma_desc *desc)
  568. {
  569. struct device *dev = desc->txd.chan->device->dev;
  570. if (!(desc->txd.flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
  571. if (desc->txd.flags & DMA_COMPL_SRC_UNMAP_SINGLE)
  572. dma_unmap_single(dev, desc->src_addr, desc->size,
  573. DMA_TO_DEVICE);
  574. else
  575. dma_unmap_page(dev, desc->src_addr, desc->size,
  576. DMA_TO_DEVICE);
  577. }
  578. if (!(desc->txd.flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
  579. if (desc->txd.flags & DMA_COMPL_DEST_UNMAP_SINGLE)
  580. dma_unmap_single(dev, desc->dst_addr, desc->size,
  581. DMA_FROM_DEVICE);
  582. else
  583. dma_unmap_page(dev, desc->dst_addr, desc->size,
  584. DMA_FROM_DEVICE);
  585. }
  586. }
  587. static void ep93xx_dma_tasklet(unsigned long data)
  588. {
  589. struct ep93xx_dma_chan *edmac = (struct ep93xx_dma_chan *)data;
  590. struct ep93xx_dma_desc *desc, *d;
  591. dma_async_tx_callback callback = NULL;
  592. void *callback_param = NULL;
  593. LIST_HEAD(list);
  594. spin_lock_irq(&edmac->lock);
  595. /*
  596. * If dma_terminate_all() was called before we get to run, the active
  597. * list has become empty. If that happens we aren't supposed to do
  598. * anything more than call ep93xx_dma_advance_work().
  599. */
  600. desc = ep93xx_dma_get_active(edmac);
  601. if (desc) {
  602. if (desc->complete) {
  603. dma_cookie_complete(&desc->txd);
  604. list_splice_init(&edmac->active, &list);
  605. }
  606. callback = desc->txd.callback;
  607. callback_param = desc->txd.callback_param;
  608. }
  609. spin_unlock_irq(&edmac->lock);
  610. /* Pick up the next descriptor from the queue */
  611. ep93xx_dma_advance_work(edmac);
  612. /* Now we can release all the chained descriptors */
  613. list_for_each_entry_safe(desc, d, &list, node) {
  614. /*
  615. * For the memcpy channels the API requires us to unmap the
  616. * buffers unless requested otherwise.
  617. */
  618. if (!edmac->chan.private)
  619. ep93xx_dma_unmap_buffers(desc);
  620. ep93xx_dma_desc_put(edmac, desc);
  621. }
  622. if (callback)
  623. callback(callback_param);
  624. }
  625. static irqreturn_t ep93xx_dma_interrupt(int irq, void *dev_id)
  626. {
  627. struct ep93xx_dma_chan *edmac = dev_id;
  628. struct ep93xx_dma_desc *desc;
  629. irqreturn_t ret = IRQ_HANDLED;
  630. spin_lock(&edmac->lock);
  631. desc = ep93xx_dma_get_active(edmac);
  632. if (!desc) {
  633. dev_warn(chan2dev(edmac),
  634. "got interrupt while active list is empty\n");
  635. spin_unlock(&edmac->lock);
  636. return IRQ_NONE;
  637. }
  638. switch (edmac->edma->hw_interrupt(edmac)) {
  639. case INTERRUPT_DONE:
  640. desc->complete = true;
  641. tasklet_schedule(&edmac->tasklet);
  642. break;
  643. case INTERRUPT_NEXT_BUFFER:
  644. if (test_bit(EP93XX_DMA_IS_CYCLIC, &edmac->flags))
  645. tasklet_schedule(&edmac->tasklet);
  646. break;
  647. default:
  648. dev_warn(chan2dev(edmac), "unknown interrupt!\n");
  649. ret = IRQ_NONE;
  650. break;
  651. }
  652. spin_unlock(&edmac->lock);
  653. return ret;
  654. }
  655. /**
  656. * ep93xx_dma_tx_submit - set the prepared descriptor(s) to be executed
  657. * @tx: descriptor to be executed
  658. *
  659. * Function will execute given descriptor on the hardware or if the hardware
  660. * is busy, queue the descriptor to be executed later on. Returns cookie which
  661. * can be used to poll the status of the descriptor.
  662. */
  663. static dma_cookie_t ep93xx_dma_tx_submit(struct dma_async_tx_descriptor *tx)
  664. {
  665. struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(tx->chan);
  666. struct ep93xx_dma_desc *desc;
  667. dma_cookie_t cookie;
  668. unsigned long flags;
  669. spin_lock_irqsave(&edmac->lock, flags);
  670. cookie = dma_cookie_assign(tx);
  671. desc = container_of(tx, struct ep93xx_dma_desc, txd);
  672. /*
  673. * If nothing is currently prosessed, we push this descriptor
  674. * directly to the hardware. Otherwise we put the descriptor
  675. * to the pending queue.
  676. */
  677. if (list_empty(&edmac->active)) {
  678. ep93xx_dma_set_active(edmac, desc);
  679. edmac->edma->hw_submit(edmac);
  680. } else {
  681. list_add_tail(&desc->node, &edmac->queue);
  682. }
  683. spin_unlock_irqrestore(&edmac->lock, flags);
  684. return cookie;
  685. }
  686. /**
  687. * ep93xx_dma_alloc_chan_resources - allocate resources for the channel
  688. * @chan: channel to allocate resources
  689. *
  690. * Function allocates necessary resources for the given DMA channel and
  691. * returns number of allocated descriptors for the channel. Negative errno
  692. * is returned in case of failure.
  693. */
  694. static int ep93xx_dma_alloc_chan_resources(struct dma_chan *chan)
  695. {
  696. struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);
  697. struct ep93xx_dma_data *data = chan->private;
  698. const char *name = dma_chan_name(chan);
  699. int ret, i;
  700. /* Sanity check the channel parameters */
  701. if (!edmac->edma->m2m) {
  702. if (!data)
  703. return -EINVAL;
  704. if (data->port < EP93XX_DMA_I2S1 ||
  705. data->port > EP93XX_DMA_IRDA)
  706. return -EINVAL;
  707. if (data->direction != ep93xx_dma_chan_direction(chan))
  708. return -EINVAL;
  709. } else {
  710. if (data) {
  711. switch (data->port) {
  712. case EP93XX_DMA_SSP:
  713. case EP93XX_DMA_IDE:
  714. if (data->direction != DMA_MEM_TO_DEV &&
  715. data->direction != DMA_DEV_TO_MEM)
  716. return -EINVAL;
  717. break;
  718. default:
  719. return -EINVAL;
  720. }
  721. }
  722. }
  723. if (data && data->name)
  724. name = data->name;
  725. ret = clk_enable(edmac->clk);
  726. if (ret)
  727. return ret;
  728. ret = request_irq(edmac->irq, ep93xx_dma_interrupt, 0, name, edmac);
  729. if (ret)
  730. goto fail_clk_disable;
  731. spin_lock_irq(&edmac->lock);
  732. edmac->chan.completed_cookie = 1;
  733. edmac->chan.cookie = 1;
  734. ret = edmac->edma->hw_setup(edmac);
  735. spin_unlock_irq(&edmac->lock);
  736. if (ret)
  737. goto fail_free_irq;
  738. for (i = 0; i < DMA_MAX_CHAN_DESCRIPTORS; i++) {
  739. struct ep93xx_dma_desc *desc;
  740. desc = kzalloc(sizeof(*desc), GFP_KERNEL);
  741. if (!desc) {
  742. dev_warn(chan2dev(edmac), "not enough descriptors\n");
  743. break;
  744. }
  745. INIT_LIST_HEAD(&desc->tx_list);
  746. dma_async_tx_descriptor_init(&desc->txd, chan);
  747. desc->txd.flags = DMA_CTRL_ACK;
  748. desc->txd.tx_submit = ep93xx_dma_tx_submit;
  749. ep93xx_dma_desc_put(edmac, desc);
  750. }
  751. return i;
  752. fail_free_irq:
  753. free_irq(edmac->irq, edmac);
  754. fail_clk_disable:
  755. clk_disable(edmac->clk);
  756. return ret;
  757. }
  758. /**
  759. * ep93xx_dma_free_chan_resources - release resources for the channel
  760. * @chan: channel
  761. *
  762. * Function releases all the resources allocated for the given channel.
  763. * The channel must be idle when this is called.
  764. */
  765. static void ep93xx_dma_free_chan_resources(struct dma_chan *chan)
  766. {
  767. struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);
  768. struct ep93xx_dma_desc *desc, *d;
  769. unsigned long flags;
  770. LIST_HEAD(list);
  771. BUG_ON(!list_empty(&edmac->active));
  772. BUG_ON(!list_empty(&edmac->queue));
  773. spin_lock_irqsave(&edmac->lock, flags);
  774. edmac->edma->hw_shutdown(edmac);
  775. edmac->runtime_addr = 0;
  776. edmac->runtime_ctrl = 0;
  777. edmac->buffer = 0;
  778. list_splice_init(&edmac->free_list, &list);
  779. spin_unlock_irqrestore(&edmac->lock, flags);
  780. list_for_each_entry_safe(desc, d, &list, node)
  781. kfree(desc);
  782. clk_disable(edmac->clk);
  783. free_irq(edmac->irq, edmac);
  784. }
  785. /**
  786. * ep93xx_dma_prep_dma_memcpy - prepare a memcpy DMA operation
  787. * @chan: channel
  788. * @dest: destination bus address
  789. * @src: source bus address
  790. * @len: size of the transaction
  791. * @flags: flags for the descriptor
  792. *
  793. * Returns a valid DMA descriptor or %NULL in case of failure.
  794. */
  795. static struct dma_async_tx_descriptor *
  796. ep93xx_dma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest,
  797. dma_addr_t src, size_t len, unsigned long flags)
  798. {
  799. struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);
  800. struct ep93xx_dma_desc *desc, *first;
  801. size_t bytes, offset;
  802. first = NULL;
  803. for (offset = 0; offset < len; offset += bytes) {
  804. desc = ep93xx_dma_desc_get(edmac);
  805. if (!desc) {
  806. dev_warn(chan2dev(edmac), "couln't get descriptor\n");
  807. goto fail;
  808. }
  809. bytes = min_t(size_t, len - offset, DMA_MAX_CHAN_BYTES);
  810. desc->src_addr = src + offset;
  811. desc->dst_addr = dest + offset;
  812. desc->size = bytes;
  813. if (!first)
  814. first = desc;
  815. else
  816. list_add_tail(&desc->node, &first->tx_list);
  817. }
  818. first->txd.cookie = -EBUSY;
  819. first->txd.flags = flags;
  820. return &first->txd;
  821. fail:
  822. ep93xx_dma_desc_put(edmac, first);
  823. return NULL;
  824. }
  825. /**
  826. * ep93xx_dma_prep_slave_sg - prepare a slave DMA operation
  827. * @chan: channel
  828. * @sgl: list of buffers to transfer
  829. * @sg_len: number of entries in @sgl
  830. * @dir: direction of tha DMA transfer
  831. * @flags: flags for the descriptor
  832. *
  833. * Returns a valid DMA descriptor or %NULL in case of failure.
  834. */
  835. static struct dma_async_tx_descriptor *
  836. ep93xx_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
  837. unsigned int sg_len, enum dma_transfer_direction dir,
  838. unsigned long flags)
  839. {
  840. struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);
  841. struct ep93xx_dma_desc *desc, *first;
  842. struct scatterlist *sg;
  843. int i;
  844. if (!edmac->edma->m2m && dir != ep93xx_dma_chan_direction(chan)) {
  845. dev_warn(chan2dev(edmac),
  846. "channel was configured with different direction\n");
  847. return NULL;
  848. }
  849. if (test_bit(EP93XX_DMA_IS_CYCLIC, &edmac->flags)) {
  850. dev_warn(chan2dev(edmac),
  851. "channel is already used for cyclic transfers\n");
  852. return NULL;
  853. }
  854. first = NULL;
  855. for_each_sg(sgl, sg, sg_len, i) {
  856. size_t sg_len = sg_dma_len(sg);
  857. if (sg_len > DMA_MAX_CHAN_BYTES) {
  858. dev_warn(chan2dev(edmac), "too big transfer size %d\n",
  859. sg_len);
  860. goto fail;
  861. }
  862. desc = ep93xx_dma_desc_get(edmac);
  863. if (!desc) {
  864. dev_warn(chan2dev(edmac), "couln't get descriptor\n");
  865. goto fail;
  866. }
  867. if (dir == DMA_MEM_TO_DEV) {
  868. desc->src_addr = sg_dma_address(sg);
  869. desc->dst_addr = edmac->runtime_addr;
  870. } else {
  871. desc->src_addr = edmac->runtime_addr;
  872. desc->dst_addr = sg_dma_address(sg);
  873. }
  874. desc->size = sg_len;
  875. if (!first)
  876. first = desc;
  877. else
  878. list_add_tail(&desc->node, &first->tx_list);
  879. }
  880. first->txd.cookie = -EBUSY;
  881. first->txd.flags = flags;
  882. return &first->txd;
  883. fail:
  884. ep93xx_dma_desc_put(edmac, first);
  885. return NULL;
  886. }
  887. /**
  888. * ep93xx_dma_prep_dma_cyclic - prepare a cyclic DMA operation
  889. * @chan: channel
  890. * @dma_addr: DMA mapped address of the buffer
  891. * @buf_len: length of the buffer (in bytes)
  892. * @period_len: lenght of a single period
  893. * @dir: direction of the operation
  894. *
  895. * Prepares a descriptor for cyclic DMA operation. This means that once the
  896. * descriptor is submitted, we will be submitting in a @period_len sized
  897. * buffers and calling callback once the period has been elapsed. Transfer
  898. * terminates only when client calls dmaengine_terminate_all() for this
  899. * channel.
  900. *
  901. * Returns a valid DMA descriptor or %NULL in case of failure.
  902. */
  903. static struct dma_async_tx_descriptor *
  904. ep93xx_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t dma_addr,
  905. size_t buf_len, size_t period_len,
  906. enum dma_transfer_direction dir)
  907. {
  908. struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);
  909. struct ep93xx_dma_desc *desc, *first;
  910. size_t offset = 0;
  911. if (!edmac->edma->m2m && dir != ep93xx_dma_chan_direction(chan)) {
  912. dev_warn(chan2dev(edmac),
  913. "channel was configured with different direction\n");
  914. return NULL;
  915. }
  916. if (test_and_set_bit(EP93XX_DMA_IS_CYCLIC, &edmac->flags)) {
  917. dev_warn(chan2dev(edmac),
  918. "channel is already used for cyclic transfers\n");
  919. return NULL;
  920. }
  921. if (period_len > DMA_MAX_CHAN_BYTES) {
  922. dev_warn(chan2dev(edmac), "too big period length %d\n",
  923. period_len);
  924. return NULL;
  925. }
  926. /* Split the buffer into period size chunks */
  927. first = NULL;
  928. for (offset = 0; offset < buf_len; offset += period_len) {
  929. desc = ep93xx_dma_desc_get(edmac);
  930. if (!desc) {
  931. dev_warn(chan2dev(edmac), "couln't get descriptor\n");
  932. goto fail;
  933. }
  934. if (dir == DMA_MEM_TO_DEV) {
  935. desc->src_addr = dma_addr + offset;
  936. desc->dst_addr = edmac->runtime_addr;
  937. } else {
  938. desc->src_addr = edmac->runtime_addr;
  939. desc->dst_addr = dma_addr + offset;
  940. }
  941. desc->size = period_len;
  942. if (!first)
  943. first = desc;
  944. else
  945. list_add_tail(&desc->node, &first->tx_list);
  946. }
  947. first->txd.cookie = -EBUSY;
  948. return &first->txd;
  949. fail:
  950. ep93xx_dma_desc_put(edmac, first);
  951. return NULL;
  952. }
  953. /**
  954. * ep93xx_dma_terminate_all - terminate all transactions
  955. * @edmac: channel
  956. *
  957. * Stops all DMA transactions. All descriptors are put back to the
  958. * @edmac->free_list and callbacks are _not_ called.
  959. */
  960. static int ep93xx_dma_terminate_all(struct ep93xx_dma_chan *edmac)
  961. {
  962. struct ep93xx_dma_desc *desc, *_d;
  963. unsigned long flags;
  964. LIST_HEAD(list);
  965. spin_lock_irqsave(&edmac->lock, flags);
  966. /* First we disable and flush the DMA channel */
  967. edmac->edma->hw_shutdown(edmac);
  968. clear_bit(EP93XX_DMA_IS_CYCLIC, &edmac->flags);
  969. list_splice_init(&edmac->active, &list);
  970. list_splice_init(&edmac->queue, &list);
  971. /*
  972. * We then re-enable the channel. This way we can continue submitting
  973. * the descriptors by just calling ->hw_submit() again.
  974. */
  975. edmac->edma->hw_setup(edmac);
  976. spin_unlock_irqrestore(&edmac->lock, flags);
  977. list_for_each_entry_safe(desc, _d, &list, node)
  978. ep93xx_dma_desc_put(edmac, desc);
  979. return 0;
  980. }
  981. static int ep93xx_dma_slave_config(struct ep93xx_dma_chan *edmac,
  982. struct dma_slave_config *config)
  983. {
  984. enum dma_slave_buswidth width;
  985. unsigned long flags;
  986. u32 addr, ctrl;
  987. if (!edmac->edma->m2m)
  988. return -EINVAL;
  989. switch (config->direction) {
  990. case DMA_DEV_TO_MEM:
  991. width = config->src_addr_width;
  992. addr = config->src_addr;
  993. break;
  994. case DMA_MEM_TO_DEV:
  995. width = config->dst_addr_width;
  996. addr = config->dst_addr;
  997. break;
  998. default:
  999. return -EINVAL;
  1000. }
  1001. switch (width) {
  1002. case DMA_SLAVE_BUSWIDTH_1_BYTE:
  1003. ctrl = 0;
  1004. break;
  1005. case DMA_SLAVE_BUSWIDTH_2_BYTES:
  1006. ctrl = M2M_CONTROL_PW_16;
  1007. break;
  1008. case DMA_SLAVE_BUSWIDTH_4_BYTES:
  1009. ctrl = M2M_CONTROL_PW_32;
  1010. break;
  1011. default:
  1012. return -EINVAL;
  1013. }
  1014. spin_lock_irqsave(&edmac->lock, flags);
  1015. edmac->runtime_addr = addr;
  1016. edmac->runtime_ctrl = ctrl;
  1017. spin_unlock_irqrestore(&edmac->lock, flags);
  1018. return 0;
  1019. }
  1020. /**
  1021. * ep93xx_dma_control - manipulate all pending operations on a channel
  1022. * @chan: channel
  1023. * @cmd: control command to perform
  1024. * @arg: optional argument
  1025. *
  1026. * Controls the channel. Function returns %0 in case of success or negative
  1027. * error in case of failure.
  1028. */
  1029. static int ep93xx_dma_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
  1030. unsigned long arg)
  1031. {
  1032. struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);
  1033. struct dma_slave_config *config;
  1034. switch (cmd) {
  1035. case DMA_TERMINATE_ALL:
  1036. return ep93xx_dma_terminate_all(edmac);
  1037. case DMA_SLAVE_CONFIG:
  1038. config = (struct dma_slave_config *)arg;
  1039. return ep93xx_dma_slave_config(edmac, config);
  1040. default:
  1041. break;
  1042. }
  1043. return -ENOSYS;
  1044. }
  1045. /**
  1046. * ep93xx_dma_tx_status - check if a transaction is completed
  1047. * @chan: channel
  1048. * @cookie: transaction specific cookie
  1049. * @state: state of the transaction is stored here if given
  1050. *
  1051. * This function can be used to query state of a given transaction.
  1052. */
  1053. static enum dma_status ep93xx_dma_tx_status(struct dma_chan *chan,
  1054. dma_cookie_t cookie,
  1055. struct dma_tx_state *state)
  1056. {
  1057. struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);
  1058. enum dma_status ret;
  1059. unsigned long flags;
  1060. spin_lock_irqsave(&edmac->lock, flags);
  1061. ret = dma_cookie_status(chan, cookie, state);
  1062. spin_unlock_irqrestore(&edmac->lock, flags);
  1063. return ret;
  1064. }
  1065. /**
  1066. * ep93xx_dma_issue_pending - push pending transactions to the hardware
  1067. * @chan: channel
  1068. *
  1069. * When this function is called, all pending transactions are pushed to the
  1070. * hardware and executed.
  1071. */
  1072. static void ep93xx_dma_issue_pending(struct dma_chan *chan)
  1073. {
  1074. ep93xx_dma_advance_work(to_ep93xx_dma_chan(chan));
  1075. }
  1076. static int __init ep93xx_dma_probe(struct platform_device *pdev)
  1077. {
  1078. struct ep93xx_dma_platform_data *pdata = dev_get_platdata(&pdev->dev);
  1079. struct ep93xx_dma_engine *edma;
  1080. struct dma_device *dma_dev;
  1081. size_t edma_size;
  1082. int ret, i;
  1083. edma_size = pdata->num_channels * sizeof(struct ep93xx_dma_chan);
  1084. edma = kzalloc(sizeof(*edma) + edma_size, GFP_KERNEL);
  1085. if (!edma)
  1086. return -ENOMEM;
  1087. dma_dev = &edma->dma_dev;
  1088. edma->m2m = platform_get_device_id(pdev)->driver_data;
  1089. edma->num_channels = pdata->num_channels;
  1090. INIT_LIST_HEAD(&dma_dev->channels);
  1091. for (i = 0; i < pdata->num_channels; i++) {
  1092. const struct ep93xx_dma_chan_data *cdata = &pdata->channels[i];
  1093. struct ep93xx_dma_chan *edmac = &edma->channels[i];
  1094. edmac->chan.device = dma_dev;
  1095. edmac->regs = cdata->base;
  1096. edmac->irq = cdata->irq;
  1097. edmac->edma = edma;
  1098. edmac->clk = clk_get(NULL, cdata->name);
  1099. if (IS_ERR(edmac->clk)) {
  1100. dev_warn(&pdev->dev, "failed to get clock for %s\n",
  1101. cdata->name);
  1102. continue;
  1103. }
  1104. spin_lock_init(&edmac->lock);
  1105. INIT_LIST_HEAD(&edmac->active);
  1106. INIT_LIST_HEAD(&edmac->queue);
  1107. INIT_LIST_HEAD(&edmac->free_list);
  1108. tasklet_init(&edmac->tasklet, ep93xx_dma_tasklet,
  1109. (unsigned long)edmac);
  1110. list_add_tail(&edmac->chan.device_node,
  1111. &dma_dev->channels);
  1112. }
  1113. dma_cap_zero(dma_dev->cap_mask);
  1114. dma_cap_set(DMA_SLAVE, dma_dev->cap_mask);
  1115. dma_cap_set(DMA_CYCLIC, dma_dev->cap_mask);
  1116. dma_dev->dev = &pdev->dev;
  1117. dma_dev->device_alloc_chan_resources = ep93xx_dma_alloc_chan_resources;
  1118. dma_dev->device_free_chan_resources = ep93xx_dma_free_chan_resources;
  1119. dma_dev->device_prep_slave_sg = ep93xx_dma_prep_slave_sg;
  1120. dma_dev->device_prep_dma_cyclic = ep93xx_dma_prep_dma_cyclic;
  1121. dma_dev->device_control = ep93xx_dma_control;
  1122. dma_dev->device_issue_pending = ep93xx_dma_issue_pending;
  1123. dma_dev->device_tx_status = ep93xx_dma_tx_status;
  1124. dma_set_max_seg_size(dma_dev->dev, DMA_MAX_CHAN_BYTES);
  1125. if (edma->m2m) {
  1126. dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
  1127. dma_dev->device_prep_dma_memcpy = ep93xx_dma_prep_dma_memcpy;
  1128. edma->hw_setup = m2m_hw_setup;
  1129. edma->hw_shutdown = m2m_hw_shutdown;
  1130. edma->hw_submit = m2m_hw_submit;
  1131. edma->hw_interrupt = m2m_hw_interrupt;
  1132. } else {
  1133. dma_cap_set(DMA_PRIVATE, dma_dev->cap_mask);
  1134. edma->hw_setup = m2p_hw_setup;
  1135. edma->hw_shutdown = m2p_hw_shutdown;
  1136. edma->hw_submit = m2p_hw_submit;
  1137. edma->hw_interrupt = m2p_hw_interrupt;
  1138. }
  1139. ret = dma_async_device_register(dma_dev);
  1140. if (unlikely(ret)) {
  1141. for (i = 0; i < edma->num_channels; i++) {
  1142. struct ep93xx_dma_chan *edmac = &edma->channels[i];
  1143. if (!IS_ERR_OR_NULL(edmac->clk))
  1144. clk_put(edmac->clk);
  1145. }
  1146. kfree(edma);
  1147. } else {
  1148. dev_info(dma_dev->dev, "EP93xx M2%s DMA ready\n",
  1149. edma->m2m ? "M" : "P");
  1150. }
  1151. return ret;
  1152. }
  1153. static struct platform_device_id ep93xx_dma_driver_ids[] = {
  1154. { "ep93xx-dma-m2p", 0 },
  1155. { "ep93xx-dma-m2m", 1 },
  1156. { },
  1157. };
  1158. static struct platform_driver ep93xx_dma_driver = {
  1159. .driver = {
  1160. .name = "ep93xx-dma",
  1161. },
  1162. .id_table = ep93xx_dma_driver_ids,
  1163. };
  1164. static int __init ep93xx_dma_module_init(void)
  1165. {
  1166. return platform_driver_probe(&ep93xx_dma_driver, ep93xx_dma_probe);
  1167. }
  1168. subsys_initcall(ep93xx_dma_module_init);
  1169. MODULE_AUTHOR("Mika Westerberg <mika.westerberg@iki.fi>");
  1170. MODULE_DESCRIPTION("EP93xx DMA driver");
  1171. MODULE_LICENSE("GPL");