tegra20-apb-dma.c 41 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525
  1. /*
  2. * DMA driver for Nvidia's Tegra20 APB DMA controller.
  3. *
  4. * Copyright (c) 2012, NVIDIA CORPORATION. All rights reserved.
  5. *
  6. * This program is free software; you can redistribute it and/or modify it
  7. * under the terms and conditions of the GNU General Public License,
  8. * version 2, as published by the Free Software Foundation.
  9. *
  10. * This program is distributed in the hope it will be useful, but WITHOUT
  11. * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12. * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
  13. * more details.
  14. *
  15. * You should have received a copy of the GNU General Public License
  16. * along with this program. If not, see <http://www.gnu.org/licenses/>.
  17. */
  18. #include <linux/bitops.h>
  19. #include <linux/clk.h>
  20. #include <linux/delay.h>
  21. #include <linux/dmaengine.h>
  22. #include <linux/dma-mapping.h>
  23. #include <linux/err.h>
  24. #include <linux/init.h>
  25. #include <linux/interrupt.h>
  26. #include <linux/io.h>
  27. #include <linux/mm.h>
  28. #include <linux/module.h>
  29. #include <linux/of.h>
  30. #include <linux/of_device.h>
  31. #include <linux/platform_device.h>
  32. #include <linux/pm.h>
  33. #include <linux/pm_runtime.h>
  34. #include <linux/slab.h>
  35. #include <linux/clk/tegra.h>
  36. #include "dmaengine.h"
  37. #define TEGRA_APBDMA_GENERAL 0x0
  38. #define TEGRA_APBDMA_GENERAL_ENABLE BIT(31)
  39. #define TEGRA_APBDMA_CONTROL 0x010
  40. #define TEGRA_APBDMA_IRQ_MASK 0x01c
  41. #define TEGRA_APBDMA_IRQ_MASK_SET 0x020
  42. /* CSR register */
  43. #define TEGRA_APBDMA_CHAN_CSR 0x00
  44. #define TEGRA_APBDMA_CSR_ENB BIT(31)
  45. #define TEGRA_APBDMA_CSR_IE_EOC BIT(30)
  46. #define TEGRA_APBDMA_CSR_HOLD BIT(29)
  47. #define TEGRA_APBDMA_CSR_DIR BIT(28)
  48. #define TEGRA_APBDMA_CSR_ONCE BIT(27)
  49. #define TEGRA_APBDMA_CSR_FLOW BIT(21)
  50. #define TEGRA_APBDMA_CSR_REQ_SEL_SHIFT 16
  51. #define TEGRA_APBDMA_CSR_WCOUNT_MASK 0xFFFC
  52. /* STATUS register */
  53. #define TEGRA_APBDMA_CHAN_STATUS 0x004
  54. #define TEGRA_APBDMA_STATUS_BUSY BIT(31)
  55. #define TEGRA_APBDMA_STATUS_ISE_EOC BIT(30)
  56. #define TEGRA_APBDMA_STATUS_HALT BIT(29)
  57. #define TEGRA_APBDMA_STATUS_PING_PONG BIT(28)
  58. #define TEGRA_APBDMA_STATUS_COUNT_SHIFT 2
  59. #define TEGRA_APBDMA_STATUS_COUNT_MASK 0xFFFC
  60. #define TEGRA_APBDMA_CHAN_CSRE 0x00C
  61. #define TEGRA_APBDMA_CHAN_CSRE_PAUSE (1 << 31)
  62. /* AHB memory address */
  63. #define TEGRA_APBDMA_CHAN_AHBPTR 0x010
  64. /* AHB sequence register */
  65. #define TEGRA_APBDMA_CHAN_AHBSEQ 0x14
  66. #define TEGRA_APBDMA_AHBSEQ_INTR_ENB BIT(31)
  67. #define TEGRA_APBDMA_AHBSEQ_BUS_WIDTH_8 (0 << 28)
  68. #define TEGRA_APBDMA_AHBSEQ_BUS_WIDTH_16 (1 << 28)
  69. #define TEGRA_APBDMA_AHBSEQ_BUS_WIDTH_32 (2 << 28)
  70. #define TEGRA_APBDMA_AHBSEQ_BUS_WIDTH_64 (3 << 28)
  71. #define TEGRA_APBDMA_AHBSEQ_BUS_WIDTH_128 (4 << 28)
  72. #define TEGRA_APBDMA_AHBSEQ_DATA_SWAP BIT(27)
  73. #define TEGRA_APBDMA_AHBSEQ_BURST_1 (4 << 24)
  74. #define TEGRA_APBDMA_AHBSEQ_BURST_4 (5 << 24)
  75. #define TEGRA_APBDMA_AHBSEQ_BURST_8 (6 << 24)
  76. #define TEGRA_APBDMA_AHBSEQ_DBL_BUF BIT(19)
  77. #define TEGRA_APBDMA_AHBSEQ_WRAP_SHIFT 16
  78. #define TEGRA_APBDMA_AHBSEQ_WRAP_NONE 0
  79. /* APB address */
  80. #define TEGRA_APBDMA_CHAN_APBPTR 0x018
  81. /* APB sequence register */
  82. #define TEGRA_APBDMA_CHAN_APBSEQ 0x01c
  83. #define TEGRA_APBDMA_APBSEQ_BUS_WIDTH_8 (0 << 28)
  84. #define TEGRA_APBDMA_APBSEQ_BUS_WIDTH_16 (1 << 28)
  85. #define TEGRA_APBDMA_APBSEQ_BUS_WIDTH_32 (2 << 28)
  86. #define TEGRA_APBDMA_APBSEQ_BUS_WIDTH_64 (3 << 28)
  87. #define TEGRA_APBDMA_APBSEQ_BUS_WIDTH_128 (4 << 28)
  88. #define TEGRA_APBDMA_APBSEQ_DATA_SWAP BIT(27)
  89. #define TEGRA_APBDMA_APBSEQ_WRAP_WORD_1 (1 << 16)
  90. /*
  91. * If any burst is in flight and DMA paused then this is the time to complete
  92. * on-flight burst and update DMA status register.
  93. */
  94. #define TEGRA_APBDMA_BURST_COMPLETE_TIME 20
  95. /* Channel base address offset from APBDMA base address */
  96. #define TEGRA_APBDMA_CHANNEL_BASE_ADD_OFFSET 0x1000
  97. /* DMA channel register space size */
  98. #define TEGRA_APBDMA_CHANNEL_REGISTER_SIZE 0x20
  99. struct tegra_dma;
  100. /*
  101. * tegra_dma_chip_data Tegra chip specific DMA data
  102. * @nr_channels: Number of channels available in the controller.
  103. * @max_dma_count: Maximum DMA transfer count supported by DMA controller.
  104. * @support_channel_pause: Support channel wise pause of dma.
  105. */
  106. struct tegra_dma_chip_data {
  107. int nr_channels;
  108. int max_dma_count;
  109. bool support_channel_pause;
  110. };
  111. /* DMA channel registers */
  112. struct tegra_dma_channel_regs {
  113. unsigned long csr;
  114. unsigned long ahb_ptr;
  115. unsigned long apb_ptr;
  116. unsigned long ahb_seq;
  117. unsigned long apb_seq;
  118. };
  119. /*
  120. * tegra_dma_sg_req: Dma request details to configure hardware. This
  121. * contains the details for one transfer to configure DMA hw.
  122. * The client's request for data transfer can be broken into multiple
  123. * sub-transfer as per requester details and hw support.
  124. * This sub transfer get added in the list of transfer and point to Tegra
  125. * DMA descriptor which manages the transfer details.
  126. */
  127. struct tegra_dma_sg_req {
  128. struct tegra_dma_channel_regs ch_regs;
  129. int req_len;
  130. bool configured;
  131. bool last_sg;
  132. bool half_done;
  133. struct list_head node;
  134. struct tegra_dma_desc *dma_desc;
  135. };
  136. /*
  137. * tegra_dma_desc: Tegra DMA descriptors which manages the client requests.
  138. * This descriptor keep track of transfer status, callbacks and request
  139. * counts etc.
  140. */
  141. struct tegra_dma_desc {
  142. struct dma_async_tx_descriptor txd;
  143. int bytes_requested;
  144. int bytes_transferred;
  145. enum dma_status dma_status;
  146. struct list_head node;
  147. struct list_head tx_list;
  148. struct list_head cb_node;
  149. int cb_count;
  150. };
  151. struct tegra_dma_channel;
  152. typedef void (*dma_isr_handler)(struct tegra_dma_channel *tdc,
  153. bool to_terminate);
  154. /* tegra_dma_channel: Channel specific information */
  155. struct tegra_dma_channel {
  156. struct dma_chan dma_chan;
  157. char name[30];
  158. bool config_init;
  159. int id;
  160. int irq;
  161. unsigned long chan_base_offset;
  162. spinlock_t lock;
  163. bool busy;
  164. struct tegra_dma *tdma;
  165. bool cyclic;
  166. /* Different lists for managing the requests */
  167. struct list_head free_sg_req;
  168. struct list_head pending_sg_req;
  169. struct list_head free_dma_desc;
  170. struct list_head cb_desc;
  171. /* ISR handler and tasklet for bottom half of isr handling */
  172. dma_isr_handler isr_handler;
  173. struct tasklet_struct tasklet;
  174. dma_async_tx_callback callback;
  175. void *callback_param;
  176. /* Channel-slave specific configuration */
  177. struct dma_slave_config dma_sconfig;
  178. struct tegra_dma_channel_regs channel_reg;
  179. };
  180. /* tegra_dma: Tegra DMA specific information */
  181. struct tegra_dma {
  182. struct dma_device dma_dev;
  183. struct device *dev;
  184. struct clk *dma_clk;
  185. spinlock_t global_lock;
  186. void __iomem *base_addr;
  187. const struct tegra_dma_chip_data *chip_data;
  188. /* Some register need to be cache before suspend */
  189. u32 reg_gen;
  190. /* Last member of the structure */
  191. struct tegra_dma_channel channels[0];
  192. };
  193. static inline void tdma_write(struct tegra_dma *tdma, u32 reg, u32 val)
  194. {
  195. writel(val, tdma->base_addr + reg);
  196. }
  197. static inline u32 tdma_read(struct tegra_dma *tdma, u32 reg)
  198. {
  199. return readl(tdma->base_addr + reg);
  200. }
  201. static inline void tdc_write(struct tegra_dma_channel *tdc,
  202. u32 reg, u32 val)
  203. {
  204. writel(val, tdc->tdma->base_addr + tdc->chan_base_offset + reg);
  205. }
  206. static inline u32 tdc_read(struct tegra_dma_channel *tdc, u32 reg)
  207. {
  208. return readl(tdc->tdma->base_addr + tdc->chan_base_offset + reg);
  209. }
  210. static inline struct tegra_dma_channel *to_tegra_dma_chan(struct dma_chan *dc)
  211. {
  212. return container_of(dc, struct tegra_dma_channel, dma_chan);
  213. }
  214. static inline struct tegra_dma_desc *txd_to_tegra_dma_desc(
  215. struct dma_async_tx_descriptor *td)
  216. {
  217. return container_of(td, struct tegra_dma_desc, txd);
  218. }
  219. static inline struct device *tdc2dev(struct tegra_dma_channel *tdc)
  220. {
  221. return &tdc->dma_chan.dev->device;
  222. }
  223. static dma_cookie_t tegra_dma_tx_submit(struct dma_async_tx_descriptor *tx);
  224. static int tegra_dma_runtime_suspend(struct device *dev);
  225. static int tegra_dma_runtime_resume(struct device *dev);
  226. /* Get DMA desc from free list, if not there then allocate it. */
  227. static struct tegra_dma_desc *tegra_dma_desc_get(
  228. struct tegra_dma_channel *tdc)
  229. {
  230. struct tegra_dma_desc *dma_desc;
  231. unsigned long flags;
  232. spin_lock_irqsave(&tdc->lock, flags);
  233. /* Do not allocate if desc are waiting for ack */
  234. list_for_each_entry(dma_desc, &tdc->free_dma_desc, node) {
  235. if (async_tx_test_ack(&dma_desc->txd)) {
  236. list_del(&dma_desc->node);
  237. spin_unlock_irqrestore(&tdc->lock, flags);
  238. dma_desc->txd.flags = 0;
  239. return dma_desc;
  240. }
  241. }
  242. spin_unlock_irqrestore(&tdc->lock, flags);
  243. /* Allocate DMA desc */
  244. dma_desc = kzalloc(sizeof(*dma_desc), GFP_ATOMIC);
  245. if (!dma_desc) {
  246. dev_err(tdc2dev(tdc), "dma_desc alloc failed\n");
  247. return NULL;
  248. }
  249. dma_async_tx_descriptor_init(&dma_desc->txd, &tdc->dma_chan);
  250. dma_desc->txd.tx_submit = tegra_dma_tx_submit;
  251. dma_desc->txd.flags = 0;
  252. return dma_desc;
  253. }
  254. static void tegra_dma_desc_put(struct tegra_dma_channel *tdc,
  255. struct tegra_dma_desc *dma_desc)
  256. {
  257. unsigned long flags;
  258. spin_lock_irqsave(&tdc->lock, flags);
  259. if (!list_empty(&dma_desc->tx_list))
  260. list_splice_init(&dma_desc->tx_list, &tdc->free_sg_req);
  261. list_add_tail(&dma_desc->node, &tdc->free_dma_desc);
  262. spin_unlock_irqrestore(&tdc->lock, flags);
  263. }
  264. static struct tegra_dma_sg_req *tegra_dma_sg_req_get(
  265. struct tegra_dma_channel *tdc)
  266. {
  267. struct tegra_dma_sg_req *sg_req = NULL;
  268. unsigned long flags;
  269. spin_lock_irqsave(&tdc->lock, flags);
  270. if (!list_empty(&tdc->free_sg_req)) {
  271. sg_req = list_first_entry(&tdc->free_sg_req,
  272. typeof(*sg_req), node);
  273. list_del(&sg_req->node);
  274. spin_unlock_irqrestore(&tdc->lock, flags);
  275. return sg_req;
  276. }
  277. spin_unlock_irqrestore(&tdc->lock, flags);
  278. sg_req = kzalloc(sizeof(struct tegra_dma_sg_req), GFP_ATOMIC);
  279. if (!sg_req)
  280. dev_err(tdc2dev(tdc), "sg_req alloc failed\n");
  281. return sg_req;
  282. }
  283. static int tegra_dma_slave_config(struct dma_chan *dc,
  284. struct dma_slave_config *sconfig)
  285. {
  286. struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
  287. if (!list_empty(&tdc->pending_sg_req)) {
  288. dev_err(tdc2dev(tdc), "Configuration not allowed\n");
  289. return -EBUSY;
  290. }
  291. memcpy(&tdc->dma_sconfig, sconfig, sizeof(*sconfig));
  292. tdc->config_init = true;
  293. return 0;
  294. }
  295. static void tegra_dma_global_pause(struct tegra_dma_channel *tdc,
  296. bool wait_for_burst_complete)
  297. {
  298. struct tegra_dma *tdma = tdc->tdma;
  299. spin_lock(&tdma->global_lock);
  300. tdma_write(tdma, TEGRA_APBDMA_GENERAL, 0);
  301. if (wait_for_burst_complete)
  302. udelay(TEGRA_APBDMA_BURST_COMPLETE_TIME);
  303. }
  304. static void tegra_dma_global_resume(struct tegra_dma_channel *tdc)
  305. {
  306. struct tegra_dma *tdma = tdc->tdma;
  307. tdma_write(tdma, TEGRA_APBDMA_GENERAL, TEGRA_APBDMA_GENERAL_ENABLE);
  308. spin_unlock(&tdma->global_lock);
  309. }
  310. static void tegra_dma_pause(struct tegra_dma_channel *tdc,
  311. bool wait_for_burst_complete)
  312. {
  313. struct tegra_dma *tdma = tdc->tdma;
  314. if (tdma->chip_data->support_channel_pause) {
  315. tdc_write(tdc, TEGRA_APBDMA_CHAN_CSRE,
  316. TEGRA_APBDMA_CHAN_CSRE_PAUSE);
  317. if (wait_for_burst_complete)
  318. udelay(TEGRA_APBDMA_BURST_COMPLETE_TIME);
  319. } else {
  320. tegra_dma_global_pause(tdc, wait_for_burst_complete);
  321. }
  322. }
  323. static void tegra_dma_resume(struct tegra_dma_channel *tdc)
  324. {
  325. struct tegra_dma *tdma = tdc->tdma;
  326. if (tdma->chip_data->support_channel_pause) {
  327. tdc_write(tdc, TEGRA_APBDMA_CHAN_CSRE, 0);
  328. } else {
  329. tegra_dma_global_resume(tdc);
  330. }
  331. }
  332. static void tegra_dma_stop(struct tegra_dma_channel *tdc)
  333. {
  334. u32 csr;
  335. u32 status;
  336. /* Disable interrupts */
  337. csr = tdc_read(tdc, TEGRA_APBDMA_CHAN_CSR);
  338. csr &= ~TEGRA_APBDMA_CSR_IE_EOC;
  339. tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR, csr);
  340. /* Disable DMA */
  341. csr &= ~TEGRA_APBDMA_CSR_ENB;
  342. tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR, csr);
  343. /* Clear interrupt status if it is there */
  344. status = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS);
  345. if (status & TEGRA_APBDMA_STATUS_ISE_EOC) {
  346. dev_dbg(tdc2dev(tdc), "%s():clearing interrupt\n", __func__);
  347. tdc_write(tdc, TEGRA_APBDMA_CHAN_STATUS, status);
  348. }
  349. tdc->busy = false;
  350. }
  351. static void tegra_dma_start(struct tegra_dma_channel *tdc,
  352. struct tegra_dma_sg_req *sg_req)
  353. {
  354. struct tegra_dma_channel_regs *ch_regs = &sg_req->ch_regs;
  355. tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR, ch_regs->csr);
  356. tdc_write(tdc, TEGRA_APBDMA_CHAN_APBSEQ, ch_regs->apb_seq);
  357. tdc_write(tdc, TEGRA_APBDMA_CHAN_APBPTR, ch_regs->apb_ptr);
  358. tdc_write(tdc, TEGRA_APBDMA_CHAN_AHBSEQ, ch_regs->ahb_seq);
  359. tdc_write(tdc, TEGRA_APBDMA_CHAN_AHBPTR, ch_regs->ahb_ptr);
  360. /* Start DMA */
  361. tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR,
  362. ch_regs->csr | TEGRA_APBDMA_CSR_ENB);
  363. }
  364. static void tegra_dma_configure_for_next(struct tegra_dma_channel *tdc,
  365. struct tegra_dma_sg_req *nsg_req)
  366. {
  367. unsigned long status;
  368. /*
  369. * The DMA controller reloads the new configuration for next transfer
  370. * after last burst of current transfer completes.
  371. * If there is no IEC status then this makes sure that last burst
  372. * has not be completed. There may be case that last burst is on
  373. * flight and so it can complete but because DMA is paused, it
  374. * will not generates interrupt as well as not reload the new
  375. * configuration.
  376. * If there is already IEC status then interrupt handler need to
  377. * load new configuration.
  378. */
  379. tegra_dma_pause(tdc, false);
  380. status = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS);
  381. /*
  382. * If interrupt is pending then do nothing as the ISR will handle
  383. * the programing for new request.
  384. */
  385. if (status & TEGRA_APBDMA_STATUS_ISE_EOC) {
  386. dev_err(tdc2dev(tdc),
  387. "Skipping new configuration as interrupt is pending\n");
  388. tegra_dma_resume(tdc);
  389. return;
  390. }
  391. /* Safe to program new configuration */
  392. tdc_write(tdc, TEGRA_APBDMA_CHAN_APBPTR, nsg_req->ch_regs.apb_ptr);
  393. tdc_write(tdc, TEGRA_APBDMA_CHAN_AHBPTR, nsg_req->ch_regs.ahb_ptr);
  394. tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR,
  395. nsg_req->ch_regs.csr | TEGRA_APBDMA_CSR_ENB);
  396. nsg_req->configured = true;
  397. tegra_dma_resume(tdc);
  398. }
  399. static void tdc_start_head_req(struct tegra_dma_channel *tdc)
  400. {
  401. struct tegra_dma_sg_req *sg_req;
  402. if (list_empty(&tdc->pending_sg_req))
  403. return;
  404. sg_req = list_first_entry(&tdc->pending_sg_req,
  405. typeof(*sg_req), node);
  406. tegra_dma_start(tdc, sg_req);
  407. sg_req->configured = true;
  408. tdc->busy = true;
  409. }
  410. static void tdc_configure_next_head_desc(struct tegra_dma_channel *tdc)
  411. {
  412. struct tegra_dma_sg_req *hsgreq;
  413. struct tegra_dma_sg_req *hnsgreq;
  414. if (list_empty(&tdc->pending_sg_req))
  415. return;
  416. hsgreq = list_first_entry(&tdc->pending_sg_req, typeof(*hsgreq), node);
  417. if (!list_is_last(&hsgreq->node, &tdc->pending_sg_req)) {
  418. hnsgreq = list_first_entry(&hsgreq->node,
  419. typeof(*hnsgreq), node);
  420. tegra_dma_configure_for_next(tdc, hnsgreq);
  421. }
  422. }
  423. static inline int get_current_xferred_count(struct tegra_dma_channel *tdc,
  424. struct tegra_dma_sg_req *sg_req, unsigned long status)
  425. {
  426. return sg_req->req_len - (status & TEGRA_APBDMA_STATUS_COUNT_MASK) - 4;
  427. }
  428. static void tegra_dma_abort_all(struct tegra_dma_channel *tdc)
  429. {
  430. struct tegra_dma_sg_req *sgreq;
  431. struct tegra_dma_desc *dma_desc;
  432. while (!list_empty(&tdc->pending_sg_req)) {
  433. sgreq = list_first_entry(&tdc->pending_sg_req,
  434. typeof(*sgreq), node);
  435. list_move_tail(&sgreq->node, &tdc->free_sg_req);
  436. if (sgreq->last_sg) {
  437. dma_desc = sgreq->dma_desc;
  438. dma_desc->dma_status = DMA_ERROR;
  439. list_add_tail(&dma_desc->node, &tdc->free_dma_desc);
  440. /* Add in cb list if it is not there. */
  441. if (!dma_desc->cb_count)
  442. list_add_tail(&dma_desc->cb_node,
  443. &tdc->cb_desc);
  444. dma_desc->cb_count++;
  445. }
  446. }
  447. tdc->isr_handler = NULL;
  448. }
  449. static bool handle_continuous_head_request(struct tegra_dma_channel *tdc,
  450. struct tegra_dma_sg_req *last_sg_req, bool to_terminate)
  451. {
  452. struct tegra_dma_sg_req *hsgreq = NULL;
  453. if (list_empty(&tdc->pending_sg_req)) {
  454. dev_err(tdc2dev(tdc), "Dma is running without req\n");
  455. tegra_dma_stop(tdc);
  456. return false;
  457. }
  458. /*
  459. * Check that head req on list should be in flight.
  460. * If it is not in flight then abort transfer as
  461. * looping of transfer can not continue.
  462. */
  463. hsgreq = list_first_entry(&tdc->pending_sg_req, typeof(*hsgreq), node);
  464. if (!hsgreq->configured) {
  465. tegra_dma_stop(tdc);
  466. dev_err(tdc2dev(tdc), "Error in dma transfer, aborting dma\n");
  467. tegra_dma_abort_all(tdc);
  468. return false;
  469. }
  470. /* Configure next request */
  471. if (!to_terminate)
  472. tdc_configure_next_head_desc(tdc);
  473. return true;
  474. }
  475. static void handle_once_dma_done(struct tegra_dma_channel *tdc,
  476. bool to_terminate)
  477. {
  478. struct tegra_dma_sg_req *sgreq;
  479. struct tegra_dma_desc *dma_desc;
  480. tdc->busy = false;
  481. sgreq = list_first_entry(&tdc->pending_sg_req, typeof(*sgreq), node);
  482. dma_desc = sgreq->dma_desc;
  483. dma_desc->bytes_transferred += sgreq->req_len;
  484. list_del(&sgreq->node);
  485. if (sgreq->last_sg) {
  486. dma_desc->dma_status = DMA_SUCCESS;
  487. dma_cookie_complete(&dma_desc->txd);
  488. if (!dma_desc->cb_count)
  489. list_add_tail(&dma_desc->cb_node, &tdc->cb_desc);
  490. dma_desc->cb_count++;
  491. list_add_tail(&dma_desc->node, &tdc->free_dma_desc);
  492. }
  493. list_add_tail(&sgreq->node, &tdc->free_sg_req);
  494. /* Do not start DMA if it is going to be terminate */
  495. if (to_terminate || list_empty(&tdc->pending_sg_req))
  496. return;
  497. tdc_start_head_req(tdc);
  498. return;
  499. }
  500. static void handle_cont_sngl_cycle_dma_done(struct tegra_dma_channel *tdc,
  501. bool to_terminate)
  502. {
  503. struct tegra_dma_sg_req *sgreq;
  504. struct tegra_dma_desc *dma_desc;
  505. bool st;
  506. sgreq = list_first_entry(&tdc->pending_sg_req, typeof(*sgreq), node);
  507. dma_desc = sgreq->dma_desc;
  508. dma_desc->bytes_transferred += sgreq->req_len;
  509. /* Callback need to be call */
  510. if (!dma_desc->cb_count)
  511. list_add_tail(&dma_desc->cb_node, &tdc->cb_desc);
  512. dma_desc->cb_count++;
  513. /* If not last req then put at end of pending list */
  514. if (!list_is_last(&sgreq->node, &tdc->pending_sg_req)) {
  515. list_move_tail(&sgreq->node, &tdc->pending_sg_req);
  516. sgreq->configured = false;
  517. st = handle_continuous_head_request(tdc, sgreq, to_terminate);
  518. if (!st)
  519. dma_desc->dma_status = DMA_ERROR;
  520. }
  521. return;
  522. }
  523. static void tegra_dma_tasklet(unsigned long data)
  524. {
  525. struct tegra_dma_channel *tdc = (struct tegra_dma_channel *)data;
  526. dma_async_tx_callback callback = NULL;
  527. void *callback_param = NULL;
  528. struct tegra_dma_desc *dma_desc;
  529. unsigned long flags;
  530. int cb_count;
  531. spin_lock_irqsave(&tdc->lock, flags);
  532. while (!list_empty(&tdc->cb_desc)) {
  533. dma_desc = list_first_entry(&tdc->cb_desc,
  534. typeof(*dma_desc), cb_node);
  535. list_del(&dma_desc->cb_node);
  536. callback = dma_desc->txd.callback;
  537. callback_param = dma_desc->txd.callback_param;
  538. cb_count = dma_desc->cb_count;
  539. dma_desc->cb_count = 0;
  540. spin_unlock_irqrestore(&tdc->lock, flags);
  541. while (cb_count-- && callback)
  542. callback(callback_param);
  543. spin_lock_irqsave(&tdc->lock, flags);
  544. }
  545. spin_unlock_irqrestore(&tdc->lock, flags);
  546. }
  547. static irqreturn_t tegra_dma_isr(int irq, void *dev_id)
  548. {
  549. struct tegra_dma_channel *tdc = dev_id;
  550. unsigned long status;
  551. unsigned long flags;
  552. spin_lock_irqsave(&tdc->lock, flags);
  553. status = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS);
  554. if (status & TEGRA_APBDMA_STATUS_ISE_EOC) {
  555. tdc_write(tdc, TEGRA_APBDMA_CHAN_STATUS, status);
  556. tdc->isr_handler(tdc, false);
  557. tasklet_schedule(&tdc->tasklet);
  558. spin_unlock_irqrestore(&tdc->lock, flags);
  559. return IRQ_HANDLED;
  560. }
  561. spin_unlock_irqrestore(&tdc->lock, flags);
  562. dev_info(tdc2dev(tdc),
  563. "Interrupt already served status 0x%08lx\n", status);
  564. return IRQ_NONE;
  565. }
  566. static dma_cookie_t tegra_dma_tx_submit(struct dma_async_tx_descriptor *txd)
  567. {
  568. struct tegra_dma_desc *dma_desc = txd_to_tegra_dma_desc(txd);
  569. struct tegra_dma_channel *tdc = to_tegra_dma_chan(txd->chan);
  570. unsigned long flags;
  571. dma_cookie_t cookie;
  572. spin_lock_irqsave(&tdc->lock, flags);
  573. dma_desc->dma_status = DMA_IN_PROGRESS;
  574. cookie = dma_cookie_assign(&dma_desc->txd);
  575. list_splice_tail_init(&dma_desc->tx_list, &tdc->pending_sg_req);
  576. spin_unlock_irqrestore(&tdc->lock, flags);
  577. return cookie;
  578. }
  579. static void tegra_dma_issue_pending(struct dma_chan *dc)
  580. {
  581. struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
  582. unsigned long flags;
  583. spin_lock_irqsave(&tdc->lock, flags);
  584. if (list_empty(&tdc->pending_sg_req)) {
  585. dev_err(tdc2dev(tdc), "No DMA request\n");
  586. goto end;
  587. }
  588. if (!tdc->busy) {
  589. tdc_start_head_req(tdc);
  590. /* Continuous single mode: Configure next req */
  591. if (tdc->cyclic) {
  592. /*
  593. * Wait for 1 burst time for configure DMA for
  594. * next transfer.
  595. */
  596. udelay(TEGRA_APBDMA_BURST_COMPLETE_TIME);
  597. tdc_configure_next_head_desc(tdc);
  598. }
  599. }
  600. end:
  601. spin_unlock_irqrestore(&tdc->lock, flags);
  602. return;
  603. }
  604. static void tegra_dma_terminate_all(struct dma_chan *dc)
  605. {
  606. struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
  607. struct tegra_dma_sg_req *sgreq;
  608. struct tegra_dma_desc *dma_desc;
  609. unsigned long flags;
  610. unsigned long status;
  611. bool was_busy;
  612. spin_lock_irqsave(&tdc->lock, flags);
  613. if (list_empty(&tdc->pending_sg_req)) {
  614. spin_unlock_irqrestore(&tdc->lock, flags);
  615. return;
  616. }
  617. if (!tdc->busy)
  618. goto skip_dma_stop;
  619. /* Pause DMA before checking the queue status */
  620. tegra_dma_pause(tdc, true);
  621. status = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS);
  622. if (status & TEGRA_APBDMA_STATUS_ISE_EOC) {
  623. dev_dbg(tdc2dev(tdc), "%s():handling isr\n", __func__);
  624. tdc->isr_handler(tdc, true);
  625. status = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS);
  626. }
  627. was_busy = tdc->busy;
  628. tegra_dma_stop(tdc);
  629. if (!list_empty(&tdc->pending_sg_req) && was_busy) {
  630. sgreq = list_first_entry(&tdc->pending_sg_req,
  631. typeof(*sgreq), node);
  632. sgreq->dma_desc->bytes_transferred +=
  633. get_current_xferred_count(tdc, sgreq, status);
  634. }
  635. tegra_dma_resume(tdc);
  636. skip_dma_stop:
  637. tegra_dma_abort_all(tdc);
  638. while (!list_empty(&tdc->cb_desc)) {
  639. dma_desc = list_first_entry(&tdc->cb_desc,
  640. typeof(*dma_desc), cb_node);
  641. list_del(&dma_desc->cb_node);
  642. dma_desc->cb_count = 0;
  643. }
  644. spin_unlock_irqrestore(&tdc->lock, flags);
  645. }
  646. static enum dma_status tegra_dma_tx_status(struct dma_chan *dc,
  647. dma_cookie_t cookie, struct dma_tx_state *txstate)
  648. {
  649. struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
  650. struct tegra_dma_desc *dma_desc;
  651. struct tegra_dma_sg_req *sg_req;
  652. enum dma_status ret;
  653. unsigned long flags;
  654. unsigned int residual;
  655. ret = dma_cookie_status(dc, cookie, txstate);
  656. if (ret == DMA_SUCCESS)
  657. return ret;
  658. spin_lock_irqsave(&tdc->lock, flags);
  659. /* Check on wait_ack desc status */
  660. list_for_each_entry(dma_desc, &tdc->free_dma_desc, node) {
  661. if (dma_desc->txd.cookie == cookie) {
  662. residual = dma_desc->bytes_requested -
  663. (dma_desc->bytes_transferred %
  664. dma_desc->bytes_requested);
  665. dma_set_residue(txstate, residual);
  666. ret = dma_desc->dma_status;
  667. spin_unlock_irqrestore(&tdc->lock, flags);
  668. return ret;
  669. }
  670. }
  671. /* Check in pending list */
  672. list_for_each_entry(sg_req, &tdc->pending_sg_req, node) {
  673. dma_desc = sg_req->dma_desc;
  674. if (dma_desc->txd.cookie == cookie) {
  675. residual = dma_desc->bytes_requested -
  676. (dma_desc->bytes_transferred %
  677. dma_desc->bytes_requested);
  678. dma_set_residue(txstate, residual);
  679. ret = dma_desc->dma_status;
  680. spin_unlock_irqrestore(&tdc->lock, flags);
  681. return ret;
  682. }
  683. }
  684. dev_dbg(tdc2dev(tdc), "cookie %d does not found\n", cookie);
  685. spin_unlock_irqrestore(&tdc->lock, flags);
  686. return ret;
  687. }
  688. static int tegra_dma_device_control(struct dma_chan *dc, enum dma_ctrl_cmd cmd,
  689. unsigned long arg)
  690. {
  691. switch (cmd) {
  692. case DMA_SLAVE_CONFIG:
  693. return tegra_dma_slave_config(dc,
  694. (struct dma_slave_config *)arg);
  695. case DMA_TERMINATE_ALL:
  696. tegra_dma_terminate_all(dc);
  697. return 0;
  698. default:
  699. break;
  700. }
  701. return -ENXIO;
  702. }
  703. static inline int get_bus_width(struct tegra_dma_channel *tdc,
  704. enum dma_slave_buswidth slave_bw)
  705. {
  706. switch (slave_bw) {
  707. case DMA_SLAVE_BUSWIDTH_1_BYTE:
  708. return TEGRA_APBDMA_APBSEQ_BUS_WIDTH_8;
  709. case DMA_SLAVE_BUSWIDTH_2_BYTES:
  710. return TEGRA_APBDMA_APBSEQ_BUS_WIDTH_16;
  711. case DMA_SLAVE_BUSWIDTH_4_BYTES:
  712. return TEGRA_APBDMA_APBSEQ_BUS_WIDTH_32;
  713. case DMA_SLAVE_BUSWIDTH_8_BYTES:
  714. return TEGRA_APBDMA_APBSEQ_BUS_WIDTH_64;
  715. default:
  716. dev_warn(tdc2dev(tdc),
  717. "slave bw is not supported, using 32bits\n");
  718. return TEGRA_APBDMA_APBSEQ_BUS_WIDTH_32;
  719. }
  720. }
  721. static inline int get_burst_size(struct tegra_dma_channel *tdc,
  722. u32 burst_size, enum dma_slave_buswidth slave_bw, int len)
  723. {
  724. int burst_byte;
  725. int burst_ahb_width;
  726. /*
  727. * burst_size from client is in terms of the bus_width.
  728. * convert them into AHB memory width which is 4 byte.
  729. */
  730. burst_byte = burst_size * slave_bw;
  731. burst_ahb_width = burst_byte / 4;
  732. /* If burst size is 0 then calculate the burst size based on length */
  733. if (!burst_ahb_width) {
  734. if (len & 0xF)
  735. return TEGRA_APBDMA_AHBSEQ_BURST_1;
  736. else if ((len >> 4) & 0x1)
  737. return TEGRA_APBDMA_AHBSEQ_BURST_4;
  738. else
  739. return TEGRA_APBDMA_AHBSEQ_BURST_8;
  740. }
  741. if (burst_ahb_width < 4)
  742. return TEGRA_APBDMA_AHBSEQ_BURST_1;
  743. else if (burst_ahb_width < 8)
  744. return TEGRA_APBDMA_AHBSEQ_BURST_4;
  745. else
  746. return TEGRA_APBDMA_AHBSEQ_BURST_8;
  747. }
  748. static int get_transfer_param(struct tegra_dma_channel *tdc,
  749. enum dma_transfer_direction direction, unsigned long *apb_addr,
  750. unsigned long *apb_seq, unsigned long *csr, unsigned int *burst_size,
  751. enum dma_slave_buswidth *slave_bw)
  752. {
  753. switch (direction) {
  754. case DMA_MEM_TO_DEV:
  755. *apb_addr = tdc->dma_sconfig.dst_addr;
  756. *apb_seq = get_bus_width(tdc, tdc->dma_sconfig.dst_addr_width);
  757. *burst_size = tdc->dma_sconfig.dst_maxburst;
  758. *slave_bw = tdc->dma_sconfig.dst_addr_width;
  759. *csr = TEGRA_APBDMA_CSR_DIR;
  760. return 0;
  761. case DMA_DEV_TO_MEM:
  762. *apb_addr = tdc->dma_sconfig.src_addr;
  763. *apb_seq = get_bus_width(tdc, tdc->dma_sconfig.src_addr_width);
  764. *burst_size = tdc->dma_sconfig.src_maxburst;
  765. *slave_bw = tdc->dma_sconfig.src_addr_width;
  766. *csr = 0;
  767. return 0;
  768. default:
  769. dev_err(tdc2dev(tdc), "Dma direction is not supported\n");
  770. return -EINVAL;
  771. }
  772. return -EINVAL;
  773. }
  774. static struct dma_async_tx_descriptor *tegra_dma_prep_slave_sg(
  775. struct dma_chan *dc, struct scatterlist *sgl, unsigned int sg_len,
  776. enum dma_transfer_direction direction, unsigned long flags,
  777. void *context)
  778. {
  779. struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
  780. struct tegra_dma_desc *dma_desc;
  781. unsigned int i;
  782. struct scatterlist *sg;
  783. unsigned long csr, ahb_seq, apb_ptr, apb_seq;
  784. struct list_head req_list;
  785. struct tegra_dma_sg_req *sg_req = NULL;
  786. u32 burst_size;
  787. enum dma_slave_buswidth slave_bw;
  788. int ret;
  789. if (!tdc->config_init) {
  790. dev_err(tdc2dev(tdc), "dma channel is not configured\n");
  791. return NULL;
  792. }
  793. if (sg_len < 1) {
  794. dev_err(tdc2dev(tdc), "Invalid segment length %d\n", sg_len);
  795. return NULL;
  796. }
  797. ret = get_transfer_param(tdc, direction, &apb_ptr, &apb_seq, &csr,
  798. &burst_size, &slave_bw);
  799. if (ret < 0)
  800. return NULL;
  801. INIT_LIST_HEAD(&req_list);
  802. ahb_seq = TEGRA_APBDMA_AHBSEQ_INTR_ENB;
  803. ahb_seq |= TEGRA_APBDMA_AHBSEQ_WRAP_NONE <<
  804. TEGRA_APBDMA_AHBSEQ_WRAP_SHIFT;
  805. ahb_seq |= TEGRA_APBDMA_AHBSEQ_BUS_WIDTH_32;
  806. csr |= TEGRA_APBDMA_CSR_ONCE | TEGRA_APBDMA_CSR_FLOW;
  807. csr |= tdc->dma_sconfig.slave_id << TEGRA_APBDMA_CSR_REQ_SEL_SHIFT;
  808. if (flags & DMA_PREP_INTERRUPT)
  809. csr |= TEGRA_APBDMA_CSR_IE_EOC;
  810. apb_seq |= TEGRA_APBDMA_APBSEQ_WRAP_WORD_1;
  811. dma_desc = tegra_dma_desc_get(tdc);
  812. if (!dma_desc) {
  813. dev_err(tdc2dev(tdc), "Dma descriptors not available\n");
  814. return NULL;
  815. }
  816. INIT_LIST_HEAD(&dma_desc->tx_list);
  817. INIT_LIST_HEAD(&dma_desc->cb_node);
  818. dma_desc->cb_count = 0;
  819. dma_desc->bytes_requested = 0;
  820. dma_desc->bytes_transferred = 0;
  821. dma_desc->dma_status = DMA_IN_PROGRESS;
  822. /* Make transfer requests */
  823. for_each_sg(sgl, sg, sg_len, i) {
  824. u32 len, mem;
  825. mem = sg_dma_address(sg);
  826. len = sg_dma_len(sg);
  827. if ((len & 3) || (mem & 3) ||
  828. (len > tdc->tdma->chip_data->max_dma_count)) {
  829. dev_err(tdc2dev(tdc),
  830. "Dma length/memory address is not supported\n");
  831. tegra_dma_desc_put(tdc, dma_desc);
  832. return NULL;
  833. }
  834. sg_req = tegra_dma_sg_req_get(tdc);
  835. if (!sg_req) {
  836. dev_err(tdc2dev(tdc), "Dma sg-req not available\n");
  837. tegra_dma_desc_put(tdc, dma_desc);
  838. return NULL;
  839. }
  840. ahb_seq |= get_burst_size(tdc, burst_size, slave_bw, len);
  841. dma_desc->bytes_requested += len;
  842. sg_req->ch_regs.apb_ptr = apb_ptr;
  843. sg_req->ch_regs.ahb_ptr = mem;
  844. sg_req->ch_regs.csr = csr | ((len - 4) & 0xFFFC);
  845. sg_req->ch_regs.apb_seq = apb_seq;
  846. sg_req->ch_regs.ahb_seq = ahb_seq;
  847. sg_req->configured = false;
  848. sg_req->last_sg = false;
  849. sg_req->dma_desc = dma_desc;
  850. sg_req->req_len = len;
  851. list_add_tail(&sg_req->node, &dma_desc->tx_list);
  852. }
  853. sg_req->last_sg = true;
  854. if (flags & DMA_CTRL_ACK)
  855. dma_desc->txd.flags = DMA_CTRL_ACK;
  856. /*
  857. * Make sure that mode should not be conflicting with currently
  858. * configured mode.
  859. */
  860. if (!tdc->isr_handler) {
  861. tdc->isr_handler = handle_once_dma_done;
  862. tdc->cyclic = false;
  863. } else {
  864. if (tdc->cyclic) {
  865. dev_err(tdc2dev(tdc), "DMA configured in cyclic mode\n");
  866. tegra_dma_desc_put(tdc, dma_desc);
  867. return NULL;
  868. }
  869. }
  870. return &dma_desc->txd;
  871. }
  872. struct dma_async_tx_descriptor *tegra_dma_prep_dma_cyclic(
  873. struct dma_chan *dc, dma_addr_t buf_addr, size_t buf_len,
  874. size_t period_len, enum dma_transfer_direction direction,
  875. unsigned long flags, void *context)
  876. {
  877. struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
  878. struct tegra_dma_desc *dma_desc = NULL;
  879. struct tegra_dma_sg_req *sg_req = NULL;
  880. unsigned long csr, ahb_seq, apb_ptr, apb_seq;
  881. int len;
  882. size_t remain_len;
  883. dma_addr_t mem = buf_addr;
  884. u32 burst_size;
  885. enum dma_slave_buswidth slave_bw;
  886. int ret;
  887. if (!buf_len || !period_len) {
  888. dev_err(tdc2dev(tdc), "Invalid buffer/period len\n");
  889. return NULL;
  890. }
  891. if (!tdc->config_init) {
  892. dev_err(tdc2dev(tdc), "DMA slave is not configured\n");
  893. return NULL;
  894. }
  895. /*
  896. * We allow to take more number of requests till DMA is
  897. * not started. The driver will loop over all requests.
  898. * Once DMA is started then new requests can be queued only after
  899. * terminating the DMA.
  900. */
  901. if (tdc->busy) {
  902. dev_err(tdc2dev(tdc), "Request not allowed when dma running\n");
  903. return NULL;
  904. }
  905. /*
  906. * We only support cycle transfer when buf_len is multiple of
  907. * period_len.
  908. */
  909. if (buf_len % period_len) {
  910. dev_err(tdc2dev(tdc), "buf_len is not multiple of period_len\n");
  911. return NULL;
  912. }
  913. len = period_len;
  914. if ((len & 3) || (buf_addr & 3) ||
  915. (len > tdc->tdma->chip_data->max_dma_count)) {
  916. dev_err(tdc2dev(tdc), "Req len/mem address is not correct\n");
  917. return NULL;
  918. }
  919. ret = get_transfer_param(tdc, direction, &apb_ptr, &apb_seq, &csr,
  920. &burst_size, &slave_bw);
  921. if (ret < 0)
  922. return NULL;
  923. ahb_seq = TEGRA_APBDMA_AHBSEQ_INTR_ENB;
  924. ahb_seq |= TEGRA_APBDMA_AHBSEQ_WRAP_NONE <<
  925. TEGRA_APBDMA_AHBSEQ_WRAP_SHIFT;
  926. ahb_seq |= TEGRA_APBDMA_AHBSEQ_BUS_WIDTH_32;
  927. csr |= TEGRA_APBDMA_CSR_FLOW;
  928. if (flags & DMA_PREP_INTERRUPT)
  929. csr |= TEGRA_APBDMA_CSR_IE_EOC;
  930. csr |= tdc->dma_sconfig.slave_id << TEGRA_APBDMA_CSR_REQ_SEL_SHIFT;
  931. apb_seq |= TEGRA_APBDMA_APBSEQ_WRAP_WORD_1;
  932. dma_desc = tegra_dma_desc_get(tdc);
  933. if (!dma_desc) {
  934. dev_err(tdc2dev(tdc), "not enough descriptors available\n");
  935. return NULL;
  936. }
  937. INIT_LIST_HEAD(&dma_desc->tx_list);
  938. INIT_LIST_HEAD(&dma_desc->cb_node);
  939. dma_desc->cb_count = 0;
  940. dma_desc->bytes_transferred = 0;
  941. dma_desc->bytes_requested = buf_len;
  942. remain_len = buf_len;
  943. /* Split transfer equal to period size */
  944. while (remain_len) {
  945. sg_req = tegra_dma_sg_req_get(tdc);
  946. if (!sg_req) {
  947. dev_err(tdc2dev(tdc), "Dma sg-req not available\n");
  948. tegra_dma_desc_put(tdc, dma_desc);
  949. return NULL;
  950. }
  951. ahb_seq |= get_burst_size(tdc, burst_size, slave_bw, len);
  952. sg_req->ch_regs.apb_ptr = apb_ptr;
  953. sg_req->ch_regs.ahb_ptr = mem;
  954. sg_req->ch_regs.csr = csr | ((len - 4) & 0xFFFC);
  955. sg_req->ch_regs.apb_seq = apb_seq;
  956. sg_req->ch_regs.ahb_seq = ahb_seq;
  957. sg_req->configured = false;
  958. sg_req->half_done = false;
  959. sg_req->last_sg = false;
  960. sg_req->dma_desc = dma_desc;
  961. sg_req->req_len = len;
  962. list_add_tail(&sg_req->node, &dma_desc->tx_list);
  963. remain_len -= len;
  964. mem += len;
  965. }
  966. sg_req->last_sg = true;
  967. if (flags & DMA_CTRL_ACK)
  968. dma_desc->txd.flags = DMA_CTRL_ACK;
  969. /*
  970. * Make sure that mode should not be conflicting with currently
  971. * configured mode.
  972. */
  973. if (!tdc->isr_handler) {
  974. tdc->isr_handler = handle_cont_sngl_cycle_dma_done;
  975. tdc->cyclic = true;
  976. } else {
  977. if (!tdc->cyclic) {
  978. dev_err(tdc2dev(tdc), "DMA configuration conflict\n");
  979. tegra_dma_desc_put(tdc, dma_desc);
  980. return NULL;
  981. }
  982. }
  983. return &dma_desc->txd;
  984. }
  985. static int tegra_dma_alloc_chan_resources(struct dma_chan *dc)
  986. {
  987. struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
  988. struct tegra_dma *tdma = tdc->tdma;
  989. int ret;
  990. dma_cookie_init(&tdc->dma_chan);
  991. tdc->config_init = false;
  992. ret = clk_prepare_enable(tdma->dma_clk);
  993. if (ret < 0)
  994. dev_err(tdc2dev(tdc), "clk_prepare_enable failed: %d\n", ret);
  995. return ret;
  996. }
  997. static void tegra_dma_free_chan_resources(struct dma_chan *dc)
  998. {
  999. struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
  1000. struct tegra_dma *tdma = tdc->tdma;
  1001. struct tegra_dma_desc *dma_desc;
  1002. struct tegra_dma_sg_req *sg_req;
  1003. struct list_head dma_desc_list;
  1004. struct list_head sg_req_list;
  1005. unsigned long flags;
  1006. INIT_LIST_HEAD(&dma_desc_list);
  1007. INIT_LIST_HEAD(&sg_req_list);
  1008. dev_dbg(tdc2dev(tdc), "Freeing channel %d\n", tdc->id);
  1009. if (tdc->busy)
  1010. tegra_dma_terminate_all(dc);
  1011. spin_lock_irqsave(&tdc->lock, flags);
  1012. list_splice_init(&tdc->pending_sg_req, &sg_req_list);
  1013. list_splice_init(&tdc->free_sg_req, &sg_req_list);
  1014. list_splice_init(&tdc->free_dma_desc, &dma_desc_list);
  1015. INIT_LIST_HEAD(&tdc->cb_desc);
  1016. tdc->config_init = false;
  1017. tdc->isr_handler = NULL;
  1018. spin_unlock_irqrestore(&tdc->lock, flags);
  1019. while (!list_empty(&dma_desc_list)) {
  1020. dma_desc = list_first_entry(&dma_desc_list,
  1021. typeof(*dma_desc), node);
  1022. list_del(&dma_desc->node);
  1023. kfree(dma_desc);
  1024. }
  1025. while (!list_empty(&sg_req_list)) {
  1026. sg_req = list_first_entry(&sg_req_list, typeof(*sg_req), node);
  1027. list_del(&sg_req->node);
  1028. kfree(sg_req);
  1029. }
  1030. clk_disable_unprepare(tdma->dma_clk);
  1031. }
  1032. /* Tegra20 specific DMA controller information */
  1033. static const struct tegra_dma_chip_data tegra20_dma_chip_data = {
  1034. .nr_channels = 16,
  1035. .max_dma_count = 1024UL * 64,
  1036. .support_channel_pause = false,
  1037. };
  1038. /* Tegra30 specific DMA controller information */
  1039. static const struct tegra_dma_chip_data tegra30_dma_chip_data = {
  1040. .nr_channels = 32,
  1041. .max_dma_count = 1024UL * 64,
  1042. .support_channel_pause = false,
  1043. };
  1044. /* Tegra114 specific DMA controller information */
  1045. static const struct tegra_dma_chip_data tegra114_dma_chip_data = {
  1046. .nr_channels = 32,
  1047. .max_dma_count = 1024UL * 64,
  1048. .support_channel_pause = true,
  1049. };
  1050. static const struct of_device_id tegra_dma_of_match[] = {
  1051. {
  1052. .compatible = "nvidia,tegra114-apbdma",
  1053. .data = &tegra114_dma_chip_data,
  1054. }, {
  1055. .compatible = "nvidia,tegra30-apbdma",
  1056. .data = &tegra30_dma_chip_data,
  1057. }, {
  1058. .compatible = "nvidia,tegra20-apbdma",
  1059. .data = &tegra20_dma_chip_data,
  1060. }, {
  1061. },
  1062. };
  1063. MODULE_DEVICE_TABLE(of, tegra_dma_of_match);
  1064. static int tegra_dma_probe(struct platform_device *pdev)
  1065. {
  1066. struct resource *res;
  1067. struct tegra_dma *tdma;
  1068. int ret;
  1069. int i;
  1070. const struct tegra_dma_chip_data *cdata = NULL;
  1071. const struct of_device_id *match;
  1072. match = of_match_device(tegra_dma_of_match, &pdev->dev);
  1073. if (!match) {
  1074. dev_err(&pdev->dev, "Error: No device match found\n");
  1075. return -ENODEV;
  1076. }
  1077. cdata = match->data;
  1078. tdma = devm_kzalloc(&pdev->dev, sizeof(*tdma) + cdata->nr_channels *
  1079. sizeof(struct tegra_dma_channel), GFP_KERNEL);
  1080. if (!tdma) {
  1081. dev_err(&pdev->dev, "Error: memory allocation failed\n");
  1082. return -ENOMEM;
  1083. }
  1084. tdma->dev = &pdev->dev;
  1085. tdma->chip_data = cdata;
  1086. platform_set_drvdata(pdev, tdma);
  1087. res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
  1088. tdma->base_addr = devm_ioremap_resource(&pdev->dev, res);
  1089. if (IS_ERR(tdma->base_addr))
  1090. return PTR_ERR(tdma->base_addr);
  1091. tdma->dma_clk = devm_clk_get(&pdev->dev, NULL);
  1092. if (IS_ERR(tdma->dma_clk)) {
  1093. dev_err(&pdev->dev, "Error: Missing controller clock\n");
  1094. return PTR_ERR(tdma->dma_clk);
  1095. }
  1096. spin_lock_init(&tdma->global_lock);
  1097. pm_runtime_enable(&pdev->dev);
  1098. if (!pm_runtime_enabled(&pdev->dev)) {
  1099. ret = tegra_dma_runtime_resume(&pdev->dev);
  1100. if (ret) {
  1101. dev_err(&pdev->dev, "dma_runtime_resume failed %d\n",
  1102. ret);
  1103. goto err_pm_disable;
  1104. }
  1105. }
  1106. /* Enable clock before accessing registers */
  1107. ret = clk_prepare_enable(tdma->dma_clk);
  1108. if (ret < 0) {
  1109. dev_err(&pdev->dev, "clk_prepare_enable failed: %d\n", ret);
  1110. goto err_pm_disable;
  1111. }
  1112. /* Reset DMA controller */
  1113. tegra_periph_reset_assert(tdma->dma_clk);
  1114. udelay(2);
  1115. tegra_periph_reset_deassert(tdma->dma_clk);
  1116. /* Enable global DMA registers */
  1117. tdma_write(tdma, TEGRA_APBDMA_GENERAL, TEGRA_APBDMA_GENERAL_ENABLE);
  1118. tdma_write(tdma, TEGRA_APBDMA_CONTROL, 0);
  1119. tdma_write(tdma, TEGRA_APBDMA_IRQ_MASK_SET, 0xFFFFFFFFul);
  1120. clk_disable_unprepare(tdma->dma_clk);
  1121. INIT_LIST_HEAD(&tdma->dma_dev.channels);
  1122. for (i = 0; i < cdata->nr_channels; i++) {
  1123. struct tegra_dma_channel *tdc = &tdma->channels[i];
  1124. tdc->chan_base_offset = TEGRA_APBDMA_CHANNEL_BASE_ADD_OFFSET +
  1125. i * TEGRA_APBDMA_CHANNEL_REGISTER_SIZE;
  1126. res = platform_get_resource(pdev, IORESOURCE_IRQ, i);
  1127. if (!res) {
  1128. ret = -EINVAL;
  1129. dev_err(&pdev->dev, "No irq resource for chan %d\n", i);
  1130. goto err_irq;
  1131. }
  1132. tdc->irq = res->start;
  1133. snprintf(tdc->name, sizeof(tdc->name), "apbdma.%d", i);
  1134. ret = devm_request_irq(&pdev->dev, tdc->irq,
  1135. tegra_dma_isr, 0, tdc->name, tdc);
  1136. if (ret) {
  1137. dev_err(&pdev->dev,
  1138. "request_irq failed with err %d channel %d\n",
  1139. ret, i);
  1140. goto err_irq;
  1141. }
  1142. tdc->dma_chan.device = &tdma->dma_dev;
  1143. dma_cookie_init(&tdc->dma_chan);
  1144. list_add_tail(&tdc->dma_chan.device_node,
  1145. &tdma->dma_dev.channels);
  1146. tdc->tdma = tdma;
  1147. tdc->id = i;
  1148. tasklet_init(&tdc->tasklet, tegra_dma_tasklet,
  1149. (unsigned long)tdc);
  1150. spin_lock_init(&tdc->lock);
  1151. INIT_LIST_HEAD(&tdc->pending_sg_req);
  1152. INIT_LIST_HEAD(&tdc->free_sg_req);
  1153. INIT_LIST_HEAD(&tdc->free_dma_desc);
  1154. INIT_LIST_HEAD(&tdc->cb_desc);
  1155. }
  1156. dma_cap_set(DMA_SLAVE, tdma->dma_dev.cap_mask);
  1157. dma_cap_set(DMA_PRIVATE, tdma->dma_dev.cap_mask);
  1158. dma_cap_set(DMA_CYCLIC, tdma->dma_dev.cap_mask);
  1159. tdma->dma_dev.dev = &pdev->dev;
  1160. tdma->dma_dev.device_alloc_chan_resources =
  1161. tegra_dma_alloc_chan_resources;
  1162. tdma->dma_dev.device_free_chan_resources =
  1163. tegra_dma_free_chan_resources;
  1164. tdma->dma_dev.device_prep_slave_sg = tegra_dma_prep_slave_sg;
  1165. tdma->dma_dev.device_prep_dma_cyclic = tegra_dma_prep_dma_cyclic;
  1166. tdma->dma_dev.device_control = tegra_dma_device_control;
  1167. tdma->dma_dev.device_tx_status = tegra_dma_tx_status;
  1168. tdma->dma_dev.device_issue_pending = tegra_dma_issue_pending;
  1169. ret = dma_async_device_register(&tdma->dma_dev);
  1170. if (ret < 0) {
  1171. dev_err(&pdev->dev,
  1172. "Tegra20 APB DMA driver registration failed %d\n", ret);
  1173. goto err_irq;
  1174. }
  1175. dev_info(&pdev->dev, "Tegra20 APB DMA driver register %d channels\n",
  1176. cdata->nr_channels);
  1177. return 0;
  1178. err_irq:
  1179. while (--i >= 0) {
  1180. struct tegra_dma_channel *tdc = &tdma->channels[i];
  1181. tasklet_kill(&tdc->tasklet);
  1182. }
  1183. err_pm_disable:
  1184. pm_runtime_disable(&pdev->dev);
  1185. if (!pm_runtime_status_suspended(&pdev->dev))
  1186. tegra_dma_runtime_suspend(&pdev->dev);
  1187. return ret;
  1188. }
  1189. static int tegra_dma_remove(struct platform_device *pdev)
  1190. {
  1191. struct tegra_dma *tdma = platform_get_drvdata(pdev);
  1192. int i;
  1193. struct tegra_dma_channel *tdc;
  1194. dma_async_device_unregister(&tdma->dma_dev);
  1195. for (i = 0; i < tdma->chip_data->nr_channels; ++i) {
  1196. tdc = &tdma->channels[i];
  1197. tasklet_kill(&tdc->tasklet);
  1198. }
  1199. pm_runtime_disable(&pdev->dev);
  1200. if (!pm_runtime_status_suspended(&pdev->dev))
  1201. tegra_dma_runtime_suspend(&pdev->dev);
  1202. return 0;
  1203. }
  1204. static int tegra_dma_runtime_suspend(struct device *dev)
  1205. {
  1206. struct platform_device *pdev = to_platform_device(dev);
  1207. struct tegra_dma *tdma = platform_get_drvdata(pdev);
  1208. clk_disable_unprepare(tdma->dma_clk);
  1209. return 0;
  1210. }
  1211. static int tegra_dma_runtime_resume(struct device *dev)
  1212. {
  1213. struct platform_device *pdev = to_platform_device(dev);
  1214. struct tegra_dma *tdma = platform_get_drvdata(pdev);
  1215. int ret;
  1216. ret = clk_prepare_enable(tdma->dma_clk);
  1217. if (ret < 0) {
  1218. dev_err(dev, "clk_enable failed: %d\n", ret);
  1219. return ret;
  1220. }
  1221. return 0;
  1222. }
  1223. #ifdef CONFIG_PM_SLEEP
  1224. static int tegra_dma_pm_suspend(struct device *dev)
  1225. {
  1226. struct tegra_dma *tdma = dev_get_drvdata(dev);
  1227. int i;
  1228. int ret;
  1229. /* Enable clock before accessing register */
  1230. ret = tegra_dma_runtime_resume(dev);
  1231. if (ret < 0)
  1232. return ret;
  1233. tdma->reg_gen = tdma_read(tdma, TEGRA_APBDMA_GENERAL);
  1234. for (i = 0; i < tdma->chip_data->nr_channels; i++) {
  1235. struct tegra_dma_channel *tdc = &tdma->channels[i];
  1236. struct tegra_dma_channel_regs *ch_reg = &tdc->channel_reg;
  1237. ch_reg->csr = tdc_read(tdc, TEGRA_APBDMA_CHAN_CSR);
  1238. ch_reg->ahb_ptr = tdc_read(tdc, TEGRA_APBDMA_CHAN_AHBPTR);
  1239. ch_reg->apb_ptr = tdc_read(tdc, TEGRA_APBDMA_CHAN_APBPTR);
  1240. ch_reg->ahb_seq = tdc_read(tdc, TEGRA_APBDMA_CHAN_AHBSEQ);
  1241. ch_reg->apb_seq = tdc_read(tdc, TEGRA_APBDMA_CHAN_APBSEQ);
  1242. }
  1243. /* Disable clock */
  1244. tegra_dma_runtime_suspend(dev);
  1245. return 0;
  1246. }
  1247. static int tegra_dma_pm_resume(struct device *dev)
  1248. {
  1249. struct tegra_dma *tdma = dev_get_drvdata(dev);
  1250. int i;
  1251. int ret;
  1252. /* Enable clock before accessing register */
  1253. ret = tegra_dma_runtime_resume(dev);
  1254. if (ret < 0)
  1255. return ret;
  1256. tdma_write(tdma, TEGRA_APBDMA_GENERAL, tdma->reg_gen);
  1257. tdma_write(tdma, TEGRA_APBDMA_CONTROL, 0);
  1258. tdma_write(tdma, TEGRA_APBDMA_IRQ_MASK_SET, 0xFFFFFFFFul);
  1259. for (i = 0; i < tdma->chip_data->nr_channels; i++) {
  1260. struct tegra_dma_channel *tdc = &tdma->channels[i];
  1261. struct tegra_dma_channel_regs *ch_reg = &tdc->channel_reg;
  1262. tdc_write(tdc, TEGRA_APBDMA_CHAN_APBSEQ, ch_reg->apb_seq);
  1263. tdc_write(tdc, TEGRA_APBDMA_CHAN_APBPTR, ch_reg->apb_ptr);
  1264. tdc_write(tdc, TEGRA_APBDMA_CHAN_AHBSEQ, ch_reg->ahb_seq);
  1265. tdc_write(tdc, TEGRA_APBDMA_CHAN_AHBPTR, ch_reg->ahb_ptr);
  1266. tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR,
  1267. (ch_reg->csr & ~TEGRA_APBDMA_CSR_ENB));
  1268. }
  1269. /* Disable clock */
  1270. tegra_dma_runtime_suspend(dev);
  1271. return 0;
  1272. }
  1273. #endif
  1274. static const struct dev_pm_ops tegra_dma_dev_pm_ops = {
  1275. #ifdef CONFIG_PM_RUNTIME
  1276. .runtime_suspend = tegra_dma_runtime_suspend,
  1277. .runtime_resume = tegra_dma_runtime_resume,
  1278. #endif
  1279. SET_SYSTEM_SLEEP_PM_OPS(tegra_dma_pm_suspend, tegra_dma_pm_resume)
  1280. };
  1281. static struct platform_driver tegra_dmac_driver = {
  1282. .driver = {
  1283. .name = "tegra-apbdma",
  1284. .owner = THIS_MODULE,
  1285. .pm = &tegra_dma_dev_pm_ops,
  1286. .of_match_table = tegra_dma_of_match,
  1287. },
  1288. .probe = tegra_dma_probe,
  1289. .remove = tegra_dma_remove,
  1290. };
  1291. module_platform_driver(tegra_dmac_driver);
  1292. MODULE_ALIAS("platform:tegra20-apbdma");
  1293. MODULE_DESCRIPTION("NVIDIA Tegra APB DMA Controller driver");
  1294. MODULE_AUTHOR("Laxman Dewangan <ldewangan@nvidia.com>");
  1295. MODULE_LICENSE("GPL v2");