ste_dma40.c 83 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327
  1. /*
  2. * Copyright (C) Ericsson AB 2007-2008
  3. * Copyright (C) ST-Ericsson SA 2008-2010
  4. * Author: Per Forlin <per.forlin@stericsson.com> for ST-Ericsson
  5. * Author: Jonas Aaberg <jonas.aberg@stericsson.com> for ST-Ericsson
  6. * License terms: GNU General Public License (GPL) version 2
  7. */
  8. #include <linux/dma-mapping.h>
  9. #include <linux/kernel.h>
  10. #include <linux/slab.h>
  11. #include <linux/export.h>
  12. #include <linux/dmaengine.h>
  13. #include <linux/platform_device.h>
  14. #include <linux/clk.h>
  15. #include <linux/delay.h>
  16. #include <linux/pm.h>
  17. #include <linux/pm_runtime.h>
  18. #include <linux/err.h>
  19. #include <linux/amba/bus.h>
  20. #include <plat/ste_dma40.h>
  21. #include "ste_dma40_ll.h"
  22. #define D40_NAME "dma40"
  23. #define D40_PHY_CHAN -1
  24. /* For masking out/in 2 bit channel positions */
  25. #define D40_CHAN_POS(chan) (2 * (chan / 2))
  26. #define D40_CHAN_POS_MASK(chan) (0x3 << D40_CHAN_POS(chan))
  27. /* Maximum iterations taken before giving up suspending a channel */
  28. #define D40_SUSPEND_MAX_IT 500
  29. /* Milliseconds */
  30. #define DMA40_AUTOSUSPEND_DELAY 100
  31. /* Hardware requirement on LCLA alignment */
  32. #define LCLA_ALIGNMENT 0x40000
  33. /* Max number of links per event group */
  34. #define D40_LCLA_LINK_PER_EVENT_GRP 128
  35. #define D40_LCLA_END D40_LCLA_LINK_PER_EVENT_GRP
  36. /* Attempts before giving up to trying to get pages that are aligned */
  37. #define MAX_LCLA_ALLOC_ATTEMPTS 256
  38. /* Bit markings for allocation map */
  39. #define D40_ALLOC_FREE (1 << 31)
  40. #define D40_ALLOC_PHY (1 << 30)
  41. #define D40_ALLOC_LOG_FREE 0
  42. /**
  43. * enum 40_command - The different commands and/or statuses.
  44. *
  45. * @D40_DMA_STOP: DMA channel command STOP or status STOPPED,
  46. * @D40_DMA_RUN: The DMA channel is RUNNING of the command RUN.
  47. * @D40_DMA_SUSPEND_REQ: Request the DMA to SUSPEND as soon as possible.
  48. * @D40_DMA_SUSPENDED: The DMA channel is SUSPENDED.
  49. */
  50. enum d40_command {
  51. D40_DMA_STOP = 0,
  52. D40_DMA_RUN = 1,
  53. D40_DMA_SUSPEND_REQ = 2,
  54. D40_DMA_SUSPENDED = 3
  55. };
  56. /*
  57. * These are the registers that has to be saved and later restored
  58. * when the DMA hw is powered off.
  59. * TODO: Add save/restore of D40_DREG_GCC on dma40 v3 or later, if that works.
  60. */
  61. static u32 d40_backup_regs[] = {
  62. D40_DREG_LCPA,
  63. D40_DREG_LCLA,
  64. D40_DREG_PRMSE,
  65. D40_DREG_PRMSO,
  66. D40_DREG_PRMOE,
  67. D40_DREG_PRMOO,
  68. };
  69. #define BACKUP_REGS_SZ ARRAY_SIZE(d40_backup_regs)
  70. /* TODO: Check if all these registers have to be saved/restored on dma40 v3 */
  71. static u32 d40_backup_regs_v3[] = {
  72. D40_DREG_PSEG1,
  73. D40_DREG_PSEG2,
  74. D40_DREG_PSEG3,
  75. D40_DREG_PSEG4,
  76. D40_DREG_PCEG1,
  77. D40_DREG_PCEG2,
  78. D40_DREG_PCEG3,
  79. D40_DREG_PCEG4,
  80. D40_DREG_RSEG1,
  81. D40_DREG_RSEG2,
  82. D40_DREG_RSEG3,
  83. D40_DREG_RSEG4,
  84. D40_DREG_RCEG1,
  85. D40_DREG_RCEG2,
  86. D40_DREG_RCEG3,
  87. D40_DREG_RCEG4,
  88. };
  89. #define BACKUP_REGS_SZ_V3 ARRAY_SIZE(d40_backup_regs_v3)
  90. static u32 d40_backup_regs_chan[] = {
  91. D40_CHAN_REG_SSCFG,
  92. D40_CHAN_REG_SSELT,
  93. D40_CHAN_REG_SSPTR,
  94. D40_CHAN_REG_SSLNK,
  95. D40_CHAN_REG_SDCFG,
  96. D40_CHAN_REG_SDELT,
  97. D40_CHAN_REG_SDPTR,
  98. D40_CHAN_REG_SDLNK,
  99. };
  100. /**
  101. * struct d40_lli_pool - Structure for keeping LLIs in memory
  102. *
  103. * @base: Pointer to memory area when the pre_alloc_lli's are not large
  104. * enough, IE bigger than the most common case, 1 dst and 1 src. NULL if
  105. * pre_alloc_lli is used.
  106. * @dma_addr: DMA address, if mapped
  107. * @size: The size in bytes of the memory at base or the size of pre_alloc_lli.
  108. * @pre_alloc_lli: Pre allocated area for the most common case of transfers,
  109. * one buffer to one buffer.
  110. */
  111. struct d40_lli_pool {
  112. void *base;
  113. int size;
  114. dma_addr_t dma_addr;
  115. /* Space for dst and src, plus an extra for padding */
  116. u8 pre_alloc_lli[3 * sizeof(struct d40_phy_lli)];
  117. };
  118. /**
  119. * struct d40_desc - A descriptor is one DMA job.
  120. *
  121. * @lli_phy: LLI settings for physical channel. Both src and dst=
  122. * points into the lli_pool, to base if lli_len > 1 or to pre_alloc_lli if
  123. * lli_len equals one.
  124. * @lli_log: Same as above but for logical channels.
  125. * @lli_pool: The pool with two entries pre-allocated.
  126. * @lli_len: Number of llis of current descriptor.
  127. * @lli_current: Number of transferred llis.
  128. * @lcla_alloc: Number of LCLA entries allocated.
  129. * @txd: DMA engine struct. Used for among other things for communication
  130. * during a transfer.
  131. * @node: List entry.
  132. * @is_in_client_list: true if the client owns this descriptor.
  133. * @cyclic: true if this is a cyclic job
  134. *
  135. * This descriptor is used for both logical and physical transfers.
  136. */
  137. struct d40_desc {
  138. /* LLI physical */
  139. struct d40_phy_lli_bidir lli_phy;
  140. /* LLI logical */
  141. struct d40_log_lli_bidir lli_log;
  142. struct d40_lli_pool lli_pool;
  143. int lli_len;
  144. int lli_current;
  145. int lcla_alloc;
  146. struct dma_async_tx_descriptor txd;
  147. struct list_head node;
  148. bool is_in_client_list;
  149. bool cyclic;
  150. };
  151. /**
  152. * struct d40_lcla_pool - LCLA pool settings and data.
  153. *
  154. * @base: The virtual address of LCLA. 18 bit aligned.
  155. * @base_unaligned: The orignal kmalloc pointer, if kmalloc is used.
  156. * This pointer is only there for clean-up on error.
  157. * @pages: The number of pages needed for all physical channels.
  158. * Only used later for clean-up on error
  159. * @lock: Lock to protect the content in this struct.
  160. * @alloc_map: big map over which LCLA entry is own by which job.
  161. */
  162. struct d40_lcla_pool {
  163. void *base;
  164. dma_addr_t dma_addr;
  165. void *base_unaligned;
  166. int pages;
  167. spinlock_t lock;
  168. struct d40_desc **alloc_map;
  169. };
  170. /**
  171. * struct d40_phy_res - struct for handling eventlines mapped to physical
  172. * channels.
  173. *
  174. * @lock: A lock protection this entity.
  175. * @reserved: True if used by secure world or otherwise.
  176. * @num: The physical channel number of this entity.
  177. * @allocated_src: Bit mapped to show which src event line's are mapped to
  178. * this physical channel. Can also be free or physically allocated.
  179. * @allocated_dst: Same as for src but is dst.
  180. * allocated_dst and allocated_src uses the D40_ALLOC* defines as well as
  181. * event line number.
  182. */
  183. struct d40_phy_res {
  184. spinlock_t lock;
  185. bool reserved;
  186. int num;
  187. u32 allocated_src;
  188. u32 allocated_dst;
  189. };
  190. struct d40_base;
  191. /**
  192. * struct d40_chan - Struct that describes a channel.
  193. *
  194. * @lock: A spinlock to protect this struct.
  195. * @log_num: The logical number, if any of this channel.
  196. * @completed: Starts with 1, after first interrupt it is set to dma engine's
  197. * current cookie.
  198. * @pending_tx: The number of pending transfers. Used between interrupt handler
  199. * and tasklet.
  200. * @busy: Set to true when transfer is ongoing on this channel.
  201. * @phy_chan: Pointer to physical channel which this instance runs on. If this
  202. * point is NULL, then the channel is not allocated.
  203. * @chan: DMA engine handle.
  204. * @tasklet: Tasklet that gets scheduled from interrupt context to complete a
  205. * transfer and call client callback.
  206. * @client: Cliented owned descriptor list.
  207. * @pending_queue: Submitted jobs, to be issued by issue_pending()
  208. * @active: Active descriptor.
  209. * @queue: Queued jobs.
  210. * @prepare_queue: Prepared jobs.
  211. * @dma_cfg: The client configuration of this dma channel.
  212. * @configured: whether the dma_cfg configuration is valid
  213. * @base: Pointer to the device instance struct.
  214. * @src_def_cfg: Default cfg register setting for src.
  215. * @dst_def_cfg: Default cfg register setting for dst.
  216. * @log_def: Default logical channel settings.
  217. * @lcpa: Pointer to dst and src lcpa settings.
  218. * @runtime_addr: runtime configured address.
  219. * @runtime_direction: runtime configured direction.
  220. *
  221. * This struct can either "be" a logical or a physical channel.
  222. */
  223. struct d40_chan {
  224. spinlock_t lock;
  225. int log_num;
  226. /* ID of the most recent completed transfer */
  227. int completed;
  228. int pending_tx;
  229. bool busy;
  230. struct d40_phy_res *phy_chan;
  231. struct dma_chan chan;
  232. struct tasklet_struct tasklet;
  233. struct list_head client;
  234. struct list_head pending_queue;
  235. struct list_head active;
  236. struct list_head queue;
  237. struct list_head prepare_queue;
  238. struct stedma40_chan_cfg dma_cfg;
  239. bool configured;
  240. struct d40_base *base;
  241. /* Default register configurations */
  242. u32 src_def_cfg;
  243. u32 dst_def_cfg;
  244. struct d40_def_lcsp log_def;
  245. struct d40_log_lli_full *lcpa;
  246. /* Runtime reconfiguration */
  247. dma_addr_t runtime_addr;
  248. enum dma_transfer_direction runtime_direction;
  249. };
  250. /**
  251. * struct d40_base - The big global struct, one for each probe'd instance.
  252. *
  253. * @interrupt_lock: Lock used to make sure one interrupt is handle a time.
  254. * @execmd_lock: Lock for execute command usage since several channels share
  255. * the same physical register.
  256. * @dev: The device structure.
  257. * @virtbase: The virtual base address of the DMA's register.
  258. * @rev: silicon revision detected.
  259. * @clk: Pointer to the DMA clock structure.
  260. * @phy_start: Physical memory start of the DMA registers.
  261. * @phy_size: Size of the DMA register map.
  262. * @irq: The IRQ number.
  263. * @num_phy_chans: The number of physical channels. Read from HW. This
  264. * is the number of available channels for this driver, not counting "Secure
  265. * mode" allocated physical channels.
  266. * @num_log_chans: The number of logical channels. Calculated from
  267. * num_phy_chans.
  268. * @dma_both: dma_device channels that can do both memcpy and slave transfers.
  269. * @dma_slave: dma_device channels that can do only do slave transfers.
  270. * @dma_memcpy: dma_device channels that can do only do memcpy transfers.
  271. * @phy_chans: Room for all possible physical channels in system.
  272. * @log_chans: Room for all possible logical channels in system.
  273. * @lookup_log_chans: Used to map interrupt number to logical channel. Points
  274. * to log_chans entries.
  275. * @lookup_phy_chans: Used to map interrupt number to physical channel. Points
  276. * to phy_chans entries.
  277. * @plat_data: Pointer to provided platform_data which is the driver
  278. * configuration.
  279. * @lcpa_regulator: Pointer to hold the regulator for the esram bank for lcla.
  280. * @phy_res: Vector containing all physical channels.
  281. * @lcla_pool: lcla pool settings and data.
  282. * @lcpa_base: The virtual mapped address of LCPA.
  283. * @phy_lcpa: The physical address of the LCPA.
  284. * @lcpa_size: The size of the LCPA area.
  285. * @desc_slab: cache for descriptors.
  286. * @reg_val_backup: Here the values of some hardware registers are stored
  287. * before the DMA is powered off. They are restored when the power is back on.
  288. * @reg_val_backup_v3: Backup of registers that only exits on dma40 v3 and
  289. * later.
  290. * @reg_val_backup_chan: Backup data for standard channel parameter registers.
  291. * @gcc_pwr_off_mask: Mask to maintain the channels that can be turned off.
  292. * @initialized: true if the dma has been initialized
  293. */
  294. struct d40_base {
  295. spinlock_t interrupt_lock;
  296. spinlock_t execmd_lock;
  297. struct device *dev;
  298. void __iomem *virtbase;
  299. u8 rev:4;
  300. struct clk *clk;
  301. phys_addr_t phy_start;
  302. resource_size_t phy_size;
  303. int irq;
  304. int num_phy_chans;
  305. int num_log_chans;
  306. struct dma_device dma_both;
  307. struct dma_device dma_slave;
  308. struct dma_device dma_memcpy;
  309. struct d40_chan *phy_chans;
  310. struct d40_chan *log_chans;
  311. struct d40_chan **lookup_log_chans;
  312. struct d40_chan **lookup_phy_chans;
  313. struct stedma40_platform_data *plat_data;
  314. struct regulator *lcpa_regulator;
  315. /* Physical half channels */
  316. struct d40_phy_res *phy_res;
  317. struct d40_lcla_pool lcla_pool;
  318. void *lcpa_base;
  319. dma_addr_t phy_lcpa;
  320. resource_size_t lcpa_size;
  321. struct kmem_cache *desc_slab;
  322. u32 reg_val_backup[BACKUP_REGS_SZ];
  323. u32 reg_val_backup_v3[BACKUP_REGS_SZ_V3];
  324. u32 *reg_val_backup_chan;
  325. u16 gcc_pwr_off_mask;
  326. bool initialized;
  327. };
  328. /**
  329. * struct d40_interrupt_lookup - lookup table for interrupt handler
  330. *
  331. * @src: Interrupt mask register.
  332. * @clr: Interrupt clear register.
  333. * @is_error: true if this is an error interrupt.
  334. * @offset: start delta in the lookup_log_chans in d40_base. If equals to
  335. * D40_PHY_CHAN, the lookup_phy_chans shall be used instead.
  336. */
  337. struct d40_interrupt_lookup {
  338. u32 src;
  339. u32 clr;
  340. bool is_error;
  341. int offset;
  342. };
  343. /**
  344. * struct d40_reg_val - simple lookup struct
  345. *
  346. * @reg: The register.
  347. * @val: The value that belongs to the register in reg.
  348. */
  349. struct d40_reg_val {
  350. unsigned int reg;
  351. unsigned int val;
  352. };
  353. static struct device *chan2dev(struct d40_chan *d40c)
  354. {
  355. return &d40c->chan.dev->device;
  356. }
  357. static bool chan_is_physical(struct d40_chan *chan)
  358. {
  359. return chan->log_num == D40_PHY_CHAN;
  360. }
  361. static bool chan_is_logical(struct d40_chan *chan)
  362. {
  363. return !chan_is_physical(chan);
  364. }
  365. static void __iomem *chan_base(struct d40_chan *chan)
  366. {
  367. return chan->base->virtbase + D40_DREG_PCBASE +
  368. chan->phy_chan->num * D40_DREG_PCDELTA;
  369. }
  370. #define d40_err(dev, format, arg...) \
  371. dev_err(dev, "[%s] " format, __func__, ## arg)
  372. #define chan_err(d40c, format, arg...) \
  373. d40_err(chan2dev(d40c), format, ## arg)
  374. static int d40_pool_lli_alloc(struct d40_chan *d40c, struct d40_desc *d40d,
  375. int lli_len)
  376. {
  377. bool is_log = chan_is_logical(d40c);
  378. u32 align;
  379. void *base;
  380. if (is_log)
  381. align = sizeof(struct d40_log_lli);
  382. else
  383. align = sizeof(struct d40_phy_lli);
  384. if (lli_len == 1) {
  385. base = d40d->lli_pool.pre_alloc_lli;
  386. d40d->lli_pool.size = sizeof(d40d->lli_pool.pre_alloc_lli);
  387. d40d->lli_pool.base = NULL;
  388. } else {
  389. d40d->lli_pool.size = lli_len * 2 * align;
  390. base = kmalloc(d40d->lli_pool.size + align, GFP_NOWAIT);
  391. d40d->lli_pool.base = base;
  392. if (d40d->lli_pool.base == NULL)
  393. return -ENOMEM;
  394. }
  395. if (is_log) {
  396. d40d->lli_log.src = PTR_ALIGN(base, align);
  397. d40d->lli_log.dst = d40d->lli_log.src + lli_len;
  398. d40d->lli_pool.dma_addr = 0;
  399. } else {
  400. d40d->lli_phy.src = PTR_ALIGN(base, align);
  401. d40d->lli_phy.dst = d40d->lli_phy.src + lli_len;
  402. d40d->lli_pool.dma_addr = dma_map_single(d40c->base->dev,
  403. d40d->lli_phy.src,
  404. d40d->lli_pool.size,
  405. DMA_TO_DEVICE);
  406. if (dma_mapping_error(d40c->base->dev,
  407. d40d->lli_pool.dma_addr)) {
  408. kfree(d40d->lli_pool.base);
  409. d40d->lli_pool.base = NULL;
  410. d40d->lli_pool.dma_addr = 0;
  411. return -ENOMEM;
  412. }
  413. }
  414. return 0;
  415. }
  416. static void d40_pool_lli_free(struct d40_chan *d40c, struct d40_desc *d40d)
  417. {
  418. if (d40d->lli_pool.dma_addr)
  419. dma_unmap_single(d40c->base->dev, d40d->lli_pool.dma_addr,
  420. d40d->lli_pool.size, DMA_TO_DEVICE);
  421. kfree(d40d->lli_pool.base);
  422. d40d->lli_pool.base = NULL;
  423. d40d->lli_pool.size = 0;
  424. d40d->lli_log.src = NULL;
  425. d40d->lli_log.dst = NULL;
  426. d40d->lli_phy.src = NULL;
  427. d40d->lli_phy.dst = NULL;
  428. }
  429. static int d40_lcla_alloc_one(struct d40_chan *d40c,
  430. struct d40_desc *d40d)
  431. {
  432. unsigned long flags;
  433. int i;
  434. int ret = -EINVAL;
  435. int p;
  436. spin_lock_irqsave(&d40c->base->lcla_pool.lock, flags);
  437. p = d40c->phy_chan->num * D40_LCLA_LINK_PER_EVENT_GRP;
  438. /*
  439. * Allocate both src and dst at the same time, therefore the half
  440. * start on 1 since 0 can't be used since zero is used as end marker.
  441. */
  442. for (i = 1 ; i < D40_LCLA_LINK_PER_EVENT_GRP / 2; i++) {
  443. if (!d40c->base->lcla_pool.alloc_map[p + i]) {
  444. d40c->base->lcla_pool.alloc_map[p + i] = d40d;
  445. d40d->lcla_alloc++;
  446. ret = i;
  447. break;
  448. }
  449. }
  450. spin_unlock_irqrestore(&d40c->base->lcla_pool.lock, flags);
  451. return ret;
  452. }
  453. static int d40_lcla_free_all(struct d40_chan *d40c,
  454. struct d40_desc *d40d)
  455. {
  456. unsigned long flags;
  457. int i;
  458. int ret = -EINVAL;
  459. if (chan_is_physical(d40c))
  460. return 0;
  461. spin_lock_irqsave(&d40c->base->lcla_pool.lock, flags);
  462. for (i = 1 ; i < D40_LCLA_LINK_PER_EVENT_GRP / 2; i++) {
  463. if (d40c->base->lcla_pool.alloc_map[d40c->phy_chan->num *
  464. D40_LCLA_LINK_PER_EVENT_GRP + i] == d40d) {
  465. d40c->base->lcla_pool.alloc_map[d40c->phy_chan->num *
  466. D40_LCLA_LINK_PER_EVENT_GRP + i] = NULL;
  467. d40d->lcla_alloc--;
  468. if (d40d->lcla_alloc == 0) {
  469. ret = 0;
  470. break;
  471. }
  472. }
  473. }
  474. spin_unlock_irqrestore(&d40c->base->lcla_pool.lock, flags);
  475. return ret;
  476. }
  477. static void d40_desc_remove(struct d40_desc *d40d)
  478. {
  479. list_del(&d40d->node);
  480. }
  481. static struct d40_desc *d40_desc_get(struct d40_chan *d40c)
  482. {
  483. struct d40_desc *desc = NULL;
  484. if (!list_empty(&d40c->client)) {
  485. struct d40_desc *d;
  486. struct d40_desc *_d;
  487. list_for_each_entry_safe(d, _d, &d40c->client, node) {
  488. if (async_tx_test_ack(&d->txd)) {
  489. d40_desc_remove(d);
  490. desc = d;
  491. memset(desc, 0, sizeof(*desc));
  492. break;
  493. }
  494. }
  495. }
  496. if (!desc)
  497. desc = kmem_cache_zalloc(d40c->base->desc_slab, GFP_NOWAIT);
  498. if (desc)
  499. INIT_LIST_HEAD(&desc->node);
  500. return desc;
  501. }
  502. static void d40_desc_free(struct d40_chan *d40c, struct d40_desc *d40d)
  503. {
  504. d40_pool_lli_free(d40c, d40d);
  505. d40_lcla_free_all(d40c, d40d);
  506. kmem_cache_free(d40c->base->desc_slab, d40d);
  507. }
  508. static void d40_desc_submit(struct d40_chan *d40c, struct d40_desc *desc)
  509. {
  510. list_add_tail(&desc->node, &d40c->active);
  511. }
  512. static void d40_phy_lli_load(struct d40_chan *chan, struct d40_desc *desc)
  513. {
  514. struct d40_phy_lli *lli_dst = desc->lli_phy.dst;
  515. struct d40_phy_lli *lli_src = desc->lli_phy.src;
  516. void __iomem *base = chan_base(chan);
  517. writel(lli_src->reg_cfg, base + D40_CHAN_REG_SSCFG);
  518. writel(lli_src->reg_elt, base + D40_CHAN_REG_SSELT);
  519. writel(lli_src->reg_ptr, base + D40_CHAN_REG_SSPTR);
  520. writel(lli_src->reg_lnk, base + D40_CHAN_REG_SSLNK);
  521. writel(lli_dst->reg_cfg, base + D40_CHAN_REG_SDCFG);
  522. writel(lli_dst->reg_elt, base + D40_CHAN_REG_SDELT);
  523. writel(lli_dst->reg_ptr, base + D40_CHAN_REG_SDPTR);
  524. writel(lli_dst->reg_lnk, base + D40_CHAN_REG_SDLNK);
  525. }
  526. static void d40_log_lli_to_lcxa(struct d40_chan *chan, struct d40_desc *desc)
  527. {
  528. struct d40_lcla_pool *pool = &chan->base->lcla_pool;
  529. struct d40_log_lli_bidir *lli = &desc->lli_log;
  530. int lli_current = desc->lli_current;
  531. int lli_len = desc->lli_len;
  532. bool cyclic = desc->cyclic;
  533. int curr_lcla = -EINVAL;
  534. int first_lcla = 0;
  535. bool use_esram_lcla = chan->base->plat_data->use_esram_lcla;
  536. bool linkback;
  537. /*
  538. * We may have partially running cyclic transfers, in case we did't get
  539. * enough LCLA entries.
  540. */
  541. linkback = cyclic && lli_current == 0;
  542. /*
  543. * For linkback, we need one LCLA even with only one link, because we
  544. * can't link back to the one in LCPA space
  545. */
  546. if (linkback || (lli_len - lli_current > 1)) {
  547. curr_lcla = d40_lcla_alloc_one(chan, desc);
  548. first_lcla = curr_lcla;
  549. }
  550. /*
  551. * For linkback, we normally load the LCPA in the loop since we need to
  552. * link it to the second LCLA and not the first. However, if we
  553. * couldn't even get a first LCLA, then we have to run in LCPA and
  554. * reload manually.
  555. */
  556. if (!linkback || curr_lcla == -EINVAL) {
  557. unsigned int flags = 0;
  558. if (curr_lcla == -EINVAL)
  559. flags |= LLI_TERM_INT;
  560. d40_log_lli_lcpa_write(chan->lcpa,
  561. &lli->dst[lli_current],
  562. &lli->src[lli_current],
  563. curr_lcla,
  564. flags);
  565. lli_current++;
  566. }
  567. if (curr_lcla < 0)
  568. goto out;
  569. for (; lli_current < lli_len; lli_current++) {
  570. unsigned int lcla_offset = chan->phy_chan->num * 1024 +
  571. 8 * curr_lcla * 2;
  572. struct d40_log_lli *lcla = pool->base + lcla_offset;
  573. unsigned int flags = 0;
  574. int next_lcla;
  575. if (lli_current + 1 < lli_len)
  576. next_lcla = d40_lcla_alloc_one(chan, desc);
  577. else
  578. next_lcla = linkback ? first_lcla : -EINVAL;
  579. if (cyclic || next_lcla == -EINVAL)
  580. flags |= LLI_TERM_INT;
  581. if (linkback && curr_lcla == first_lcla) {
  582. /* First link goes in both LCPA and LCLA */
  583. d40_log_lli_lcpa_write(chan->lcpa,
  584. &lli->dst[lli_current],
  585. &lli->src[lli_current],
  586. next_lcla, flags);
  587. }
  588. /*
  589. * One unused LCLA in the cyclic case if the very first
  590. * next_lcla fails...
  591. */
  592. d40_log_lli_lcla_write(lcla,
  593. &lli->dst[lli_current],
  594. &lli->src[lli_current],
  595. next_lcla, flags);
  596. /*
  597. * Cache maintenance is not needed if lcla is
  598. * mapped in esram
  599. */
  600. if (!use_esram_lcla) {
  601. dma_sync_single_range_for_device(chan->base->dev,
  602. pool->dma_addr, lcla_offset,
  603. 2 * sizeof(struct d40_log_lli),
  604. DMA_TO_DEVICE);
  605. }
  606. curr_lcla = next_lcla;
  607. if (curr_lcla == -EINVAL || curr_lcla == first_lcla) {
  608. lli_current++;
  609. break;
  610. }
  611. }
  612. out:
  613. desc->lli_current = lli_current;
  614. }
  615. static void d40_desc_load(struct d40_chan *d40c, struct d40_desc *d40d)
  616. {
  617. if (chan_is_physical(d40c)) {
  618. d40_phy_lli_load(d40c, d40d);
  619. d40d->lli_current = d40d->lli_len;
  620. } else
  621. d40_log_lli_to_lcxa(d40c, d40d);
  622. }
  623. static struct d40_desc *d40_first_active_get(struct d40_chan *d40c)
  624. {
  625. struct d40_desc *d;
  626. if (list_empty(&d40c->active))
  627. return NULL;
  628. d = list_first_entry(&d40c->active,
  629. struct d40_desc,
  630. node);
  631. return d;
  632. }
  633. /* remove desc from current queue and add it to the pending_queue */
  634. static void d40_desc_queue(struct d40_chan *d40c, struct d40_desc *desc)
  635. {
  636. d40_desc_remove(desc);
  637. desc->is_in_client_list = false;
  638. list_add_tail(&desc->node, &d40c->pending_queue);
  639. }
  640. static struct d40_desc *d40_first_pending(struct d40_chan *d40c)
  641. {
  642. struct d40_desc *d;
  643. if (list_empty(&d40c->pending_queue))
  644. return NULL;
  645. d = list_first_entry(&d40c->pending_queue,
  646. struct d40_desc,
  647. node);
  648. return d;
  649. }
  650. static struct d40_desc *d40_first_queued(struct d40_chan *d40c)
  651. {
  652. struct d40_desc *d;
  653. if (list_empty(&d40c->queue))
  654. return NULL;
  655. d = list_first_entry(&d40c->queue,
  656. struct d40_desc,
  657. node);
  658. return d;
  659. }
  660. static int d40_psize_2_burst_size(bool is_log, int psize)
  661. {
  662. if (is_log) {
  663. if (psize == STEDMA40_PSIZE_LOG_1)
  664. return 1;
  665. } else {
  666. if (psize == STEDMA40_PSIZE_PHY_1)
  667. return 1;
  668. }
  669. return 2 << psize;
  670. }
  671. /*
  672. * The dma only supports transmitting packages up to
  673. * STEDMA40_MAX_SEG_SIZE << data_width. Calculate the total number of
  674. * dma elements required to send the entire sg list
  675. */
  676. static int d40_size_2_dmalen(int size, u32 data_width1, u32 data_width2)
  677. {
  678. int dmalen;
  679. u32 max_w = max(data_width1, data_width2);
  680. u32 min_w = min(data_width1, data_width2);
  681. u32 seg_max = ALIGN(STEDMA40_MAX_SEG_SIZE << min_w, 1 << max_w);
  682. if (seg_max > STEDMA40_MAX_SEG_SIZE)
  683. seg_max -= (1 << max_w);
  684. if (!IS_ALIGNED(size, 1 << max_w))
  685. return -EINVAL;
  686. if (size <= seg_max)
  687. dmalen = 1;
  688. else {
  689. dmalen = size / seg_max;
  690. if (dmalen * seg_max < size)
  691. dmalen++;
  692. }
  693. return dmalen;
  694. }
  695. static int d40_sg_2_dmalen(struct scatterlist *sgl, int sg_len,
  696. u32 data_width1, u32 data_width2)
  697. {
  698. struct scatterlist *sg;
  699. int i;
  700. int len = 0;
  701. int ret;
  702. for_each_sg(sgl, sg, sg_len, i) {
  703. ret = d40_size_2_dmalen(sg_dma_len(sg),
  704. data_width1, data_width2);
  705. if (ret < 0)
  706. return ret;
  707. len += ret;
  708. }
  709. return len;
  710. }
  711. #ifdef CONFIG_PM
  712. static void dma40_backup(void __iomem *baseaddr, u32 *backup,
  713. u32 *regaddr, int num, bool save)
  714. {
  715. int i;
  716. for (i = 0; i < num; i++) {
  717. void __iomem *addr = baseaddr + regaddr[i];
  718. if (save)
  719. backup[i] = readl_relaxed(addr);
  720. else
  721. writel_relaxed(backup[i], addr);
  722. }
  723. }
  724. static void d40_save_restore_registers(struct d40_base *base, bool save)
  725. {
  726. int i;
  727. /* Save/Restore channel specific registers */
  728. for (i = 0; i < base->num_phy_chans; i++) {
  729. void __iomem *addr;
  730. int idx;
  731. if (base->phy_res[i].reserved)
  732. continue;
  733. addr = base->virtbase + D40_DREG_PCBASE + i * D40_DREG_PCDELTA;
  734. idx = i * ARRAY_SIZE(d40_backup_regs_chan);
  735. dma40_backup(addr, &base->reg_val_backup_chan[idx],
  736. d40_backup_regs_chan,
  737. ARRAY_SIZE(d40_backup_regs_chan),
  738. save);
  739. }
  740. /* Save/Restore global registers */
  741. dma40_backup(base->virtbase, base->reg_val_backup,
  742. d40_backup_regs, ARRAY_SIZE(d40_backup_regs),
  743. save);
  744. /* Save/Restore registers only existing on dma40 v3 and later */
  745. if (base->rev >= 3)
  746. dma40_backup(base->virtbase, base->reg_val_backup_v3,
  747. d40_backup_regs_v3,
  748. ARRAY_SIZE(d40_backup_regs_v3),
  749. save);
  750. }
  751. #else
  752. static void d40_save_restore_registers(struct d40_base *base, bool save)
  753. {
  754. }
  755. #endif
  756. static int d40_channel_execute_command(struct d40_chan *d40c,
  757. enum d40_command command)
  758. {
  759. u32 status;
  760. int i;
  761. void __iomem *active_reg;
  762. int ret = 0;
  763. unsigned long flags;
  764. u32 wmask;
  765. spin_lock_irqsave(&d40c->base->execmd_lock, flags);
  766. if (d40c->phy_chan->num % 2 == 0)
  767. active_reg = d40c->base->virtbase + D40_DREG_ACTIVE;
  768. else
  769. active_reg = d40c->base->virtbase + D40_DREG_ACTIVO;
  770. if (command == D40_DMA_SUSPEND_REQ) {
  771. status = (readl(active_reg) &
  772. D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
  773. D40_CHAN_POS(d40c->phy_chan->num);
  774. if (status == D40_DMA_SUSPENDED || status == D40_DMA_STOP)
  775. goto done;
  776. }
  777. wmask = 0xffffffff & ~(D40_CHAN_POS_MASK(d40c->phy_chan->num));
  778. writel(wmask | (command << D40_CHAN_POS(d40c->phy_chan->num)),
  779. active_reg);
  780. if (command == D40_DMA_SUSPEND_REQ) {
  781. for (i = 0 ; i < D40_SUSPEND_MAX_IT; i++) {
  782. status = (readl(active_reg) &
  783. D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
  784. D40_CHAN_POS(d40c->phy_chan->num);
  785. cpu_relax();
  786. /*
  787. * Reduce the number of bus accesses while
  788. * waiting for the DMA to suspend.
  789. */
  790. udelay(3);
  791. if (status == D40_DMA_STOP ||
  792. status == D40_DMA_SUSPENDED)
  793. break;
  794. }
  795. if (i == D40_SUSPEND_MAX_IT) {
  796. chan_err(d40c,
  797. "unable to suspend the chl %d (log: %d) status %x\n",
  798. d40c->phy_chan->num, d40c->log_num,
  799. status);
  800. dump_stack();
  801. ret = -EBUSY;
  802. }
  803. }
  804. done:
  805. spin_unlock_irqrestore(&d40c->base->execmd_lock, flags);
  806. return ret;
  807. }
  808. static void d40_term_all(struct d40_chan *d40c)
  809. {
  810. struct d40_desc *d40d;
  811. struct d40_desc *_d;
  812. /* Release active descriptors */
  813. while ((d40d = d40_first_active_get(d40c))) {
  814. d40_desc_remove(d40d);
  815. d40_desc_free(d40c, d40d);
  816. }
  817. /* Release queued descriptors waiting for transfer */
  818. while ((d40d = d40_first_queued(d40c))) {
  819. d40_desc_remove(d40d);
  820. d40_desc_free(d40c, d40d);
  821. }
  822. /* Release pending descriptors */
  823. while ((d40d = d40_first_pending(d40c))) {
  824. d40_desc_remove(d40d);
  825. d40_desc_free(d40c, d40d);
  826. }
  827. /* Release client owned descriptors */
  828. if (!list_empty(&d40c->client))
  829. list_for_each_entry_safe(d40d, _d, &d40c->client, node) {
  830. d40_desc_remove(d40d);
  831. d40_desc_free(d40c, d40d);
  832. }
  833. /* Release descriptors in prepare queue */
  834. if (!list_empty(&d40c->prepare_queue))
  835. list_for_each_entry_safe(d40d, _d,
  836. &d40c->prepare_queue, node) {
  837. d40_desc_remove(d40d);
  838. d40_desc_free(d40c, d40d);
  839. }
  840. d40c->pending_tx = 0;
  841. d40c->busy = false;
  842. }
  843. static void __d40_config_set_event(struct d40_chan *d40c, bool enable,
  844. u32 event, int reg)
  845. {
  846. void __iomem *addr = chan_base(d40c) + reg;
  847. int tries;
  848. if (!enable) {
  849. writel((D40_DEACTIVATE_EVENTLINE << D40_EVENTLINE_POS(event))
  850. | ~D40_EVENTLINE_MASK(event), addr);
  851. return;
  852. }
  853. /*
  854. * The hardware sometimes doesn't register the enable when src and dst
  855. * event lines are active on the same logical channel. Retry to ensure
  856. * it does. Usually only one retry is sufficient.
  857. */
  858. tries = 100;
  859. while (--tries) {
  860. writel((D40_ACTIVATE_EVENTLINE << D40_EVENTLINE_POS(event))
  861. | ~D40_EVENTLINE_MASK(event), addr);
  862. if (readl(addr) & D40_EVENTLINE_MASK(event))
  863. break;
  864. }
  865. if (tries != 99)
  866. dev_dbg(chan2dev(d40c),
  867. "[%s] workaround enable S%cLNK (%d tries)\n",
  868. __func__, reg == D40_CHAN_REG_SSLNK ? 'S' : 'D',
  869. 100 - tries);
  870. WARN_ON(!tries);
  871. }
  872. static void d40_config_set_event(struct d40_chan *d40c, bool do_enable)
  873. {
  874. unsigned long flags;
  875. spin_lock_irqsave(&d40c->phy_chan->lock, flags);
  876. /* Enable event line connected to device (or memcpy) */
  877. if ((d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) ||
  878. (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH)) {
  879. u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type);
  880. __d40_config_set_event(d40c, do_enable, event,
  881. D40_CHAN_REG_SSLNK);
  882. }
  883. if (d40c->dma_cfg.dir != STEDMA40_PERIPH_TO_MEM) {
  884. u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type);
  885. __d40_config_set_event(d40c, do_enable, event,
  886. D40_CHAN_REG_SDLNK);
  887. }
  888. spin_unlock_irqrestore(&d40c->phy_chan->lock, flags);
  889. }
  890. static u32 d40_chan_has_events(struct d40_chan *d40c)
  891. {
  892. void __iomem *chanbase = chan_base(d40c);
  893. u32 val;
  894. val = readl(chanbase + D40_CHAN_REG_SSLNK);
  895. val |= readl(chanbase + D40_CHAN_REG_SDLNK);
  896. return val;
  897. }
  898. static u32 d40_get_prmo(struct d40_chan *d40c)
  899. {
  900. static const unsigned int phy_map[] = {
  901. [STEDMA40_PCHAN_BASIC_MODE]
  902. = D40_DREG_PRMO_PCHAN_BASIC,
  903. [STEDMA40_PCHAN_MODULO_MODE]
  904. = D40_DREG_PRMO_PCHAN_MODULO,
  905. [STEDMA40_PCHAN_DOUBLE_DST_MODE]
  906. = D40_DREG_PRMO_PCHAN_DOUBLE_DST,
  907. };
  908. static const unsigned int log_map[] = {
  909. [STEDMA40_LCHAN_SRC_PHY_DST_LOG]
  910. = D40_DREG_PRMO_LCHAN_SRC_PHY_DST_LOG,
  911. [STEDMA40_LCHAN_SRC_LOG_DST_PHY]
  912. = D40_DREG_PRMO_LCHAN_SRC_LOG_DST_PHY,
  913. [STEDMA40_LCHAN_SRC_LOG_DST_LOG]
  914. = D40_DREG_PRMO_LCHAN_SRC_LOG_DST_LOG,
  915. };
  916. if (chan_is_physical(d40c))
  917. return phy_map[d40c->dma_cfg.mode_opt];
  918. else
  919. return log_map[d40c->dma_cfg.mode_opt];
  920. }
  921. static void d40_config_write(struct d40_chan *d40c)
  922. {
  923. u32 addr_base;
  924. u32 var;
  925. /* Odd addresses are even addresses + 4 */
  926. addr_base = (d40c->phy_chan->num % 2) * 4;
  927. /* Setup channel mode to logical or physical */
  928. var = ((u32)(chan_is_logical(d40c)) + 1) <<
  929. D40_CHAN_POS(d40c->phy_chan->num);
  930. writel(var, d40c->base->virtbase + D40_DREG_PRMSE + addr_base);
  931. /* Setup operational mode option register */
  932. var = d40_get_prmo(d40c) << D40_CHAN_POS(d40c->phy_chan->num);
  933. writel(var, d40c->base->virtbase + D40_DREG_PRMOE + addr_base);
  934. if (chan_is_logical(d40c)) {
  935. int lidx = (d40c->phy_chan->num << D40_SREG_ELEM_LOG_LIDX_POS)
  936. & D40_SREG_ELEM_LOG_LIDX_MASK;
  937. void __iomem *chanbase = chan_base(d40c);
  938. /* Set default config for CFG reg */
  939. writel(d40c->src_def_cfg, chanbase + D40_CHAN_REG_SSCFG);
  940. writel(d40c->dst_def_cfg, chanbase + D40_CHAN_REG_SDCFG);
  941. /* Set LIDX for lcla */
  942. writel(lidx, chanbase + D40_CHAN_REG_SSELT);
  943. writel(lidx, chanbase + D40_CHAN_REG_SDELT);
  944. }
  945. }
  946. static u32 d40_residue(struct d40_chan *d40c)
  947. {
  948. u32 num_elt;
  949. if (chan_is_logical(d40c))
  950. num_elt = (readl(&d40c->lcpa->lcsp2) & D40_MEM_LCSP2_ECNT_MASK)
  951. >> D40_MEM_LCSP2_ECNT_POS;
  952. else {
  953. u32 val = readl(chan_base(d40c) + D40_CHAN_REG_SDELT);
  954. num_elt = (val & D40_SREG_ELEM_PHY_ECNT_MASK)
  955. >> D40_SREG_ELEM_PHY_ECNT_POS;
  956. }
  957. return num_elt * (1 << d40c->dma_cfg.dst_info.data_width);
  958. }
  959. static bool d40_tx_is_linked(struct d40_chan *d40c)
  960. {
  961. bool is_link;
  962. if (chan_is_logical(d40c))
  963. is_link = readl(&d40c->lcpa->lcsp3) & D40_MEM_LCSP3_DLOS_MASK;
  964. else
  965. is_link = readl(chan_base(d40c) + D40_CHAN_REG_SDLNK)
  966. & D40_SREG_LNK_PHYS_LNK_MASK;
  967. return is_link;
  968. }
  969. static int d40_pause(struct d40_chan *d40c)
  970. {
  971. int res = 0;
  972. unsigned long flags;
  973. if (!d40c->busy)
  974. return 0;
  975. pm_runtime_get_sync(d40c->base->dev);
  976. spin_lock_irqsave(&d40c->lock, flags);
  977. res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ);
  978. if (res == 0) {
  979. if (chan_is_logical(d40c)) {
  980. d40_config_set_event(d40c, false);
  981. /* Resume the other logical channels if any */
  982. if (d40_chan_has_events(d40c))
  983. res = d40_channel_execute_command(d40c,
  984. D40_DMA_RUN);
  985. }
  986. }
  987. pm_runtime_mark_last_busy(d40c->base->dev);
  988. pm_runtime_put_autosuspend(d40c->base->dev);
  989. spin_unlock_irqrestore(&d40c->lock, flags);
  990. return res;
  991. }
  992. static int d40_resume(struct d40_chan *d40c)
  993. {
  994. int res = 0;
  995. unsigned long flags;
  996. if (!d40c->busy)
  997. return 0;
  998. spin_lock_irqsave(&d40c->lock, flags);
  999. pm_runtime_get_sync(d40c->base->dev);
  1000. if (d40c->base->rev == 0)
  1001. if (chan_is_logical(d40c)) {
  1002. res = d40_channel_execute_command(d40c,
  1003. D40_DMA_SUSPEND_REQ);
  1004. goto no_suspend;
  1005. }
  1006. /* If bytes left to transfer or linked tx resume job */
  1007. if (d40_residue(d40c) || d40_tx_is_linked(d40c)) {
  1008. if (chan_is_logical(d40c))
  1009. d40_config_set_event(d40c, true);
  1010. res = d40_channel_execute_command(d40c, D40_DMA_RUN);
  1011. }
  1012. no_suspend:
  1013. pm_runtime_mark_last_busy(d40c->base->dev);
  1014. pm_runtime_put_autosuspend(d40c->base->dev);
  1015. spin_unlock_irqrestore(&d40c->lock, flags);
  1016. return res;
  1017. }
  1018. static int d40_terminate_all(struct d40_chan *chan)
  1019. {
  1020. unsigned long flags;
  1021. int ret = 0;
  1022. ret = d40_pause(chan);
  1023. if (!ret && chan_is_physical(chan))
  1024. ret = d40_channel_execute_command(chan, D40_DMA_STOP);
  1025. spin_lock_irqsave(&chan->lock, flags);
  1026. d40_term_all(chan);
  1027. spin_unlock_irqrestore(&chan->lock, flags);
  1028. return ret;
  1029. }
  1030. static dma_cookie_t d40_tx_submit(struct dma_async_tx_descriptor *tx)
  1031. {
  1032. struct d40_chan *d40c = container_of(tx->chan,
  1033. struct d40_chan,
  1034. chan);
  1035. struct d40_desc *d40d = container_of(tx, struct d40_desc, txd);
  1036. unsigned long flags;
  1037. spin_lock_irqsave(&d40c->lock, flags);
  1038. d40c->chan.cookie++;
  1039. if (d40c->chan.cookie < 0)
  1040. d40c->chan.cookie = 1;
  1041. d40d->txd.cookie = d40c->chan.cookie;
  1042. d40_desc_queue(d40c, d40d);
  1043. spin_unlock_irqrestore(&d40c->lock, flags);
  1044. return tx->cookie;
  1045. }
  1046. static int d40_start(struct d40_chan *d40c)
  1047. {
  1048. if (d40c->base->rev == 0) {
  1049. int err;
  1050. if (chan_is_logical(d40c)) {
  1051. err = d40_channel_execute_command(d40c,
  1052. D40_DMA_SUSPEND_REQ);
  1053. if (err)
  1054. return err;
  1055. }
  1056. }
  1057. if (chan_is_logical(d40c))
  1058. d40_config_set_event(d40c, true);
  1059. return d40_channel_execute_command(d40c, D40_DMA_RUN);
  1060. }
  1061. static struct d40_desc *d40_queue_start(struct d40_chan *d40c)
  1062. {
  1063. struct d40_desc *d40d;
  1064. int err;
  1065. /* Start queued jobs, if any */
  1066. d40d = d40_first_queued(d40c);
  1067. if (d40d != NULL) {
  1068. if (!d40c->busy)
  1069. d40c->busy = true;
  1070. pm_runtime_get_sync(d40c->base->dev);
  1071. /* Remove from queue */
  1072. d40_desc_remove(d40d);
  1073. /* Add to active queue */
  1074. d40_desc_submit(d40c, d40d);
  1075. /* Initiate DMA job */
  1076. d40_desc_load(d40c, d40d);
  1077. /* Start dma job */
  1078. err = d40_start(d40c);
  1079. if (err)
  1080. return NULL;
  1081. }
  1082. return d40d;
  1083. }
  1084. /* called from interrupt context */
  1085. static void dma_tc_handle(struct d40_chan *d40c)
  1086. {
  1087. struct d40_desc *d40d;
  1088. /* Get first active entry from list */
  1089. d40d = d40_first_active_get(d40c);
  1090. if (d40d == NULL)
  1091. return;
  1092. if (d40d->cyclic) {
  1093. /*
  1094. * If this was a paritially loaded list, we need to reloaded
  1095. * it, and only when the list is completed. We need to check
  1096. * for done because the interrupt will hit for every link, and
  1097. * not just the last one.
  1098. */
  1099. if (d40d->lli_current < d40d->lli_len
  1100. && !d40_tx_is_linked(d40c)
  1101. && !d40_residue(d40c)) {
  1102. d40_lcla_free_all(d40c, d40d);
  1103. d40_desc_load(d40c, d40d);
  1104. (void) d40_start(d40c);
  1105. if (d40d->lli_current == d40d->lli_len)
  1106. d40d->lli_current = 0;
  1107. }
  1108. } else {
  1109. d40_lcla_free_all(d40c, d40d);
  1110. if (d40d->lli_current < d40d->lli_len) {
  1111. d40_desc_load(d40c, d40d);
  1112. /* Start dma job */
  1113. (void) d40_start(d40c);
  1114. return;
  1115. }
  1116. if (d40_queue_start(d40c) == NULL)
  1117. d40c->busy = false;
  1118. pm_runtime_mark_last_busy(d40c->base->dev);
  1119. pm_runtime_put_autosuspend(d40c->base->dev);
  1120. }
  1121. d40c->pending_tx++;
  1122. tasklet_schedule(&d40c->tasklet);
  1123. }
  1124. static void dma_tasklet(unsigned long data)
  1125. {
  1126. struct d40_chan *d40c = (struct d40_chan *) data;
  1127. struct d40_desc *d40d;
  1128. unsigned long flags;
  1129. dma_async_tx_callback callback;
  1130. void *callback_param;
  1131. spin_lock_irqsave(&d40c->lock, flags);
  1132. /* Get first active entry from list */
  1133. d40d = d40_first_active_get(d40c);
  1134. if (d40d == NULL)
  1135. goto err;
  1136. if (!d40d->cyclic)
  1137. d40c->completed = d40d->txd.cookie;
  1138. /*
  1139. * If terminating a channel pending_tx is set to zero.
  1140. * This prevents any finished active jobs to return to the client.
  1141. */
  1142. if (d40c->pending_tx == 0) {
  1143. spin_unlock_irqrestore(&d40c->lock, flags);
  1144. return;
  1145. }
  1146. /* Callback to client */
  1147. callback = d40d->txd.callback;
  1148. callback_param = d40d->txd.callback_param;
  1149. if (!d40d->cyclic) {
  1150. if (async_tx_test_ack(&d40d->txd)) {
  1151. d40_desc_remove(d40d);
  1152. d40_desc_free(d40c, d40d);
  1153. } else {
  1154. if (!d40d->is_in_client_list) {
  1155. d40_desc_remove(d40d);
  1156. d40_lcla_free_all(d40c, d40d);
  1157. list_add_tail(&d40d->node, &d40c->client);
  1158. d40d->is_in_client_list = true;
  1159. }
  1160. }
  1161. }
  1162. d40c->pending_tx--;
  1163. if (d40c->pending_tx)
  1164. tasklet_schedule(&d40c->tasklet);
  1165. spin_unlock_irqrestore(&d40c->lock, flags);
  1166. if (callback && (d40d->txd.flags & DMA_PREP_INTERRUPT))
  1167. callback(callback_param);
  1168. return;
  1169. err:
  1170. /* Rescue manoeuvre if receiving double interrupts */
  1171. if (d40c->pending_tx > 0)
  1172. d40c->pending_tx--;
  1173. spin_unlock_irqrestore(&d40c->lock, flags);
  1174. }
  1175. static irqreturn_t d40_handle_interrupt(int irq, void *data)
  1176. {
  1177. static const struct d40_interrupt_lookup il[] = {
  1178. {D40_DREG_LCTIS0, D40_DREG_LCICR0, false, 0},
  1179. {D40_DREG_LCTIS1, D40_DREG_LCICR1, false, 32},
  1180. {D40_DREG_LCTIS2, D40_DREG_LCICR2, false, 64},
  1181. {D40_DREG_LCTIS3, D40_DREG_LCICR3, false, 96},
  1182. {D40_DREG_LCEIS0, D40_DREG_LCICR0, true, 0},
  1183. {D40_DREG_LCEIS1, D40_DREG_LCICR1, true, 32},
  1184. {D40_DREG_LCEIS2, D40_DREG_LCICR2, true, 64},
  1185. {D40_DREG_LCEIS3, D40_DREG_LCICR3, true, 96},
  1186. {D40_DREG_PCTIS, D40_DREG_PCICR, false, D40_PHY_CHAN},
  1187. {D40_DREG_PCEIS, D40_DREG_PCICR, true, D40_PHY_CHAN},
  1188. };
  1189. int i;
  1190. u32 regs[ARRAY_SIZE(il)];
  1191. u32 idx;
  1192. u32 row;
  1193. long chan = -1;
  1194. struct d40_chan *d40c;
  1195. unsigned long flags;
  1196. struct d40_base *base = data;
  1197. spin_lock_irqsave(&base->interrupt_lock, flags);
  1198. /* Read interrupt status of both logical and physical channels */
  1199. for (i = 0; i < ARRAY_SIZE(il); i++)
  1200. regs[i] = readl(base->virtbase + il[i].src);
  1201. for (;;) {
  1202. chan = find_next_bit((unsigned long *)regs,
  1203. BITS_PER_LONG * ARRAY_SIZE(il), chan + 1);
  1204. /* No more set bits found? */
  1205. if (chan == BITS_PER_LONG * ARRAY_SIZE(il))
  1206. break;
  1207. row = chan / BITS_PER_LONG;
  1208. idx = chan & (BITS_PER_LONG - 1);
  1209. /* ACK interrupt */
  1210. writel(1 << idx, base->virtbase + il[row].clr);
  1211. if (il[row].offset == D40_PHY_CHAN)
  1212. d40c = base->lookup_phy_chans[idx];
  1213. else
  1214. d40c = base->lookup_log_chans[il[row].offset + idx];
  1215. spin_lock(&d40c->lock);
  1216. if (!il[row].is_error)
  1217. dma_tc_handle(d40c);
  1218. else
  1219. d40_err(base->dev, "IRQ chan: %ld offset %d idx %d\n",
  1220. chan, il[row].offset, idx);
  1221. spin_unlock(&d40c->lock);
  1222. }
  1223. spin_unlock_irqrestore(&base->interrupt_lock, flags);
  1224. return IRQ_HANDLED;
  1225. }
  1226. static int d40_validate_conf(struct d40_chan *d40c,
  1227. struct stedma40_chan_cfg *conf)
  1228. {
  1229. int res = 0;
  1230. u32 dst_event_group = D40_TYPE_TO_GROUP(conf->dst_dev_type);
  1231. u32 src_event_group = D40_TYPE_TO_GROUP(conf->src_dev_type);
  1232. bool is_log = conf->mode == STEDMA40_MODE_LOGICAL;
  1233. if (!conf->dir) {
  1234. chan_err(d40c, "Invalid direction.\n");
  1235. res = -EINVAL;
  1236. }
  1237. if (conf->dst_dev_type != STEDMA40_DEV_DST_MEMORY &&
  1238. d40c->base->plat_data->dev_tx[conf->dst_dev_type] == 0 &&
  1239. d40c->runtime_addr == 0) {
  1240. chan_err(d40c, "Invalid TX channel address (%d)\n",
  1241. conf->dst_dev_type);
  1242. res = -EINVAL;
  1243. }
  1244. if (conf->src_dev_type != STEDMA40_DEV_SRC_MEMORY &&
  1245. d40c->base->plat_data->dev_rx[conf->src_dev_type] == 0 &&
  1246. d40c->runtime_addr == 0) {
  1247. chan_err(d40c, "Invalid RX channel address (%d)\n",
  1248. conf->src_dev_type);
  1249. res = -EINVAL;
  1250. }
  1251. if (conf->dir == STEDMA40_MEM_TO_PERIPH &&
  1252. dst_event_group == STEDMA40_DEV_DST_MEMORY) {
  1253. chan_err(d40c, "Invalid dst\n");
  1254. res = -EINVAL;
  1255. }
  1256. if (conf->dir == STEDMA40_PERIPH_TO_MEM &&
  1257. src_event_group == STEDMA40_DEV_SRC_MEMORY) {
  1258. chan_err(d40c, "Invalid src\n");
  1259. res = -EINVAL;
  1260. }
  1261. if (src_event_group == STEDMA40_DEV_SRC_MEMORY &&
  1262. dst_event_group == STEDMA40_DEV_DST_MEMORY && is_log) {
  1263. chan_err(d40c, "No event line\n");
  1264. res = -EINVAL;
  1265. }
  1266. if (conf->dir == STEDMA40_PERIPH_TO_PERIPH &&
  1267. (src_event_group != dst_event_group)) {
  1268. chan_err(d40c, "Invalid event group\n");
  1269. res = -EINVAL;
  1270. }
  1271. if (conf->dir == STEDMA40_PERIPH_TO_PERIPH) {
  1272. /*
  1273. * DMAC HW supports it. Will be added to this driver,
  1274. * in case any dma client requires it.
  1275. */
  1276. chan_err(d40c, "periph to periph not supported\n");
  1277. res = -EINVAL;
  1278. }
  1279. if (d40_psize_2_burst_size(is_log, conf->src_info.psize) *
  1280. (1 << conf->src_info.data_width) !=
  1281. d40_psize_2_burst_size(is_log, conf->dst_info.psize) *
  1282. (1 << conf->dst_info.data_width)) {
  1283. /*
  1284. * The DMAC hardware only supports
  1285. * src (burst x width) == dst (burst x width)
  1286. */
  1287. chan_err(d40c, "src (burst x width) != dst (burst x width)\n");
  1288. res = -EINVAL;
  1289. }
  1290. return res;
  1291. }
  1292. static bool d40_alloc_mask_set(struct d40_phy_res *phy, bool is_src,
  1293. int log_event_line, bool is_log)
  1294. {
  1295. unsigned long flags;
  1296. spin_lock_irqsave(&phy->lock, flags);
  1297. if (!is_log) {
  1298. /* Physical interrupts are masked per physical full channel */
  1299. if (phy->allocated_src == D40_ALLOC_FREE &&
  1300. phy->allocated_dst == D40_ALLOC_FREE) {
  1301. phy->allocated_dst = D40_ALLOC_PHY;
  1302. phy->allocated_src = D40_ALLOC_PHY;
  1303. goto found;
  1304. } else
  1305. goto not_found;
  1306. }
  1307. /* Logical channel */
  1308. if (is_src) {
  1309. if (phy->allocated_src == D40_ALLOC_PHY)
  1310. goto not_found;
  1311. if (phy->allocated_src == D40_ALLOC_FREE)
  1312. phy->allocated_src = D40_ALLOC_LOG_FREE;
  1313. if (!(phy->allocated_src & (1 << log_event_line))) {
  1314. phy->allocated_src |= 1 << log_event_line;
  1315. goto found;
  1316. } else
  1317. goto not_found;
  1318. } else {
  1319. if (phy->allocated_dst == D40_ALLOC_PHY)
  1320. goto not_found;
  1321. if (phy->allocated_dst == D40_ALLOC_FREE)
  1322. phy->allocated_dst = D40_ALLOC_LOG_FREE;
  1323. if (!(phy->allocated_dst & (1 << log_event_line))) {
  1324. phy->allocated_dst |= 1 << log_event_line;
  1325. goto found;
  1326. } else
  1327. goto not_found;
  1328. }
  1329. not_found:
  1330. spin_unlock_irqrestore(&phy->lock, flags);
  1331. return false;
  1332. found:
  1333. spin_unlock_irqrestore(&phy->lock, flags);
  1334. return true;
  1335. }
  1336. static bool d40_alloc_mask_free(struct d40_phy_res *phy, bool is_src,
  1337. int log_event_line)
  1338. {
  1339. unsigned long flags;
  1340. bool is_free = false;
  1341. spin_lock_irqsave(&phy->lock, flags);
  1342. if (!log_event_line) {
  1343. phy->allocated_dst = D40_ALLOC_FREE;
  1344. phy->allocated_src = D40_ALLOC_FREE;
  1345. is_free = true;
  1346. goto out;
  1347. }
  1348. /* Logical channel */
  1349. if (is_src) {
  1350. phy->allocated_src &= ~(1 << log_event_line);
  1351. if (phy->allocated_src == D40_ALLOC_LOG_FREE)
  1352. phy->allocated_src = D40_ALLOC_FREE;
  1353. } else {
  1354. phy->allocated_dst &= ~(1 << log_event_line);
  1355. if (phy->allocated_dst == D40_ALLOC_LOG_FREE)
  1356. phy->allocated_dst = D40_ALLOC_FREE;
  1357. }
  1358. is_free = ((phy->allocated_src | phy->allocated_dst) ==
  1359. D40_ALLOC_FREE);
  1360. out:
  1361. spin_unlock_irqrestore(&phy->lock, flags);
  1362. return is_free;
  1363. }
  1364. static int d40_allocate_channel(struct d40_chan *d40c)
  1365. {
  1366. int dev_type;
  1367. int event_group;
  1368. int event_line;
  1369. struct d40_phy_res *phys;
  1370. int i;
  1371. int j;
  1372. int log_num;
  1373. bool is_src;
  1374. bool is_log = d40c->dma_cfg.mode == STEDMA40_MODE_LOGICAL;
  1375. phys = d40c->base->phy_res;
  1376. if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) {
  1377. dev_type = d40c->dma_cfg.src_dev_type;
  1378. log_num = 2 * dev_type;
  1379. is_src = true;
  1380. } else if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
  1381. d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
  1382. /* dst event lines are used for logical memcpy */
  1383. dev_type = d40c->dma_cfg.dst_dev_type;
  1384. log_num = 2 * dev_type + 1;
  1385. is_src = false;
  1386. } else
  1387. return -EINVAL;
  1388. event_group = D40_TYPE_TO_GROUP(dev_type);
  1389. event_line = D40_TYPE_TO_EVENT(dev_type);
  1390. if (!is_log) {
  1391. if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
  1392. /* Find physical half channel */
  1393. for (i = 0; i < d40c->base->num_phy_chans; i++) {
  1394. if (d40_alloc_mask_set(&phys[i], is_src,
  1395. 0, is_log))
  1396. goto found_phy;
  1397. }
  1398. } else
  1399. for (j = 0; j < d40c->base->num_phy_chans; j += 8) {
  1400. int phy_num = j + event_group * 2;
  1401. for (i = phy_num; i < phy_num + 2; i++) {
  1402. if (d40_alloc_mask_set(&phys[i],
  1403. is_src,
  1404. 0,
  1405. is_log))
  1406. goto found_phy;
  1407. }
  1408. }
  1409. return -EINVAL;
  1410. found_phy:
  1411. d40c->phy_chan = &phys[i];
  1412. d40c->log_num = D40_PHY_CHAN;
  1413. goto out;
  1414. }
  1415. if (dev_type == -1)
  1416. return -EINVAL;
  1417. /* Find logical channel */
  1418. for (j = 0; j < d40c->base->num_phy_chans; j += 8) {
  1419. int phy_num = j + event_group * 2;
  1420. /*
  1421. * Spread logical channels across all available physical rather
  1422. * than pack every logical channel at the first available phy
  1423. * channels.
  1424. */
  1425. if (is_src) {
  1426. for (i = phy_num; i < phy_num + 2; i++) {
  1427. if (d40_alloc_mask_set(&phys[i], is_src,
  1428. event_line, is_log))
  1429. goto found_log;
  1430. }
  1431. } else {
  1432. for (i = phy_num + 1; i >= phy_num; i--) {
  1433. if (d40_alloc_mask_set(&phys[i], is_src,
  1434. event_line, is_log))
  1435. goto found_log;
  1436. }
  1437. }
  1438. }
  1439. return -EINVAL;
  1440. found_log:
  1441. d40c->phy_chan = &phys[i];
  1442. d40c->log_num = log_num;
  1443. out:
  1444. if (is_log)
  1445. d40c->base->lookup_log_chans[d40c->log_num] = d40c;
  1446. else
  1447. d40c->base->lookup_phy_chans[d40c->phy_chan->num] = d40c;
  1448. return 0;
  1449. }
  1450. static int d40_config_memcpy(struct d40_chan *d40c)
  1451. {
  1452. dma_cap_mask_t cap = d40c->chan.device->cap_mask;
  1453. if (dma_has_cap(DMA_MEMCPY, cap) && !dma_has_cap(DMA_SLAVE, cap)) {
  1454. d40c->dma_cfg = *d40c->base->plat_data->memcpy_conf_log;
  1455. d40c->dma_cfg.src_dev_type = STEDMA40_DEV_SRC_MEMORY;
  1456. d40c->dma_cfg.dst_dev_type = d40c->base->plat_data->
  1457. memcpy[d40c->chan.chan_id];
  1458. } else if (dma_has_cap(DMA_MEMCPY, cap) &&
  1459. dma_has_cap(DMA_SLAVE, cap)) {
  1460. d40c->dma_cfg = *d40c->base->plat_data->memcpy_conf_phy;
  1461. } else {
  1462. chan_err(d40c, "No memcpy\n");
  1463. return -EINVAL;
  1464. }
  1465. return 0;
  1466. }
  1467. static int d40_free_dma(struct d40_chan *d40c)
  1468. {
  1469. int res = 0;
  1470. u32 event;
  1471. struct d40_phy_res *phy = d40c->phy_chan;
  1472. bool is_src;
  1473. /* Terminate all queued and active transfers */
  1474. d40_term_all(d40c);
  1475. if (phy == NULL) {
  1476. chan_err(d40c, "phy == null\n");
  1477. return -EINVAL;
  1478. }
  1479. if (phy->allocated_src == D40_ALLOC_FREE &&
  1480. phy->allocated_dst == D40_ALLOC_FREE) {
  1481. chan_err(d40c, "channel already free\n");
  1482. return -EINVAL;
  1483. }
  1484. if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
  1485. d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
  1486. event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type);
  1487. is_src = false;
  1488. } else if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) {
  1489. event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type);
  1490. is_src = true;
  1491. } else {
  1492. chan_err(d40c, "Unknown direction\n");
  1493. return -EINVAL;
  1494. }
  1495. pm_runtime_get_sync(d40c->base->dev);
  1496. res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ);
  1497. if (res) {
  1498. chan_err(d40c, "suspend failed\n");
  1499. goto out;
  1500. }
  1501. if (chan_is_logical(d40c)) {
  1502. /* Release logical channel, deactivate the event line */
  1503. d40_config_set_event(d40c, false);
  1504. d40c->base->lookup_log_chans[d40c->log_num] = NULL;
  1505. /*
  1506. * Check if there are more logical allocation
  1507. * on this phy channel.
  1508. */
  1509. if (!d40_alloc_mask_free(phy, is_src, event)) {
  1510. /* Resume the other logical channels if any */
  1511. if (d40_chan_has_events(d40c)) {
  1512. res = d40_channel_execute_command(d40c,
  1513. D40_DMA_RUN);
  1514. if (res)
  1515. chan_err(d40c,
  1516. "Executing RUN command\n");
  1517. }
  1518. goto out;
  1519. }
  1520. } else {
  1521. (void) d40_alloc_mask_free(phy, is_src, 0);
  1522. }
  1523. /* Release physical channel */
  1524. res = d40_channel_execute_command(d40c, D40_DMA_STOP);
  1525. if (res) {
  1526. chan_err(d40c, "Failed to stop channel\n");
  1527. goto out;
  1528. }
  1529. if (d40c->busy) {
  1530. pm_runtime_mark_last_busy(d40c->base->dev);
  1531. pm_runtime_put_autosuspend(d40c->base->dev);
  1532. }
  1533. d40c->busy = false;
  1534. d40c->phy_chan = NULL;
  1535. d40c->configured = false;
  1536. d40c->base->lookup_phy_chans[phy->num] = NULL;
  1537. out:
  1538. pm_runtime_mark_last_busy(d40c->base->dev);
  1539. pm_runtime_put_autosuspend(d40c->base->dev);
  1540. return res;
  1541. }
  1542. static bool d40_is_paused(struct d40_chan *d40c)
  1543. {
  1544. void __iomem *chanbase = chan_base(d40c);
  1545. bool is_paused = false;
  1546. unsigned long flags;
  1547. void __iomem *active_reg;
  1548. u32 status;
  1549. u32 event;
  1550. spin_lock_irqsave(&d40c->lock, flags);
  1551. if (chan_is_physical(d40c)) {
  1552. if (d40c->phy_chan->num % 2 == 0)
  1553. active_reg = d40c->base->virtbase + D40_DREG_ACTIVE;
  1554. else
  1555. active_reg = d40c->base->virtbase + D40_DREG_ACTIVO;
  1556. status = (readl(active_reg) &
  1557. D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
  1558. D40_CHAN_POS(d40c->phy_chan->num);
  1559. if (status == D40_DMA_SUSPENDED || status == D40_DMA_STOP)
  1560. is_paused = true;
  1561. goto _exit;
  1562. }
  1563. if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
  1564. d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
  1565. event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type);
  1566. status = readl(chanbase + D40_CHAN_REG_SDLNK);
  1567. } else if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) {
  1568. event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type);
  1569. status = readl(chanbase + D40_CHAN_REG_SSLNK);
  1570. } else {
  1571. chan_err(d40c, "Unknown direction\n");
  1572. goto _exit;
  1573. }
  1574. status = (status & D40_EVENTLINE_MASK(event)) >>
  1575. D40_EVENTLINE_POS(event);
  1576. if (status != D40_DMA_RUN)
  1577. is_paused = true;
  1578. _exit:
  1579. spin_unlock_irqrestore(&d40c->lock, flags);
  1580. return is_paused;
  1581. }
  1582. static u32 stedma40_residue(struct dma_chan *chan)
  1583. {
  1584. struct d40_chan *d40c =
  1585. container_of(chan, struct d40_chan, chan);
  1586. u32 bytes_left;
  1587. unsigned long flags;
  1588. spin_lock_irqsave(&d40c->lock, flags);
  1589. bytes_left = d40_residue(d40c);
  1590. spin_unlock_irqrestore(&d40c->lock, flags);
  1591. return bytes_left;
  1592. }
  1593. static int
  1594. d40_prep_sg_log(struct d40_chan *chan, struct d40_desc *desc,
  1595. struct scatterlist *sg_src, struct scatterlist *sg_dst,
  1596. unsigned int sg_len, dma_addr_t src_dev_addr,
  1597. dma_addr_t dst_dev_addr)
  1598. {
  1599. struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
  1600. struct stedma40_half_channel_info *src_info = &cfg->src_info;
  1601. struct stedma40_half_channel_info *dst_info = &cfg->dst_info;
  1602. int ret;
  1603. ret = d40_log_sg_to_lli(sg_src, sg_len,
  1604. src_dev_addr,
  1605. desc->lli_log.src,
  1606. chan->log_def.lcsp1,
  1607. src_info->data_width,
  1608. dst_info->data_width);
  1609. ret = d40_log_sg_to_lli(sg_dst, sg_len,
  1610. dst_dev_addr,
  1611. desc->lli_log.dst,
  1612. chan->log_def.lcsp3,
  1613. dst_info->data_width,
  1614. src_info->data_width);
  1615. return ret < 0 ? ret : 0;
  1616. }
  1617. static int
  1618. d40_prep_sg_phy(struct d40_chan *chan, struct d40_desc *desc,
  1619. struct scatterlist *sg_src, struct scatterlist *sg_dst,
  1620. unsigned int sg_len, dma_addr_t src_dev_addr,
  1621. dma_addr_t dst_dev_addr)
  1622. {
  1623. struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
  1624. struct stedma40_half_channel_info *src_info = &cfg->src_info;
  1625. struct stedma40_half_channel_info *dst_info = &cfg->dst_info;
  1626. unsigned long flags = 0;
  1627. int ret;
  1628. if (desc->cyclic)
  1629. flags |= LLI_CYCLIC | LLI_TERM_INT;
  1630. ret = d40_phy_sg_to_lli(sg_src, sg_len, src_dev_addr,
  1631. desc->lli_phy.src,
  1632. virt_to_phys(desc->lli_phy.src),
  1633. chan->src_def_cfg,
  1634. src_info, dst_info, flags);
  1635. ret = d40_phy_sg_to_lli(sg_dst, sg_len, dst_dev_addr,
  1636. desc->lli_phy.dst,
  1637. virt_to_phys(desc->lli_phy.dst),
  1638. chan->dst_def_cfg,
  1639. dst_info, src_info, flags);
  1640. dma_sync_single_for_device(chan->base->dev, desc->lli_pool.dma_addr,
  1641. desc->lli_pool.size, DMA_TO_DEVICE);
  1642. return ret < 0 ? ret : 0;
  1643. }
  1644. static struct d40_desc *
  1645. d40_prep_desc(struct d40_chan *chan, struct scatterlist *sg,
  1646. unsigned int sg_len, unsigned long dma_flags)
  1647. {
  1648. struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
  1649. struct d40_desc *desc;
  1650. int ret;
  1651. desc = d40_desc_get(chan);
  1652. if (!desc)
  1653. return NULL;
  1654. desc->lli_len = d40_sg_2_dmalen(sg, sg_len, cfg->src_info.data_width,
  1655. cfg->dst_info.data_width);
  1656. if (desc->lli_len < 0) {
  1657. chan_err(chan, "Unaligned size\n");
  1658. goto err;
  1659. }
  1660. ret = d40_pool_lli_alloc(chan, desc, desc->lli_len);
  1661. if (ret < 0) {
  1662. chan_err(chan, "Could not allocate lli\n");
  1663. goto err;
  1664. }
  1665. desc->lli_current = 0;
  1666. desc->txd.flags = dma_flags;
  1667. desc->txd.tx_submit = d40_tx_submit;
  1668. dma_async_tx_descriptor_init(&desc->txd, &chan->chan);
  1669. return desc;
  1670. err:
  1671. d40_desc_free(chan, desc);
  1672. return NULL;
  1673. }
  1674. static dma_addr_t
  1675. d40_get_dev_addr(struct d40_chan *chan, enum dma_transfer_direction direction)
  1676. {
  1677. struct stedma40_platform_data *plat = chan->base->plat_data;
  1678. struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
  1679. dma_addr_t addr = 0;
  1680. if (chan->runtime_addr)
  1681. return chan->runtime_addr;
  1682. if (direction == DMA_DEV_TO_MEM)
  1683. addr = plat->dev_rx[cfg->src_dev_type];
  1684. else if (direction == DMA_MEM_TO_DEV)
  1685. addr = plat->dev_tx[cfg->dst_dev_type];
  1686. return addr;
  1687. }
  1688. static struct dma_async_tx_descriptor *
  1689. d40_prep_sg(struct dma_chan *dchan, struct scatterlist *sg_src,
  1690. struct scatterlist *sg_dst, unsigned int sg_len,
  1691. enum dma_transfer_direction direction, unsigned long dma_flags)
  1692. {
  1693. struct d40_chan *chan = container_of(dchan, struct d40_chan, chan);
  1694. dma_addr_t src_dev_addr = 0;
  1695. dma_addr_t dst_dev_addr = 0;
  1696. struct d40_desc *desc;
  1697. unsigned long flags;
  1698. int ret;
  1699. if (!chan->phy_chan) {
  1700. chan_err(chan, "Cannot prepare unallocated channel\n");
  1701. return NULL;
  1702. }
  1703. spin_lock_irqsave(&chan->lock, flags);
  1704. desc = d40_prep_desc(chan, sg_src, sg_len, dma_flags);
  1705. if (desc == NULL)
  1706. goto err;
  1707. if (sg_next(&sg_src[sg_len - 1]) == sg_src)
  1708. desc->cyclic = true;
  1709. if (direction != DMA_NONE) {
  1710. dma_addr_t dev_addr = d40_get_dev_addr(chan, direction);
  1711. if (direction == DMA_DEV_TO_MEM)
  1712. src_dev_addr = dev_addr;
  1713. else if (direction == DMA_MEM_TO_DEV)
  1714. dst_dev_addr = dev_addr;
  1715. }
  1716. if (chan_is_logical(chan))
  1717. ret = d40_prep_sg_log(chan, desc, sg_src, sg_dst,
  1718. sg_len, src_dev_addr, dst_dev_addr);
  1719. else
  1720. ret = d40_prep_sg_phy(chan, desc, sg_src, sg_dst,
  1721. sg_len, src_dev_addr, dst_dev_addr);
  1722. if (ret) {
  1723. chan_err(chan, "Failed to prepare %s sg job: %d\n",
  1724. chan_is_logical(chan) ? "log" : "phy", ret);
  1725. goto err;
  1726. }
  1727. /*
  1728. * add descriptor to the prepare queue in order to be able
  1729. * to free them later in terminate_all
  1730. */
  1731. list_add_tail(&desc->node, &chan->prepare_queue);
  1732. spin_unlock_irqrestore(&chan->lock, flags);
  1733. return &desc->txd;
  1734. err:
  1735. if (desc)
  1736. d40_desc_free(chan, desc);
  1737. spin_unlock_irqrestore(&chan->lock, flags);
  1738. return NULL;
  1739. }
  1740. bool stedma40_filter(struct dma_chan *chan, void *data)
  1741. {
  1742. struct stedma40_chan_cfg *info = data;
  1743. struct d40_chan *d40c =
  1744. container_of(chan, struct d40_chan, chan);
  1745. int err;
  1746. if (data) {
  1747. err = d40_validate_conf(d40c, info);
  1748. if (!err)
  1749. d40c->dma_cfg = *info;
  1750. } else
  1751. err = d40_config_memcpy(d40c);
  1752. if (!err)
  1753. d40c->configured = true;
  1754. return err == 0;
  1755. }
  1756. EXPORT_SYMBOL(stedma40_filter);
  1757. static void __d40_set_prio_rt(struct d40_chan *d40c, int dev_type, bool src)
  1758. {
  1759. bool realtime = d40c->dma_cfg.realtime;
  1760. bool highprio = d40c->dma_cfg.high_priority;
  1761. u32 prioreg = highprio ? D40_DREG_PSEG1 : D40_DREG_PCEG1;
  1762. u32 rtreg = realtime ? D40_DREG_RSEG1 : D40_DREG_RCEG1;
  1763. u32 event = D40_TYPE_TO_EVENT(dev_type);
  1764. u32 group = D40_TYPE_TO_GROUP(dev_type);
  1765. u32 bit = 1 << event;
  1766. /* Destination event lines are stored in the upper halfword */
  1767. if (!src)
  1768. bit <<= 16;
  1769. writel(bit, d40c->base->virtbase + prioreg + group * 4);
  1770. writel(bit, d40c->base->virtbase + rtreg + group * 4);
  1771. }
  1772. static void d40_set_prio_realtime(struct d40_chan *d40c)
  1773. {
  1774. if (d40c->base->rev < 3)
  1775. return;
  1776. if ((d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) ||
  1777. (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH))
  1778. __d40_set_prio_rt(d40c, d40c->dma_cfg.src_dev_type, true);
  1779. if ((d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH) ||
  1780. (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH))
  1781. __d40_set_prio_rt(d40c, d40c->dma_cfg.dst_dev_type, false);
  1782. }
  1783. /* DMA ENGINE functions */
  1784. static int d40_alloc_chan_resources(struct dma_chan *chan)
  1785. {
  1786. int err;
  1787. unsigned long flags;
  1788. struct d40_chan *d40c =
  1789. container_of(chan, struct d40_chan, chan);
  1790. bool is_free_phy;
  1791. spin_lock_irqsave(&d40c->lock, flags);
  1792. d40c->completed = chan->cookie = 1;
  1793. /* If no dma configuration is set use default configuration (memcpy) */
  1794. if (!d40c->configured) {
  1795. err = d40_config_memcpy(d40c);
  1796. if (err) {
  1797. chan_err(d40c, "Failed to configure memcpy channel\n");
  1798. goto fail;
  1799. }
  1800. }
  1801. is_free_phy = (d40c->phy_chan == NULL);
  1802. err = d40_allocate_channel(d40c);
  1803. if (err) {
  1804. chan_err(d40c, "Failed to allocate channel\n");
  1805. d40c->configured = false;
  1806. goto fail;
  1807. }
  1808. pm_runtime_get_sync(d40c->base->dev);
  1809. /* Fill in basic CFG register values */
  1810. d40_phy_cfg(&d40c->dma_cfg, &d40c->src_def_cfg,
  1811. &d40c->dst_def_cfg, chan_is_logical(d40c));
  1812. d40_set_prio_realtime(d40c);
  1813. if (chan_is_logical(d40c)) {
  1814. d40_log_cfg(&d40c->dma_cfg,
  1815. &d40c->log_def.lcsp1, &d40c->log_def.lcsp3);
  1816. if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM)
  1817. d40c->lcpa = d40c->base->lcpa_base +
  1818. d40c->dma_cfg.src_dev_type * D40_LCPA_CHAN_SIZE;
  1819. else
  1820. d40c->lcpa = d40c->base->lcpa_base +
  1821. d40c->dma_cfg.dst_dev_type *
  1822. D40_LCPA_CHAN_SIZE + D40_LCPA_CHAN_DST_DELTA;
  1823. }
  1824. /*
  1825. * Only write channel configuration to the DMA if the physical
  1826. * resource is free. In case of multiple logical channels
  1827. * on the same physical resource, only the first write is necessary.
  1828. */
  1829. if (is_free_phy)
  1830. d40_config_write(d40c);
  1831. fail:
  1832. pm_runtime_mark_last_busy(d40c->base->dev);
  1833. pm_runtime_put_autosuspend(d40c->base->dev);
  1834. spin_unlock_irqrestore(&d40c->lock, flags);
  1835. return err;
  1836. }
  1837. static void d40_free_chan_resources(struct dma_chan *chan)
  1838. {
  1839. struct d40_chan *d40c =
  1840. container_of(chan, struct d40_chan, chan);
  1841. int err;
  1842. unsigned long flags;
  1843. if (d40c->phy_chan == NULL) {
  1844. chan_err(d40c, "Cannot free unallocated channel\n");
  1845. return;
  1846. }
  1847. spin_lock_irqsave(&d40c->lock, flags);
  1848. err = d40_free_dma(d40c);
  1849. if (err)
  1850. chan_err(d40c, "Failed to free channel\n");
  1851. spin_unlock_irqrestore(&d40c->lock, flags);
  1852. }
  1853. static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan,
  1854. dma_addr_t dst,
  1855. dma_addr_t src,
  1856. size_t size,
  1857. unsigned long dma_flags)
  1858. {
  1859. struct scatterlist dst_sg;
  1860. struct scatterlist src_sg;
  1861. sg_init_table(&dst_sg, 1);
  1862. sg_init_table(&src_sg, 1);
  1863. sg_dma_address(&dst_sg) = dst;
  1864. sg_dma_address(&src_sg) = src;
  1865. sg_dma_len(&dst_sg) = size;
  1866. sg_dma_len(&src_sg) = size;
  1867. return d40_prep_sg(chan, &src_sg, &dst_sg, 1, DMA_NONE, dma_flags);
  1868. }
  1869. static struct dma_async_tx_descriptor *
  1870. d40_prep_memcpy_sg(struct dma_chan *chan,
  1871. struct scatterlist *dst_sg, unsigned int dst_nents,
  1872. struct scatterlist *src_sg, unsigned int src_nents,
  1873. unsigned long dma_flags)
  1874. {
  1875. if (dst_nents != src_nents)
  1876. return NULL;
  1877. return d40_prep_sg(chan, src_sg, dst_sg, src_nents, DMA_NONE, dma_flags);
  1878. }
  1879. static struct dma_async_tx_descriptor *d40_prep_slave_sg(struct dma_chan *chan,
  1880. struct scatterlist *sgl,
  1881. unsigned int sg_len,
  1882. enum dma_transfer_direction direction,
  1883. unsigned long dma_flags)
  1884. {
  1885. if (direction != DMA_DEV_TO_MEM && direction != DMA_MEM_TO_DEV)
  1886. return NULL;
  1887. return d40_prep_sg(chan, sgl, sgl, sg_len, direction, dma_flags);
  1888. }
  1889. static struct dma_async_tx_descriptor *
  1890. dma40_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t dma_addr,
  1891. size_t buf_len, size_t period_len,
  1892. enum dma_transfer_direction direction)
  1893. {
  1894. unsigned int periods = buf_len / period_len;
  1895. struct dma_async_tx_descriptor *txd;
  1896. struct scatterlist *sg;
  1897. int i;
  1898. sg = kcalloc(periods + 1, sizeof(struct scatterlist), GFP_NOWAIT);
  1899. for (i = 0; i < periods; i++) {
  1900. sg_dma_address(&sg[i]) = dma_addr;
  1901. sg_dma_len(&sg[i]) = period_len;
  1902. dma_addr += period_len;
  1903. }
  1904. sg[periods].offset = 0;
  1905. sg[periods].length = 0;
  1906. sg[periods].page_link =
  1907. ((unsigned long)sg | 0x01) & ~0x02;
  1908. txd = d40_prep_sg(chan, sg, sg, periods, direction,
  1909. DMA_PREP_INTERRUPT);
  1910. kfree(sg);
  1911. return txd;
  1912. }
  1913. static enum dma_status d40_tx_status(struct dma_chan *chan,
  1914. dma_cookie_t cookie,
  1915. struct dma_tx_state *txstate)
  1916. {
  1917. struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
  1918. dma_cookie_t last_used;
  1919. dma_cookie_t last_complete;
  1920. int ret;
  1921. if (d40c->phy_chan == NULL) {
  1922. chan_err(d40c, "Cannot read status of unallocated channel\n");
  1923. return -EINVAL;
  1924. }
  1925. last_complete = d40c->completed;
  1926. last_used = chan->cookie;
  1927. if (d40_is_paused(d40c))
  1928. ret = DMA_PAUSED;
  1929. else
  1930. ret = dma_async_is_complete(cookie, last_complete, last_used);
  1931. dma_set_tx_state(txstate, last_complete, last_used,
  1932. stedma40_residue(chan));
  1933. return ret;
  1934. }
  1935. static void d40_issue_pending(struct dma_chan *chan)
  1936. {
  1937. struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
  1938. unsigned long flags;
  1939. if (d40c->phy_chan == NULL) {
  1940. chan_err(d40c, "Channel is not allocated!\n");
  1941. return;
  1942. }
  1943. spin_lock_irqsave(&d40c->lock, flags);
  1944. list_splice_tail_init(&d40c->pending_queue, &d40c->queue);
  1945. /* Busy means that queued jobs are already being processed */
  1946. if (!d40c->busy)
  1947. (void) d40_queue_start(d40c);
  1948. spin_unlock_irqrestore(&d40c->lock, flags);
  1949. }
  1950. static int
  1951. dma40_config_to_halfchannel(struct d40_chan *d40c,
  1952. struct stedma40_half_channel_info *info,
  1953. enum dma_slave_buswidth width,
  1954. u32 maxburst)
  1955. {
  1956. enum stedma40_periph_data_width addr_width;
  1957. int psize;
  1958. switch (width) {
  1959. case DMA_SLAVE_BUSWIDTH_1_BYTE:
  1960. addr_width = STEDMA40_BYTE_WIDTH;
  1961. break;
  1962. case DMA_SLAVE_BUSWIDTH_2_BYTES:
  1963. addr_width = STEDMA40_HALFWORD_WIDTH;
  1964. break;
  1965. case DMA_SLAVE_BUSWIDTH_4_BYTES:
  1966. addr_width = STEDMA40_WORD_WIDTH;
  1967. break;
  1968. case DMA_SLAVE_BUSWIDTH_8_BYTES:
  1969. addr_width = STEDMA40_DOUBLEWORD_WIDTH;
  1970. break;
  1971. default:
  1972. dev_err(d40c->base->dev,
  1973. "illegal peripheral address width "
  1974. "requested (%d)\n",
  1975. width);
  1976. return -EINVAL;
  1977. }
  1978. if (chan_is_logical(d40c)) {
  1979. if (maxburst >= 16)
  1980. psize = STEDMA40_PSIZE_LOG_16;
  1981. else if (maxburst >= 8)
  1982. psize = STEDMA40_PSIZE_LOG_8;
  1983. else if (maxburst >= 4)
  1984. psize = STEDMA40_PSIZE_LOG_4;
  1985. else
  1986. psize = STEDMA40_PSIZE_LOG_1;
  1987. } else {
  1988. if (maxburst >= 16)
  1989. psize = STEDMA40_PSIZE_PHY_16;
  1990. else if (maxburst >= 8)
  1991. psize = STEDMA40_PSIZE_PHY_8;
  1992. else if (maxburst >= 4)
  1993. psize = STEDMA40_PSIZE_PHY_4;
  1994. else
  1995. psize = STEDMA40_PSIZE_PHY_1;
  1996. }
  1997. info->data_width = addr_width;
  1998. info->psize = psize;
  1999. info->flow_ctrl = STEDMA40_NO_FLOW_CTRL;
  2000. return 0;
  2001. }
  2002. /* Runtime reconfiguration extension */
  2003. static int d40_set_runtime_config(struct dma_chan *chan,
  2004. struct dma_slave_config *config)
  2005. {
  2006. struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
  2007. struct stedma40_chan_cfg *cfg = &d40c->dma_cfg;
  2008. enum dma_slave_buswidth src_addr_width, dst_addr_width;
  2009. dma_addr_t config_addr;
  2010. u32 src_maxburst, dst_maxburst;
  2011. int ret;
  2012. src_addr_width = config->src_addr_width;
  2013. src_maxburst = config->src_maxburst;
  2014. dst_addr_width = config->dst_addr_width;
  2015. dst_maxburst = config->dst_maxburst;
  2016. if (config->direction == DMA_DEV_TO_MEM) {
  2017. dma_addr_t dev_addr_rx =
  2018. d40c->base->plat_data->dev_rx[cfg->src_dev_type];
  2019. config_addr = config->src_addr;
  2020. if (dev_addr_rx)
  2021. dev_dbg(d40c->base->dev,
  2022. "channel has a pre-wired RX address %08x "
  2023. "overriding with %08x\n",
  2024. dev_addr_rx, config_addr);
  2025. if (cfg->dir != STEDMA40_PERIPH_TO_MEM)
  2026. dev_dbg(d40c->base->dev,
  2027. "channel was not configured for peripheral "
  2028. "to memory transfer (%d) overriding\n",
  2029. cfg->dir);
  2030. cfg->dir = STEDMA40_PERIPH_TO_MEM;
  2031. /* Configure the memory side */
  2032. if (dst_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED)
  2033. dst_addr_width = src_addr_width;
  2034. if (dst_maxburst == 0)
  2035. dst_maxburst = src_maxburst;
  2036. } else if (config->direction == DMA_MEM_TO_DEV) {
  2037. dma_addr_t dev_addr_tx =
  2038. d40c->base->plat_data->dev_tx[cfg->dst_dev_type];
  2039. config_addr = config->dst_addr;
  2040. if (dev_addr_tx)
  2041. dev_dbg(d40c->base->dev,
  2042. "channel has a pre-wired TX address %08x "
  2043. "overriding with %08x\n",
  2044. dev_addr_tx, config_addr);
  2045. if (cfg->dir != STEDMA40_MEM_TO_PERIPH)
  2046. dev_dbg(d40c->base->dev,
  2047. "channel was not configured for memory "
  2048. "to peripheral transfer (%d) overriding\n",
  2049. cfg->dir);
  2050. cfg->dir = STEDMA40_MEM_TO_PERIPH;
  2051. /* Configure the memory side */
  2052. if (src_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED)
  2053. src_addr_width = dst_addr_width;
  2054. if (src_maxburst == 0)
  2055. src_maxburst = dst_maxburst;
  2056. } else {
  2057. dev_err(d40c->base->dev,
  2058. "unrecognized channel direction %d\n",
  2059. config->direction);
  2060. return -EINVAL;
  2061. }
  2062. if (src_maxburst * src_addr_width != dst_maxburst * dst_addr_width) {
  2063. dev_err(d40c->base->dev,
  2064. "src/dst width/maxburst mismatch: %d*%d != %d*%d\n",
  2065. src_maxburst,
  2066. src_addr_width,
  2067. dst_maxburst,
  2068. dst_addr_width);
  2069. return -EINVAL;
  2070. }
  2071. ret = dma40_config_to_halfchannel(d40c, &cfg->src_info,
  2072. src_addr_width,
  2073. src_maxburst);
  2074. if (ret)
  2075. return ret;
  2076. ret = dma40_config_to_halfchannel(d40c, &cfg->dst_info,
  2077. dst_addr_width,
  2078. dst_maxburst);
  2079. if (ret)
  2080. return ret;
  2081. /* Fill in register values */
  2082. if (chan_is_logical(d40c))
  2083. d40_log_cfg(cfg, &d40c->log_def.lcsp1, &d40c->log_def.lcsp3);
  2084. else
  2085. d40_phy_cfg(cfg, &d40c->src_def_cfg,
  2086. &d40c->dst_def_cfg, false);
  2087. /* These settings will take precedence later */
  2088. d40c->runtime_addr = config_addr;
  2089. d40c->runtime_direction = config->direction;
  2090. dev_dbg(d40c->base->dev,
  2091. "configured channel %s for %s, data width %d/%d, "
  2092. "maxburst %d/%d elements, LE, no flow control\n",
  2093. dma_chan_name(chan),
  2094. (config->direction == DMA_DEV_TO_MEM) ? "RX" : "TX",
  2095. src_addr_width, dst_addr_width,
  2096. src_maxburst, dst_maxburst);
  2097. return 0;
  2098. }
  2099. static int d40_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
  2100. unsigned long arg)
  2101. {
  2102. struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
  2103. if (d40c->phy_chan == NULL) {
  2104. chan_err(d40c, "Channel is not allocated!\n");
  2105. return -EINVAL;
  2106. }
  2107. switch (cmd) {
  2108. case DMA_TERMINATE_ALL:
  2109. return d40_terminate_all(d40c);
  2110. case DMA_PAUSE:
  2111. return d40_pause(d40c);
  2112. case DMA_RESUME:
  2113. return d40_resume(d40c);
  2114. case DMA_SLAVE_CONFIG:
  2115. return d40_set_runtime_config(chan,
  2116. (struct dma_slave_config *) arg);
  2117. default:
  2118. break;
  2119. }
  2120. /* Other commands are unimplemented */
  2121. return -ENXIO;
  2122. }
  2123. /* Initialization functions */
  2124. static void __init d40_chan_init(struct d40_base *base, struct dma_device *dma,
  2125. struct d40_chan *chans, int offset,
  2126. int num_chans)
  2127. {
  2128. int i = 0;
  2129. struct d40_chan *d40c;
  2130. INIT_LIST_HEAD(&dma->channels);
  2131. for (i = offset; i < offset + num_chans; i++) {
  2132. d40c = &chans[i];
  2133. d40c->base = base;
  2134. d40c->chan.device = dma;
  2135. spin_lock_init(&d40c->lock);
  2136. d40c->log_num = D40_PHY_CHAN;
  2137. INIT_LIST_HEAD(&d40c->active);
  2138. INIT_LIST_HEAD(&d40c->queue);
  2139. INIT_LIST_HEAD(&d40c->pending_queue);
  2140. INIT_LIST_HEAD(&d40c->client);
  2141. INIT_LIST_HEAD(&d40c->prepare_queue);
  2142. tasklet_init(&d40c->tasklet, dma_tasklet,
  2143. (unsigned long) d40c);
  2144. list_add_tail(&d40c->chan.device_node,
  2145. &dma->channels);
  2146. }
  2147. }
  2148. static void d40_ops_init(struct d40_base *base, struct dma_device *dev)
  2149. {
  2150. if (dma_has_cap(DMA_SLAVE, dev->cap_mask))
  2151. dev->device_prep_slave_sg = d40_prep_slave_sg;
  2152. if (dma_has_cap(DMA_MEMCPY, dev->cap_mask)) {
  2153. dev->device_prep_dma_memcpy = d40_prep_memcpy;
  2154. /*
  2155. * This controller can only access address at even
  2156. * 32bit boundaries, i.e. 2^2
  2157. */
  2158. dev->copy_align = 2;
  2159. }
  2160. if (dma_has_cap(DMA_SG, dev->cap_mask))
  2161. dev->device_prep_dma_sg = d40_prep_memcpy_sg;
  2162. if (dma_has_cap(DMA_CYCLIC, dev->cap_mask))
  2163. dev->device_prep_dma_cyclic = dma40_prep_dma_cyclic;
  2164. dev->device_alloc_chan_resources = d40_alloc_chan_resources;
  2165. dev->device_free_chan_resources = d40_free_chan_resources;
  2166. dev->device_issue_pending = d40_issue_pending;
  2167. dev->device_tx_status = d40_tx_status;
  2168. dev->device_control = d40_control;
  2169. dev->dev = base->dev;
  2170. }
  2171. static int __init d40_dmaengine_init(struct d40_base *base,
  2172. int num_reserved_chans)
  2173. {
  2174. int err ;
  2175. d40_chan_init(base, &base->dma_slave, base->log_chans,
  2176. 0, base->num_log_chans);
  2177. dma_cap_zero(base->dma_slave.cap_mask);
  2178. dma_cap_set(DMA_SLAVE, base->dma_slave.cap_mask);
  2179. dma_cap_set(DMA_CYCLIC, base->dma_slave.cap_mask);
  2180. d40_ops_init(base, &base->dma_slave);
  2181. err = dma_async_device_register(&base->dma_slave);
  2182. if (err) {
  2183. d40_err(base->dev, "Failed to register slave channels\n");
  2184. goto failure1;
  2185. }
  2186. d40_chan_init(base, &base->dma_memcpy, base->log_chans,
  2187. base->num_log_chans, base->plat_data->memcpy_len);
  2188. dma_cap_zero(base->dma_memcpy.cap_mask);
  2189. dma_cap_set(DMA_MEMCPY, base->dma_memcpy.cap_mask);
  2190. dma_cap_set(DMA_SG, base->dma_memcpy.cap_mask);
  2191. d40_ops_init(base, &base->dma_memcpy);
  2192. err = dma_async_device_register(&base->dma_memcpy);
  2193. if (err) {
  2194. d40_err(base->dev,
  2195. "Failed to regsiter memcpy only channels\n");
  2196. goto failure2;
  2197. }
  2198. d40_chan_init(base, &base->dma_both, base->phy_chans,
  2199. 0, num_reserved_chans);
  2200. dma_cap_zero(base->dma_both.cap_mask);
  2201. dma_cap_set(DMA_SLAVE, base->dma_both.cap_mask);
  2202. dma_cap_set(DMA_MEMCPY, base->dma_both.cap_mask);
  2203. dma_cap_set(DMA_SG, base->dma_both.cap_mask);
  2204. dma_cap_set(DMA_CYCLIC, base->dma_slave.cap_mask);
  2205. d40_ops_init(base, &base->dma_both);
  2206. err = dma_async_device_register(&base->dma_both);
  2207. if (err) {
  2208. d40_err(base->dev,
  2209. "Failed to register logical and physical capable channels\n");
  2210. goto failure3;
  2211. }
  2212. return 0;
  2213. failure3:
  2214. dma_async_device_unregister(&base->dma_memcpy);
  2215. failure2:
  2216. dma_async_device_unregister(&base->dma_slave);
  2217. failure1:
  2218. return err;
  2219. }
  2220. /* Suspend resume functionality */
  2221. #ifdef CONFIG_PM
  2222. static int dma40_pm_suspend(struct device *dev)
  2223. {
  2224. struct platform_device *pdev = to_platform_device(dev);
  2225. struct d40_base *base = platform_get_drvdata(pdev);
  2226. int ret = 0;
  2227. if (!pm_runtime_suspended(dev))
  2228. return -EBUSY;
  2229. if (base->lcpa_regulator)
  2230. ret = regulator_disable(base->lcpa_regulator);
  2231. return ret;
  2232. }
  2233. static int dma40_runtime_suspend(struct device *dev)
  2234. {
  2235. struct platform_device *pdev = to_platform_device(dev);
  2236. struct d40_base *base = platform_get_drvdata(pdev);
  2237. d40_save_restore_registers(base, true);
  2238. /* Don't disable/enable clocks for v1 due to HW bugs */
  2239. if (base->rev != 1)
  2240. writel_relaxed(base->gcc_pwr_off_mask,
  2241. base->virtbase + D40_DREG_GCC);
  2242. return 0;
  2243. }
  2244. static int dma40_runtime_resume(struct device *dev)
  2245. {
  2246. struct platform_device *pdev = to_platform_device(dev);
  2247. struct d40_base *base = platform_get_drvdata(pdev);
  2248. if (base->initialized)
  2249. d40_save_restore_registers(base, false);
  2250. writel_relaxed(D40_DREG_GCC_ENABLE_ALL,
  2251. base->virtbase + D40_DREG_GCC);
  2252. return 0;
  2253. }
  2254. static int dma40_resume(struct device *dev)
  2255. {
  2256. struct platform_device *pdev = to_platform_device(dev);
  2257. struct d40_base *base = platform_get_drvdata(pdev);
  2258. int ret = 0;
  2259. if (base->lcpa_regulator)
  2260. ret = regulator_enable(base->lcpa_regulator);
  2261. return ret;
  2262. }
  2263. static const struct dev_pm_ops dma40_pm_ops = {
  2264. .suspend = dma40_pm_suspend,
  2265. .runtime_suspend = dma40_runtime_suspend,
  2266. .runtime_resume = dma40_runtime_resume,
  2267. .resume = dma40_resume,
  2268. };
  2269. #define DMA40_PM_OPS (&dma40_pm_ops)
  2270. #else
  2271. #define DMA40_PM_OPS NULL
  2272. #endif
  2273. /* Initialization functions. */
  2274. static int __init d40_phy_res_init(struct d40_base *base)
  2275. {
  2276. int i;
  2277. int num_phy_chans_avail = 0;
  2278. u32 val[2];
  2279. int odd_even_bit = -2;
  2280. int gcc = D40_DREG_GCC_ENA;
  2281. val[0] = readl(base->virtbase + D40_DREG_PRSME);
  2282. val[1] = readl(base->virtbase + D40_DREG_PRSMO);
  2283. for (i = 0; i < base->num_phy_chans; i++) {
  2284. base->phy_res[i].num = i;
  2285. odd_even_bit += 2 * ((i % 2) == 0);
  2286. if (((val[i % 2] >> odd_even_bit) & 3) == 1) {
  2287. /* Mark security only channels as occupied */
  2288. base->phy_res[i].allocated_src = D40_ALLOC_PHY;
  2289. base->phy_res[i].allocated_dst = D40_ALLOC_PHY;
  2290. base->phy_res[i].reserved = true;
  2291. gcc |= D40_DREG_GCC_EVTGRP_ENA(D40_PHYS_TO_GROUP(i),
  2292. D40_DREG_GCC_SRC);
  2293. gcc |= D40_DREG_GCC_EVTGRP_ENA(D40_PHYS_TO_GROUP(i),
  2294. D40_DREG_GCC_DST);
  2295. } else {
  2296. base->phy_res[i].allocated_src = D40_ALLOC_FREE;
  2297. base->phy_res[i].allocated_dst = D40_ALLOC_FREE;
  2298. base->phy_res[i].reserved = false;
  2299. num_phy_chans_avail++;
  2300. }
  2301. spin_lock_init(&base->phy_res[i].lock);
  2302. }
  2303. /* Mark disabled channels as occupied */
  2304. for (i = 0; base->plat_data->disabled_channels[i] != -1; i++) {
  2305. int chan = base->plat_data->disabled_channels[i];
  2306. base->phy_res[chan].allocated_src = D40_ALLOC_PHY;
  2307. base->phy_res[chan].allocated_dst = D40_ALLOC_PHY;
  2308. base->phy_res[chan].reserved = true;
  2309. gcc |= D40_DREG_GCC_EVTGRP_ENA(D40_PHYS_TO_GROUP(chan),
  2310. D40_DREG_GCC_SRC);
  2311. gcc |= D40_DREG_GCC_EVTGRP_ENA(D40_PHYS_TO_GROUP(chan),
  2312. D40_DREG_GCC_DST);
  2313. num_phy_chans_avail--;
  2314. }
  2315. dev_info(base->dev, "%d of %d physical DMA channels available\n",
  2316. num_phy_chans_avail, base->num_phy_chans);
  2317. /* Verify settings extended vs standard */
  2318. val[0] = readl(base->virtbase + D40_DREG_PRTYP);
  2319. for (i = 0; i < base->num_phy_chans; i++) {
  2320. if (base->phy_res[i].allocated_src == D40_ALLOC_FREE &&
  2321. (val[0] & 0x3) != 1)
  2322. dev_info(base->dev,
  2323. "[%s] INFO: channel %d is misconfigured (%d)\n",
  2324. __func__, i, val[0] & 0x3);
  2325. val[0] = val[0] >> 2;
  2326. }
  2327. /*
  2328. * To keep things simple, Enable all clocks initially.
  2329. * The clocks will get managed later post channel allocation.
  2330. * The clocks for the event lines on which reserved channels exists
  2331. * are not managed here.
  2332. */
  2333. writel(D40_DREG_GCC_ENABLE_ALL, base->virtbase + D40_DREG_GCC);
  2334. base->gcc_pwr_off_mask = gcc;
  2335. return num_phy_chans_avail;
  2336. }
  2337. static struct d40_base * __init d40_hw_detect_init(struct platform_device *pdev)
  2338. {
  2339. struct stedma40_platform_data *plat_data;
  2340. struct clk *clk = NULL;
  2341. void __iomem *virtbase = NULL;
  2342. struct resource *res = NULL;
  2343. struct d40_base *base = NULL;
  2344. int num_log_chans = 0;
  2345. int num_phy_chans;
  2346. int i;
  2347. u32 pid;
  2348. u32 cid;
  2349. u8 rev;
  2350. clk = clk_get(&pdev->dev, NULL);
  2351. if (IS_ERR(clk)) {
  2352. d40_err(&pdev->dev, "No matching clock found\n");
  2353. goto failure;
  2354. }
  2355. clk_enable(clk);
  2356. /* Get IO for DMAC base address */
  2357. res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "base");
  2358. if (!res)
  2359. goto failure;
  2360. if (request_mem_region(res->start, resource_size(res),
  2361. D40_NAME " I/O base") == NULL)
  2362. goto failure;
  2363. virtbase = ioremap(res->start, resource_size(res));
  2364. if (!virtbase)
  2365. goto failure;
  2366. /* This is just a regular AMBA PrimeCell ID actually */
  2367. for (pid = 0, i = 0; i < 4; i++)
  2368. pid |= (readl(virtbase + resource_size(res) - 0x20 + 4 * i)
  2369. & 255) << (i * 8);
  2370. for (cid = 0, i = 0; i < 4; i++)
  2371. cid |= (readl(virtbase + resource_size(res) - 0x10 + 4 * i)
  2372. & 255) << (i * 8);
  2373. if (cid != AMBA_CID) {
  2374. d40_err(&pdev->dev, "Unknown hardware! No PrimeCell ID\n");
  2375. goto failure;
  2376. }
  2377. if (AMBA_MANF_BITS(pid) != AMBA_VENDOR_ST) {
  2378. d40_err(&pdev->dev, "Unknown designer! Got %x wanted %x\n",
  2379. AMBA_MANF_BITS(pid),
  2380. AMBA_VENDOR_ST);
  2381. goto failure;
  2382. }
  2383. /*
  2384. * HW revision:
  2385. * DB8500ed has revision 0
  2386. * ? has revision 1
  2387. * DB8500v1 has revision 2
  2388. * DB8500v2 has revision 3
  2389. */
  2390. rev = AMBA_REV_BITS(pid);
  2391. /* The number of physical channels on this HW */
  2392. num_phy_chans = 4 * (readl(virtbase + D40_DREG_ICFG) & 0x7) + 4;
  2393. dev_info(&pdev->dev, "hardware revision: %d @ 0x%x\n",
  2394. rev, res->start);
  2395. plat_data = pdev->dev.platform_data;
  2396. /* Count the number of logical channels in use */
  2397. for (i = 0; i < plat_data->dev_len; i++)
  2398. if (plat_data->dev_rx[i] != 0)
  2399. num_log_chans++;
  2400. for (i = 0; i < plat_data->dev_len; i++)
  2401. if (plat_data->dev_tx[i] != 0)
  2402. num_log_chans++;
  2403. base = kzalloc(ALIGN(sizeof(struct d40_base), 4) +
  2404. (num_phy_chans + num_log_chans + plat_data->memcpy_len) *
  2405. sizeof(struct d40_chan), GFP_KERNEL);
  2406. if (base == NULL) {
  2407. d40_err(&pdev->dev, "Out of memory\n");
  2408. goto failure;
  2409. }
  2410. base->rev = rev;
  2411. base->clk = clk;
  2412. base->num_phy_chans = num_phy_chans;
  2413. base->num_log_chans = num_log_chans;
  2414. base->phy_start = res->start;
  2415. base->phy_size = resource_size(res);
  2416. base->virtbase = virtbase;
  2417. base->plat_data = plat_data;
  2418. base->dev = &pdev->dev;
  2419. base->phy_chans = ((void *)base) + ALIGN(sizeof(struct d40_base), 4);
  2420. base->log_chans = &base->phy_chans[num_phy_chans];
  2421. base->phy_res = kzalloc(num_phy_chans * sizeof(struct d40_phy_res),
  2422. GFP_KERNEL);
  2423. if (!base->phy_res)
  2424. goto failure;
  2425. base->lookup_phy_chans = kzalloc(num_phy_chans *
  2426. sizeof(struct d40_chan *),
  2427. GFP_KERNEL);
  2428. if (!base->lookup_phy_chans)
  2429. goto failure;
  2430. if (num_log_chans + plat_data->memcpy_len) {
  2431. /*
  2432. * The max number of logical channels are event lines for all
  2433. * src devices and dst devices
  2434. */
  2435. base->lookup_log_chans = kzalloc(plat_data->dev_len * 2 *
  2436. sizeof(struct d40_chan *),
  2437. GFP_KERNEL);
  2438. if (!base->lookup_log_chans)
  2439. goto failure;
  2440. }
  2441. base->reg_val_backup_chan = kmalloc(base->num_phy_chans *
  2442. sizeof(d40_backup_regs_chan),
  2443. GFP_KERNEL);
  2444. if (!base->reg_val_backup_chan)
  2445. goto failure;
  2446. base->lcla_pool.alloc_map =
  2447. kzalloc(num_phy_chans * sizeof(struct d40_desc *)
  2448. * D40_LCLA_LINK_PER_EVENT_GRP, GFP_KERNEL);
  2449. if (!base->lcla_pool.alloc_map)
  2450. goto failure;
  2451. base->desc_slab = kmem_cache_create(D40_NAME, sizeof(struct d40_desc),
  2452. 0, SLAB_HWCACHE_ALIGN,
  2453. NULL);
  2454. if (base->desc_slab == NULL)
  2455. goto failure;
  2456. return base;
  2457. failure:
  2458. if (!IS_ERR(clk)) {
  2459. clk_disable(clk);
  2460. clk_put(clk);
  2461. }
  2462. if (virtbase)
  2463. iounmap(virtbase);
  2464. if (res)
  2465. release_mem_region(res->start,
  2466. resource_size(res));
  2467. if (virtbase)
  2468. iounmap(virtbase);
  2469. if (base) {
  2470. kfree(base->lcla_pool.alloc_map);
  2471. kfree(base->lookup_log_chans);
  2472. kfree(base->lookup_phy_chans);
  2473. kfree(base->phy_res);
  2474. kfree(base);
  2475. }
  2476. return NULL;
  2477. }
  2478. static void __init d40_hw_init(struct d40_base *base)
  2479. {
  2480. static struct d40_reg_val dma_init_reg[] = {
  2481. /* Clock every part of the DMA block from start */
  2482. { .reg = D40_DREG_GCC, .val = D40_DREG_GCC_ENABLE_ALL},
  2483. /* Interrupts on all logical channels */
  2484. { .reg = D40_DREG_LCMIS0, .val = 0xFFFFFFFF},
  2485. { .reg = D40_DREG_LCMIS1, .val = 0xFFFFFFFF},
  2486. { .reg = D40_DREG_LCMIS2, .val = 0xFFFFFFFF},
  2487. { .reg = D40_DREG_LCMIS3, .val = 0xFFFFFFFF},
  2488. { .reg = D40_DREG_LCICR0, .val = 0xFFFFFFFF},
  2489. { .reg = D40_DREG_LCICR1, .val = 0xFFFFFFFF},
  2490. { .reg = D40_DREG_LCICR2, .val = 0xFFFFFFFF},
  2491. { .reg = D40_DREG_LCICR3, .val = 0xFFFFFFFF},
  2492. { .reg = D40_DREG_LCTIS0, .val = 0xFFFFFFFF},
  2493. { .reg = D40_DREG_LCTIS1, .val = 0xFFFFFFFF},
  2494. { .reg = D40_DREG_LCTIS2, .val = 0xFFFFFFFF},
  2495. { .reg = D40_DREG_LCTIS3, .val = 0xFFFFFFFF}
  2496. };
  2497. int i;
  2498. u32 prmseo[2] = {0, 0};
  2499. u32 activeo[2] = {0xFFFFFFFF, 0xFFFFFFFF};
  2500. u32 pcmis = 0;
  2501. u32 pcicr = 0;
  2502. for (i = 0; i < ARRAY_SIZE(dma_init_reg); i++)
  2503. writel(dma_init_reg[i].val,
  2504. base->virtbase + dma_init_reg[i].reg);
  2505. /* Configure all our dma channels to default settings */
  2506. for (i = 0; i < base->num_phy_chans; i++) {
  2507. activeo[i % 2] = activeo[i % 2] << 2;
  2508. if (base->phy_res[base->num_phy_chans - i - 1].allocated_src
  2509. == D40_ALLOC_PHY) {
  2510. activeo[i % 2] |= 3;
  2511. continue;
  2512. }
  2513. /* Enable interrupt # */
  2514. pcmis = (pcmis << 1) | 1;
  2515. /* Clear interrupt # */
  2516. pcicr = (pcicr << 1) | 1;
  2517. /* Set channel to physical mode */
  2518. prmseo[i % 2] = prmseo[i % 2] << 2;
  2519. prmseo[i % 2] |= 1;
  2520. }
  2521. writel(prmseo[1], base->virtbase + D40_DREG_PRMSE);
  2522. writel(prmseo[0], base->virtbase + D40_DREG_PRMSO);
  2523. writel(activeo[1], base->virtbase + D40_DREG_ACTIVE);
  2524. writel(activeo[0], base->virtbase + D40_DREG_ACTIVO);
  2525. /* Write which interrupt to enable */
  2526. writel(pcmis, base->virtbase + D40_DREG_PCMIS);
  2527. /* Write which interrupt to clear */
  2528. writel(pcicr, base->virtbase + D40_DREG_PCICR);
  2529. }
  2530. static int __init d40_lcla_allocate(struct d40_base *base)
  2531. {
  2532. struct d40_lcla_pool *pool = &base->lcla_pool;
  2533. unsigned long *page_list;
  2534. int i, j;
  2535. int ret = 0;
  2536. /*
  2537. * This is somewhat ugly. We need 8192 bytes that are 18 bit aligned,
  2538. * To full fill this hardware requirement without wasting 256 kb
  2539. * we allocate pages until we get an aligned one.
  2540. */
  2541. page_list = kmalloc(sizeof(unsigned long) * MAX_LCLA_ALLOC_ATTEMPTS,
  2542. GFP_KERNEL);
  2543. if (!page_list) {
  2544. ret = -ENOMEM;
  2545. goto failure;
  2546. }
  2547. /* Calculating how many pages that are required */
  2548. base->lcla_pool.pages = SZ_1K * base->num_phy_chans / PAGE_SIZE;
  2549. for (i = 0; i < MAX_LCLA_ALLOC_ATTEMPTS; i++) {
  2550. page_list[i] = __get_free_pages(GFP_KERNEL,
  2551. base->lcla_pool.pages);
  2552. if (!page_list[i]) {
  2553. d40_err(base->dev, "Failed to allocate %d pages.\n",
  2554. base->lcla_pool.pages);
  2555. for (j = 0; j < i; j++)
  2556. free_pages(page_list[j], base->lcla_pool.pages);
  2557. goto failure;
  2558. }
  2559. if ((virt_to_phys((void *)page_list[i]) &
  2560. (LCLA_ALIGNMENT - 1)) == 0)
  2561. break;
  2562. }
  2563. for (j = 0; j < i; j++)
  2564. free_pages(page_list[j], base->lcla_pool.pages);
  2565. if (i < MAX_LCLA_ALLOC_ATTEMPTS) {
  2566. base->lcla_pool.base = (void *)page_list[i];
  2567. } else {
  2568. /*
  2569. * After many attempts and no succees with finding the correct
  2570. * alignment, try with allocating a big buffer.
  2571. */
  2572. dev_warn(base->dev,
  2573. "[%s] Failed to get %d pages @ 18 bit align.\n",
  2574. __func__, base->lcla_pool.pages);
  2575. base->lcla_pool.base_unaligned = kmalloc(SZ_1K *
  2576. base->num_phy_chans +
  2577. LCLA_ALIGNMENT,
  2578. GFP_KERNEL);
  2579. if (!base->lcla_pool.base_unaligned) {
  2580. ret = -ENOMEM;
  2581. goto failure;
  2582. }
  2583. base->lcla_pool.base = PTR_ALIGN(base->lcla_pool.base_unaligned,
  2584. LCLA_ALIGNMENT);
  2585. }
  2586. pool->dma_addr = dma_map_single(base->dev, pool->base,
  2587. SZ_1K * base->num_phy_chans,
  2588. DMA_TO_DEVICE);
  2589. if (dma_mapping_error(base->dev, pool->dma_addr)) {
  2590. pool->dma_addr = 0;
  2591. ret = -ENOMEM;
  2592. goto failure;
  2593. }
  2594. writel(virt_to_phys(base->lcla_pool.base),
  2595. base->virtbase + D40_DREG_LCLA);
  2596. failure:
  2597. kfree(page_list);
  2598. return ret;
  2599. }
  2600. static int __init d40_probe(struct platform_device *pdev)
  2601. {
  2602. int err;
  2603. int ret = -ENOENT;
  2604. struct d40_base *base;
  2605. struct resource *res = NULL;
  2606. int num_reserved_chans;
  2607. u32 val;
  2608. base = d40_hw_detect_init(pdev);
  2609. if (!base)
  2610. goto failure;
  2611. num_reserved_chans = d40_phy_res_init(base);
  2612. platform_set_drvdata(pdev, base);
  2613. spin_lock_init(&base->interrupt_lock);
  2614. spin_lock_init(&base->execmd_lock);
  2615. /* Get IO for logical channel parameter address */
  2616. res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "lcpa");
  2617. if (!res) {
  2618. ret = -ENOENT;
  2619. d40_err(&pdev->dev, "No \"lcpa\" memory resource\n");
  2620. goto failure;
  2621. }
  2622. base->lcpa_size = resource_size(res);
  2623. base->phy_lcpa = res->start;
  2624. if (request_mem_region(res->start, resource_size(res),
  2625. D40_NAME " I/O lcpa") == NULL) {
  2626. ret = -EBUSY;
  2627. d40_err(&pdev->dev,
  2628. "Failed to request LCPA region 0x%x-0x%x\n",
  2629. res->start, res->end);
  2630. goto failure;
  2631. }
  2632. /* We make use of ESRAM memory for this. */
  2633. val = readl(base->virtbase + D40_DREG_LCPA);
  2634. if (res->start != val && val != 0) {
  2635. dev_warn(&pdev->dev,
  2636. "[%s] Mismatch LCPA dma 0x%x, def 0x%x\n",
  2637. __func__, val, res->start);
  2638. } else
  2639. writel(res->start, base->virtbase + D40_DREG_LCPA);
  2640. base->lcpa_base = ioremap(res->start, resource_size(res));
  2641. if (!base->lcpa_base) {
  2642. ret = -ENOMEM;
  2643. d40_err(&pdev->dev, "Failed to ioremap LCPA region\n");
  2644. goto failure;
  2645. }
  2646. /* If lcla has to be located in ESRAM we don't need to allocate */
  2647. if (base->plat_data->use_esram_lcla) {
  2648. res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
  2649. "lcla_esram");
  2650. if (!res) {
  2651. ret = -ENOENT;
  2652. d40_err(&pdev->dev,
  2653. "No \"lcla_esram\" memory resource\n");
  2654. goto failure;
  2655. }
  2656. base->lcla_pool.base = ioremap(res->start,
  2657. resource_size(res));
  2658. if (!base->lcla_pool.base) {
  2659. ret = -ENOMEM;
  2660. d40_err(&pdev->dev, "Failed to ioremap LCLA region\n");
  2661. goto failure;
  2662. }
  2663. writel(res->start, base->virtbase + D40_DREG_LCLA);
  2664. } else {
  2665. ret = d40_lcla_allocate(base);
  2666. if (ret) {
  2667. d40_err(&pdev->dev, "Failed to allocate LCLA area\n");
  2668. goto failure;
  2669. }
  2670. }
  2671. spin_lock_init(&base->lcla_pool.lock);
  2672. base->irq = platform_get_irq(pdev, 0);
  2673. ret = request_irq(base->irq, d40_handle_interrupt, 0, D40_NAME, base);
  2674. if (ret) {
  2675. d40_err(&pdev->dev, "No IRQ defined\n");
  2676. goto failure;
  2677. }
  2678. pm_runtime_irq_safe(base->dev);
  2679. pm_runtime_set_autosuspend_delay(base->dev, DMA40_AUTOSUSPEND_DELAY);
  2680. pm_runtime_use_autosuspend(base->dev);
  2681. pm_runtime_enable(base->dev);
  2682. pm_runtime_resume(base->dev);
  2683. if (base->plat_data->use_esram_lcla) {
  2684. base->lcpa_regulator = regulator_get(base->dev, "lcla_esram");
  2685. if (IS_ERR(base->lcpa_regulator)) {
  2686. d40_err(&pdev->dev, "Failed to get lcpa_regulator\n");
  2687. base->lcpa_regulator = NULL;
  2688. goto failure;
  2689. }
  2690. ret = regulator_enable(base->lcpa_regulator);
  2691. if (ret) {
  2692. d40_err(&pdev->dev,
  2693. "Failed to enable lcpa_regulator\n");
  2694. regulator_put(base->lcpa_regulator);
  2695. base->lcpa_regulator = NULL;
  2696. goto failure;
  2697. }
  2698. }
  2699. base->initialized = true;
  2700. err = d40_dmaengine_init(base, num_reserved_chans);
  2701. if (err)
  2702. goto failure;
  2703. d40_hw_init(base);
  2704. dev_info(base->dev, "initialized\n");
  2705. return 0;
  2706. failure:
  2707. if (base) {
  2708. if (base->desc_slab)
  2709. kmem_cache_destroy(base->desc_slab);
  2710. if (base->virtbase)
  2711. iounmap(base->virtbase);
  2712. if (base->lcla_pool.base && base->plat_data->use_esram_lcla) {
  2713. iounmap(base->lcla_pool.base);
  2714. base->lcla_pool.base = NULL;
  2715. }
  2716. if (base->lcla_pool.dma_addr)
  2717. dma_unmap_single(base->dev, base->lcla_pool.dma_addr,
  2718. SZ_1K * base->num_phy_chans,
  2719. DMA_TO_DEVICE);
  2720. if (!base->lcla_pool.base_unaligned && base->lcla_pool.base)
  2721. free_pages((unsigned long)base->lcla_pool.base,
  2722. base->lcla_pool.pages);
  2723. kfree(base->lcla_pool.base_unaligned);
  2724. if (base->phy_lcpa)
  2725. release_mem_region(base->phy_lcpa,
  2726. base->lcpa_size);
  2727. if (base->phy_start)
  2728. release_mem_region(base->phy_start,
  2729. base->phy_size);
  2730. if (base->clk) {
  2731. clk_disable(base->clk);
  2732. clk_put(base->clk);
  2733. }
  2734. if (base->lcpa_regulator) {
  2735. regulator_disable(base->lcpa_regulator);
  2736. regulator_put(base->lcpa_regulator);
  2737. }
  2738. kfree(base->lcla_pool.alloc_map);
  2739. kfree(base->lookup_log_chans);
  2740. kfree(base->lookup_phy_chans);
  2741. kfree(base->phy_res);
  2742. kfree(base);
  2743. }
  2744. d40_err(&pdev->dev, "probe failed\n");
  2745. return ret;
  2746. }
  2747. static struct platform_driver d40_driver = {
  2748. .driver = {
  2749. .owner = THIS_MODULE,
  2750. .name = D40_NAME,
  2751. .pm = DMA40_PM_OPS,
  2752. },
  2753. };
  2754. static int __init stedma40_init(void)
  2755. {
  2756. return platform_driver_probe(&d40_driver, d40_probe);
  2757. }
  2758. subsys_initcall(stedma40_init);