ste_dma40.c 95 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713
  1. /*
  2. * Copyright (C) Ericsson AB 2007-2008
  3. * Copyright (C) ST-Ericsson SA 2008-2010
  4. * Author: Per Forlin <per.forlin@stericsson.com> for ST-Ericsson
  5. * Author: Jonas Aaberg <jonas.aberg@stericsson.com> for ST-Ericsson
  6. * License terms: GNU General Public License (GPL) version 2
  7. */
  8. #include <linux/dma-mapping.h>
  9. #include <linux/kernel.h>
  10. #include <linux/slab.h>
  11. #include <linux/export.h>
  12. #include <linux/dmaengine.h>
  13. #include <linux/platform_device.h>
  14. #include <linux/clk.h>
  15. #include <linux/delay.h>
  16. #include <linux/pm.h>
  17. #include <linux/pm_runtime.h>
  18. #include <linux/err.h>
  19. #include <linux/amba/bus.h>
  20. #include <linux/regulator/consumer.h>
  21. #include <linux/platform_data/dma-ste-dma40.h>
  22. #include "dmaengine.h"
  23. #include "ste_dma40_ll.h"
  24. #define D40_NAME "dma40"
  25. #define D40_PHY_CHAN -1
  26. /* For masking out/in 2 bit channel positions */
  27. #define D40_CHAN_POS(chan) (2 * (chan / 2))
  28. #define D40_CHAN_POS_MASK(chan) (0x3 << D40_CHAN_POS(chan))
  29. /* Maximum iterations taken before giving up suspending a channel */
  30. #define D40_SUSPEND_MAX_IT 500
  31. /* Milliseconds */
  32. #define DMA40_AUTOSUSPEND_DELAY 100
  33. /* Hardware requirement on LCLA alignment */
  34. #define LCLA_ALIGNMENT 0x40000
  35. /* Max number of links per event group */
  36. #define D40_LCLA_LINK_PER_EVENT_GRP 128
  37. #define D40_LCLA_END D40_LCLA_LINK_PER_EVENT_GRP
  38. /* Attempts before giving up to trying to get pages that are aligned */
  39. #define MAX_LCLA_ALLOC_ATTEMPTS 256
  40. /* Bit markings for allocation map */
  41. #define D40_ALLOC_FREE (1 << 31)
  42. #define D40_ALLOC_PHY (1 << 30)
  43. #define D40_ALLOC_LOG_FREE 0
  44. #define MAX(a, b) (((a) < (b)) ? (b) : (a))
  45. /* Reserved event lines for memcpy only. */
  46. #define DB8500_DMA_MEMCPY_EV_0 51
  47. #define DB8500_DMA_MEMCPY_EV_1 56
  48. #define DB8500_DMA_MEMCPY_EV_2 57
  49. #define DB8500_DMA_MEMCPY_EV_3 58
  50. #define DB8500_DMA_MEMCPY_EV_4 59
  51. #define DB8500_DMA_MEMCPY_EV_5 60
  52. static int dma40_memcpy_channels[] = {
  53. DB8500_DMA_MEMCPY_EV_0,
  54. DB8500_DMA_MEMCPY_EV_1,
  55. DB8500_DMA_MEMCPY_EV_2,
  56. DB8500_DMA_MEMCPY_EV_3,
  57. DB8500_DMA_MEMCPY_EV_4,
  58. DB8500_DMA_MEMCPY_EV_5,
  59. };
  60. /* Default configuration for physcial memcpy */
  61. struct stedma40_chan_cfg dma40_memcpy_conf_phy = {
  62. .mode = STEDMA40_MODE_PHYSICAL,
  63. .dir = STEDMA40_MEM_TO_MEM,
  64. .src_info.data_width = STEDMA40_BYTE_WIDTH,
  65. .src_info.psize = STEDMA40_PSIZE_PHY_1,
  66. .src_info.flow_ctrl = STEDMA40_NO_FLOW_CTRL,
  67. .dst_info.data_width = STEDMA40_BYTE_WIDTH,
  68. .dst_info.psize = STEDMA40_PSIZE_PHY_1,
  69. .dst_info.flow_ctrl = STEDMA40_NO_FLOW_CTRL,
  70. };
  71. /* Default configuration for logical memcpy */
  72. struct stedma40_chan_cfg dma40_memcpy_conf_log = {
  73. .mode = STEDMA40_MODE_LOGICAL,
  74. .dir = STEDMA40_MEM_TO_MEM,
  75. .src_info.data_width = STEDMA40_BYTE_WIDTH,
  76. .src_info.psize = STEDMA40_PSIZE_LOG_1,
  77. .src_info.flow_ctrl = STEDMA40_NO_FLOW_CTRL,
  78. .dst_info.data_width = STEDMA40_BYTE_WIDTH,
  79. .dst_info.psize = STEDMA40_PSIZE_LOG_1,
  80. .dst_info.flow_ctrl = STEDMA40_NO_FLOW_CTRL,
  81. };
  82. /**
  83. * enum 40_command - The different commands and/or statuses.
  84. *
  85. * @D40_DMA_STOP: DMA channel command STOP or status STOPPED,
  86. * @D40_DMA_RUN: The DMA channel is RUNNING of the command RUN.
  87. * @D40_DMA_SUSPEND_REQ: Request the DMA to SUSPEND as soon as possible.
  88. * @D40_DMA_SUSPENDED: The DMA channel is SUSPENDED.
  89. */
  90. enum d40_command {
  91. D40_DMA_STOP = 0,
  92. D40_DMA_RUN = 1,
  93. D40_DMA_SUSPEND_REQ = 2,
  94. D40_DMA_SUSPENDED = 3
  95. };
  96. /*
  97. * enum d40_events - The different Event Enables for the event lines.
  98. *
  99. * @D40_DEACTIVATE_EVENTLINE: De-activate Event line, stopping the logical chan.
  100. * @D40_ACTIVATE_EVENTLINE: Activate the Event line, to start a logical chan.
  101. * @D40_SUSPEND_REQ_EVENTLINE: Requesting for suspending a event line.
  102. * @D40_ROUND_EVENTLINE: Status check for event line.
  103. */
  104. enum d40_events {
  105. D40_DEACTIVATE_EVENTLINE = 0,
  106. D40_ACTIVATE_EVENTLINE = 1,
  107. D40_SUSPEND_REQ_EVENTLINE = 2,
  108. D40_ROUND_EVENTLINE = 3
  109. };
  110. /*
  111. * These are the registers that has to be saved and later restored
  112. * when the DMA hw is powered off.
  113. * TODO: Add save/restore of D40_DREG_GCC on dma40 v3 or later, if that works.
  114. */
  115. static u32 d40_backup_regs[] = {
  116. D40_DREG_LCPA,
  117. D40_DREG_LCLA,
  118. D40_DREG_PRMSE,
  119. D40_DREG_PRMSO,
  120. D40_DREG_PRMOE,
  121. D40_DREG_PRMOO,
  122. };
  123. #define BACKUP_REGS_SZ ARRAY_SIZE(d40_backup_regs)
  124. /*
  125. * since 9540 and 8540 has the same HW revision
  126. * use v4a for 9540 or ealier
  127. * use v4b for 8540 or later
  128. * HW revision:
  129. * DB8500ed has revision 0
  130. * DB8500v1 has revision 2
  131. * DB8500v2 has revision 3
  132. * AP9540v1 has revision 4
  133. * DB8540v1 has revision 4
  134. * TODO: Check if all these registers have to be saved/restored on dma40 v4a
  135. */
  136. static u32 d40_backup_regs_v4a[] = {
  137. D40_DREG_PSEG1,
  138. D40_DREG_PSEG2,
  139. D40_DREG_PSEG3,
  140. D40_DREG_PSEG4,
  141. D40_DREG_PCEG1,
  142. D40_DREG_PCEG2,
  143. D40_DREG_PCEG3,
  144. D40_DREG_PCEG4,
  145. D40_DREG_RSEG1,
  146. D40_DREG_RSEG2,
  147. D40_DREG_RSEG3,
  148. D40_DREG_RSEG4,
  149. D40_DREG_RCEG1,
  150. D40_DREG_RCEG2,
  151. D40_DREG_RCEG3,
  152. D40_DREG_RCEG4,
  153. };
  154. #define BACKUP_REGS_SZ_V4A ARRAY_SIZE(d40_backup_regs_v4a)
  155. static u32 d40_backup_regs_v4b[] = {
  156. D40_DREG_CPSEG1,
  157. D40_DREG_CPSEG2,
  158. D40_DREG_CPSEG3,
  159. D40_DREG_CPSEG4,
  160. D40_DREG_CPSEG5,
  161. D40_DREG_CPCEG1,
  162. D40_DREG_CPCEG2,
  163. D40_DREG_CPCEG3,
  164. D40_DREG_CPCEG4,
  165. D40_DREG_CPCEG5,
  166. D40_DREG_CRSEG1,
  167. D40_DREG_CRSEG2,
  168. D40_DREG_CRSEG3,
  169. D40_DREG_CRSEG4,
  170. D40_DREG_CRSEG5,
  171. D40_DREG_CRCEG1,
  172. D40_DREG_CRCEG2,
  173. D40_DREG_CRCEG3,
  174. D40_DREG_CRCEG4,
  175. D40_DREG_CRCEG5,
  176. };
  177. #define BACKUP_REGS_SZ_V4B ARRAY_SIZE(d40_backup_regs_v4b)
  178. static u32 d40_backup_regs_chan[] = {
  179. D40_CHAN_REG_SSCFG,
  180. D40_CHAN_REG_SSELT,
  181. D40_CHAN_REG_SSPTR,
  182. D40_CHAN_REG_SSLNK,
  183. D40_CHAN_REG_SDCFG,
  184. D40_CHAN_REG_SDELT,
  185. D40_CHAN_REG_SDPTR,
  186. D40_CHAN_REG_SDLNK,
  187. };
  188. /**
  189. * struct d40_interrupt_lookup - lookup table for interrupt handler
  190. *
  191. * @src: Interrupt mask register.
  192. * @clr: Interrupt clear register.
  193. * @is_error: true if this is an error interrupt.
  194. * @offset: start delta in the lookup_log_chans in d40_base. If equals to
  195. * D40_PHY_CHAN, the lookup_phy_chans shall be used instead.
  196. */
  197. struct d40_interrupt_lookup {
  198. u32 src;
  199. u32 clr;
  200. bool is_error;
  201. int offset;
  202. };
  203. static struct d40_interrupt_lookup il_v4a[] = {
  204. {D40_DREG_LCTIS0, D40_DREG_LCICR0, false, 0},
  205. {D40_DREG_LCTIS1, D40_DREG_LCICR1, false, 32},
  206. {D40_DREG_LCTIS2, D40_DREG_LCICR2, false, 64},
  207. {D40_DREG_LCTIS3, D40_DREG_LCICR3, false, 96},
  208. {D40_DREG_LCEIS0, D40_DREG_LCICR0, true, 0},
  209. {D40_DREG_LCEIS1, D40_DREG_LCICR1, true, 32},
  210. {D40_DREG_LCEIS2, D40_DREG_LCICR2, true, 64},
  211. {D40_DREG_LCEIS3, D40_DREG_LCICR3, true, 96},
  212. {D40_DREG_PCTIS, D40_DREG_PCICR, false, D40_PHY_CHAN},
  213. {D40_DREG_PCEIS, D40_DREG_PCICR, true, D40_PHY_CHAN},
  214. };
  215. static struct d40_interrupt_lookup il_v4b[] = {
  216. {D40_DREG_CLCTIS1, D40_DREG_CLCICR1, false, 0},
  217. {D40_DREG_CLCTIS2, D40_DREG_CLCICR2, false, 32},
  218. {D40_DREG_CLCTIS3, D40_DREG_CLCICR3, false, 64},
  219. {D40_DREG_CLCTIS4, D40_DREG_CLCICR4, false, 96},
  220. {D40_DREG_CLCTIS5, D40_DREG_CLCICR5, false, 128},
  221. {D40_DREG_CLCEIS1, D40_DREG_CLCICR1, true, 0},
  222. {D40_DREG_CLCEIS2, D40_DREG_CLCICR2, true, 32},
  223. {D40_DREG_CLCEIS3, D40_DREG_CLCICR3, true, 64},
  224. {D40_DREG_CLCEIS4, D40_DREG_CLCICR4, true, 96},
  225. {D40_DREG_CLCEIS5, D40_DREG_CLCICR5, true, 128},
  226. {D40_DREG_CPCTIS, D40_DREG_CPCICR, false, D40_PHY_CHAN},
  227. {D40_DREG_CPCEIS, D40_DREG_CPCICR, true, D40_PHY_CHAN},
  228. };
  229. /**
  230. * struct d40_reg_val - simple lookup struct
  231. *
  232. * @reg: The register.
  233. * @val: The value that belongs to the register in reg.
  234. */
  235. struct d40_reg_val {
  236. unsigned int reg;
  237. unsigned int val;
  238. };
  239. static __initdata struct d40_reg_val dma_init_reg_v4a[] = {
  240. /* Clock every part of the DMA block from start */
  241. { .reg = D40_DREG_GCC, .val = D40_DREG_GCC_ENABLE_ALL},
  242. /* Interrupts on all logical channels */
  243. { .reg = D40_DREG_LCMIS0, .val = 0xFFFFFFFF},
  244. { .reg = D40_DREG_LCMIS1, .val = 0xFFFFFFFF},
  245. { .reg = D40_DREG_LCMIS2, .val = 0xFFFFFFFF},
  246. { .reg = D40_DREG_LCMIS3, .val = 0xFFFFFFFF},
  247. { .reg = D40_DREG_LCICR0, .val = 0xFFFFFFFF},
  248. { .reg = D40_DREG_LCICR1, .val = 0xFFFFFFFF},
  249. { .reg = D40_DREG_LCICR2, .val = 0xFFFFFFFF},
  250. { .reg = D40_DREG_LCICR3, .val = 0xFFFFFFFF},
  251. { .reg = D40_DREG_LCTIS0, .val = 0xFFFFFFFF},
  252. { .reg = D40_DREG_LCTIS1, .val = 0xFFFFFFFF},
  253. { .reg = D40_DREG_LCTIS2, .val = 0xFFFFFFFF},
  254. { .reg = D40_DREG_LCTIS3, .val = 0xFFFFFFFF}
  255. };
  256. static __initdata struct d40_reg_val dma_init_reg_v4b[] = {
  257. /* Clock every part of the DMA block from start */
  258. { .reg = D40_DREG_GCC, .val = D40_DREG_GCC_ENABLE_ALL},
  259. /* Interrupts on all logical channels */
  260. { .reg = D40_DREG_CLCMIS1, .val = 0xFFFFFFFF},
  261. { .reg = D40_DREG_CLCMIS2, .val = 0xFFFFFFFF},
  262. { .reg = D40_DREG_CLCMIS3, .val = 0xFFFFFFFF},
  263. { .reg = D40_DREG_CLCMIS4, .val = 0xFFFFFFFF},
  264. { .reg = D40_DREG_CLCMIS5, .val = 0xFFFFFFFF},
  265. { .reg = D40_DREG_CLCICR1, .val = 0xFFFFFFFF},
  266. { .reg = D40_DREG_CLCICR2, .val = 0xFFFFFFFF},
  267. { .reg = D40_DREG_CLCICR3, .val = 0xFFFFFFFF},
  268. { .reg = D40_DREG_CLCICR4, .val = 0xFFFFFFFF},
  269. { .reg = D40_DREG_CLCICR5, .val = 0xFFFFFFFF},
  270. { .reg = D40_DREG_CLCTIS1, .val = 0xFFFFFFFF},
  271. { .reg = D40_DREG_CLCTIS2, .val = 0xFFFFFFFF},
  272. { .reg = D40_DREG_CLCTIS3, .val = 0xFFFFFFFF},
  273. { .reg = D40_DREG_CLCTIS4, .val = 0xFFFFFFFF},
  274. { .reg = D40_DREG_CLCTIS5, .val = 0xFFFFFFFF}
  275. };
  276. /**
  277. * struct d40_lli_pool - Structure for keeping LLIs in memory
  278. *
  279. * @base: Pointer to memory area when the pre_alloc_lli's are not large
  280. * enough, IE bigger than the most common case, 1 dst and 1 src. NULL if
  281. * pre_alloc_lli is used.
  282. * @dma_addr: DMA address, if mapped
  283. * @size: The size in bytes of the memory at base or the size of pre_alloc_lli.
  284. * @pre_alloc_lli: Pre allocated area for the most common case of transfers,
  285. * one buffer to one buffer.
  286. */
  287. struct d40_lli_pool {
  288. void *base;
  289. int size;
  290. dma_addr_t dma_addr;
  291. /* Space for dst and src, plus an extra for padding */
  292. u8 pre_alloc_lli[3 * sizeof(struct d40_phy_lli)];
  293. };
  294. /**
  295. * struct d40_desc - A descriptor is one DMA job.
  296. *
  297. * @lli_phy: LLI settings for physical channel. Both src and dst=
  298. * points into the lli_pool, to base if lli_len > 1 or to pre_alloc_lli if
  299. * lli_len equals one.
  300. * @lli_log: Same as above but for logical channels.
  301. * @lli_pool: The pool with two entries pre-allocated.
  302. * @lli_len: Number of llis of current descriptor.
  303. * @lli_current: Number of transferred llis.
  304. * @lcla_alloc: Number of LCLA entries allocated.
  305. * @txd: DMA engine struct. Used for among other things for communication
  306. * during a transfer.
  307. * @node: List entry.
  308. * @is_in_client_list: true if the client owns this descriptor.
  309. * @cyclic: true if this is a cyclic job
  310. *
  311. * This descriptor is used for both logical and physical transfers.
  312. */
  313. struct d40_desc {
  314. /* LLI physical */
  315. struct d40_phy_lli_bidir lli_phy;
  316. /* LLI logical */
  317. struct d40_log_lli_bidir lli_log;
  318. struct d40_lli_pool lli_pool;
  319. int lli_len;
  320. int lli_current;
  321. int lcla_alloc;
  322. struct dma_async_tx_descriptor txd;
  323. struct list_head node;
  324. bool is_in_client_list;
  325. bool cyclic;
  326. };
  327. /**
  328. * struct d40_lcla_pool - LCLA pool settings and data.
  329. *
  330. * @base: The virtual address of LCLA. 18 bit aligned.
  331. * @base_unaligned: The orignal kmalloc pointer, if kmalloc is used.
  332. * This pointer is only there for clean-up on error.
  333. * @pages: The number of pages needed for all physical channels.
  334. * Only used later for clean-up on error
  335. * @lock: Lock to protect the content in this struct.
  336. * @alloc_map: big map over which LCLA entry is own by which job.
  337. */
  338. struct d40_lcla_pool {
  339. void *base;
  340. dma_addr_t dma_addr;
  341. void *base_unaligned;
  342. int pages;
  343. spinlock_t lock;
  344. struct d40_desc **alloc_map;
  345. };
  346. /**
  347. * struct d40_phy_res - struct for handling eventlines mapped to physical
  348. * channels.
  349. *
  350. * @lock: A lock protection this entity.
  351. * @reserved: True if used by secure world or otherwise.
  352. * @num: The physical channel number of this entity.
  353. * @allocated_src: Bit mapped to show which src event line's are mapped to
  354. * this physical channel. Can also be free or physically allocated.
  355. * @allocated_dst: Same as for src but is dst.
  356. * allocated_dst and allocated_src uses the D40_ALLOC* defines as well as
  357. * event line number.
  358. * @use_soft_lli: To mark if the linked lists of channel are managed by SW.
  359. */
  360. struct d40_phy_res {
  361. spinlock_t lock;
  362. bool reserved;
  363. int num;
  364. u32 allocated_src;
  365. u32 allocated_dst;
  366. bool use_soft_lli;
  367. };
  368. struct d40_base;
  369. /**
  370. * struct d40_chan - Struct that describes a channel.
  371. *
  372. * @lock: A spinlock to protect this struct.
  373. * @log_num: The logical number, if any of this channel.
  374. * @pending_tx: The number of pending transfers. Used between interrupt handler
  375. * and tasklet.
  376. * @busy: Set to true when transfer is ongoing on this channel.
  377. * @phy_chan: Pointer to physical channel which this instance runs on. If this
  378. * point is NULL, then the channel is not allocated.
  379. * @chan: DMA engine handle.
  380. * @tasklet: Tasklet that gets scheduled from interrupt context to complete a
  381. * transfer and call client callback.
  382. * @client: Cliented owned descriptor list.
  383. * @pending_queue: Submitted jobs, to be issued by issue_pending()
  384. * @active: Active descriptor.
  385. * @done: Completed jobs
  386. * @queue: Queued jobs.
  387. * @prepare_queue: Prepared jobs.
  388. * @dma_cfg: The client configuration of this dma channel.
  389. * @configured: whether the dma_cfg configuration is valid
  390. * @base: Pointer to the device instance struct.
  391. * @src_def_cfg: Default cfg register setting for src.
  392. * @dst_def_cfg: Default cfg register setting for dst.
  393. * @log_def: Default logical channel settings.
  394. * @lcpa: Pointer to dst and src lcpa settings.
  395. * @runtime_addr: runtime configured address.
  396. * @runtime_direction: runtime configured direction.
  397. *
  398. * This struct can either "be" a logical or a physical channel.
  399. */
  400. struct d40_chan {
  401. spinlock_t lock;
  402. int log_num;
  403. int pending_tx;
  404. bool busy;
  405. struct d40_phy_res *phy_chan;
  406. struct dma_chan chan;
  407. struct tasklet_struct tasklet;
  408. struct list_head client;
  409. struct list_head pending_queue;
  410. struct list_head active;
  411. struct list_head done;
  412. struct list_head queue;
  413. struct list_head prepare_queue;
  414. struct stedma40_chan_cfg dma_cfg;
  415. bool configured;
  416. struct d40_base *base;
  417. /* Default register configurations */
  418. u32 src_def_cfg;
  419. u32 dst_def_cfg;
  420. struct d40_def_lcsp log_def;
  421. struct d40_log_lli_full *lcpa;
  422. /* Runtime reconfiguration */
  423. dma_addr_t runtime_addr;
  424. enum dma_transfer_direction runtime_direction;
  425. };
  426. /**
  427. * struct d40_gen_dmac - generic values to represent u8500/u8540 DMA
  428. * controller
  429. *
  430. * @backup: the pointer to the registers address array for backup
  431. * @backup_size: the size of the registers address array for backup
  432. * @realtime_en: the realtime enable register
  433. * @realtime_clear: the realtime clear register
  434. * @high_prio_en: the high priority enable register
  435. * @high_prio_clear: the high priority clear register
  436. * @interrupt_en: the interrupt enable register
  437. * @interrupt_clear: the interrupt clear register
  438. * @il: the pointer to struct d40_interrupt_lookup
  439. * @il_size: the size of d40_interrupt_lookup array
  440. * @init_reg: the pointer to the struct d40_reg_val
  441. * @init_reg_size: the size of d40_reg_val array
  442. */
  443. struct d40_gen_dmac {
  444. u32 *backup;
  445. u32 backup_size;
  446. u32 realtime_en;
  447. u32 realtime_clear;
  448. u32 high_prio_en;
  449. u32 high_prio_clear;
  450. u32 interrupt_en;
  451. u32 interrupt_clear;
  452. struct d40_interrupt_lookup *il;
  453. u32 il_size;
  454. struct d40_reg_val *init_reg;
  455. u32 init_reg_size;
  456. };
  457. /**
  458. * struct d40_base - The big global struct, one for each probe'd instance.
  459. *
  460. * @interrupt_lock: Lock used to make sure one interrupt is handle a time.
  461. * @execmd_lock: Lock for execute command usage since several channels share
  462. * the same physical register.
  463. * @dev: The device structure.
  464. * @virtbase: The virtual base address of the DMA's register.
  465. * @rev: silicon revision detected.
  466. * @clk: Pointer to the DMA clock structure.
  467. * @phy_start: Physical memory start of the DMA registers.
  468. * @phy_size: Size of the DMA register map.
  469. * @irq: The IRQ number.
  470. * @num_phy_chans: The number of physical channels. Read from HW. This
  471. * is the number of available channels for this driver, not counting "Secure
  472. * mode" allocated physical channels.
  473. * @num_log_chans: The number of logical channels. Calculated from
  474. * num_phy_chans.
  475. * @dma_both: dma_device channels that can do both memcpy and slave transfers.
  476. * @dma_slave: dma_device channels that can do only do slave transfers.
  477. * @dma_memcpy: dma_device channels that can do only do memcpy transfers.
  478. * @phy_chans: Room for all possible physical channels in system.
  479. * @log_chans: Room for all possible logical channels in system.
  480. * @lookup_log_chans: Used to map interrupt number to logical channel. Points
  481. * to log_chans entries.
  482. * @lookup_phy_chans: Used to map interrupt number to physical channel. Points
  483. * to phy_chans entries.
  484. * @plat_data: Pointer to provided platform_data which is the driver
  485. * configuration.
  486. * @lcpa_regulator: Pointer to hold the regulator for the esram bank for lcla.
  487. * @phy_res: Vector containing all physical channels.
  488. * @lcla_pool: lcla pool settings and data.
  489. * @lcpa_base: The virtual mapped address of LCPA.
  490. * @phy_lcpa: The physical address of the LCPA.
  491. * @lcpa_size: The size of the LCPA area.
  492. * @desc_slab: cache for descriptors.
  493. * @reg_val_backup: Here the values of some hardware registers are stored
  494. * before the DMA is powered off. They are restored when the power is back on.
  495. * @reg_val_backup_v4: Backup of registers that only exits on dma40 v3 and
  496. * later
  497. * @reg_val_backup_chan: Backup data for standard channel parameter registers.
  498. * @gcc_pwr_off_mask: Mask to maintain the channels that can be turned off.
  499. * @initialized: true if the dma has been initialized
  500. * @gen_dmac: the struct for generic registers values to represent u8500/8540
  501. * DMA controller
  502. */
  503. struct d40_base {
  504. spinlock_t interrupt_lock;
  505. spinlock_t execmd_lock;
  506. struct device *dev;
  507. void __iomem *virtbase;
  508. u8 rev:4;
  509. struct clk *clk;
  510. phys_addr_t phy_start;
  511. resource_size_t phy_size;
  512. int irq;
  513. int num_phy_chans;
  514. int num_log_chans;
  515. struct device_dma_parameters dma_parms;
  516. struct dma_device dma_both;
  517. struct dma_device dma_slave;
  518. struct dma_device dma_memcpy;
  519. struct d40_chan *phy_chans;
  520. struct d40_chan *log_chans;
  521. struct d40_chan **lookup_log_chans;
  522. struct d40_chan **lookup_phy_chans;
  523. struct stedma40_platform_data *plat_data;
  524. struct regulator *lcpa_regulator;
  525. /* Physical half channels */
  526. struct d40_phy_res *phy_res;
  527. struct d40_lcla_pool lcla_pool;
  528. void *lcpa_base;
  529. dma_addr_t phy_lcpa;
  530. resource_size_t lcpa_size;
  531. struct kmem_cache *desc_slab;
  532. u32 reg_val_backup[BACKUP_REGS_SZ];
  533. u32 reg_val_backup_v4[MAX(BACKUP_REGS_SZ_V4A, BACKUP_REGS_SZ_V4B)];
  534. u32 *reg_val_backup_chan;
  535. u16 gcc_pwr_off_mask;
  536. bool initialized;
  537. struct d40_gen_dmac gen_dmac;
  538. };
  539. static struct device *chan2dev(struct d40_chan *d40c)
  540. {
  541. return &d40c->chan.dev->device;
  542. }
  543. static bool chan_is_physical(struct d40_chan *chan)
  544. {
  545. return chan->log_num == D40_PHY_CHAN;
  546. }
  547. static bool chan_is_logical(struct d40_chan *chan)
  548. {
  549. return !chan_is_physical(chan);
  550. }
  551. static void __iomem *chan_base(struct d40_chan *chan)
  552. {
  553. return chan->base->virtbase + D40_DREG_PCBASE +
  554. chan->phy_chan->num * D40_DREG_PCDELTA;
  555. }
  556. #define d40_err(dev, format, arg...) \
  557. dev_err(dev, "[%s] " format, __func__, ## arg)
  558. #define chan_err(d40c, format, arg...) \
  559. d40_err(chan2dev(d40c), format, ## arg)
  560. static int d40_pool_lli_alloc(struct d40_chan *d40c, struct d40_desc *d40d,
  561. int lli_len)
  562. {
  563. bool is_log = chan_is_logical(d40c);
  564. u32 align;
  565. void *base;
  566. if (is_log)
  567. align = sizeof(struct d40_log_lli);
  568. else
  569. align = sizeof(struct d40_phy_lli);
  570. if (lli_len == 1) {
  571. base = d40d->lli_pool.pre_alloc_lli;
  572. d40d->lli_pool.size = sizeof(d40d->lli_pool.pre_alloc_lli);
  573. d40d->lli_pool.base = NULL;
  574. } else {
  575. d40d->lli_pool.size = lli_len * 2 * align;
  576. base = kmalloc(d40d->lli_pool.size + align, GFP_NOWAIT);
  577. d40d->lli_pool.base = base;
  578. if (d40d->lli_pool.base == NULL)
  579. return -ENOMEM;
  580. }
  581. if (is_log) {
  582. d40d->lli_log.src = PTR_ALIGN(base, align);
  583. d40d->lli_log.dst = d40d->lli_log.src + lli_len;
  584. d40d->lli_pool.dma_addr = 0;
  585. } else {
  586. d40d->lli_phy.src = PTR_ALIGN(base, align);
  587. d40d->lli_phy.dst = d40d->lli_phy.src + lli_len;
  588. d40d->lli_pool.dma_addr = dma_map_single(d40c->base->dev,
  589. d40d->lli_phy.src,
  590. d40d->lli_pool.size,
  591. DMA_TO_DEVICE);
  592. if (dma_mapping_error(d40c->base->dev,
  593. d40d->lli_pool.dma_addr)) {
  594. kfree(d40d->lli_pool.base);
  595. d40d->lli_pool.base = NULL;
  596. d40d->lli_pool.dma_addr = 0;
  597. return -ENOMEM;
  598. }
  599. }
  600. return 0;
  601. }
  602. static void d40_pool_lli_free(struct d40_chan *d40c, struct d40_desc *d40d)
  603. {
  604. if (d40d->lli_pool.dma_addr)
  605. dma_unmap_single(d40c->base->dev, d40d->lli_pool.dma_addr,
  606. d40d->lli_pool.size, DMA_TO_DEVICE);
  607. kfree(d40d->lli_pool.base);
  608. d40d->lli_pool.base = NULL;
  609. d40d->lli_pool.size = 0;
  610. d40d->lli_log.src = NULL;
  611. d40d->lli_log.dst = NULL;
  612. d40d->lli_phy.src = NULL;
  613. d40d->lli_phy.dst = NULL;
  614. }
  615. static int d40_lcla_alloc_one(struct d40_chan *d40c,
  616. struct d40_desc *d40d)
  617. {
  618. unsigned long flags;
  619. int i;
  620. int ret = -EINVAL;
  621. spin_lock_irqsave(&d40c->base->lcla_pool.lock, flags);
  622. /*
  623. * Allocate both src and dst at the same time, therefore the half
  624. * start on 1 since 0 can't be used since zero is used as end marker.
  625. */
  626. for (i = 1 ; i < D40_LCLA_LINK_PER_EVENT_GRP / 2; i++) {
  627. int idx = d40c->phy_chan->num * D40_LCLA_LINK_PER_EVENT_GRP + i;
  628. if (!d40c->base->lcla_pool.alloc_map[idx]) {
  629. d40c->base->lcla_pool.alloc_map[idx] = d40d;
  630. d40d->lcla_alloc++;
  631. ret = i;
  632. break;
  633. }
  634. }
  635. spin_unlock_irqrestore(&d40c->base->lcla_pool.lock, flags);
  636. return ret;
  637. }
  638. static int d40_lcla_free_all(struct d40_chan *d40c,
  639. struct d40_desc *d40d)
  640. {
  641. unsigned long flags;
  642. int i;
  643. int ret = -EINVAL;
  644. if (chan_is_physical(d40c))
  645. return 0;
  646. spin_lock_irqsave(&d40c->base->lcla_pool.lock, flags);
  647. for (i = 1 ; i < D40_LCLA_LINK_PER_EVENT_GRP / 2; i++) {
  648. int idx = d40c->phy_chan->num * D40_LCLA_LINK_PER_EVENT_GRP + i;
  649. if (d40c->base->lcla_pool.alloc_map[idx] == d40d) {
  650. d40c->base->lcla_pool.alloc_map[idx] = NULL;
  651. d40d->lcla_alloc--;
  652. if (d40d->lcla_alloc == 0) {
  653. ret = 0;
  654. break;
  655. }
  656. }
  657. }
  658. spin_unlock_irqrestore(&d40c->base->lcla_pool.lock, flags);
  659. return ret;
  660. }
  661. static void d40_desc_remove(struct d40_desc *d40d)
  662. {
  663. list_del(&d40d->node);
  664. }
  665. static struct d40_desc *d40_desc_get(struct d40_chan *d40c)
  666. {
  667. struct d40_desc *desc = NULL;
  668. if (!list_empty(&d40c->client)) {
  669. struct d40_desc *d;
  670. struct d40_desc *_d;
  671. list_for_each_entry_safe(d, _d, &d40c->client, node) {
  672. if (async_tx_test_ack(&d->txd)) {
  673. d40_desc_remove(d);
  674. desc = d;
  675. memset(desc, 0, sizeof(*desc));
  676. break;
  677. }
  678. }
  679. }
  680. if (!desc)
  681. desc = kmem_cache_zalloc(d40c->base->desc_slab, GFP_NOWAIT);
  682. if (desc)
  683. INIT_LIST_HEAD(&desc->node);
  684. return desc;
  685. }
  686. static void d40_desc_free(struct d40_chan *d40c, struct d40_desc *d40d)
  687. {
  688. d40_pool_lli_free(d40c, d40d);
  689. d40_lcla_free_all(d40c, d40d);
  690. kmem_cache_free(d40c->base->desc_slab, d40d);
  691. }
  692. static void d40_desc_submit(struct d40_chan *d40c, struct d40_desc *desc)
  693. {
  694. list_add_tail(&desc->node, &d40c->active);
  695. }
  696. static void d40_phy_lli_load(struct d40_chan *chan, struct d40_desc *desc)
  697. {
  698. struct d40_phy_lli *lli_dst = desc->lli_phy.dst;
  699. struct d40_phy_lli *lli_src = desc->lli_phy.src;
  700. void __iomem *base = chan_base(chan);
  701. writel(lli_src->reg_cfg, base + D40_CHAN_REG_SSCFG);
  702. writel(lli_src->reg_elt, base + D40_CHAN_REG_SSELT);
  703. writel(lli_src->reg_ptr, base + D40_CHAN_REG_SSPTR);
  704. writel(lli_src->reg_lnk, base + D40_CHAN_REG_SSLNK);
  705. writel(lli_dst->reg_cfg, base + D40_CHAN_REG_SDCFG);
  706. writel(lli_dst->reg_elt, base + D40_CHAN_REG_SDELT);
  707. writel(lli_dst->reg_ptr, base + D40_CHAN_REG_SDPTR);
  708. writel(lli_dst->reg_lnk, base + D40_CHAN_REG_SDLNK);
  709. }
  710. static void d40_desc_done(struct d40_chan *d40c, struct d40_desc *desc)
  711. {
  712. list_add_tail(&desc->node, &d40c->done);
  713. }
  714. static void d40_log_lli_to_lcxa(struct d40_chan *chan, struct d40_desc *desc)
  715. {
  716. struct d40_lcla_pool *pool = &chan->base->lcla_pool;
  717. struct d40_log_lli_bidir *lli = &desc->lli_log;
  718. int lli_current = desc->lli_current;
  719. int lli_len = desc->lli_len;
  720. bool cyclic = desc->cyclic;
  721. int curr_lcla = -EINVAL;
  722. int first_lcla = 0;
  723. bool use_esram_lcla = chan->base->plat_data->use_esram_lcla;
  724. bool linkback;
  725. /*
  726. * We may have partially running cyclic transfers, in case we did't get
  727. * enough LCLA entries.
  728. */
  729. linkback = cyclic && lli_current == 0;
  730. /*
  731. * For linkback, we need one LCLA even with only one link, because we
  732. * can't link back to the one in LCPA space
  733. */
  734. if (linkback || (lli_len - lli_current > 1)) {
  735. /*
  736. * If the channel is expected to use only soft_lli don't
  737. * allocate a lcla. This is to avoid a HW issue that exists
  738. * in some controller during a peripheral to memory transfer
  739. * that uses linked lists.
  740. */
  741. if (!(chan->phy_chan->use_soft_lli &&
  742. chan->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM))
  743. curr_lcla = d40_lcla_alloc_one(chan, desc);
  744. first_lcla = curr_lcla;
  745. }
  746. /*
  747. * For linkback, we normally load the LCPA in the loop since we need to
  748. * link it to the second LCLA and not the first. However, if we
  749. * couldn't even get a first LCLA, then we have to run in LCPA and
  750. * reload manually.
  751. */
  752. if (!linkback || curr_lcla == -EINVAL) {
  753. unsigned int flags = 0;
  754. if (curr_lcla == -EINVAL)
  755. flags |= LLI_TERM_INT;
  756. d40_log_lli_lcpa_write(chan->lcpa,
  757. &lli->dst[lli_current],
  758. &lli->src[lli_current],
  759. curr_lcla,
  760. flags);
  761. lli_current++;
  762. }
  763. if (curr_lcla < 0)
  764. goto out;
  765. for (; lli_current < lli_len; lli_current++) {
  766. unsigned int lcla_offset = chan->phy_chan->num * 1024 +
  767. 8 * curr_lcla * 2;
  768. struct d40_log_lli *lcla = pool->base + lcla_offset;
  769. unsigned int flags = 0;
  770. int next_lcla;
  771. if (lli_current + 1 < lli_len)
  772. next_lcla = d40_lcla_alloc_one(chan, desc);
  773. else
  774. next_lcla = linkback ? first_lcla : -EINVAL;
  775. if (cyclic || next_lcla == -EINVAL)
  776. flags |= LLI_TERM_INT;
  777. if (linkback && curr_lcla == first_lcla) {
  778. /* First link goes in both LCPA and LCLA */
  779. d40_log_lli_lcpa_write(chan->lcpa,
  780. &lli->dst[lli_current],
  781. &lli->src[lli_current],
  782. next_lcla, flags);
  783. }
  784. /*
  785. * One unused LCLA in the cyclic case if the very first
  786. * next_lcla fails...
  787. */
  788. d40_log_lli_lcla_write(lcla,
  789. &lli->dst[lli_current],
  790. &lli->src[lli_current],
  791. next_lcla, flags);
  792. /*
  793. * Cache maintenance is not needed if lcla is
  794. * mapped in esram
  795. */
  796. if (!use_esram_lcla) {
  797. dma_sync_single_range_for_device(chan->base->dev,
  798. pool->dma_addr, lcla_offset,
  799. 2 * sizeof(struct d40_log_lli),
  800. DMA_TO_DEVICE);
  801. }
  802. curr_lcla = next_lcla;
  803. if (curr_lcla == -EINVAL || curr_lcla == first_lcla) {
  804. lli_current++;
  805. break;
  806. }
  807. }
  808. out:
  809. desc->lli_current = lli_current;
  810. }
  811. static void d40_desc_load(struct d40_chan *d40c, struct d40_desc *d40d)
  812. {
  813. if (chan_is_physical(d40c)) {
  814. d40_phy_lli_load(d40c, d40d);
  815. d40d->lli_current = d40d->lli_len;
  816. } else
  817. d40_log_lli_to_lcxa(d40c, d40d);
  818. }
  819. static struct d40_desc *d40_first_active_get(struct d40_chan *d40c)
  820. {
  821. struct d40_desc *d;
  822. if (list_empty(&d40c->active))
  823. return NULL;
  824. d = list_first_entry(&d40c->active,
  825. struct d40_desc,
  826. node);
  827. return d;
  828. }
  829. /* remove desc from current queue and add it to the pending_queue */
  830. static void d40_desc_queue(struct d40_chan *d40c, struct d40_desc *desc)
  831. {
  832. d40_desc_remove(desc);
  833. desc->is_in_client_list = false;
  834. list_add_tail(&desc->node, &d40c->pending_queue);
  835. }
  836. static struct d40_desc *d40_first_pending(struct d40_chan *d40c)
  837. {
  838. struct d40_desc *d;
  839. if (list_empty(&d40c->pending_queue))
  840. return NULL;
  841. d = list_first_entry(&d40c->pending_queue,
  842. struct d40_desc,
  843. node);
  844. return d;
  845. }
  846. static struct d40_desc *d40_first_queued(struct d40_chan *d40c)
  847. {
  848. struct d40_desc *d;
  849. if (list_empty(&d40c->queue))
  850. return NULL;
  851. d = list_first_entry(&d40c->queue,
  852. struct d40_desc,
  853. node);
  854. return d;
  855. }
  856. static struct d40_desc *d40_first_done(struct d40_chan *d40c)
  857. {
  858. if (list_empty(&d40c->done))
  859. return NULL;
  860. return list_first_entry(&d40c->done, struct d40_desc, node);
  861. }
  862. static int d40_psize_2_burst_size(bool is_log, int psize)
  863. {
  864. if (is_log) {
  865. if (psize == STEDMA40_PSIZE_LOG_1)
  866. return 1;
  867. } else {
  868. if (psize == STEDMA40_PSIZE_PHY_1)
  869. return 1;
  870. }
  871. return 2 << psize;
  872. }
  873. /*
  874. * The dma only supports transmitting packages up to
  875. * STEDMA40_MAX_SEG_SIZE << data_width. Calculate the total number of
  876. * dma elements required to send the entire sg list
  877. */
  878. static int d40_size_2_dmalen(int size, u32 data_width1, u32 data_width2)
  879. {
  880. int dmalen;
  881. u32 max_w = max(data_width1, data_width2);
  882. u32 min_w = min(data_width1, data_width2);
  883. u32 seg_max = ALIGN(STEDMA40_MAX_SEG_SIZE << min_w, 1 << max_w);
  884. if (seg_max > STEDMA40_MAX_SEG_SIZE)
  885. seg_max -= (1 << max_w);
  886. if (!IS_ALIGNED(size, 1 << max_w))
  887. return -EINVAL;
  888. if (size <= seg_max)
  889. dmalen = 1;
  890. else {
  891. dmalen = size / seg_max;
  892. if (dmalen * seg_max < size)
  893. dmalen++;
  894. }
  895. return dmalen;
  896. }
  897. static int d40_sg_2_dmalen(struct scatterlist *sgl, int sg_len,
  898. u32 data_width1, u32 data_width2)
  899. {
  900. struct scatterlist *sg;
  901. int i;
  902. int len = 0;
  903. int ret;
  904. for_each_sg(sgl, sg, sg_len, i) {
  905. ret = d40_size_2_dmalen(sg_dma_len(sg),
  906. data_width1, data_width2);
  907. if (ret < 0)
  908. return ret;
  909. len += ret;
  910. }
  911. return len;
  912. }
  913. #ifdef CONFIG_PM
  914. static void dma40_backup(void __iomem *baseaddr, u32 *backup,
  915. u32 *regaddr, int num, bool save)
  916. {
  917. int i;
  918. for (i = 0; i < num; i++) {
  919. void __iomem *addr = baseaddr + regaddr[i];
  920. if (save)
  921. backup[i] = readl_relaxed(addr);
  922. else
  923. writel_relaxed(backup[i], addr);
  924. }
  925. }
  926. static void d40_save_restore_registers(struct d40_base *base, bool save)
  927. {
  928. int i;
  929. /* Save/Restore channel specific registers */
  930. for (i = 0; i < base->num_phy_chans; i++) {
  931. void __iomem *addr;
  932. int idx;
  933. if (base->phy_res[i].reserved)
  934. continue;
  935. addr = base->virtbase + D40_DREG_PCBASE + i * D40_DREG_PCDELTA;
  936. idx = i * ARRAY_SIZE(d40_backup_regs_chan);
  937. dma40_backup(addr, &base->reg_val_backup_chan[idx],
  938. d40_backup_regs_chan,
  939. ARRAY_SIZE(d40_backup_regs_chan),
  940. save);
  941. }
  942. /* Save/Restore global registers */
  943. dma40_backup(base->virtbase, base->reg_val_backup,
  944. d40_backup_regs, ARRAY_SIZE(d40_backup_regs),
  945. save);
  946. /* Save/Restore registers only existing on dma40 v3 and later */
  947. if (base->gen_dmac.backup)
  948. dma40_backup(base->virtbase, base->reg_val_backup_v4,
  949. base->gen_dmac.backup,
  950. base->gen_dmac.backup_size,
  951. save);
  952. }
  953. #else
  954. static void d40_save_restore_registers(struct d40_base *base, bool save)
  955. {
  956. }
  957. #endif
  958. static int __d40_execute_command_phy(struct d40_chan *d40c,
  959. enum d40_command command)
  960. {
  961. u32 status;
  962. int i;
  963. void __iomem *active_reg;
  964. int ret = 0;
  965. unsigned long flags;
  966. u32 wmask;
  967. if (command == D40_DMA_STOP) {
  968. ret = __d40_execute_command_phy(d40c, D40_DMA_SUSPEND_REQ);
  969. if (ret)
  970. return ret;
  971. }
  972. spin_lock_irqsave(&d40c->base->execmd_lock, flags);
  973. if (d40c->phy_chan->num % 2 == 0)
  974. active_reg = d40c->base->virtbase + D40_DREG_ACTIVE;
  975. else
  976. active_reg = d40c->base->virtbase + D40_DREG_ACTIVO;
  977. if (command == D40_DMA_SUSPEND_REQ) {
  978. status = (readl(active_reg) &
  979. D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
  980. D40_CHAN_POS(d40c->phy_chan->num);
  981. if (status == D40_DMA_SUSPENDED || status == D40_DMA_STOP)
  982. goto done;
  983. }
  984. wmask = 0xffffffff & ~(D40_CHAN_POS_MASK(d40c->phy_chan->num));
  985. writel(wmask | (command << D40_CHAN_POS(d40c->phy_chan->num)),
  986. active_reg);
  987. if (command == D40_DMA_SUSPEND_REQ) {
  988. for (i = 0 ; i < D40_SUSPEND_MAX_IT; i++) {
  989. status = (readl(active_reg) &
  990. D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
  991. D40_CHAN_POS(d40c->phy_chan->num);
  992. cpu_relax();
  993. /*
  994. * Reduce the number of bus accesses while
  995. * waiting for the DMA to suspend.
  996. */
  997. udelay(3);
  998. if (status == D40_DMA_STOP ||
  999. status == D40_DMA_SUSPENDED)
  1000. break;
  1001. }
  1002. if (i == D40_SUSPEND_MAX_IT) {
  1003. chan_err(d40c,
  1004. "unable to suspend the chl %d (log: %d) status %x\n",
  1005. d40c->phy_chan->num, d40c->log_num,
  1006. status);
  1007. dump_stack();
  1008. ret = -EBUSY;
  1009. }
  1010. }
  1011. done:
  1012. spin_unlock_irqrestore(&d40c->base->execmd_lock, flags);
  1013. return ret;
  1014. }
  1015. static void d40_term_all(struct d40_chan *d40c)
  1016. {
  1017. struct d40_desc *d40d;
  1018. struct d40_desc *_d;
  1019. /* Release completed descriptors */
  1020. while ((d40d = d40_first_done(d40c))) {
  1021. d40_desc_remove(d40d);
  1022. d40_desc_free(d40c, d40d);
  1023. }
  1024. /* Release active descriptors */
  1025. while ((d40d = d40_first_active_get(d40c))) {
  1026. d40_desc_remove(d40d);
  1027. d40_desc_free(d40c, d40d);
  1028. }
  1029. /* Release queued descriptors waiting for transfer */
  1030. while ((d40d = d40_first_queued(d40c))) {
  1031. d40_desc_remove(d40d);
  1032. d40_desc_free(d40c, d40d);
  1033. }
  1034. /* Release pending descriptors */
  1035. while ((d40d = d40_first_pending(d40c))) {
  1036. d40_desc_remove(d40d);
  1037. d40_desc_free(d40c, d40d);
  1038. }
  1039. /* Release client owned descriptors */
  1040. if (!list_empty(&d40c->client))
  1041. list_for_each_entry_safe(d40d, _d, &d40c->client, node) {
  1042. d40_desc_remove(d40d);
  1043. d40_desc_free(d40c, d40d);
  1044. }
  1045. /* Release descriptors in prepare queue */
  1046. if (!list_empty(&d40c->prepare_queue))
  1047. list_for_each_entry_safe(d40d, _d,
  1048. &d40c->prepare_queue, node) {
  1049. d40_desc_remove(d40d);
  1050. d40_desc_free(d40c, d40d);
  1051. }
  1052. d40c->pending_tx = 0;
  1053. }
  1054. static void __d40_config_set_event(struct d40_chan *d40c,
  1055. enum d40_events event_type, u32 event,
  1056. int reg)
  1057. {
  1058. void __iomem *addr = chan_base(d40c) + reg;
  1059. int tries;
  1060. u32 status;
  1061. switch (event_type) {
  1062. case D40_DEACTIVATE_EVENTLINE:
  1063. writel((D40_DEACTIVATE_EVENTLINE << D40_EVENTLINE_POS(event))
  1064. | ~D40_EVENTLINE_MASK(event), addr);
  1065. break;
  1066. case D40_SUSPEND_REQ_EVENTLINE:
  1067. status = (readl(addr) & D40_EVENTLINE_MASK(event)) >>
  1068. D40_EVENTLINE_POS(event);
  1069. if (status == D40_DEACTIVATE_EVENTLINE ||
  1070. status == D40_SUSPEND_REQ_EVENTLINE)
  1071. break;
  1072. writel((D40_SUSPEND_REQ_EVENTLINE << D40_EVENTLINE_POS(event))
  1073. | ~D40_EVENTLINE_MASK(event), addr);
  1074. for (tries = 0 ; tries < D40_SUSPEND_MAX_IT; tries++) {
  1075. status = (readl(addr) & D40_EVENTLINE_MASK(event)) >>
  1076. D40_EVENTLINE_POS(event);
  1077. cpu_relax();
  1078. /*
  1079. * Reduce the number of bus accesses while
  1080. * waiting for the DMA to suspend.
  1081. */
  1082. udelay(3);
  1083. if (status == D40_DEACTIVATE_EVENTLINE)
  1084. break;
  1085. }
  1086. if (tries == D40_SUSPEND_MAX_IT) {
  1087. chan_err(d40c,
  1088. "unable to stop the event_line chl %d (log: %d)"
  1089. "status %x\n", d40c->phy_chan->num,
  1090. d40c->log_num, status);
  1091. }
  1092. break;
  1093. case D40_ACTIVATE_EVENTLINE:
  1094. /*
  1095. * The hardware sometimes doesn't register the enable when src and dst
  1096. * event lines are active on the same logical channel. Retry to ensure
  1097. * it does. Usually only one retry is sufficient.
  1098. */
  1099. tries = 100;
  1100. while (--tries) {
  1101. writel((D40_ACTIVATE_EVENTLINE <<
  1102. D40_EVENTLINE_POS(event)) |
  1103. ~D40_EVENTLINE_MASK(event), addr);
  1104. if (readl(addr) & D40_EVENTLINE_MASK(event))
  1105. break;
  1106. }
  1107. if (tries != 99)
  1108. dev_dbg(chan2dev(d40c),
  1109. "[%s] workaround enable S%cLNK (%d tries)\n",
  1110. __func__, reg == D40_CHAN_REG_SSLNK ? 'S' : 'D',
  1111. 100 - tries);
  1112. WARN_ON(!tries);
  1113. break;
  1114. case D40_ROUND_EVENTLINE:
  1115. BUG();
  1116. break;
  1117. }
  1118. }
  1119. static void d40_config_set_event(struct d40_chan *d40c,
  1120. enum d40_events event_type)
  1121. {
  1122. /* Enable event line connected to device (or memcpy) */
  1123. if ((d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) ||
  1124. (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH)) {
  1125. u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type);
  1126. __d40_config_set_event(d40c, event_type, event,
  1127. D40_CHAN_REG_SSLNK);
  1128. }
  1129. if (d40c->dma_cfg.dir != STEDMA40_PERIPH_TO_MEM) {
  1130. u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type);
  1131. __d40_config_set_event(d40c, event_type, event,
  1132. D40_CHAN_REG_SDLNK);
  1133. }
  1134. }
  1135. static u32 d40_chan_has_events(struct d40_chan *d40c)
  1136. {
  1137. void __iomem *chanbase = chan_base(d40c);
  1138. u32 val;
  1139. val = readl(chanbase + D40_CHAN_REG_SSLNK);
  1140. val |= readl(chanbase + D40_CHAN_REG_SDLNK);
  1141. return val;
  1142. }
  1143. static int
  1144. __d40_execute_command_log(struct d40_chan *d40c, enum d40_command command)
  1145. {
  1146. unsigned long flags;
  1147. int ret = 0;
  1148. u32 active_status;
  1149. void __iomem *active_reg;
  1150. if (d40c->phy_chan->num % 2 == 0)
  1151. active_reg = d40c->base->virtbase + D40_DREG_ACTIVE;
  1152. else
  1153. active_reg = d40c->base->virtbase + D40_DREG_ACTIVO;
  1154. spin_lock_irqsave(&d40c->phy_chan->lock, flags);
  1155. switch (command) {
  1156. case D40_DMA_STOP:
  1157. case D40_DMA_SUSPEND_REQ:
  1158. active_status = (readl(active_reg) &
  1159. D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
  1160. D40_CHAN_POS(d40c->phy_chan->num);
  1161. if (active_status == D40_DMA_RUN)
  1162. d40_config_set_event(d40c, D40_SUSPEND_REQ_EVENTLINE);
  1163. else
  1164. d40_config_set_event(d40c, D40_DEACTIVATE_EVENTLINE);
  1165. if (!d40_chan_has_events(d40c) && (command == D40_DMA_STOP))
  1166. ret = __d40_execute_command_phy(d40c, command);
  1167. break;
  1168. case D40_DMA_RUN:
  1169. d40_config_set_event(d40c, D40_ACTIVATE_EVENTLINE);
  1170. ret = __d40_execute_command_phy(d40c, command);
  1171. break;
  1172. case D40_DMA_SUSPENDED:
  1173. BUG();
  1174. break;
  1175. }
  1176. spin_unlock_irqrestore(&d40c->phy_chan->lock, flags);
  1177. return ret;
  1178. }
  1179. static int d40_channel_execute_command(struct d40_chan *d40c,
  1180. enum d40_command command)
  1181. {
  1182. if (chan_is_logical(d40c))
  1183. return __d40_execute_command_log(d40c, command);
  1184. else
  1185. return __d40_execute_command_phy(d40c, command);
  1186. }
  1187. static u32 d40_get_prmo(struct d40_chan *d40c)
  1188. {
  1189. static const unsigned int phy_map[] = {
  1190. [STEDMA40_PCHAN_BASIC_MODE]
  1191. = D40_DREG_PRMO_PCHAN_BASIC,
  1192. [STEDMA40_PCHAN_MODULO_MODE]
  1193. = D40_DREG_PRMO_PCHAN_MODULO,
  1194. [STEDMA40_PCHAN_DOUBLE_DST_MODE]
  1195. = D40_DREG_PRMO_PCHAN_DOUBLE_DST,
  1196. };
  1197. static const unsigned int log_map[] = {
  1198. [STEDMA40_LCHAN_SRC_PHY_DST_LOG]
  1199. = D40_DREG_PRMO_LCHAN_SRC_PHY_DST_LOG,
  1200. [STEDMA40_LCHAN_SRC_LOG_DST_PHY]
  1201. = D40_DREG_PRMO_LCHAN_SRC_LOG_DST_PHY,
  1202. [STEDMA40_LCHAN_SRC_LOG_DST_LOG]
  1203. = D40_DREG_PRMO_LCHAN_SRC_LOG_DST_LOG,
  1204. };
  1205. if (chan_is_physical(d40c))
  1206. return phy_map[d40c->dma_cfg.mode_opt];
  1207. else
  1208. return log_map[d40c->dma_cfg.mode_opt];
  1209. }
  1210. static void d40_config_write(struct d40_chan *d40c)
  1211. {
  1212. u32 addr_base;
  1213. u32 var;
  1214. /* Odd addresses are even addresses + 4 */
  1215. addr_base = (d40c->phy_chan->num % 2) * 4;
  1216. /* Setup channel mode to logical or physical */
  1217. var = ((u32)(chan_is_logical(d40c)) + 1) <<
  1218. D40_CHAN_POS(d40c->phy_chan->num);
  1219. writel(var, d40c->base->virtbase + D40_DREG_PRMSE + addr_base);
  1220. /* Setup operational mode option register */
  1221. var = d40_get_prmo(d40c) << D40_CHAN_POS(d40c->phy_chan->num);
  1222. writel(var, d40c->base->virtbase + D40_DREG_PRMOE + addr_base);
  1223. if (chan_is_logical(d40c)) {
  1224. int lidx = (d40c->phy_chan->num << D40_SREG_ELEM_LOG_LIDX_POS)
  1225. & D40_SREG_ELEM_LOG_LIDX_MASK;
  1226. void __iomem *chanbase = chan_base(d40c);
  1227. /* Set default config for CFG reg */
  1228. writel(d40c->src_def_cfg, chanbase + D40_CHAN_REG_SSCFG);
  1229. writel(d40c->dst_def_cfg, chanbase + D40_CHAN_REG_SDCFG);
  1230. /* Set LIDX for lcla */
  1231. writel(lidx, chanbase + D40_CHAN_REG_SSELT);
  1232. writel(lidx, chanbase + D40_CHAN_REG_SDELT);
  1233. /* Clear LNK which will be used by d40_chan_has_events() */
  1234. writel(0, chanbase + D40_CHAN_REG_SSLNK);
  1235. writel(0, chanbase + D40_CHAN_REG_SDLNK);
  1236. }
  1237. }
  1238. static u32 d40_residue(struct d40_chan *d40c)
  1239. {
  1240. u32 num_elt;
  1241. if (chan_is_logical(d40c))
  1242. num_elt = (readl(&d40c->lcpa->lcsp2) & D40_MEM_LCSP2_ECNT_MASK)
  1243. >> D40_MEM_LCSP2_ECNT_POS;
  1244. else {
  1245. u32 val = readl(chan_base(d40c) + D40_CHAN_REG_SDELT);
  1246. num_elt = (val & D40_SREG_ELEM_PHY_ECNT_MASK)
  1247. >> D40_SREG_ELEM_PHY_ECNT_POS;
  1248. }
  1249. return num_elt * (1 << d40c->dma_cfg.dst_info.data_width);
  1250. }
  1251. static bool d40_tx_is_linked(struct d40_chan *d40c)
  1252. {
  1253. bool is_link;
  1254. if (chan_is_logical(d40c))
  1255. is_link = readl(&d40c->lcpa->lcsp3) & D40_MEM_LCSP3_DLOS_MASK;
  1256. else
  1257. is_link = readl(chan_base(d40c) + D40_CHAN_REG_SDLNK)
  1258. & D40_SREG_LNK_PHYS_LNK_MASK;
  1259. return is_link;
  1260. }
  1261. static int d40_pause(struct d40_chan *d40c)
  1262. {
  1263. int res = 0;
  1264. unsigned long flags;
  1265. if (!d40c->busy)
  1266. return 0;
  1267. pm_runtime_get_sync(d40c->base->dev);
  1268. spin_lock_irqsave(&d40c->lock, flags);
  1269. res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ);
  1270. pm_runtime_mark_last_busy(d40c->base->dev);
  1271. pm_runtime_put_autosuspend(d40c->base->dev);
  1272. spin_unlock_irqrestore(&d40c->lock, flags);
  1273. return res;
  1274. }
  1275. static int d40_resume(struct d40_chan *d40c)
  1276. {
  1277. int res = 0;
  1278. unsigned long flags;
  1279. if (!d40c->busy)
  1280. return 0;
  1281. spin_lock_irqsave(&d40c->lock, flags);
  1282. pm_runtime_get_sync(d40c->base->dev);
  1283. /* If bytes left to transfer or linked tx resume job */
  1284. if (d40_residue(d40c) || d40_tx_is_linked(d40c))
  1285. res = d40_channel_execute_command(d40c, D40_DMA_RUN);
  1286. pm_runtime_mark_last_busy(d40c->base->dev);
  1287. pm_runtime_put_autosuspend(d40c->base->dev);
  1288. spin_unlock_irqrestore(&d40c->lock, flags);
  1289. return res;
  1290. }
  1291. static dma_cookie_t d40_tx_submit(struct dma_async_tx_descriptor *tx)
  1292. {
  1293. struct d40_chan *d40c = container_of(tx->chan,
  1294. struct d40_chan,
  1295. chan);
  1296. struct d40_desc *d40d = container_of(tx, struct d40_desc, txd);
  1297. unsigned long flags;
  1298. dma_cookie_t cookie;
  1299. spin_lock_irqsave(&d40c->lock, flags);
  1300. cookie = dma_cookie_assign(tx);
  1301. d40_desc_queue(d40c, d40d);
  1302. spin_unlock_irqrestore(&d40c->lock, flags);
  1303. return cookie;
  1304. }
  1305. static int d40_start(struct d40_chan *d40c)
  1306. {
  1307. return d40_channel_execute_command(d40c, D40_DMA_RUN);
  1308. }
  1309. static struct d40_desc *d40_queue_start(struct d40_chan *d40c)
  1310. {
  1311. struct d40_desc *d40d;
  1312. int err;
  1313. /* Start queued jobs, if any */
  1314. d40d = d40_first_queued(d40c);
  1315. if (d40d != NULL) {
  1316. if (!d40c->busy) {
  1317. d40c->busy = true;
  1318. pm_runtime_get_sync(d40c->base->dev);
  1319. }
  1320. /* Remove from queue */
  1321. d40_desc_remove(d40d);
  1322. /* Add to active queue */
  1323. d40_desc_submit(d40c, d40d);
  1324. /* Initiate DMA job */
  1325. d40_desc_load(d40c, d40d);
  1326. /* Start dma job */
  1327. err = d40_start(d40c);
  1328. if (err)
  1329. return NULL;
  1330. }
  1331. return d40d;
  1332. }
  1333. /* called from interrupt context */
  1334. static void dma_tc_handle(struct d40_chan *d40c)
  1335. {
  1336. struct d40_desc *d40d;
  1337. /* Get first active entry from list */
  1338. d40d = d40_first_active_get(d40c);
  1339. if (d40d == NULL)
  1340. return;
  1341. if (d40d->cyclic) {
  1342. /*
  1343. * If this was a paritially loaded list, we need to reloaded
  1344. * it, and only when the list is completed. We need to check
  1345. * for done because the interrupt will hit for every link, and
  1346. * not just the last one.
  1347. */
  1348. if (d40d->lli_current < d40d->lli_len
  1349. && !d40_tx_is_linked(d40c)
  1350. && !d40_residue(d40c)) {
  1351. d40_lcla_free_all(d40c, d40d);
  1352. d40_desc_load(d40c, d40d);
  1353. (void) d40_start(d40c);
  1354. if (d40d->lli_current == d40d->lli_len)
  1355. d40d->lli_current = 0;
  1356. }
  1357. } else {
  1358. d40_lcla_free_all(d40c, d40d);
  1359. if (d40d->lli_current < d40d->lli_len) {
  1360. d40_desc_load(d40c, d40d);
  1361. /* Start dma job */
  1362. (void) d40_start(d40c);
  1363. return;
  1364. }
  1365. if (d40_queue_start(d40c) == NULL)
  1366. d40c->busy = false;
  1367. pm_runtime_mark_last_busy(d40c->base->dev);
  1368. pm_runtime_put_autosuspend(d40c->base->dev);
  1369. d40_desc_remove(d40d);
  1370. d40_desc_done(d40c, d40d);
  1371. }
  1372. d40c->pending_tx++;
  1373. tasklet_schedule(&d40c->tasklet);
  1374. }
  1375. static void dma_tasklet(unsigned long data)
  1376. {
  1377. struct d40_chan *d40c = (struct d40_chan *) data;
  1378. struct d40_desc *d40d;
  1379. unsigned long flags;
  1380. dma_async_tx_callback callback;
  1381. void *callback_param;
  1382. spin_lock_irqsave(&d40c->lock, flags);
  1383. /* Get first entry from the done list */
  1384. d40d = d40_first_done(d40c);
  1385. if (d40d == NULL) {
  1386. /* Check if we have reached here for cyclic job */
  1387. d40d = d40_first_active_get(d40c);
  1388. if (d40d == NULL || !d40d->cyclic)
  1389. goto err;
  1390. }
  1391. if (!d40d->cyclic)
  1392. dma_cookie_complete(&d40d->txd);
  1393. /*
  1394. * If terminating a channel pending_tx is set to zero.
  1395. * This prevents any finished active jobs to return to the client.
  1396. */
  1397. if (d40c->pending_tx == 0) {
  1398. spin_unlock_irqrestore(&d40c->lock, flags);
  1399. return;
  1400. }
  1401. /* Callback to client */
  1402. callback = d40d->txd.callback;
  1403. callback_param = d40d->txd.callback_param;
  1404. if (!d40d->cyclic) {
  1405. if (async_tx_test_ack(&d40d->txd)) {
  1406. d40_desc_remove(d40d);
  1407. d40_desc_free(d40c, d40d);
  1408. } else if (!d40d->is_in_client_list) {
  1409. d40_desc_remove(d40d);
  1410. d40_lcla_free_all(d40c, d40d);
  1411. list_add_tail(&d40d->node, &d40c->client);
  1412. d40d->is_in_client_list = true;
  1413. }
  1414. }
  1415. d40c->pending_tx--;
  1416. if (d40c->pending_tx)
  1417. tasklet_schedule(&d40c->tasklet);
  1418. spin_unlock_irqrestore(&d40c->lock, flags);
  1419. if (callback && (d40d->txd.flags & DMA_PREP_INTERRUPT))
  1420. callback(callback_param);
  1421. return;
  1422. err:
  1423. /* Rescue manouver if receiving double interrupts */
  1424. if (d40c->pending_tx > 0)
  1425. d40c->pending_tx--;
  1426. spin_unlock_irqrestore(&d40c->lock, flags);
  1427. }
  1428. static irqreturn_t d40_handle_interrupt(int irq, void *data)
  1429. {
  1430. int i;
  1431. u32 idx;
  1432. u32 row;
  1433. long chan = -1;
  1434. struct d40_chan *d40c;
  1435. unsigned long flags;
  1436. struct d40_base *base = data;
  1437. u32 regs[base->gen_dmac.il_size];
  1438. struct d40_interrupt_lookup *il = base->gen_dmac.il;
  1439. u32 il_size = base->gen_dmac.il_size;
  1440. spin_lock_irqsave(&base->interrupt_lock, flags);
  1441. /* Read interrupt status of both logical and physical channels */
  1442. for (i = 0; i < il_size; i++)
  1443. regs[i] = readl(base->virtbase + il[i].src);
  1444. for (;;) {
  1445. chan = find_next_bit((unsigned long *)regs,
  1446. BITS_PER_LONG * il_size, chan + 1);
  1447. /* No more set bits found? */
  1448. if (chan == BITS_PER_LONG * il_size)
  1449. break;
  1450. row = chan / BITS_PER_LONG;
  1451. idx = chan & (BITS_PER_LONG - 1);
  1452. if (il[row].offset == D40_PHY_CHAN)
  1453. d40c = base->lookup_phy_chans[idx];
  1454. else
  1455. d40c = base->lookup_log_chans[il[row].offset + idx];
  1456. if (!d40c) {
  1457. /*
  1458. * No error because this can happen if something else
  1459. * in the system is using the channel.
  1460. */
  1461. continue;
  1462. }
  1463. /* ACK interrupt */
  1464. writel(1 << idx, base->virtbase + il[row].clr);
  1465. spin_lock(&d40c->lock);
  1466. if (!il[row].is_error)
  1467. dma_tc_handle(d40c);
  1468. else
  1469. d40_err(base->dev, "IRQ chan: %ld offset %d idx %d\n",
  1470. chan, il[row].offset, idx);
  1471. spin_unlock(&d40c->lock);
  1472. }
  1473. spin_unlock_irqrestore(&base->interrupt_lock, flags);
  1474. return IRQ_HANDLED;
  1475. }
  1476. static int d40_validate_conf(struct d40_chan *d40c,
  1477. struct stedma40_chan_cfg *conf)
  1478. {
  1479. int res = 0;
  1480. u32 dst_event_group = D40_TYPE_TO_GROUP(conf->dst_dev_type);
  1481. u32 src_event_group = D40_TYPE_TO_GROUP(conf->src_dev_type);
  1482. bool is_log = conf->mode == STEDMA40_MODE_LOGICAL;
  1483. if (!conf->dir) {
  1484. chan_err(d40c, "Invalid direction.\n");
  1485. res = -EINVAL;
  1486. }
  1487. if (conf->dst_dev_type != STEDMA40_DEV_DST_MEMORY &&
  1488. d40c->base->plat_data->dev_tx[conf->dst_dev_type] == 0 &&
  1489. d40c->runtime_addr == 0) {
  1490. chan_err(d40c, "Invalid TX channel address (%d)\n",
  1491. conf->dst_dev_type);
  1492. res = -EINVAL;
  1493. }
  1494. if (conf->src_dev_type != STEDMA40_DEV_SRC_MEMORY &&
  1495. d40c->base->plat_data->dev_rx[conf->src_dev_type] == 0 &&
  1496. d40c->runtime_addr == 0) {
  1497. chan_err(d40c, "Invalid RX channel address (%d)\n",
  1498. conf->src_dev_type);
  1499. res = -EINVAL;
  1500. }
  1501. if (conf->dir == STEDMA40_MEM_TO_PERIPH &&
  1502. dst_event_group == STEDMA40_DEV_DST_MEMORY) {
  1503. chan_err(d40c, "Invalid dst\n");
  1504. res = -EINVAL;
  1505. }
  1506. if (conf->dir == STEDMA40_PERIPH_TO_MEM &&
  1507. src_event_group == STEDMA40_DEV_SRC_MEMORY) {
  1508. chan_err(d40c, "Invalid src\n");
  1509. res = -EINVAL;
  1510. }
  1511. if (src_event_group == STEDMA40_DEV_SRC_MEMORY &&
  1512. dst_event_group == STEDMA40_DEV_DST_MEMORY && is_log) {
  1513. chan_err(d40c, "No event line\n");
  1514. res = -EINVAL;
  1515. }
  1516. if (conf->dir == STEDMA40_PERIPH_TO_PERIPH &&
  1517. (src_event_group != dst_event_group)) {
  1518. chan_err(d40c, "Invalid event group\n");
  1519. res = -EINVAL;
  1520. }
  1521. if (conf->dir == STEDMA40_PERIPH_TO_PERIPH) {
  1522. /*
  1523. * DMAC HW supports it. Will be added to this driver,
  1524. * in case any dma client requires it.
  1525. */
  1526. chan_err(d40c, "periph to periph not supported\n");
  1527. res = -EINVAL;
  1528. }
  1529. if (d40_psize_2_burst_size(is_log, conf->src_info.psize) *
  1530. (1 << conf->src_info.data_width) !=
  1531. d40_psize_2_burst_size(is_log, conf->dst_info.psize) *
  1532. (1 << conf->dst_info.data_width)) {
  1533. /*
  1534. * The DMAC hardware only supports
  1535. * src (burst x width) == dst (burst x width)
  1536. */
  1537. chan_err(d40c, "src (burst x width) != dst (burst x width)\n");
  1538. res = -EINVAL;
  1539. }
  1540. return res;
  1541. }
  1542. static bool d40_alloc_mask_set(struct d40_phy_res *phy,
  1543. bool is_src, int log_event_line, bool is_log,
  1544. bool *first_user)
  1545. {
  1546. unsigned long flags;
  1547. spin_lock_irqsave(&phy->lock, flags);
  1548. *first_user = ((phy->allocated_src | phy->allocated_dst)
  1549. == D40_ALLOC_FREE);
  1550. if (!is_log) {
  1551. /* Physical interrupts are masked per physical full channel */
  1552. if (phy->allocated_src == D40_ALLOC_FREE &&
  1553. phy->allocated_dst == D40_ALLOC_FREE) {
  1554. phy->allocated_dst = D40_ALLOC_PHY;
  1555. phy->allocated_src = D40_ALLOC_PHY;
  1556. goto found;
  1557. } else
  1558. goto not_found;
  1559. }
  1560. /* Logical channel */
  1561. if (is_src) {
  1562. if (phy->allocated_src == D40_ALLOC_PHY)
  1563. goto not_found;
  1564. if (phy->allocated_src == D40_ALLOC_FREE)
  1565. phy->allocated_src = D40_ALLOC_LOG_FREE;
  1566. if (!(phy->allocated_src & (1 << log_event_line))) {
  1567. phy->allocated_src |= 1 << log_event_line;
  1568. goto found;
  1569. } else
  1570. goto not_found;
  1571. } else {
  1572. if (phy->allocated_dst == D40_ALLOC_PHY)
  1573. goto not_found;
  1574. if (phy->allocated_dst == D40_ALLOC_FREE)
  1575. phy->allocated_dst = D40_ALLOC_LOG_FREE;
  1576. if (!(phy->allocated_dst & (1 << log_event_line))) {
  1577. phy->allocated_dst |= 1 << log_event_line;
  1578. goto found;
  1579. } else
  1580. goto not_found;
  1581. }
  1582. not_found:
  1583. spin_unlock_irqrestore(&phy->lock, flags);
  1584. return false;
  1585. found:
  1586. spin_unlock_irqrestore(&phy->lock, flags);
  1587. return true;
  1588. }
  1589. static bool d40_alloc_mask_free(struct d40_phy_res *phy, bool is_src,
  1590. int log_event_line)
  1591. {
  1592. unsigned long flags;
  1593. bool is_free = false;
  1594. spin_lock_irqsave(&phy->lock, flags);
  1595. if (!log_event_line) {
  1596. phy->allocated_dst = D40_ALLOC_FREE;
  1597. phy->allocated_src = D40_ALLOC_FREE;
  1598. is_free = true;
  1599. goto out;
  1600. }
  1601. /* Logical channel */
  1602. if (is_src) {
  1603. phy->allocated_src &= ~(1 << log_event_line);
  1604. if (phy->allocated_src == D40_ALLOC_LOG_FREE)
  1605. phy->allocated_src = D40_ALLOC_FREE;
  1606. } else {
  1607. phy->allocated_dst &= ~(1 << log_event_line);
  1608. if (phy->allocated_dst == D40_ALLOC_LOG_FREE)
  1609. phy->allocated_dst = D40_ALLOC_FREE;
  1610. }
  1611. is_free = ((phy->allocated_src | phy->allocated_dst) ==
  1612. D40_ALLOC_FREE);
  1613. out:
  1614. spin_unlock_irqrestore(&phy->lock, flags);
  1615. return is_free;
  1616. }
  1617. static int d40_allocate_channel(struct d40_chan *d40c, bool *first_phy_user)
  1618. {
  1619. int dev_type;
  1620. int event_group;
  1621. int event_line;
  1622. struct d40_phy_res *phys;
  1623. int i;
  1624. int j;
  1625. int log_num;
  1626. int num_phy_chans;
  1627. bool is_src;
  1628. bool is_log = d40c->dma_cfg.mode == STEDMA40_MODE_LOGICAL;
  1629. phys = d40c->base->phy_res;
  1630. num_phy_chans = d40c->base->num_phy_chans;
  1631. if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) {
  1632. dev_type = d40c->dma_cfg.src_dev_type;
  1633. log_num = 2 * dev_type;
  1634. is_src = true;
  1635. } else if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
  1636. d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
  1637. /* dst event lines are used for logical memcpy */
  1638. dev_type = d40c->dma_cfg.dst_dev_type;
  1639. log_num = 2 * dev_type + 1;
  1640. is_src = false;
  1641. } else
  1642. return -EINVAL;
  1643. event_group = D40_TYPE_TO_GROUP(dev_type);
  1644. event_line = D40_TYPE_TO_EVENT(dev_type);
  1645. if (!is_log) {
  1646. if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
  1647. /* Find physical half channel */
  1648. if (d40c->dma_cfg.use_fixed_channel) {
  1649. i = d40c->dma_cfg.phy_channel;
  1650. if (d40_alloc_mask_set(&phys[i], is_src,
  1651. 0, is_log,
  1652. first_phy_user))
  1653. goto found_phy;
  1654. } else {
  1655. for (i = 0; i < num_phy_chans; i++) {
  1656. if (d40_alloc_mask_set(&phys[i], is_src,
  1657. 0, is_log,
  1658. first_phy_user))
  1659. goto found_phy;
  1660. }
  1661. }
  1662. } else
  1663. for (j = 0; j < d40c->base->num_phy_chans; j += 8) {
  1664. int phy_num = j + event_group * 2;
  1665. for (i = phy_num; i < phy_num + 2; i++) {
  1666. if (d40_alloc_mask_set(&phys[i],
  1667. is_src,
  1668. 0,
  1669. is_log,
  1670. first_phy_user))
  1671. goto found_phy;
  1672. }
  1673. }
  1674. return -EINVAL;
  1675. found_phy:
  1676. d40c->phy_chan = &phys[i];
  1677. d40c->log_num = D40_PHY_CHAN;
  1678. goto out;
  1679. }
  1680. if (dev_type == -1)
  1681. return -EINVAL;
  1682. /* Find logical channel */
  1683. for (j = 0; j < d40c->base->num_phy_chans; j += 8) {
  1684. int phy_num = j + event_group * 2;
  1685. if (d40c->dma_cfg.use_fixed_channel) {
  1686. i = d40c->dma_cfg.phy_channel;
  1687. if ((i != phy_num) && (i != phy_num + 1)) {
  1688. dev_err(chan2dev(d40c),
  1689. "invalid fixed phy channel %d\n", i);
  1690. return -EINVAL;
  1691. }
  1692. if (d40_alloc_mask_set(&phys[i], is_src, event_line,
  1693. is_log, first_phy_user))
  1694. goto found_log;
  1695. dev_err(chan2dev(d40c),
  1696. "could not allocate fixed phy channel %d\n", i);
  1697. return -EINVAL;
  1698. }
  1699. /*
  1700. * Spread logical channels across all available physical rather
  1701. * than pack every logical channel at the first available phy
  1702. * channels.
  1703. */
  1704. if (is_src) {
  1705. for (i = phy_num; i < phy_num + 2; i++) {
  1706. if (d40_alloc_mask_set(&phys[i], is_src,
  1707. event_line, is_log,
  1708. first_phy_user))
  1709. goto found_log;
  1710. }
  1711. } else {
  1712. for (i = phy_num + 1; i >= phy_num; i--) {
  1713. if (d40_alloc_mask_set(&phys[i], is_src,
  1714. event_line, is_log,
  1715. first_phy_user))
  1716. goto found_log;
  1717. }
  1718. }
  1719. }
  1720. return -EINVAL;
  1721. found_log:
  1722. d40c->phy_chan = &phys[i];
  1723. d40c->log_num = log_num;
  1724. out:
  1725. if (is_log)
  1726. d40c->base->lookup_log_chans[d40c->log_num] = d40c;
  1727. else
  1728. d40c->base->lookup_phy_chans[d40c->phy_chan->num] = d40c;
  1729. return 0;
  1730. }
  1731. static int d40_config_memcpy(struct d40_chan *d40c)
  1732. {
  1733. dma_cap_mask_t cap = d40c->chan.device->cap_mask;
  1734. if (dma_has_cap(DMA_MEMCPY, cap) && !dma_has_cap(DMA_SLAVE, cap)) {
  1735. d40c->dma_cfg = dma40_memcpy_conf_log;
  1736. d40c->dma_cfg.src_dev_type = STEDMA40_DEV_SRC_MEMORY;
  1737. d40c->dma_cfg.dst_dev_type = dma40_memcpy_channels[d40c->chan.chan_id];
  1738. } else if (dma_has_cap(DMA_MEMCPY, cap) &&
  1739. dma_has_cap(DMA_SLAVE, cap)) {
  1740. d40c->dma_cfg = dma40_memcpy_conf_phy;
  1741. } else {
  1742. chan_err(d40c, "No memcpy\n");
  1743. return -EINVAL;
  1744. }
  1745. return 0;
  1746. }
  1747. static int d40_free_dma(struct d40_chan *d40c)
  1748. {
  1749. int res = 0;
  1750. u32 event;
  1751. struct d40_phy_res *phy = d40c->phy_chan;
  1752. bool is_src;
  1753. /* Terminate all queued and active transfers */
  1754. d40_term_all(d40c);
  1755. if (phy == NULL) {
  1756. chan_err(d40c, "phy == null\n");
  1757. return -EINVAL;
  1758. }
  1759. if (phy->allocated_src == D40_ALLOC_FREE &&
  1760. phy->allocated_dst == D40_ALLOC_FREE) {
  1761. chan_err(d40c, "channel already free\n");
  1762. return -EINVAL;
  1763. }
  1764. if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
  1765. d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
  1766. event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type);
  1767. is_src = false;
  1768. } else if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) {
  1769. event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type);
  1770. is_src = true;
  1771. } else {
  1772. chan_err(d40c, "Unknown direction\n");
  1773. return -EINVAL;
  1774. }
  1775. pm_runtime_get_sync(d40c->base->dev);
  1776. res = d40_channel_execute_command(d40c, D40_DMA_STOP);
  1777. if (res) {
  1778. chan_err(d40c, "stop failed\n");
  1779. goto out;
  1780. }
  1781. d40_alloc_mask_free(phy, is_src, chan_is_logical(d40c) ? event : 0);
  1782. if (chan_is_logical(d40c))
  1783. d40c->base->lookup_log_chans[d40c->log_num] = NULL;
  1784. else
  1785. d40c->base->lookup_phy_chans[phy->num] = NULL;
  1786. if (d40c->busy) {
  1787. pm_runtime_mark_last_busy(d40c->base->dev);
  1788. pm_runtime_put_autosuspend(d40c->base->dev);
  1789. }
  1790. d40c->busy = false;
  1791. d40c->phy_chan = NULL;
  1792. d40c->configured = false;
  1793. out:
  1794. pm_runtime_mark_last_busy(d40c->base->dev);
  1795. pm_runtime_put_autosuspend(d40c->base->dev);
  1796. return res;
  1797. }
  1798. static bool d40_is_paused(struct d40_chan *d40c)
  1799. {
  1800. void __iomem *chanbase = chan_base(d40c);
  1801. bool is_paused = false;
  1802. unsigned long flags;
  1803. void __iomem *active_reg;
  1804. u32 status;
  1805. u32 event;
  1806. spin_lock_irqsave(&d40c->lock, flags);
  1807. if (chan_is_physical(d40c)) {
  1808. if (d40c->phy_chan->num % 2 == 0)
  1809. active_reg = d40c->base->virtbase + D40_DREG_ACTIVE;
  1810. else
  1811. active_reg = d40c->base->virtbase + D40_DREG_ACTIVO;
  1812. status = (readl(active_reg) &
  1813. D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
  1814. D40_CHAN_POS(d40c->phy_chan->num);
  1815. if (status == D40_DMA_SUSPENDED || status == D40_DMA_STOP)
  1816. is_paused = true;
  1817. goto _exit;
  1818. }
  1819. if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
  1820. d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
  1821. event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type);
  1822. status = readl(chanbase + D40_CHAN_REG_SDLNK);
  1823. } else if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) {
  1824. event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type);
  1825. status = readl(chanbase + D40_CHAN_REG_SSLNK);
  1826. } else {
  1827. chan_err(d40c, "Unknown direction\n");
  1828. goto _exit;
  1829. }
  1830. status = (status & D40_EVENTLINE_MASK(event)) >>
  1831. D40_EVENTLINE_POS(event);
  1832. if (status != D40_DMA_RUN)
  1833. is_paused = true;
  1834. _exit:
  1835. spin_unlock_irqrestore(&d40c->lock, flags);
  1836. return is_paused;
  1837. }
  1838. static u32 stedma40_residue(struct dma_chan *chan)
  1839. {
  1840. struct d40_chan *d40c =
  1841. container_of(chan, struct d40_chan, chan);
  1842. u32 bytes_left;
  1843. unsigned long flags;
  1844. spin_lock_irqsave(&d40c->lock, flags);
  1845. bytes_left = d40_residue(d40c);
  1846. spin_unlock_irqrestore(&d40c->lock, flags);
  1847. return bytes_left;
  1848. }
  1849. static int
  1850. d40_prep_sg_log(struct d40_chan *chan, struct d40_desc *desc,
  1851. struct scatterlist *sg_src, struct scatterlist *sg_dst,
  1852. unsigned int sg_len, dma_addr_t src_dev_addr,
  1853. dma_addr_t dst_dev_addr)
  1854. {
  1855. struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
  1856. struct stedma40_half_channel_info *src_info = &cfg->src_info;
  1857. struct stedma40_half_channel_info *dst_info = &cfg->dst_info;
  1858. int ret;
  1859. ret = d40_log_sg_to_lli(sg_src, sg_len,
  1860. src_dev_addr,
  1861. desc->lli_log.src,
  1862. chan->log_def.lcsp1,
  1863. src_info->data_width,
  1864. dst_info->data_width);
  1865. ret = d40_log_sg_to_lli(sg_dst, sg_len,
  1866. dst_dev_addr,
  1867. desc->lli_log.dst,
  1868. chan->log_def.lcsp3,
  1869. dst_info->data_width,
  1870. src_info->data_width);
  1871. return ret < 0 ? ret : 0;
  1872. }
  1873. static int
  1874. d40_prep_sg_phy(struct d40_chan *chan, struct d40_desc *desc,
  1875. struct scatterlist *sg_src, struct scatterlist *sg_dst,
  1876. unsigned int sg_len, dma_addr_t src_dev_addr,
  1877. dma_addr_t dst_dev_addr)
  1878. {
  1879. struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
  1880. struct stedma40_half_channel_info *src_info = &cfg->src_info;
  1881. struct stedma40_half_channel_info *dst_info = &cfg->dst_info;
  1882. unsigned long flags = 0;
  1883. int ret;
  1884. if (desc->cyclic)
  1885. flags |= LLI_CYCLIC | LLI_TERM_INT;
  1886. ret = d40_phy_sg_to_lli(sg_src, sg_len, src_dev_addr,
  1887. desc->lli_phy.src,
  1888. virt_to_phys(desc->lli_phy.src),
  1889. chan->src_def_cfg,
  1890. src_info, dst_info, flags);
  1891. ret = d40_phy_sg_to_lli(sg_dst, sg_len, dst_dev_addr,
  1892. desc->lli_phy.dst,
  1893. virt_to_phys(desc->lli_phy.dst),
  1894. chan->dst_def_cfg,
  1895. dst_info, src_info, flags);
  1896. dma_sync_single_for_device(chan->base->dev, desc->lli_pool.dma_addr,
  1897. desc->lli_pool.size, DMA_TO_DEVICE);
  1898. return ret < 0 ? ret : 0;
  1899. }
  1900. static struct d40_desc *
  1901. d40_prep_desc(struct d40_chan *chan, struct scatterlist *sg,
  1902. unsigned int sg_len, unsigned long dma_flags)
  1903. {
  1904. struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
  1905. struct d40_desc *desc;
  1906. int ret;
  1907. desc = d40_desc_get(chan);
  1908. if (!desc)
  1909. return NULL;
  1910. desc->lli_len = d40_sg_2_dmalen(sg, sg_len, cfg->src_info.data_width,
  1911. cfg->dst_info.data_width);
  1912. if (desc->lli_len < 0) {
  1913. chan_err(chan, "Unaligned size\n");
  1914. goto err;
  1915. }
  1916. ret = d40_pool_lli_alloc(chan, desc, desc->lli_len);
  1917. if (ret < 0) {
  1918. chan_err(chan, "Could not allocate lli\n");
  1919. goto err;
  1920. }
  1921. desc->lli_current = 0;
  1922. desc->txd.flags = dma_flags;
  1923. desc->txd.tx_submit = d40_tx_submit;
  1924. dma_async_tx_descriptor_init(&desc->txd, &chan->chan);
  1925. return desc;
  1926. err:
  1927. d40_desc_free(chan, desc);
  1928. return NULL;
  1929. }
  1930. static dma_addr_t
  1931. d40_get_dev_addr(struct d40_chan *chan, enum dma_transfer_direction direction)
  1932. {
  1933. struct stedma40_platform_data *plat = chan->base->plat_data;
  1934. struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
  1935. dma_addr_t addr = 0;
  1936. if (chan->runtime_addr)
  1937. return chan->runtime_addr;
  1938. if (direction == DMA_DEV_TO_MEM)
  1939. addr = plat->dev_rx[cfg->src_dev_type];
  1940. else if (direction == DMA_MEM_TO_DEV)
  1941. addr = plat->dev_tx[cfg->dst_dev_type];
  1942. return addr;
  1943. }
  1944. static struct dma_async_tx_descriptor *
  1945. d40_prep_sg(struct dma_chan *dchan, struct scatterlist *sg_src,
  1946. struct scatterlist *sg_dst, unsigned int sg_len,
  1947. enum dma_transfer_direction direction, unsigned long dma_flags)
  1948. {
  1949. struct d40_chan *chan = container_of(dchan, struct d40_chan, chan);
  1950. dma_addr_t src_dev_addr = 0;
  1951. dma_addr_t dst_dev_addr = 0;
  1952. struct d40_desc *desc;
  1953. unsigned long flags;
  1954. int ret;
  1955. if (!chan->phy_chan) {
  1956. chan_err(chan, "Cannot prepare unallocated channel\n");
  1957. return NULL;
  1958. }
  1959. spin_lock_irqsave(&chan->lock, flags);
  1960. desc = d40_prep_desc(chan, sg_src, sg_len, dma_flags);
  1961. if (desc == NULL)
  1962. goto err;
  1963. if (sg_next(&sg_src[sg_len - 1]) == sg_src)
  1964. desc->cyclic = true;
  1965. if (direction != DMA_TRANS_NONE) {
  1966. dma_addr_t dev_addr = d40_get_dev_addr(chan, direction);
  1967. if (direction == DMA_DEV_TO_MEM)
  1968. src_dev_addr = dev_addr;
  1969. else if (direction == DMA_MEM_TO_DEV)
  1970. dst_dev_addr = dev_addr;
  1971. }
  1972. if (chan_is_logical(chan))
  1973. ret = d40_prep_sg_log(chan, desc, sg_src, sg_dst,
  1974. sg_len, src_dev_addr, dst_dev_addr);
  1975. else
  1976. ret = d40_prep_sg_phy(chan, desc, sg_src, sg_dst,
  1977. sg_len, src_dev_addr, dst_dev_addr);
  1978. if (ret) {
  1979. chan_err(chan, "Failed to prepare %s sg job: %d\n",
  1980. chan_is_logical(chan) ? "log" : "phy", ret);
  1981. goto err;
  1982. }
  1983. /*
  1984. * add descriptor to the prepare queue in order to be able
  1985. * to free them later in terminate_all
  1986. */
  1987. list_add_tail(&desc->node, &chan->prepare_queue);
  1988. spin_unlock_irqrestore(&chan->lock, flags);
  1989. return &desc->txd;
  1990. err:
  1991. if (desc)
  1992. d40_desc_free(chan, desc);
  1993. spin_unlock_irqrestore(&chan->lock, flags);
  1994. return NULL;
  1995. }
  1996. bool stedma40_filter(struct dma_chan *chan, void *data)
  1997. {
  1998. struct stedma40_chan_cfg *info = data;
  1999. struct d40_chan *d40c =
  2000. container_of(chan, struct d40_chan, chan);
  2001. int err;
  2002. if (data) {
  2003. err = d40_validate_conf(d40c, info);
  2004. if (!err)
  2005. d40c->dma_cfg = *info;
  2006. } else
  2007. err = d40_config_memcpy(d40c);
  2008. if (!err)
  2009. d40c->configured = true;
  2010. return err == 0;
  2011. }
  2012. EXPORT_SYMBOL(stedma40_filter);
  2013. static void __d40_set_prio_rt(struct d40_chan *d40c, int dev_type, bool src)
  2014. {
  2015. bool realtime = d40c->dma_cfg.realtime;
  2016. bool highprio = d40c->dma_cfg.high_priority;
  2017. u32 rtreg;
  2018. u32 event = D40_TYPE_TO_EVENT(dev_type);
  2019. u32 group = D40_TYPE_TO_GROUP(dev_type);
  2020. u32 bit = 1 << event;
  2021. u32 prioreg;
  2022. struct d40_gen_dmac *dmac = &d40c->base->gen_dmac;
  2023. rtreg = realtime ? dmac->realtime_en : dmac->realtime_clear;
  2024. /*
  2025. * Due to a hardware bug, in some cases a logical channel triggered by
  2026. * a high priority destination event line can generate extra packet
  2027. * transactions.
  2028. *
  2029. * The workaround is to not set the high priority level for the
  2030. * destination event lines that trigger logical channels.
  2031. */
  2032. if (!src && chan_is_logical(d40c))
  2033. highprio = false;
  2034. prioreg = highprio ? dmac->high_prio_en : dmac->high_prio_clear;
  2035. /* Destination event lines are stored in the upper halfword */
  2036. if (!src)
  2037. bit <<= 16;
  2038. writel(bit, d40c->base->virtbase + prioreg + group * 4);
  2039. writel(bit, d40c->base->virtbase + rtreg + group * 4);
  2040. }
  2041. static void d40_set_prio_realtime(struct d40_chan *d40c)
  2042. {
  2043. if (d40c->base->rev < 3)
  2044. return;
  2045. if ((d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) ||
  2046. (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH))
  2047. __d40_set_prio_rt(d40c, d40c->dma_cfg.src_dev_type, true);
  2048. if ((d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH) ||
  2049. (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH))
  2050. __d40_set_prio_rt(d40c, d40c->dma_cfg.dst_dev_type, false);
  2051. }
  2052. /* DMA ENGINE functions */
  2053. static int d40_alloc_chan_resources(struct dma_chan *chan)
  2054. {
  2055. int err;
  2056. unsigned long flags;
  2057. struct d40_chan *d40c =
  2058. container_of(chan, struct d40_chan, chan);
  2059. bool is_free_phy;
  2060. spin_lock_irqsave(&d40c->lock, flags);
  2061. dma_cookie_init(chan);
  2062. /* If no dma configuration is set use default configuration (memcpy) */
  2063. if (!d40c->configured) {
  2064. err = d40_config_memcpy(d40c);
  2065. if (err) {
  2066. chan_err(d40c, "Failed to configure memcpy channel\n");
  2067. goto fail;
  2068. }
  2069. }
  2070. err = d40_allocate_channel(d40c, &is_free_phy);
  2071. if (err) {
  2072. chan_err(d40c, "Failed to allocate channel\n");
  2073. d40c->configured = false;
  2074. goto fail;
  2075. }
  2076. pm_runtime_get_sync(d40c->base->dev);
  2077. /* Fill in basic CFG register values */
  2078. d40_phy_cfg(&d40c->dma_cfg, &d40c->src_def_cfg,
  2079. &d40c->dst_def_cfg, chan_is_logical(d40c));
  2080. d40_set_prio_realtime(d40c);
  2081. if (chan_is_logical(d40c)) {
  2082. d40_log_cfg(&d40c->dma_cfg,
  2083. &d40c->log_def.lcsp1, &d40c->log_def.lcsp3);
  2084. if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM)
  2085. d40c->lcpa = d40c->base->lcpa_base +
  2086. d40c->dma_cfg.src_dev_type * D40_LCPA_CHAN_SIZE;
  2087. else
  2088. d40c->lcpa = d40c->base->lcpa_base +
  2089. d40c->dma_cfg.dst_dev_type *
  2090. D40_LCPA_CHAN_SIZE + D40_LCPA_CHAN_DST_DELTA;
  2091. }
  2092. dev_dbg(chan2dev(d40c), "allocated %s channel (phy %d%s)\n",
  2093. chan_is_logical(d40c) ? "logical" : "physical",
  2094. d40c->phy_chan->num,
  2095. d40c->dma_cfg.use_fixed_channel ? ", fixed" : "");
  2096. /*
  2097. * Only write channel configuration to the DMA if the physical
  2098. * resource is free. In case of multiple logical channels
  2099. * on the same physical resource, only the first write is necessary.
  2100. */
  2101. if (is_free_phy)
  2102. d40_config_write(d40c);
  2103. fail:
  2104. pm_runtime_mark_last_busy(d40c->base->dev);
  2105. pm_runtime_put_autosuspend(d40c->base->dev);
  2106. spin_unlock_irqrestore(&d40c->lock, flags);
  2107. return err;
  2108. }
  2109. static void d40_free_chan_resources(struct dma_chan *chan)
  2110. {
  2111. struct d40_chan *d40c =
  2112. container_of(chan, struct d40_chan, chan);
  2113. int err;
  2114. unsigned long flags;
  2115. if (d40c->phy_chan == NULL) {
  2116. chan_err(d40c, "Cannot free unallocated channel\n");
  2117. return;
  2118. }
  2119. spin_lock_irqsave(&d40c->lock, flags);
  2120. err = d40_free_dma(d40c);
  2121. if (err)
  2122. chan_err(d40c, "Failed to free channel\n");
  2123. spin_unlock_irqrestore(&d40c->lock, flags);
  2124. }
  2125. static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan,
  2126. dma_addr_t dst,
  2127. dma_addr_t src,
  2128. size_t size,
  2129. unsigned long dma_flags)
  2130. {
  2131. struct scatterlist dst_sg;
  2132. struct scatterlist src_sg;
  2133. sg_init_table(&dst_sg, 1);
  2134. sg_init_table(&src_sg, 1);
  2135. sg_dma_address(&dst_sg) = dst;
  2136. sg_dma_address(&src_sg) = src;
  2137. sg_dma_len(&dst_sg) = size;
  2138. sg_dma_len(&src_sg) = size;
  2139. return d40_prep_sg(chan, &src_sg, &dst_sg, 1, DMA_NONE, dma_flags);
  2140. }
  2141. static struct dma_async_tx_descriptor *
  2142. d40_prep_memcpy_sg(struct dma_chan *chan,
  2143. struct scatterlist *dst_sg, unsigned int dst_nents,
  2144. struct scatterlist *src_sg, unsigned int src_nents,
  2145. unsigned long dma_flags)
  2146. {
  2147. if (dst_nents != src_nents)
  2148. return NULL;
  2149. return d40_prep_sg(chan, src_sg, dst_sg, src_nents, DMA_NONE, dma_flags);
  2150. }
  2151. static struct dma_async_tx_descriptor *
  2152. d40_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
  2153. unsigned int sg_len, enum dma_transfer_direction direction,
  2154. unsigned long dma_flags, void *context)
  2155. {
  2156. if (!is_slave_direction(direction))
  2157. return NULL;
  2158. return d40_prep_sg(chan, sgl, sgl, sg_len, direction, dma_flags);
  2159. }
  2160. static struct dma_async_tx_descriptor *
  2161. dma40_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t dma_addr,
  2162. size_t buf_len, size_t period_len,
  2163. enum dma_transfer_direction direction, unsigned long flags,
  2164. void *context)
  2165. {
  2166. unsigned int periods = buf_len / period_len;
  2167. struct dma_async_tx_descriptor *txd;
  2168. struct scatterlist *sg;
  2169. int i;
  2170. sg = kcalloc(periods + 1, sizeof(struct scatterlist), GFP_NOWAIT);
  2171. for (i = 0; i < periods; i++) {
  2172. sg_dma_address(&sg[i]) = dma_addr;
  2173. sg_dma_len(&sg[i]) = period_len;
  2174. dma_addr += period_len;
  2175. }
  2176. sg[periods].offset = 0;
  2177. sg_dma_len(&sg[periods]) = 0;
  2178. sg[periods].page_link =
  2179. ((unsigned long)sg | 0x01) & ~0x02;
  2180. txd = d40_prep_sg(chan, sg, sg, periods, direction,
  2181. DMA_PREP_INTERRUPT);
  2182. kfree(sg);
  2183. return txd;
  2184. }
  2185. static enum dma_status d40_tx_status(struct dma_chan *chan,
  2186. dma_cookie_t cookie,
  2187. struct dma_tx_state *txstate)
  2188. {
  2189. struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
  2190. enum dma_status ret;
  2191. if (d40c->phy_chan == NULL) {
  2192. chan_err(d40c, "Cannot read status of unallocated channel\n");
  2193. return -EINVAL;
  2194. }
  2195. ret = dma_cookie_status(chan, cookie, txstate);
  2196. if (ret != DMA_SUCCESS)
  2197. dma_set_residue(txstate, stedma40_residue(chan));
  2198. if (d40_is_paused(d40c))
  2199. ret = DMA_PAUSED;
  2200. return ret;
  2201. }
  2202. static void d40_issue_pending(struct dma_chan *chan)
  2203. {
  2204. struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
  2205. unsigned long flags;
  2206. if (d40c->phy_chan == NULL) {
  2207. chan_err(d40c, "Channel is not allocated!\n");
  2208. return;
  2209. }
  2210. spin_lock_irqsave(&d40c->lock, flags);
  2211. list_splice_tail_init(&d40c->pending_queue, &d40c->queue);
  2212. /* Busy means that queued jobs are already being processed */
  2213. if (!d40c->busy)
  2214. (void) d40_queue_start(d40c);
  2215. spin_unlock_irqrestore(&d40c->lock, flags);
  2216. }
  2217. static void d40_terminate_all(struct dma_chan *chan)
  2218. {
  2219. unsigned long flags;
  2220. struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
  2221. int ret;
  2222. spin_lock_irqsave(&d40c->lock, flags);
  2223. pm_runtime_get_sync(d40c->base->dev);
  2224. ret = d40_channel_execute_command(d40c, D40_DMA_STOP);
  2225. if (ret)
  2226. chan_err(d40c, "Failed to stop channel\n");
  2227. d40_term_all(d40c);
  2228. pm_runtime_mark_last_busy(d40c->base->dev);
  2229. pm_runtime_put_autosuspend(d40c->base->dev);
  2230. if (d40c->busy) {
  2231. pm_runtime_mark_last_busy(d40c->base->dev);
  2232. pm_runtime_put_autosuspend(d40c->base->dev);
  2233. }
  2234. d40c->busy = false;
  2235. spin_unlock_irqrestore(&d40c->lock, flags);
  2236. }
  2237. static int
  2238. dma40_config_to_halfchannel(struct d40_chan *d40c,
  2239. struct stedma40_half_channel_info *info,
  2240. enum dma_slave_buswidth width,
  2241. u32 maxburst)
  2242. {
  2243. enum stedma40_periph_data_width addr_width;
  2244. int psize;
  2245. switch (width) {
  2246. case DMA_SLAVE_BUSWIDTH_1_BYTE:
  2247. addr_width = STEDMA40_BYTE_WIDTH;
  2248. break;
  2249. case DMA_SLAVE_BUSWIDTH_2_BYTES:
  2250. addr_width = STEDMA40_HALFWORD_WIDTH;
  2251. break;
  2252. case DMA_SLAVE_BUSWIDTH_4_BYTES:
  2253. addr_width = STEDMA40_WORD_WIDTH;
  2254. break;
  2255. case DMA_SLAVE_BUSWIDTH_8_BYTES:
  2256. addr_width = STEDMA40_DOUBLEWORD_WIDTH;
  2257. break;
  2258. default:
  2259. dev_err(d40c->base->dev,
  2260. "illegal peripheral address width "
  2261. "requested (%d)\n",
  2262. width);
  2263. return -EINVAL;
  2264. }
  2265. if (chan_is_logical(d40c)) {
  2266. if (maxburst >= 16)
  2267. psize = STEDMA40_PSIZE_LOG_16;
  2268. else if (maxburst >= 8)
  2269. psize = STEDMA40_PSIZE_LOG_8;
  2270. else if (maxburst >= 4)
  2271. psize = STEDMA40_PSIZE_LOG_4;
  2272. else
  2273. psize = STEDMA40_PSIZE_LOG_1;
  2274. } else {
  2275. if (maxburst >= 16)
  2276. psize = STEDMA40_PSIZE_PHY_16;
  2277. else if (maxburst >= 8)
  2278. psize = STEDMA40_PSIZE_PHY_8;
  2279. else if (maxburst >= 4)
  2280. psize = STEDMA40_PSIZE_PHY_4;
  2281. else
  2282. psize = STEDMA40_PSIZE_PHY_1;
  2283. }
  2284. info->data_width = addr_width;
  2285. info->psize = psize;
  2286. info->flow_ctrl = STEDMA40_NO_FLOW_CTRL;
  2287. return 0;
  2288. }
  2289. /* Runtime reconfiguration extension */
  2290. static int d40_set_runtime_config(struct dma_chan *chan,
  2291. struct dma_slave_config *config)
  2292. {
  2293. struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
  2294. struct stedma40_chan_cfg *cfg = &d40c->dma_cfg;
  2295. enum dma_slave_buswidth src_addr_width, dst_addr_width;
  2296. dma_addr_t config_addr;
  2297. u32 src_maxburst, dst_maxburst;
  2298. int ret;
  2299. src_addr_width = config->src_addr_width;
  2300. src_maxburst = config->src_maxburst;
  2301. dst_addr_width = config->dst_addr_width;
  2302. dst_maxburst = config->dst_maxburst;
  2303. if (config->direction == DMA_DEV_TO_MEM) {
  2304. dma_addr_t dev_addr_rx =
  2305. d40c->base->plat_data->dev_rx[cfg->src_dev_type];
  2306. config_addr = config->src_addr;
  2307. if (dev_addr_rx)
  2308. dev_dbg(d40c->base->dev,
  2309. "channel has a pre-wired RX address %08x "
  2310. "overriding with %08x\n",
  2311. dev_addr_rx, config_addr);
  2312. if (cfg->dir != STEDMA40_PERIPH_TO_MEM)
  2313. dev_dbg(d40c->base->dev,
  2314. "channel was not configured for peripheral "
  2315. "to memory transfer (%d) overriding\n",
  2316. cfg->dir);
  2317. cfg->dir = STEDMA40_PERIPH_TO_MEM;
  2318. /* Configure the memory side */
  2319. if (dst_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED)
  2320. dst_addr_width = src_addr_width;
  2321. if (dst_maxburst == 0)
  2322. dst_maxburst = src_maxburst;
  2323. } else if (config->direction == DMA_MEM_TO_DEV) {
  2324. dma_addr_t dev_addr_tx =
  2325. d40c->base->plat_data->dev_tx[cfg->dst_dev_type];
  2326. config_addr = config->dst_addr;
  2327. if (dev_addr_tx)
  2328. dev_dbg(d40c->base->dev,
  2329. "channel has a pre-wired TX address %08x "
  2330. "overriding with %08x\n",
  2331. dev_addr_tx, config_addr);
  2332. if (cfg->dir != STEDMA40_MEM_TO_PERIPH)
  2333. dev_dbg(d40c->base->dev,
  2334. "channel was not configured for memory "
  2335. "to peripheral transfer (%d) overriding\n",
  2336. cfg->dir);
  2337. cfg->dir = STEDMA40_MEM_TO_PERIPH;
  2338. /* Configure the memory side */
  2339. if (src_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED)
  2340. src_addr_width = dst_addr_width;
  2341. if (src_maxburst == 0)
  2342. src_maxburst = dst_maxburst;
  2343. } else {
  2344. dev_err(d40c->base->dev,
  2345. "unrecognized channel direction %d\n",
  2346. config->direction);
  2347. return -EINVAL;
  2348. }
  2349. if (src_maxburst * src_addr_width != dst_maxburst * dst_addr_width) {
  2350. dev_err(d40c->base->dev,
  2351. "src/dst width/maxburst mismatch: %d*%d != %d*%d\n",
  2352. src_maxburst,
  2353. src_addr_width,
  2354. dst_maxburst,
  2355. dst_addr_width);
  2356. return -EINVAL;
  2357. }
  2358. if (src_maxburst > 16) {
  2359. src_maxburst = 16;
  2360. dst_maxburst = src_maxburst * src_addr_width / dst_addr_width;
  2361. } else if (dst_maxburst > 16) {
  2362. dst_maxburst = 16;
  2363. src_maxburst = dst_maxburst * dst_addr_width / src_addr_width;
  2364. }
  2365. ret = dma40_config_to_halfchannel(d40c, &cfg->src_info,
  2366. src_addr_width,
  2367. src_maxburst);
  2368. if (ret)
  2369. return ret;
  2370. ret = dma40_config_to_halfchannel(d40c, &cfg->dst_info,
  2371. dst_addr_width,
  2372. dst_maxburst);
  2373. if (ret)
  2374. return ret;
  2375. /* Fill in register values */
  2376. if (chan_is_logical(d40c))
  2377. d40_log_cfg(cfg, &d40c->log_def.lcsp1, &d40c->log_def.lcsp3);
  2378. else
  2379. d40_phy_cfg(cfg, &d40c->src_def_cfg,
  2380. &d40c->dst_def_cfg, false);
  2381. /* These settings will take precedence later */
  2382. d40c->runtime_addr = config_addr;
  2383. d40c->runtime_direction = config->direction;
  2384. dev_dbg(d40c->base->dev,
  2385. "configured channel %s for %s, data width %d/%d, "
  2386. "maxburst %d/%d elements, LE, no flow control\n",
  2387. dma_chan_name(chan),
  2388. (config->direction == DMA_DEV_TO_MEM) ? "RX" : "TX",
  2389. src_addr_width, dst_addr_width,
  2390. src_maxburst, dst_maxburst);
  2391. return 0;
  2392. }
  2393. static int d40_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
  2394. unsigned long arg)
  2395. {
  2396. struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
  2397. if (d40c->phy_chan == NULL) {
  2398. chan_err(d40c, "Channel is not allocated!\n");
  2399. return -EINVAL;
  2400. }
  2401. switch (cmd) {
  2402. case DMA_TERMINATE_ALL:
  2403. d40_terminate_all(chan);
  2404. return 0;
  2405. case DMA_PAUSE:
  2406. return d40_pause(d40c);
  2407. case DMA_RESUME:
  2408. return d40_resume(d40c);
  2409. case DMA_SLAVE_CONFIG:
  2410. return d40_set_runtime_config(chan,
  2411. (struct dma_slave_config *) arg);
  2412. default:
  2413. break;
  2414. }
  2415. /* Other commands are unimplemented */
  2416. return -ENXIO;
  2417. }
  2418. /* Initialization functions */
  2419. static void __init d40_chan_init(struct d40_base *base, struct dma_device *dma,
  2420. struct d40_chan *chans, int offset,
  2421. int num_chans)
  2422. {
  2423. int i = 0;
  2424. struct d40_chan *d40c;
  2425. INIT_LIST_HEAD(&dma->channels);
  2426. for (i = offset; i < offset + num_chans; i++) {
  2427. d40c = &chans[i];
  2428. d40c->base = base;
  2429. d40c->chan.device = dma;
  2430. spin_lock_init(&d40c->lock);
  2431. d40c->log_num = D40_PHY_CHAN;
  2432. INIT_LIST_HEAD(&d40c->done);
  2433. INIT_LIST_HEAD(&d40c->active);
  2434. INIT_LIST_HEAD(&d40c->queue);
  2435. INIT_LIST_HEAD(&d40c->pending_queue);
  2436. INIT_LIST_HEAD(&d40c->client);
  2437. INIT_LIST_HEAD(&d40c->prepare_queue);
  2438. tasklet_init(&d40c->tasklet, dma_tasklet,
  2439. (unsigned long) d40c);
  2440. list_add_tail(&d40c->chan.device_node,
  2441. &dma->channels);
  2442. }
  2443. }
  2444. static void d40_ops_init(struct d40_base *base, struct dma_device *dev)
  2445. {
  2446. if (dma_has_cap(DMA_SLAVE, dev->cap_mask))
  2447. dev->device_prep_slave_sg = d40_prep_slave_sg;
  2448. if (dma_has_cap(DMA_MEMCPY, dev->cap_mask)) {
  2449. dev->device_prep_dma_memcpy = d40_prep_memcpy;
  2450. /*
  2451. * This controller can only access address at even
  2452. * 32bit boundaries, i.e. 2^2
  2453. */
  2454. dev->copy_align = 2;
  2455. }
  2456. if (dma_has_cap(DMA_SG, dev->cap_mask))
  2457. dev->device_prep_dma_sg = d40_prep_memcpy_sg;
  2458. if (dma_has_cap(DMA_CYCLIC, dev->cap_mask))
  2459. dev->device_prep_dma_cyclic = dma40_prep_dma_cyclic;
  2460. dev->device_alloc_chan_resources = d40_alloc_chan_resources;
  2461. dev->device_free_chan_resources = d40_free_chan_resources;
  2462. dev->device_issue_pending = d40_issue_pending;
  2463. dev->device_tx_status = d40_tx_status;
  2464. dev->device_control = d40_control;
  2465. dev->dev = base->dev;
  2466. }
  2467. static int __init d40_dmaengine_init(struct d40_base *base,
  2468. int num_reserved_chans)
  2469. {
  2470. int err ;
  2471. d40_chan_init(base, &base->dma_slave, base->log_chans,
  2472. 0, base->num_log_chans);
  2473. dma_cap_zero(base->dma_slave.cap_mask);
  2474. dma_cap_set(DMA_SLAVE, base->dma_slave.cap_mask);
  2475. dma_cap_set(DMA_CYCLIC, base->dma_slave.cap_mask);
  2476. d40_ops_init(base, &base->dma_slave);
  2477. err = dma_async_device_register(&base->dma_slave);
  2478. if (err) {
  2479. d40_err(base->dev, "Failed to register slave channels\n");
  2480. goto failure1;
  2481. }
  2482. d40_chan_init(base, &base->dma_memcpy, base->log_chans,
  2483. base->num_log_chans, ARRAY_SIZE(dma40_memcpy_channels));
  2484. dma_cap_zero(base->dma_memcpy.cap_mask);
  2485. dma_cap_set(DMA_MEMCPY, base->dma_memcpy.cap_mask);
  2486. dma_cap_set(DMA_SG, base->dma_memcpy.cap_mask);
  2487. d40_ops_init(base, &base->dma_memcpy);
  2488. err = dma_async_device_register(&base->dma_memcpy);
  2489. if (err) {
  2490. d40_err(base->dev,
  2491. "Failed to regsiter memcpy only channels\n");
  2492. goto failure2;
  2493. }
  2494. d40_chan_init(base, &base->dma_both, base->phy_chans,
  2495. 0, num_reserved_chans);
  2496. dma_cap_zero(base->dma_both.cap_mask);
  2497. dma_cap_set(DMA_SLAVE, base->dma_both.cap_mask);
  2498. dma_cap_set(DMA_MEMCPY, base->dma_both.cap_mask);
  2499. dma_cap_set(DMA_SG, base->dma_both.cap_mask);
  2500. dma_cap_set(DMA_CYCLIC, base->dma_slave.cap_mask);
  2501. d40_ops_init(base, &base->dma_both);
  2502. err = dma_async_device_register(&base->dma_both);
  2503. if (err) {
  2504. d40_err(base->dev,
  2505. "Failed to register logical and physical capable channels\n");
  2506. goto failure3;
  2507. }
  2508. return 0;
  2509. failure3:
  2510. dma_async_device_unregister(&base->dma_memcpy);
  2511. failure2:
  2512. dma_async_device_unregister(&base->dma_slave);
  2513. failure1:
  2514. return err;
  2515. }
  2516. /* Suspend resume functionality */
  2517. #ifdef CONFIG_PM
  2518. static int dma40_pm_suspend(struct device *dev)
  2519. {
  2520. struct platform_device *pdev = to_platform_device(dev);
  2521. struct d40_base *base = platform_get_drvdata(pdev);
  2522. int ret = 0;
  2523. if (base->lcpa_regulator)
  2524. ret = regulator_disable(base->lcpa_regulator);
  2525. return ret;
  2526. }
  2527. static int dma40_runtime_suspend(struct device *dev)
  2528. {
  2529. struct platform_device *pdev = to_platform_device(dev);
  2530. struct d40_base *base = platform_get_drvdata(pdev);
  2531. d40_save_restore_registers(base, true);
  2532. /* Don't disable/enable clocks for v1 due to HW bugs */
  2533. if (base->rev != 1)
  2534. writel_relaxed(base->gcc_pwr_off_mask,
  2535. base->virtbase + D40_DREG_GCC);
  2536. return 0;
  2537. }
  2538. static int dma40_runtime_resume(struct device *dev)
  2539. {
  2540. struct platform_device *pdev = to_platform_device(dev);
  2541. struct d40_base *base = platform_get_drvdata(pdev);
  2542. if (base->initialized)
  2543. d40_save_restore_registers(base, false);
  2544. writel_relaxed(D40_DREG_GCC_ENABLE_ALL,
  2545. base->virtbase + D40_DREG_GCC);
  2546. return 0;
  2547. }
  2548. static int dma40_resume(struct device *dev)
  2549. {
  2550. struct platform_device *pdev = to_platform_device(dev);
  2551. struct d40_base *base = platform_get_drvdata(pdev);
  2552. int ret = 0;
  2553. if (base->lcpa_regulator)
  2554. ret = regulator_enable(base->lcpa_regulator);
  2555. return ret;
  2556. }
  2557. static const struct dev_pm_ops dma40_pm_ops = {
  2558. .suspend = dma40_pm_suspend,
  2559. .runtime_suspend = dma40_runtime_suspend,
  2560. .runtime_resume = dma40_runtime_resume,
  2561. .resume = dma40_resume,
  2562. };
  2563. #define DMA40_PM_OPS (&dma40_pm_ops)
  2564. #else
  2565. #define DMA40_PM_OPS NULL
  2566. #endif
  2567. /* Initialization functions. */
  2568. static int __init d40_phy_res_init(struct d40_base *base)
  2569. {
  2570. int i;
  2571. int num_phy_chans_avail = 0;
  2572. u32 val[2];
  2573. int odd_even_bit = -2;
  2574. int gcc = D40_DREG_GCC_ENA;
  2575. val[0] = readl(base->virtbase + D40_DREG_PRSME);
  2576. val[1] = readl(base->virtbase + D40_DREG_PRSMO);
  2577. for (i = 0; i < base->num_phy_chans; i++) {
  2578. base->phy_res[i].num = i;
  2579. odd_even_bit += 2 * ((i % 2) == 0);
  2580. if (((val[i % 2] >> odd_even_bit) & 3) == 1) {
  2581. /* Mark security only channels as occupied */
  2582. base->phy_res[i].allocated_src = D40_ALLOC_PHY;
  2583. base->phy_res[i].allocated_dst = D40_ALLOC_PHY;
  2584. base->phy_res[i].reserved = true;
  2585. gcc |= D40_DREG_GCC_EVTGRP_ENA(D40_PHYS_TO_GROUP(i),
  2586. D40_DREG_GCC_SRC);
  2587. gcc |= D40_DREG_GCC_EVTGRP_ENA(D40_PHYS_TO_GROUP(i),
  2588. D40_DREG_GCC_DST);
  2589. } else {
  2590. base->phy_res[i].allocated_src = D40_ALLOC_FREE;
  2591. base->phy_res[i].allocated_dst = D40_ALLOC_FREE;
  2592. base->phy_res[i].reserved = false;
  2593. num_phy_chans_avail++;
  2594. }
  2595. spin_lock_init(&base->phy_res[i].lock);
  2596. }
  2597. /* Mark disabled channels as occupied */
  2598. for (i = 0; base->plat_data->disabled_channels[i] != -1; i++) {
  2599. int chan = base->plat_data->disabled_channels[i];
  2600. base->phy_res[chan].allocated_src = D40_ALLOC_PHY;
  2601. base->phy_res[chan].allocated_dst = D40_ALLOC_PHY;
  2602. base->phy_res[chan].reserved = true;
  2603. gcc |= D40_DREG_GCC_EVTGRP_ENA(D40_PHYS_TO_GROUP(chan),
  2604. D40_DREG_GCC_SRC);
  2605. gcc |= D40_DREG_GCC_EVTGRP_ENA(D40_PHYS_TO_GROUP(chan),
  2606. D40_DREG_GCC_DST);
  2607. num_phy_chans_avail--;
  2608. }
  2609. /* Mark soft_lli channels */
  2610. for (i = 0; i < base->plat_data->num_of_soft_lli_chans; i++) {
  2611. int chan = base->plat_data->soft_lli_chans[i];
  2612. base->phy_res[chan].use_soft_lli = true;
  2613. }
  2614. dev_info(base->dev, "%d of %d physical DMA channels available\n",
  2615. num_phy_chans_avail, base->num_phy_chans);
  2616. /* Verify settings extended vs standard */
  2617. val[0] = readl(base->virtbase + D40_DREG_PRTYP);
  2618. for (i = 0; i < base->num_phy_chans; i++) {
  2619. if (base->phy_res[i].allocated_src == D40_ALLOC_FREE &&
  2620. (val[0] & 0x3) != 1)
  2621. dev_info(base->dev,
  2622. "[%s] INFO: channel %d is misconfigured (%d)\n",
  2623. __func__, i, val[0] & 0x3);
  2624. val[0] = val[0] >> 2;
  2625. }
  2626. /*
  2627. * To keep things simple, Enable all clocks initially.
  2628. * The clocks will get managed later post channel allocation.
  2629. * The clocks for the event lines on which reserved channels exists
  2630. * are not managed here.
  2631. */
  2632. writel(D40_DREG_GCC_ENABLE_ALL, base->virtbase + D40_DREG_GCC);
  2633. base->gcc_pwr_off_mask = gcc;
  2634. return num_phy_chans_avail;
  2635. }
  2636. static struct d40_base * __init d40_hw_detect_init(struct platform_device *pdev)
  2637. {
  2638. struct stedma40_platform_data *plat_data;
  2639. struct clk *clk = NULL;
  2640. void __iomem *virtbase = NULL;
  2641. struct resource *res = NULL;
  2642. struct d40_base *base = NULL;
  2643. int num_log_chans = 0;
  2644. int num_phy_chans;
  2645. int clk_ret = -EINVAL;
  2646. int i;
  2647. u32 pid;
  2648. u32 cid;
  2649. u8 rev;
  2650. clk = clk_get(&pdev->dev, NULL);
  2651. if (IS_ERR(clk)) {
  2652. d40_err(&pdev->dev, "No matching clock found\n");
  2653. goto failure;
  2654. }
  2655. clk_ret = clk_prepare_enable(clk);
  2656. if (clk_ret) {
  2657. d40_err(&pdev->dev, "Failed to prepare/enable clock\n");
  2658. goto failure;
  2659. }
  2660. /* Get IO for DMAC base address */
  2661. res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "base");
  2662. if (!res)
  2663. goto failure;
  2664. if (request_mem_region(res->start, resource_size(res),
  2665. D40_NAME " I/O base") == NULL)
  2666. goto failure;
  2667. virtbase = ioremap(res->start, resource_size(res));
  2668. if (!virtbase)
  2669. goto failure;
  2670. /* This is just a regular AMBA PrimeCell ID actually */
  2671. for (pid = 0, i = 0; i < 4; i++)
  2672. pid |= (readl(virtbase + resource_size(res) - 0x20 + 4 * i)
  2673. & 255) << (i * 8);
  2674. for (cid = 0, i = 0; i < 4; i++)
  2675. cid |= (readl(virtbase + resource_size(res) - 0x10 + 4 * i)
  2676. & 255) << (i * 8);
  2677. if (cid != AMBA_CID) {
  2678. d40_err(&pdev->dev, "Unknown hardware! No PrimeCell ID\n");
  2679. goto failure;
  2680. }
  2681. if (AMBA_MANF_BITS(pid) != AMBA_VENDOR_ST) {
  2682. d40_err(&pdev->dev, "Unknown designer! Got %x wanted %x\n",
  2683. AMBA_MANF_BITS(pid),
  2684. AMBA_VENDOR_ST);
  2685. goto failure;
  2686. }
  2687. /*
  2688. * HW revision:
  2689. * DB8500ed has revision 0
  2690. * ? has revision 1
  2691. * DB8500v1 has revision 2
  2692. * DB8500v2 has revision 3
  2693. * AP9540v1 has revision 4
  2694. * DB8540v1 has revision 4
  2695. */
  2696. rev = AMBA_REV_BITS(pid);
  2697. plat_data = pdev->dev.platform_data;
  2698. /* The number of physical channels on this HW */
  2699. if (plat_data->num_of_phy_chans)
  2700. num_phy_chans = plat_data->num_of_phy_chans;
  2701. else
  2702. num_phy_chans = 4 * (readl(virtbase + D40_DREG_ICFG) & 0x7) + 4;
  2703. dev_info(&pdev->dev, "hardware revision: %d @ 0x%x with %d physical channels\n",
  2704. rev, res->start, num_phy_chans);
  2705. if (rev < 2) {
  2706. d40_err(&pdev->dev, "hardware revision: %d is not supported",
  2707. rev);
  2708. goto failure;
  2709. }
  2710. /* Count the number of logical channels in use */
  2711. for (i = 0; i < plat_data->dev_len; i++)
  2712. if (plat_data->dev_rx[i] != 0)
  2713. num_log_chans++;
  2714. for (i = 0; i < plat_data->dev_len; i++)
  2715. if (plat_data->dev_tx[i] != 0)
  2716. num_log_chans++;
  2717. base = kzalloc(ALIGN(sizeof(struct d40_base), 4) +
  2718. (num_phy_chans + num_log_chans + ARRAY_SIZE(dma40_memcpy_channels)) *
  2719. sizeof(struct d40_chan), GFP_KERNEL);
  2720. if (base == NULL) {
  2721. d40_err(&pdev->dev, "Out of memory\n");
  2722. goto failure;
  2723. }
  2724. base->rev = rev;
  2725. base->clk = clk;
  2726. base->num_phy_chans = num_phy_chans;
  2727. base->num_log_chans = num_log_chans;
  2728. base->phy_start = res->start;
  2729. base->phy_size = resource_size(res);
  2730. base->virtbase = virtbase;
  2731. base->plat_data = plat_data;
  2732. base->dev = &pdev->dev;
  2733. base->phy_chans = ((void *)base) + ALIGN(sizeof(struct d40_base), 4);
  2734. base->log_chans = &base->phy_chans[num_phy_chans];
  2735. if (base->plat_data->num_of_phy_chans == 14) {
  2736. base->gen_dmac.backup = d40_backup_regs_v4b;
  2737. base->gen_dmac.backup_size = BACKUP_REGS_SZ_V4B;
  2738. base->gen_dmac.interrupt_en = D40_DREG_CPCMIS;
  2739. base->gen_dmac.interrupt_clear = D40_DREG_CPCICR;
  2740. base->gen_dmac.realtime_en = D40_DREG_CRSEG1;
  2741. base->gen_dmac.realtime_clear = D40_DREG_CRCEG1;
  2742. base->gen_dmac.high_prio_en = D40_DREG_CPSEG1;
  2743. base->gen_dmac.high_prio_clear = D40_DREG_CPCEG1;
  2744. base->gen_dmac.il = il_v4b;
  2745. base->gen_dmac.il_size = ARRAY_SIZE(il_v4b);
  2746. base->gen_dmac.init_reg = dma_init_reg_v4b;
  2747. base->gen_dmac.init_reg_size = ARRAY_SIZE(dma_init_reg_v4b);
  2748. } else {
  2749. if (base->rev >= 3) {
  2750. base->gen_dmac.backup = d40_backup_regs_v4a;
  2751. base->gen_dmac.backup_size = BACKUP_REGS_SZ_V4A;
  2752. }
  2753. base->gen_dmac.interrupt_en = D40_DREG_PCMIS;
  2754. base->gen_dmac.interrupt_clear = D40_DREG_PCICR;
  2755. base->gen_dmac.realtime_en = D40_DREG_RSEG1;
  2756. base->gen_dmac.realtime_clear = D40_DREG_RCEG1;
  2757. base->gen_dmac.high_prio_en = D40_DREG_PSEG1;
  2758. base->gen_dmac.high_prio_clear = D40_DREG_PCEG1;
  2759. base->gen_dmac.il = il_v4a;
  2760. base->gen_dmac.il_size = ARRAY_SIZE(il_v4a);
  2761. base->gen_dmac.init_reg = dma_init_reg_v4a;
  2762. base->gen_dmac.init_reg_size = ARRAY_SIZE(dma_init_reg_v4a);
  2763. }
  2764. base->phy_res = kzalloc(num_phy_chans * sizeof(struct d40_phy_res),
  2765. GFP_KERNEL);
  2766. if (!base->phy_res)
  2767. goto failure;
  2768. base->lookup_phy_chans = kzalloc(num_phy_chans *
  2769. sizeof(struct d40_chan *),
  2770. GFP_KERNEL);
  2771. if (!base->lookup_phy_chans)
  2772. goto failure;
  2773. if (num_log_chans + ARRAY_SIZE(dma40_memcpy_channels)) {
  2774. /*
  2775. * The max number of logical channels are event lines for all
  2776. * src devices and dst devices
  2777. */
  2778. base->lookup_log_chans = kzalloc(plat_data->dev_len * 2 *
  2779. sizeof(struct d40_chan *),
  2780. GFP_KERNEL);
  2781. if (!base->lookup_log_chans)
  2782. goto failure;
  2783. }
  2784. base->reg_val_backup_chan = kmalloc(base->num_phy_chans *
  2785. sizeof(d40_backup_regs_chan),
  2786. GFP_KERNEL);
  2787. if (!base->reg_val_backup_chan)
  2788. goto failure;
  2789. base->lcla_pool.alloc_map =
  2790. kzalloc(num_phy_chans * sizeof(struct d40_desc *)
  2791. * D40_LCLA_LINK_PER_EVENT_GRP, GFP_KERNEL);
  2792. if (!base->lcla_pool.alloc_map)
  2793. goto failure;
  2794. base->desc_slab = kmem_cache_create(D40_NAME, sizeof(struct d40_desc),
  2795. 0, SLAB_HWCACHE_ALIGN,
  2796. NULL);
  2797. if (base->desc_slab == NULL)
  2798. goto failure;
  2799. return base;
  2800. failure:
  2801. if (!clk_ret)
  2802. clk_disable_unprepare(clk);
  2803. if (!IS_ERR(clk))
  2804. clk_put(clk);
  2805. if (virtbase)
  2806. iounmap(virtbase);
  2807. if (res)
  2808. release_mem_region(res->start,
  2809. resource_size(res));
  2810. if (virtbase)
  2811. iounmap(virtbase);
  2812. if (base) {
  2813. kfree(base->lcla_pool.alloc_map);
  2814. kfree(base->reg_val_backup_chan);
  2815. kfree(base->lookup_log_chans);
  2816. kfree(base->lookup_phy_chans);
  2817. kfree(base->phy_res);
  2818. kfree(base);
  2819. }
  2820. return NULL;
  2821. }
  2822. static void __init d40_hw_init(struct d40_base *base)
  2823. {
  2824. int i;
  2825. u32 prmseo[2] = {0, 0};
  2826. u32 activeo[2] = {0xFFFFFFFF, 0xFFFFFFFF};
  2827. u32 pcmis = 0;
  2828. u32 pcicr = 0;
  2829. struct d40_reg_val *dma_init_reg = base->gen_dmac.init_reg;
  2830. u32 reg_size = base->gen_dmac.init_reg_size;
  2831. for (i = 0; i < reg_size; i++)
  2832. writel(dma_init_reg[i].val,
  2833. base->virtbase + dma_init_reg[i].reg);
  2834. /* Configure all our dma channels to default settings */
  2835. for (i = 0; i < base->num_phy_chans; i++) {
  2836. activeo[i % 2] = activeo[i % 2] << 2;
  2837. if (base->phy_res[base->num_phy_chans - i - 1].allocated_src
  2838. == D40_ALLOC_PHY) {
  2839. activeo[i % 2] |= 3;
  2840. continue;
  2841. }
  2842. /* Enable interrupt # */
  2843. pcmis = (pcmis << 1) | 1;
  2844. /* Clear interrupt # */
  2845. pcicr = (pcicr << 1) | 1;
  2846. /* Set channel to physical mode */
  2847. prmseo[i % 2] = prmseo[i % 2] << 2;
  2848. prmseo[i % 2] |= 1;
  2849. }
  2850. writel(prmseo[1], base->virtbase + D40_DREG_PRMSE);
  2851. writel(prmseo[0], base->virtbase + D40_DREG_PRMSO);
  2852. writel(activeo[1], base->virtbase + D40_DREG_ACTIVE);
  2853. writel(activeo[0], base->virtbase + D40_DREG_ACTIVO);
  2854. /* Write which interrupt to enable */
  2855. writel(pcmis, base->virtbase + base->gen_dmac.interrupt_en);
  2856. /* Write which interrupt to clear */
  2857. writel(pcicr, base->virtbase + base->gen_dmac.interrupt_clear);
  2858. /* These are __initdata and cannot be accessed after init */
  2859. base->gen_dmac.init_reg = NULL;
  2860. base->gen_dmac.init_reg_size = 0;
  2861. }
  2862. static int __init d40_lcla_allocate(struct d40_base *base)
  2863. {
  2864. struct d40_lcla_pool *pool = &base->lcla_pool;
  2865. unsigned long *page_list;
  2866. int i, j;
  2867. int ret = 0;
  2868. /*
  2869. * This is somewhat ugly. We need 8192 bytes that are 18 bit aligned,
  2870. * To full fill this hardware requirement without wasting 256 kb
  2871. * we allocate pages until we get an aligned one.
  2872. */
  2873. page_list = kmalloc(sizeof(unsigned long) * MAX_LCLA_ALLOC_ATTEMPTS,
  2874. GFP_KERNEL);
  2875. if (!page_list) {
  2876. ret = -ENOMEM;
  2877. goto failure;
  2878. }
  2879. /* Calculating how many pages that are required */
  2880. base->lcla_pool.pages = SZ_1K * base->num_phy_chans / PAGE_SIZE;
  2881. for (i = 0; i < MAX_LCLA_ALLOC_ATTEMPTS; i++) {
  2882. page_list[i] = __get_free_pages(GFP_KERNEL,
  2883. base->lcla_pool.pages);
  2884. if (!page_list[i]) {
  2885. d40_err(base->dev, "Failed to allocate %d pages.\n",
  2886. base->lcla_pool.pages);
  2887. for (j = 0; j < i; j++)
  2888. free_pages(page_list[j], base->lcla_pool.pages);
  2889. goto failure;
  2890. }
  2891. if ((virt_to_phys((void *)page_list[i]) &
  2892. (LCLA_ALIGNMENT - 1)) == 0)
  2893. break;
  2894. }
  2895. for (j = 0; j < i; j++)
  2896. free_pages(page_list[j], base->lcla_pool.pages);
  2897. if (i < MAX_LCLA_ALLOC_ATTEMPTS) {
  2898. base->lcla_pool.base = (void *)page_list[i];
  2899. } else {
  2900. /*
  2901. * After many attempts and no succees with finding the correct
  2902. * alignment, try with allocating a big buffer.
  2903. */
  2904. dev_warn(base->dev,
  2905. "[%s] Failed to get %d pages @ 18 bit align.\n",
  2906. __func__, base->lcla_pool.pages);
  2907. base->lcla_pool.base_unaligned = kmalloc(SZ_1K *
  2908. base->num_phy_chans +
  2909. LCLA_ALIGNMENT,
  2910. GFP_KERNEL);
  2911. if (!base->lcla_pool.base_unaligned) {
  2912. ret = -ENOMEM;
  2913. goto failure;
  2914. }
  2915. base->lcla_pool.base = PTR_ALIGN(base->lcla_pool.base_unaligned,
  2916. LCLA_ALIGNMENT);
  2917. }
  2918. pool->dma_addr = dma_map_single(base->dev, pool->base,
  2919. SZ_1K * base->num_phy_chans,
  2920. DMA_TO_DEVICE);
  2921. if (dma_mapping_error(base->dev, pool->dma_addr)) {
  2922. pool->dma_addr = 0;
  2923. ret = -ENOMEM;
  2924. goto failure;
  2925. }
  2926. writel(virt_to_phys(base->lcla_pool.base),
  2927. base->virtbase + D40_DREG_LCLA);
  2928. failure:
  2929. kfree(page_list);
  2930. return ret;
  2931. }
  2932. static int __init d40_probe(struct platform_device *pdev)
  2933. {
  2934. int err;
  2935. int ret = -ENOENT;
  2936. struct d40_base *base;
  2937. struct resource *res = NULL;
  2938. int num_reserved_chans;
  2939. u32 val;
  2940. base = d40_hw_detect_init(pdev);
  2941. if (!base)
  2942. goto failure;
  2943. num_reserved_chans = d40_phy_res_init(base);
  2944. platform_set_drvdata(pdev, base);
  2945. spin_lock_init(&base->interrupt_lock);
  2946. spin_lock_init(&base->execmd_lock);
  2947. /* Get IO for logical channel parameter address */
  2948. res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "lcpa");
  2949. if (!res) {
  2950. ret = -ENOENT;
  2951. d40_err(&pdev->dev, "No \"lcpa\" memory resource\n");
  2952. goto failure;
  2953. }
  2954. base->lcpa_size = resource_size(res);
  2955. base->phy_lcpa = res->start;
  2956. if (request_mem_region(res->start, resource_size(res),
  2957. D40_NAME " I/O lcpa") == NULL) {
  2958. ret = -EBUSY;
  2959. d40_err(&pdev->dev,
  2960. "Failed to request LCPA region 0x%x-0x%x\n",
  2961. res->start, res->end);
  2962. goto failure;
  2963. }
  2964. /* We make use of ESRAM memory for this. */
  2965. val = readl(base->virtbase + D40_DREG_LCPA);
  2966. if (res->start != val && val != 0) {
  2967. dev_warn(&pdev->dev,
  2968. "[%s] Mismatch LCPA dma 0x%x, def 0x%x\n",
  2969. __func__, val, res->start);
  2970. } else
  2971. writel(res->start, base->virtbase + D40_DREG_LCPA);
  2972. base->lcpa_base = ioremap(res->start, resource_size(res));
  2973. if (!base->lcpa_base) {
  2974. ret = -ENOMEM;
  2975. d40_err(&pdev->dev, "Failed to ioremap LCPA region\n");
  2976. goto failure;
  2977. }
  2978. /* If lcla has to be located in ESRAM we don't need to allocate */
  2979. if (base->plat_data->use_esram_lcla) {
  2980. res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
  2981. "lcla_esram");
  2982. if (!res) {
  2983. ret = -ENOENT;
  2984. d40_err(&pdev->dev,
  2985. "No \"lcla_esram\" memory resource\n");
  2986. goto failure;
  2987. }
  2988. base->lcla_pool.base = ioremap(res->start,
  2989. resource_size(res));
  2990. if (!base->lcla_pool.base) {
  2991. ret = -ENOMEM;
  2992. d40_err(&pdev->dev, "Failed to ioremap LCLA region\n");
  2993. goto failure;
  2994. }
  2995. writel(res->start, base->virtbase + D40_DREG_LCLA);
  2996. } else {
  2997. ret = d40_lcla_allocate(base);
  2998. if (ret) {
  2999. d40_err(&pdev->dev, "Failed to allocate LCLA area\n");
  3000. goto failure;
  3001. }
  3002. }
  3003. spin_lock_init(&base->lcla_pool.lock);
  3004. base->irq = platform_get_irq(pdev, 0);
  3005. ret = request_irq(base->irq, d40_handle_interrupt, 0, D40_NAME, base);
  3006. if (ret) {
  3007. d40_err(&pdev->dev, "No IRQ defined\n");
  3008. goto failure;
  3009. }
  3010. pm_runtime_irq_safe(base->dev);
  3011. pm_runtime_set_autosuspend_delay(base->dev, DMA40_AUTOSUSPEND_DELAY);
  3012. pm_runtime_use_autosuspend(base->dev);
  3013. pm_runtime_enable(base->dev);
  3014. pm_runtime_resume(base->dev);
  3015. if (base->plat_data->use_esram_lcla) {
  3016. base->lcpa_regulator = regulator_get(base->dev, "lcla_esram");
  3017. if (IS_ERR(base->lcpa_regulator)) {
  3018. d40_err(&pdev->dev, "Failed to get lcpa_regulator\n");
  3019. base->lcpa_regulator = NULL;
  3020. goto failure;
  3021. }
  3022. ret = regulator_enable(base->lcpa_regulator);
  3023. if (ret) {
  3024. d40_err(&pdev->dev,
  3025. "Failed to enable lcpa_regulator\n");
  3026. regulator_put(base->lcpa_regulator);
  3027. base->lcpa_regulator = NULL;
  3028. goto failure;
  3029. }
  3030. }
  3031. base->initialized = true;
  3032. err = d40_dmaengine_init(base, num_reserved_chans);
  3033. if (err)
  3034. goto failure;
  3035. base->dev->dma_parms = &base->dma_parms;
  3036. err = dma_set_max_seg_size(base->dev, STEDMA40_MAX_SEG_SIZE);
  3037. if (err) {
  3038. d40_err(&pdev->dev, "Failed to set dma max seg size\n");
  3039. goto failure;
  3040. }
  3041. d40_hw_init(base);
  3042. dev_info(base->dev, "initialized\n");
  3043. return 0;
  3044. failure:
  3045. if (base) {
  3046. if (base->desc_slab)
  3047. kmem_cache_destroy(base->desc_slab);
  3048. if (base->virtbase)
  3049. iounmap(base->virtbase);
  3050. if (base->lcla_pool.base && base->plat_data->use_esram_lcla) {
  3051. iounmap(base->lcla_pool.base);
  3052. base->lcla_pool.base = NULL;
  3053. }
  3054. if (base->lcla_pool.dma_addr)
  3055. dma_unmap_single(base->dev, base->lcla_pool.dma_addr,
  3056. SZ_1K * base->num_phy_chans,
  3057. DMA_TO_DEVICE);
  3058. if (!base->lcla_pool.base_unaligned && base->lcla_pool.base)
  3059. free_pages((unsigned long)base->lcla_pool.base,
  3060. base->lcla_pool.pages);
  3061. kfree(base->lcla_pool.base_unaligned);
  3062. if (base->phy_lcpa)
  3063. release_mem_region(base->phy_lcpa,
  3064. base->lcpa_size);
  3065. if (base->phy_start)
  3066. release_mem_region(base->phy_start,
  3067. base->phy_size);
  3068. if (base->clk) {
  3069. clk_disable_unprepare(base->clk);
  3070. clk_put(base->clk);
  3071. }
  3072. if (base->lcpa_regulator) {
  3073. regulator_disable(base->lcpa_regulator);
  3074. regulator_put(base->lcpa_regulator);
  3075. }
  3076. kfree(base->lcla_pool.alloc_map);
  3077. kfree(base->lookup_log_chans);
  3078. kfree(base->lookup_phy_chans);
  3079. kfree(base->phy_res);
  3080. kfree(base);
  3081. }
  3082. d40_err(&pdev->dev, "probe failed\n");
  3083. return ret;
  3084. }
  3085. static struct platform_driver d40_driver = {
  3086. .driver = {
  3087. .owner = THIS_MODULE,
  3088. .name = D40_NAME,
  3089. .pm = DMA40_PM_OPS,
  3090. },
  3091. };
  3092. static int __init stedma40_init(void)
  3093. {
  3094. return platform_driver_probe(&d40_driver, d40_probe);
  3095. }
  3096. subsys_initcall(stedma40_init);