cik.c 166 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615461646174618461946204621462246234624462546264627462846294630463146324633463446354636463746384639464046414642464346444645464646474648464946504651465246534654465546564657465846594660466146624663466446654666466746684669467046714672467346744675467646774678467946804681468246834684468546864687468846894690469146924693469446954696469746984699470047014702470347044705470647074708470947104711471247134714471547164717471847194720472147224723472447254726472747284729473047314732473347344735473647374738473947404741474247434744474547464747474847494750475147524753475447554756475747584759476047614762476347644765476647674768476947704771477247734774477547764777477847794780478147824783478447854786478747884789479047914792479347944795479647974798479948004801480248034804480548064807480848094810481148124813481448154816481748184819482048214822482348244825482648274828482948304831483248334834483548364837483848394840484148424843484448454846484748484849485048514852485348544855485648574858485948604861486248634864486548664867486848694870487148724873487448754876487748784879488048814882488348844885488648874888488948904891489248934894489548964897489848994900490149024903490449054906490749084909491049114912491349144915491649174918491949204921492249234924492549264927492849294930493149324933493449354936493749384939494049414942494349444945494649474948494949504951495249534954495549564957495849594960496149624963496449654966496749684969497049714972497349744975497649774978497949804981498249834984498549864987498849894990499149924993499449954996499749984999500050015002500350045005500650075008500950105011501250135014501550165017501850195020502150225023502450255026502750285029503050315032503350345035503650375038503950405041504250435044504550465047504850495050505150525053505450555056505750585059506050615062506350645065506650675068506950705071507250735074507550765077507850795080508150825083508450855086508750885089509050915092509350945095509650975098509951005101510251035104510551065107510851095110511151125113511451155116511751185119512051215122512351245125512651275128512951305131513251335134513551365137513851395140514151425143514451455146514751485149515051515152515351545155515651575158515951605161516251635164516551665167516851695170517151725173517451755176517751785179518051815182518351845185518651875188518951905191519251935194519551965197519851995200520152025203520452055206520752085209521052115212521352145215521652175218521952205221522252235224522552265227522852295230523152325233523452355236523752385239524052415242524352445245524652475248524952505251525252535254525552565257525852595260526152625263526452655266526752685269527052715272527352745275527652775278527952805281528252835284528552865287528852895290529152925293529452955296529752985299530053015302530353045305530653075308530953105311531253135314531553165317531853195320532153225323532453255326532753285329533053315332533353345335533653375338533953405341534253435344534553465347534853495350535153525353535453555356535753585359536053615362536353645365536653675368536953705371537253735374537553765377537853795380538153825383538453855386538753885389539053915392539353945395539653975398539954005401540254035404540554065407540854095410541154125413541454155416541754185419542054215422542354245425542654275428542954305431543254335434543554365437543854395440544154425443544454455446544754485449545054515452545354545455545654575458545954605461546254635464546554665467546854695470547154725473547454755476547754785479548054815482548354845485548654875488548954905491549254935494549554965497549854995500550155025503550455055506550755085509551055115512551355145515551655175518551955205521552255235524552555265527552855295530553155325533553455355536553755385539554055415542554355445545554655475548554955505551555255535554555555565557555855595560556155625563556455655566556755685569557055715572557355745575557655775578557955805581558255835584558555865587558855895590559155925593559455955596559755985599560056015602560356045605560656075608560956105611561256135614561556165617561856195620562156225623562456255626562756285629563056315632563356345635563656375638563956405641564256435644564556465647564856495650565156525653565456555656565756585659566056615662566356645665566656675668566956705671567256735674567556765677567856795680568156825683568456855686568756885689569056915692569356945695569656975698569957005701570257035704570557065707570857095710571157125713571457155716571757185719572057215722572357245725572657275728572957305731573257335734573557365737573857395740574157425743574457455746574757485749575057515752575357545755575657575758575957605761576257635764576557665767576857695770577157725773577457755776577757785779578057815782578357845785578657875788578957905791579257935794579557965797579857995800580158025803580458055806580758085809581058115812581358145815581658175818581958205821582258235824582558265827582858295830583158325833583458355836583758385839584058415842584358445845584658475848584958505851585258535854585558565857585858595860586158625863586458655866586758685869587058715872587358745875587658775878587958805881588258835884588558865887
  1. /*
  2. * Copyright 2012 Advanced Micro Devices, Inc.
  3. *
  4. * Permission is hereby granted, free of charge, to any person obtaining a
  5. * copy of this software and associated documentation files (the "Software"),
  6. * to deal in the Software without restriction, including without limitation
  7. * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8. * and/or sell copies of the Software, and to permit persons to whom the
  9. * Software is furnished to do so, subject to the following conditions:
  10. *
  11. * The above copyright notice and this permission notice shall be included in
  12. * all copies or substantial portions of the Software.
  13. *
  14. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16. * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  17. * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18. * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19. * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20. * OTHER DEALINGS IN THE SOFTWARE.
  21. *
  22. * Authors: Alex Deucher
  23. */
  24. #include <linux/firmware.h>
  25. #include <linux/platform_device.h>
  26. #include <linux/slab.h>
  27. #include <linux/module.h>
  28. #include "drmP.h"
  29. #include "radeon.h"
  30. #include "radeon_asic.h"
  31. #include "cikd.h"
  32. #include "atom.h"
  33. #include "cik_blit_shaders.h"
  34. /* GFX */
  35. #define CIK_PFP_UCODE_SIZE 2144
  36. #define CIK_ME_UCODE_SIZE 2144
  37. #define CIK_CE_UCODE_SIZE 2144
  38. /* compute */
  39. #define CIK_MEC_UCODE_SIZE 4192
  40. /* interrupts */
  41. #define BONAIRE_RLC_UCODE_SIZE 2048
  42. #define KB_RLC_UCODE_SIZE 2560
  43. #define KV_RLC_UCODE_SIZE 2560
  44. /* gddr controller */
  45. #define CIK_MC_UCODE_SIZE 7866
  46. /* sdma */
  47. #define CIK_SDMA_UCODE_SIZE 1050
  48. #define CIK_SDMA_UCODE_VERSION 64
  49. MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
  50. MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
  51. MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
  52. MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
  53. MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
  54. MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
  55. MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
  56. MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
  57. MODULE_FIRMWARE("radeon/KAVERI_me.bin");
  58. MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
  59. MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
  60. MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
  61. MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
  62. MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
  63. MODULE_FIRMWARE("radeon/KABINI_me.bin");
  64. MODULE_FIRMWARE("radeon/KABINI_ce.bin");
  65. MODULE_FIRMWARE("radeon/KABINI_mec.bin");
  66. MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
  67. MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
  68. extern int r600_ih_ring_alloc(struct radeon_device *rdev);
  69. extern void r600_ih_ring_fini(struct radeon_device *rdev);
  70. extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
  71. extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
  72. extern bool evergreen_is_display_hung(struct radeon_device *rdev);
  73. extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
  74. extern void si_rlc_fini(struct radeon_device *rdev);
  75. extern int si_rlc_init(struct radeon_device *rdev);
  76. static void cik_rlc_stop(struct radeon_device *rdev);
  77. /*
  78. * Indirect registers accessor
  79. */
  80. u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
  81. {
  82. u32 r;
  83. WREG32(PCIE_INDEX, reg);
  84. (void)RREG32(PCIE_INDEX);
  85. r = RREG32(PCIE_DATA);
  86. return r;
  87. }
  88. void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  89. {
  90. WREG32(PCIE_INDEX, reg);
  91. (void)RREG32(PCIE_INDEX);
  92. WREG32(PCIE_DATA, v);
  93. (void)RREG32(PCIE_DATA);
  94. }
  95. /**
  96. * cik_get_xclk - get the xclk
  97. *
  98. * @rdev: radeon_device pointer
  99. *
  100. * Returns the reference clock used by the gfx engine
  101. * (CIK).
  102. */
  103. u32 cik_get_xclk(struct radeon_device *rdev)
  104. {
  105. u32 reference_clock = rdev->clock.spll.reference_freq;
  106. if (rdev->flags & RADEON_IS_IGP) {
  107. if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
  108. return reference_clock / 2;
  109. } else {
  110. if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
  111. return reference_clock / 4;
  112. }
  113. return reference_clock;
  114. }
  115. /**
  116. * cik_mm_rdoorbell - read a doorbell dword
  117. *
  118. * @rdev: radeon_device pointer
  119. * @offset: byte offset into the aperture
  120. *
  121. * Returns the value in the doorbell aperture at the
  122. * requested offset (CIK).
  123. */
  124. u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 offset)
  125. {
  126. if (offset < rdev->doorbell.size) {
  127. return readl(((void __iomem *)rdev->doorbell.ptr) + offset);
  128. } else {
  129. DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", offset);
  130. return 0;
  131. }
  132. }
  133. /**
  134. * cik_mm_wdoorbell - write a doorbell dword
  135. *
  136. * @rdev: radeon_device pointer
  137. * @offset: byte offset into the aperture
  138. * @v: value to write
  139. *
  140. * Writes @v to the doorbell aperture at the
  141. * requested offset (CIK).
  142. */
  143. void cik_mm_wdoorbell(struct radeon_device *rdev, u32 offset, u32 v)
  144. {
  145. if (offset < rdev->doorbell.size) {
  146. writel(v, ((void __iomem *)rdev->doorbell.ptr) + offset);
  147. } else {
  148. DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", offset);
  149. }
  150. }
  151. #define BONAIRE_IO_MC_REGS_SIZE 36
  152. static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
  153. {
  154. {0x00000070, 0x04400000},
  155. {0x00000071, 0x80c01803},
  156. {0x00000072, 0x00004004},
  157. {0x00000073, 0x00000100},
  158. {0x00000074, 0x00ff0000},
  159. {0x00000075, 0x34000000},
  160. {0x00000076, 0x08000014},
  161. {0x00000077, 0x00cc08ec},
  162. {0x00000078, 0x00000400},
  163. {0x00000079, 0x00000000},
  164. {0x0000007a, 0x04090000},
  165. {0x0000007c, 0x00000000},
  166. {0x0000007e, 0x4408a8e8},
  167. {0x0000007f, 0x00000304},
  168. {0x00000080, 0x00000000},
  169. {0x00000082, 0x00000001},
  170. {0x00000083, 0x00000002},
  171. {0x00000084, 0xf3e4f400},
  172. {0x00000085, 0x052024e3},
  173. {0x00000087, 0x00000000},
  174. {0x00000088, 0x01000000},
  175. {0x0000008a, 0x1c0a0000},
  176. {0x0000008b, 0xff010000},
  177. {0x0000008d, 0xffffefff},
  178. {0x0000008e, 0xfff3efff},
  179. {0x0000008f, 0xfff3efbf},
  180. {0x00000092, 0xf7ffffff},
  181. {0x00000093, 0xffffff7f},
  182. {0x00000095, 0x00101101},
  183. {0x00000096, 0x00000fff},
  184. {0x00000097, 0x00116fff},
  185. {0x00000098, 0x60010000},
  186. {0x00000099, 0x10010000},
  187. {0x0000009a, 0x00006000},
  188. {0x0000009b, 0x00001000},
  189. {0x0000009f, 0x00b48000}
  190. };
  191. /**
  192. * cik_srbm_select - select specific register instances
  193. *
  194. * @rdev: radeon_device pointer
  195. * @me: selected ME (micro engine)
  196. * @pipe: pipe
  197. * @queue: queue
  198. * @vmid: VMID
  199. *
  200. * Switches the currently active registers instances. Some
  201. * registers are instanced per VMID, others are instanced per
  202. * me/pipe/queue combination.
  203. */
  204. static void cik_srbm_select(struct radeon_device *rdev,
  205. u32 me, u32 pipe, u32 queue, u32 vmid)
  206. {
  207. u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
  208. MEID(me & 0x3) |
  209. VMID(vmid & 0xf) |
  210. QUEUEID(queue & 0x7));
  211. WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
  212. }
  213. /* ucode loading */
  214. /**
  215. * ci_mc_load_microcode - load MC ucode into the hw
  216. *
  217. * @rdev: radeon_device pointer
  218. *
  219. * Load the GDDR MC ucode into the hw (CIK).
  220. * Returns 0 on success, error on failure.
  221. */
  222. static int ci_mc_load_microcode(struct radeon_device *rdev)
  223. {
  224. const __be32 *fw_data;
  225. u32 running, blackout = 0;
  226. u32 *io_mc_regs;
  227. int i, ucode_size, regs_size;
  228. if (!rdev->mc_fw)
  229. return -EINVAL;
  230. switch (rdev->family) {
  231. case CHIP_BONAIRE:
  232. default:
  233. io_mc_regs = (u32 *)&bonaire_io_mc_regs;
  234. ucode_size = CIK_MC_UCODE_SIZE;
  235. regs_size = BONAIRE_IO_MC_REGS_SIZE;
  236. break;
  237. }
  238. running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
  239. if (running == 0) {
  240. if (running) {
  241. blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
  242. WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
  243. }
  244. /* reset the engine and set to writable */
  245. WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
  246. WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
  247. /* load mc io regs */
  248. for (i = 0; i < regs_size; i++) {
  249. WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
  250. WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
  251. }
  252. /* load the MC ucode */
  253. fw_data = (const __be32 *)rdev->mc_fw->data;
  254. for (i = 0; i < ucode_size; i++)
  255. WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
  256. /* put the engine back into the active state */
  257. WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
  258. WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
  259. WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
  260. /* wait for training to complete */
  261. for (i = 0; i < rdev->usec_timeout; i++) {
  262. if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
  263. break;
  264. udelay(1);
  265. }
  266. for (i = 0; i < rdev->usec_timeout; i++) {
  267. if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
  268. break;
  269. udelay(1);
  270. }
  271. if (running)
  272. WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
  273. }
  274. return 0;
  275. }
  276. /**
  277. * cik_init_microcode - load ucode images from disk
  278. *
  279. * @rdev: radeon_device pointer
  280. *
  281. * Use the firmware interface to load the ucode images into
  282. * the driver (not loaded into hw).
  283. * Returns 0 on success, error on failure.
  284. */
  285. static int cik_init_microcode(struct radeon_device *rdev)
  286. {
  287. struct platform_device *pdev;
  288. const char *chip_name;
  289. size_t pfp_req_size, me_req_size, ce_req_size,
  290. mec_req_size, rlc_req_size, mc_req_size,
  291. sdma_req_size;
  292. char fw_name[30];
  293. int err;
  294. DRM_DEBUG("\n");
  295. pdev = platform_device_register_simple("radeon_cp", 0, NULL, 0);
  296. err = IS_ERR(pdev);
  297. if (err) {
  298. printk(KERN_ERR "radeon_cp: Failed to register firmware\n");
  299. return -EINVAL;
  300. }
  301. switch (rdev->family) {
  302. case CHIP_BONAIRE:
  303. chip_name = "BONAIRE";
  304. pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
  305. me_req_size = CIK_ME_UCODE_SIZE * 4;
  306. ce_req_size = CIK_CE_UCODE_SIZE * 4;
  307. mec_req_size = CIK_MEC_UCODE_SIZE * 4;
  308. rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
  309. mc_req_size = CIK_MC_UCODE_SIZE * 4;
  310. sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
  311. break;
  312. case CHIP_KAVERI:
  313. chip_name = "KAVERI";
  314. pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
  315. me_req_size = CIK_ME_UCODE_SIZE * 4;
  316. ce_req_size = CIK_CE_UCODE_SIZE * 4;
  317. mec_req_size = CIK_MEC_UCODE_SIZE * 4;
  318. rlc_req_size = KV_RLC_UCODE_SIZE * 4;
  319. sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
  320. break;
  321. case CHIP_KABINI:
  322. chip_name = "KABINI";
  323. pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
  324. me_req_size = CIK_ME_UCODE_SIZE * 4;
  325. ce_req_size = CIK_CE_UCODE_SIZE * 4;
  326. mec_req_size = CIK_MEC_UCODE_SIZE * 4;
  327. rlc_req_size = KB_RLC_UCODE_SIZE * 4;
  328. sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
  329. break;
  330. default: BUG();
  331. }
  332. DRM_INFO("Loading %s Microcode\n", chip_name);
  333. snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
  334. err = request_firmware(&rdev->pfp_fw, fw_name, &pdev->dev);
  335. if (err)
  336. goto out;
  337. if (rdev->pfp_fw->size != pfp_req_size) {
  338. printk(KERN_ERR
  339. "cik_cp: Bogus length %zu in firmware \"%s\"\n",
  340. rdev->pfp_fw->size, fw_name);
  341. err = -EINVAL;
  342. goto out;
  343. }
  344. snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
  345. err = request_firmware(&rdev->me_fw, fw_name, &pdev->dev);
  346. if (err)
  347. goto out;
  348. if (rdev->me_fw->size != me_req_size) {
  349. printk(KERN_ERR
  350. "cik_cp: Bogus length %zu in firmware \"%s\"\n",
  351. rdev->me_fw->size, fw_name);
  352. err = -EINVAL;
  353. }
  354. snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
  355. err = request_firmware(&rdev->ce_fw, fw_name, &pdev->dev);
  356. if (err)
  357. goto out;
  358. if (rdev->ce_fw->size != ce_req_size) {
  359. printk(KERN_ERR
  360. "cik_cp: Bogus length %zu in firmware \"%s\"\n",
  361. rdev->ce_fw->size, fw_name);
  362. err = -EINVAL;
  363. }
  364. snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
  365. err = request_firmware(&rdev->mec_fw, fw_name, &pdev->dev);
  366. if (err)
  367. goto out;
  368. if (rdev->mec_fw->size != mec_req_size) {
  369. printk(KERN_ERR
  370. "cik_cp: Bogus length %zu in firmware \"%s\"\n",
  371. rdev->mec_fw->size, fw_name);
  372. err = -EINVAL;
  373. }
  374. snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
  375. err = request_firmware(&rdev->rlc_fw, fw_name, &pdev->dev);
  376. if (err)
  377. goto out;
  378. if (rdev->rlc_fw->size != rlc_req_size) {
  379. printk(KERN_ERR
  380. "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
  381. rdev->rlc_fw->size, fw_name);
  382. err = -EINVAL;
  383. }
  384. snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
  385. err = request_firmware(&rdev->sdma_fw, fw_name, &pdev->dev);
  386. if (err)
  387. goto out;
  388. if (rdev->sdma_fw->size != sdma_req_size) {
  389. printk(KERN_ERR
  390. "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
  391. rdev->sdma_fw->size, fw_name);
  392. err = -EINVAL;
  393. }
  394. /* No MC ucode on APUs */
  395. if (!(rdev->flags & RADEON_IS_IGP)) {
  396. snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
  397. err = request_firmware(&rdev->mc_fw, fw_name, &pdev->dev);
  398. if (err)
  399. goto out;
  400. if (rdev->mc_fw->size != mc_req_size) {
  401. printk(KERN_ERR
  402. "cik_mc: Bogus length %zu in firmware \"%s\"\n",
  403. rdev->mc_fw->size, fw_name);
  404. err = -EINVAL;
  405. }
  406. }
  407. out:
  408. platform_device_unregister(pdev);
  409. if (err) {
  410. if (err != -EINVAL)
  411. printk(KERN_ERR
  412. "cik_cp: Failed to load firmware \"%s\"\n",
  413. fw_name);
  414. release_firmware(rdev->pfp_fw);
  415. rdev->pfp_fw = NULL;
  416. release_firmware(rdev->me_fw);
  417. rdev->me_fw = NULL;
  418. release_firmware(rdev->ce_fw);
  419. rdev->ce_fw = NULL;
  420. release_firmware(rdev->rlc_fw);
  421. rdev->rlc_fw = NULL;
  422. release_firmware(rdev->mc_fw);
  423. rdev->mc_fw = NULL;
  424. }
  425. return err;
  426. }
  427. /*
  428. * Core functions
  429. */
  430. /**
  431. * cik_tiling_mode_table_init - init the hw tiling table
  432. *
  433. * @rdev: radeon_device pointer
  434. *
  435. * Starting with SI, the tiling setup is done globally in a
  436. * set of 32 tiling modes. Rather than selecting each set of
  437. * parameters per surface as on older asics, we just select
  438. * which index in the tiling table we want to use, and the
  439. * surface uses those parameters (CIK).
  440. */
  441. static void cik_tiling_mode_table_init(struct radeon_device *rdev)
  442. {
  443. const u32 num_tile_mode_states = 32;
  444. const u32 num_secondary_tile_mode_states = 16;
  445. u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
  446. u32 num_pipe_configs;
  447. u32 num_rbs = rdev->config.cik.max_backends_per_se *
  448. rdev->config.cik.max_shader_engines;
  449. switch (rdev->config.cik.mem_row_size_in_kb) {
  450. case 1:
  451. split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
  452. break;
  453. case 2:
  454. default:
  455. split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
  456. break;
  457. case 4:
  458. split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
  459. break;
  460. }
  461. num_pipe_configs = rdev->config.cik.max_tile_pipes;
  462. if (num_pipe_configs > 8)
  463. num_pipe_configs = 8; /* ??? */
  464. if (num_pipe_configs == 8) {
  465. for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
  466. switch (reg_offset) {
  467. case 0:
  468. gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
  469. MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
  470. PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
  471. TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
  472. break;
  473. case 1:
  474. gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
  475. MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
  476. PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
  477. TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
  478. break;
  479. case 2:
  480. gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
  481. MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
  482. PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
  483. TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
  484. break;
  485. case 3:
  486. gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
  487. MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
  488. PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
  489. TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
  490. break;
  491. case 4:
  492. gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
  493. MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
  494. PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
  495. TILE_SPLIT(split_equal_to_row_size));
  496. break;
  497. case 5:
  498. gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
  499. MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
  500. break;
  501. case 6:
  502. gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
  503. MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
  504. PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
  505. TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
  506. break;
  507. case 7:
  508. gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
  509. MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
  510. PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
  511. TILE_SPLIT(split_equal_to_row_size));
  512. break;
  513. case 8:
  514. gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
  515. PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
  516. break;
  517. case 9:
  518. gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
  519. MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
  520. break;
  521. case 10:
  522. gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
  523. MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
  524. PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
  525. SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
  526. break;
  527. case 11:
  528. gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
  529. MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
  530. PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
  531. SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
  532. break;
  533. case 12:
  534. gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
  535. MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
  536. PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
  537. SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
  538. break;
  539. case 13:
  540. gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
  541. MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
  542. break;
  543. case 14:
  544. gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
  545. MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
  546. PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
  547. SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
  548. break;
  549. case 16:
  550. gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
  551. MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
  552. PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
  553. SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
  554. break;
  555. case 17:
  556. gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
  557. MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
  558. PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
  559. SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
  560. break;
  561. case 27:
  562. gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
  563. MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
  564. break;
  565. case 28:
  566. gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
  567. MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
  568. PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
  569. SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
  570. break;
  571. case 29:
  572. gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
  573. MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
  574. PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
  575. SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
  576. break;
  577. case 30:
  578. gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
  579. MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
  580. PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
  581. SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
  582. break;
  583. default:
  584. gb_tile_moden = 0;
  585. break;
  586. }
  587. WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
  588. }
  589. for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
  590. switch (reg_offset) {
  591. case 0:
  592. gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
  593. BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
  594. MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
  595. NUM_BANKS(ADDR_SURF_16_BANK));
  596. break;
  597. case 1:
  598. gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
  599. BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
  600. MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
  601. NUM_BANKS(ADDR_SURF_16_BANK));
  602. break;
  603. case 2:
  604. gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
  605. BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
  606. MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
  607. NUM_BANKS(ADDR_SURF_16_BANK));
  608. break;
  609. case 3:
  610. gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
  611. BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
  612. MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
  613. NUM_BANKS(ADDR_SURF_16_BANK));
  614. break;
  615. case 4:
  616. gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
  617. BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
  618. MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
  619. NUM_BANKS(ADDR_SURF_8_BANK));
  620. break;
  621. case 5:
  622. gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
  623. BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
  624. MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
  625. NUM_BANKS(ADDR_SURF_4_BANK));
  626. break;
  627. case 6:
  628. gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
  629. BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
  630. MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
  631. NUM_BANKS(ADDR_SURF_2_BANK));
  632. break;
  633. case 8:
  634. gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
  635. BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
  636. MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
  637. NUM_BANKS(ADDR_SURF_16_BANK));
  638. break;
  639. case 9:
  640. gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
  641. BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
  642. MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
  643. NUM_BANKS(ADDR_SURF_16_BANK));
  644. break;
  645. case 10:
  646. gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
  647. BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
  648. MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
  649. NUM_BANKS(ADDR_SURF_16_BANK));
  650. break;
  651. case 11:
  652. gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
  653. BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
  654. MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
  655. NUM_BANKS(ADDR_SURF_16_BANK));
  656. break;
  657. case 12:
  658. gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
  659. BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
  660. MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
  661. NUM_BANKS(ADDR_SURF_8_BANK));
  662. break;
  663. case 13:
  664. gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
  665. BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
  666. MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
  667. NUM_BANKS(ADDR_SURF_4_BANK));
  668. break;
  669. case 14:
  670. gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
  671. BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
  672. MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
  673. NUM_BANKS(ADDR_SURF_2_BANK));
  674. break;
  675. default:
  676. gb_tile_moden = 0;
  677. break;
  678. }
  679. WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
  680. }
  681. } else if (num_pipe_configs == 4) {
  682. if (num_rbs == 4) {
  683. for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
  684. switch (reg_offset) {
  685. case 0:
  686. gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
  687. MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
  688. PIPE_CONFIG(ADDR_SURF_P4_16x16) |
  689. TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
  690. break;
  691. case 1:
  692. gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
  693. MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
  694. PIPE_CONFIG(ADDR_SURF_P4_16x16) |
  695. TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
  696. break;
  697. case 2:
  698. gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
  699. MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
  700. PIPE_CONFIG(ADDR_SURF_P4_16x16) |
  701. TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
  702. break;
  703. case 3:
  704. gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
  705. MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
  706. PIPE_CONFIG(ADDR_SURF_P4_16x16) |
  707. TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
  708. break;
  709. case 4:
  710. gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
  711. MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
  712. PIPE_CONFIG(ADDR_SURF_P4_16x16) |
  713. TILE_SPLIT(split_equal_to_row_size));
  714. break;
  715. case 5:
  716. gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
  717. MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
  718. break;
  719. case 6:
  720. gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
  721. MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
  722. PIPE_CONFIG(ADDR_SURF_P4_16x16) |
  723. TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
  724. break;
  725. case 7:
  726. gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
  727. MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
  728. PIPE_CONFIG(ADDR_SURF_P4_16x16) |
  729. TILE_SPLIT(split_equal_to_row_size));
  730. break;
  731. case 8:
  732. gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
  733. PIPE_CONFIG(ADDR_SURF_P4_16x16));
  734. break;
  735. case 9:
  736. gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
  737. MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
  738. break;
  739. case 10:
  740. gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
  741. MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
  742. PIPE_CONFIG(ADDR_SURF_P4_16x16) |
  743. SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
  744. break;
  745. case 11:
  746. gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
  747. MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
  748. PIPE_CONFIG(ADDR_SURF_P4_8x16) |
  749. SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
  750. break;
  751. case 12:
  752. gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
  753. MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
  754. PIPE_CONFIG(ADDR_SURF_P4_16x16) |
  755. SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
  756. break;
  757. case 13:
  758. gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
  759. MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
  760. break;
  761. case 14:
  762. gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
  763. MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
  764. PIPE_CONFIG(ADDR_SURF_P4_16x16) |
  765. SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
  766. break;
  767. case 16:
  768. gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
  769. MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
  770. PIPE_CONFIG(ADDR_SURF_P4_8x16) |
  771. SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
  772. break;
  773. case 17:
  774. gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
  775. MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
  776. PIPE_CONFIG(ADDR_SURF_P4_16x16) |
  777. SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
  778. break;
  779. case 27:
  780. gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
  781. MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
  782. break;
  783. case 28:
  784. gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
  785. MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
  786. PIPE_CONFIG(ADDR_SURF_P4_16x16) |
  787. SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
  788. break;
  789. case 29:
  790. gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
  791. MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
  792. PIPE_CONFIG(ADDR_SURF_P4_8x16) |
  793. SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
  794. break;
  795. case 30:
  796. gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
  797. MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
  798. PIPE_CONFIG(ADDR_SURF_P4_16x16) |
  799. SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
  800. break;
  801. default:
  802. gb_tile_moden = 0;
  803. break;
  804. }
  805. WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
  806. }
  807. } else if (num_rbs < 4) {
  808. for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
  809. switch (reg_offset) {
  810. case 0:
  811. gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
  812. MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
  813. PIPE_CONFIG(ADDR_SURF_P4_8x16) |
  814. TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
  815. break;
  816. case 1:
  817. gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
  818. MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
  819. PIPE_CONFIG(ADDR_SURF_P4_8x16) |
  820. TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
  821. break;
  822. case 2:
  823. gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
  824. MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
  825. PIPE_CONFIG(ADDR_SURF_P4_8x16) |
  826. TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
  827. break;
  828. case 3:
  829. gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
  830. MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
  831. PIPE_CONFIG(ADDR_SURF_P4_8x16) |
  832. TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
  833. break;
  834. case 4:
  835. gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
  836. MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
  837. PIPE_CONFIG(ADDR_SURF_P4_8x16) |
  838. TILE_SPLIT(split_equal_to_row_size));
  839. break;
  840. case 5:
  841. gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
  842. MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
  843. break;
  844. case 6:
  845. gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
  846. MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
  847. PIPE_CONFIG(ADDR_SURF_P4_8x16) |
  848. TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
  849. break;
  850. case 7:
  851. gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
  852. MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
  853. PIPE_CONFIG(ADDR_SURF_P4_8x16) |
  854. TILE_SPLIT(split_equal_to_row_size));
  855. break;
  856. case 8:
  857. gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
  858. PIPE_CONFIG(ADDR_SURF_P4_8x16));
  859. break;
  860. case 9:
  861. gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
  862. MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
  863. break;
  864. case 10:
  865. gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
  866. MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
  867. PIPE_CONFIG(ADDR_SURF_P4_8x16) |
  868. SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
  869. break;
  870. case 11:
  871. gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
  872. MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
  873. PIPE_CONFIG(ADDR_SURF_P4_8x16) |
  874. SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
  875. break;
  876. case 12:
  877. gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
  878. MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
  879. PIPE_CONFIG(ADDR_SURF_P4_8x16) |
  880. SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
  881. break;
  882. case 13:
  883. gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
  884. MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
  885. break;
  886. case 14:
  887. gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
  888. MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
  889. PIPE_CONFIG(ADDR_SURF_P4_8x16) |
  890. SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
  891. break;
  892. case 16:
  893. gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
  894. MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
  895. PIPE_CONFIG(ADDR_SURF_P4_8x16) |
  896. SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
  897. break;
  898. case 17:
  899. gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
  900. MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
  901. PIPE_CONFIG(ADDR_SURF_P4_8x16) |
  902. SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
  903. break;
  904. case 27:
  905. gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
  906. MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
  907. break;
  908. case 28:
  909. gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
  910. MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
  911. PIPE_CONFIG(ADDR_SURF_P4_8x16) |
  912. SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
  913. break;
  914. case 29:
  915. gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
  916. MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
  917. PIPE_CONFIG(ADDR_SURF_P4_8x16) |
  918. SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
  919. break;
  920. case 30:
  921. gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
  922. MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
  923. PIPE_CONFIG(ADDR_SURF_P4_8x16) |
  924. SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
  925. break;
  926. default:
  927. gb_tile_moden = 0;
  928. break;
  929. }
  930. WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
  931. }
  932. }
  933. for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
  934. switch (reg_offset) {
  935. case 0:
  936. gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
  937. BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
  938. MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
  939. NUM_BANKS(ADDR_SURF_16_BANK));
  940. break;
  941. case 1:
  942. gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
  943. BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
  944. MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
  945. NUM_BANKS(ADDR_SURF_16_BANK));
  946. break;
  947. case 2:
  948. gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
  949. BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
  950. MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
  951. NUM_BANKS(ADDR_SURF_16_BANK));
  952. break;
  953. case 3:
  954. gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
  955. BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
  956. MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
  957. NUM_BANKS(ADDR_SURF_16_BANK));
  958. break;
  959. case 4:
  960. gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
  961. BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
  962. MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
  963. NUM_BANKS(ADDR_SURF_16_BANK));
  964. break;
  965. case 5:
  966. gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
  967. BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
  968. MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
  969. NUM_BANKS(ADDR_SURF_8_BANK));
  970. break;
  971. case 6:
  972. gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
  973. BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
  974. MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
  975. NUM_BANKS(ADDR_SURF_4_BANK));
  976. break;
  977. case 8:
  978. gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
  979. BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
  980. MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
  981. NUM_BANKS(ADDR_SURF_16_BANK));
  982. break;
  983. case 9:
  984. gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
  985. BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
  986. MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
  987. NUM_BANKS(ADDR_SURF_16_BANK));
  988. break;
  989. case 10:
  990. gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
  991. BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
  992. MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
  993. NUM_BANKS(ADDR_SURF_16_BANK));
  994. break;
  995. case 11:
  996. gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
  997. BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
  998. MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
  999. NUM_BANKS(ADDR_SURF_16_BANK));
  1000. break;
  1001. case 12:
  1002. gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
  1003. BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
  1004. MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
  1005. NUM_BANKS(ADDR_SURF_16_BANK));
  1006. break;
  1007. case 13:
  1008. gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
  1009. BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
  1010. MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
  1011. NUM_BANKS(ADDR_SURF_8_BANK));
  1012. break;
  1013. case 14:
  1014. gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
  1015. BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
  1016. MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
  1017. NUM_BANKS(ADDR_SURF_4_BANK));
  1018. break;
  1019. default:
  1020. gb_tile_moden = 0;
  1021. break;
  1022. }
  1023. WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
  1024. }
  1025. } else if (num_pipe_configs == 2) {
  1026. for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
  1027. switch (reg_offset) {
  1028. case 0:
  1029. gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
  1030. MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
  1031. PIPE_CONFIG(ADDR_SURF_P2) |
  1032. TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
  1033. break;
  1034. case 1:
  1035. gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
  1036. MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
  1037. PIPE_CONFIG(ADDR_SURF_P2) |
  1038. TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
  1039. break;
  1040. case 2:
  1041. gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
  1042. MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
  1043. PIPE_CONFIG(ADDR_SURF_P2) |
  1044. TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
  1045. break;
  1046. case 3:
  1047. gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
  1048. MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
  1049. PIPE_CONFIG(ADDR_SURF_P2) |
  1050. TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
  1051. break;
  1052. case 4:
  1053. gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
  1054. MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
  1055. PIPE_CONFIG(ADDR_SURF_P2) |
  1056. TILE_SPLIT(split_equal_to_row_size));
  1057. break;
  1058. case 5:
  1059. gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
  1060. MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
  1061. break;
  1062. case 6:
  1063. gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
  1064. MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
  1065. PIPE_CONFIG(ADDR_SURF_P2) |
  1066. TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
  1067. break;
  1068. case 7:
  1069. gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
  1070. MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
  1071. PIPE_CONFIG(ADDR_SURF_P2) |
  1072. TILE_SPLIT(split_equal_to_row_size));
  1073. break;
  1074. case 8:
  1075. gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED);
  1076. break;
  1077. case 9:
  1078. gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
  1079. MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
  1080. break;
  1081. case 10:
  1082. gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
  1083. MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
  1084. PIPE_CONFIG(ADDR_SURF_P2) |
  1085. SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
  1086. break;
  1087. case 11:
  1088. gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
  1089. MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
  1090. PIPE_CONFIG(ADDR_SURF_P2) |
  1091. SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
  1092. break;
  1093. case 12:
  1094. gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
  1095. MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
  1096. PIPE_CONFIG(ADDR_SURF_P2) |
  1097. SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
  1098. break;
  1099. case 13:
  1100. gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
  1101. MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
  1102. break;
  1103. case 14:
  1104. gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
  1105. MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
  1106. PIPE_CONFIG(ADDR_SURF_P2) |
  1107. SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
  1108. break;
  1109. case 16:
  1110. gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
  1111. MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
  1112. PIPE_CONFIG(ADDR_SURF_P2) |
  1113. SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
  1114. break;
  1115. case 17:
  1116. gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
  1117. MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
  1118. PIPE_CONFIG(ADDR_SURF_P2) |
  1119. SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
  1120. break;
  1121. case 27:
  1122. gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
  1123. MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
  1124. break;
  1125. case 28:
  1126. gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
  1127. MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
  1128. PIPE_CONFIG(ADDR_SURF_P2) |
  1129. SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
  1130. break;
  1131. case 29:
  1132. gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
  1133. MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
  1134. PIPE_CONFIG(ADDR_SURF_P2) |
  1135. SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
  1136. break;
  1137. case 30:
  1138. gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
  1139. MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
  1140. PIPE_CONFIG(ADDR_SURF_P2) |
  1141. SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
  1142. break;
  1143. default:
  1144. gb_tile_moden = 0;
  1145. break;
  1146. }
  1147. WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
  1148. }
  1149. for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
  1150. switch (reg_offset) {
  1151. case 0:
  1152. gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
  1153. BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
  1154. MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
  1155. NUM_BANKS(ADDR_SURF_16_BANK));
  1156. break;
  1157. case 1:
  1158. gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
  1159. BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
  1160. MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
  1161. NUM_BANKS(ADDR_SURF_16_BANK));
  1162. break;
  1163. case 2:
  1164. gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
  1165. BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
  1166. MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
  1167. NUM_BANKS(ADDR_SURF_16_BANK));
  1168. break;
  1169. case 3:
  1170. gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
  1171. BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
  1172. MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
  1173. NUM_BANKS(ADDR_SURF_16_BANK));
  1174. break;
  1175. case 4:
  1176. gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
  1177. BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
  1178. MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
  1179. NUM_BANKS(ADDR_SURF_16_BANK));
  1180. break;
  1181. case 5:
  1182. gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
  1183. BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
  1184. MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
  1185. NUM_BANKS(ADDR_SURF_16_BANK));
  1186. break;
  1187. case 6:
  1188. gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
  1189. BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
  1190. MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
  1191. NUM_BANKS(ADDR_SURF_8_BANK));
  1192. break;
  1193. case 8:
  1194. gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
  1195. BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
  1196. MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
  1197. NUM_BANKS(ADDR_SURF_16_BANK));
  1198. break;
  1199. case 9:
  1200. gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
  1201. BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
  1202. MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
  1203. NUM_BANKS(ADDR_SURF_16_BANK));
  1204. break;
  1205. case 10:
  1206. gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
  1207. BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
  1208. MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
  1209. NUM_BANKS(ADDR_SURF_16_BANK));
  1210. break;
  1211. case 11:
  1212. gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
  1213. BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
  1214. MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
  1215. NUM_BANKS(ADDR_SURF_16_BANK));
  1216. break;
  1217. case 12:
  1218. gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
  1219. BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
  1220. MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
  1221. NUM_BANKS(ADDR_SURF_16_BANK));
  1222. break;
  1223. case 13:
  1224. gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
  1225. BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
  1226. MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
  1227. NUM_BANKS(ADDR_SURF_16_BANK));
  1228. break;
  1229. case 14:
  1230. gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
  1231. BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
  1232. MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
  1233. NUM_BANKS(ADDR_SURF_8_BANK));
  1234. break;
  1235. default:
  1236. gb_tile_moden = 0;
  1237. break;
  1238. }
  1239. WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
  1240. }
  1241. } else
  1242. DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
  1243. }
  1244. /**
  1245. * cik_select_se_sh - select which SE, SH to address
  1246. *
  1247. * @rdev: radeon_device pointer
  1248. * @se_num: shader engine to address
  1249. * @sh_num: sh block to address
  1250. *
  1251. * Select which SE, SH combinations to address. Certain
  1252. * registers are instanced per SE or SH. 0xffffffff means
  1253. * broadcast to all SEs or SHs (CIK).
  1254. */
  1255. static void cik_select_se_sh(struct radeon_device *rdev,
  1256. u32 se_num, u32 sh_num)
  1257. {
  1258. u32 data = INSTANCE_BROADCAST_WRITES;
  1259. if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
  1260. data = SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
  1261. else if (se_num == 0xffffffff)
  1262. data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
  1263. else if (sh_num == 0xffffffff)
  1264. data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
  1265. else
  1266. data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
  1267. WREG32(GRBM_GFX_INDEX, data);
  1268. }
  1269. /**
  1270. * cik_create_bitmask - create a bitmask
  1271. *
  1272. * @bit_width: length of the mask
  1273. *
  1274. * create a variable length bit mask (CIK).
  1275. * Returns the bitmask.
  1276. */
  1277. static u32 cik_create_bitmask(u32 bit_width)
  1278. {
  1279. u32 i, mask = 0;
  1280. for (i = 0; i < bit_width; i++) {
  1281. mask <<= 1;
  1282. mask |= 1;
  1283. }
  1284. return mask;
  1285. }
  1286. /**
  1287. * cik_select_se_sh - select which SE, SH to address
  1288. *
  1289. * @rdev: radeon_device pointer
  1290. * @max_rb_num: max RBs (render backends) for the asic
  1291. * @se_num: number of SEs (shader engines) for the asic
  1292. * @sh_per_se: number of SH blocks per SE for the asic
  1293. *
  1294. * Calculates the bitmask of disabled RBs (CIK).
  1295. * Returns the disabled RB bitmask.
  1296. */
  1297. static u32 cik_get_rb_disabled(struct radeon_device *rdev,
  1298. u32 max_rb_num, u32 se_num,
  1299. u32 sh_per_se)
  1300. {
  1301. u32 data, mask;
  1302. data = RREG32(CC_RB_BACKEND_DISABLE);
  1303. if (data & 1)
  1304. data &= BACKEND_DISABLE_MASK;
  1305. else
  1306. data = 0;
  1307. data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
  1308. data >>= BACKEND_DISABLE_SHIFT;
  1309. mask = cik_create_bitmask(max_rb_num / se_num / sh_per_se);
  1310. return data & mask;
  1311. }
  1312. /**
  1313. * cik_setup_rb - setup the RBs on the asic
  1314. *
  1315. * @rdev: radeon_device pointer
  1316. * @se_num: number of SEs (shader engines) for the asic
  1317. * @sh_per_se: number of SH blocks per SE for the asic
  1318. * @max_rb_num: max RBs (render backends) for the asic
  1319. *
  1320. * Configures per-SE/SH RB registers (CIK).
  1321. */
  1322. static void cik_setup_rb(struct radeon_device *rdev,
  1323. u32 se_num, u32 sh_per_se,
  1324. u32 max_rb_num)
  1325. {
  1326. int i, j;
  1327. u32 data, mask;
  1328. u32 disabled_rbs = 0;
  1329. u32 enabled_rbs = 0;
  1330. for (i = 0; i < se_num; i++) {
  1331. for (j = 0; j < sh_per_se; j++) {
  1332. cik_select_se_sh(rdev, i, j);
  1333. data = cik_get_rb_disabled(rdev, max_rb_num, se_num, sh_per_se);
  1334. disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
  1335. }
  1336. }
  1337. cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
  1338. mask = 1;
  1339. for (i = 0; i < max_rb_num; i++) {
  1340. if (!(disabled_rbs & mask))
  1341. enabled_rbs |= mask;
  1342. mask <<= 1;
  1343. }
  1344. for (i = 0; i < se_num; i++) {
  1345. cik_select_se_sh(rdev, i, 0xffffffff);
  1346. data = 0;
  1347. for (j = 0; j < sh_per_se; j++) {
  1348. switch (enabled_rbs & 3) {
  1349. case 1:
  1350. data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
  1351. break;
  1352. case 2:
  1353. data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
  1354. break;
  1355. case 3:
  1356. default:
  1357. data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
  1358. break;
  1359. }
  1360. enabled_rbs >>= 2;
  1361. }
  1362. WREG32(PA_SC_RASTER_CONFIG, data);
  1363. }
  1364. cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
  1365. }
  1366. /**
  1367. * cik_gpu_init - setup the 3D engine
  1368. *
  1369. * @rdev: radeon_device pointer
  1370. *
  1371. * Configures the 3D engine and tiling configuration
  1372. * registers so that the 3D engine is usable.
  1373. */
  1374. static void cik_gpu_init(struct radeon_device *rdev)
  1375. {
  1376. u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
  1377. u32 mc_shared_chmap, mc_arb_ramcfg;
  1378. u32 hdp_host_path_cntl;
  1379. u32 tmp;
  1380. int i, j;
  1381. switch (rdev->family) {
  1382. case CHIP_BONAIRE:
  1383. rdev->config.cik.max_shader_engines = 2;
  1384. rdev->config.cik.max_tile_pipes = 4;
  1385. rdev->config.cik.max_cu_per_sh = 7;
  1386. rdev->config.cik.max_sh_per_se = 1;
  1387. rdev->config.cik.max_backends_per_se = 2;
  1388. rdev->config.cik.max_texture_channel_caches = 4;
  1389. rdev->config.cik.max_gprs = 256;
  1390. rdev->config.cik.max_gs_threads = 32;
  1391. rdev->config.cik.max_hw_contexts = 8;
  1392. rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
  1393. rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
  1394. rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
  1395. rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
  1396. gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
  1397. break;
  1398. case CHIP_KAVERI:
  1399. /* TODO */
  1400. break;
  1401. case CHIP_KABINI:
  1402. default:
  1403. rdev->config.cik.max_shader_engines = 1;
  1404. rdev->config.cik.max_tile_pipes = 2;
  1405. rdev->config.cik.max_cu_per_sh = 2;
  1406. rdev->config.cik.max_sh_per_se = 1;
  1407. rdev->config.cik.max_backends_per_se = 1;
  1408. rdev->config.cik.max_texture_channel_caches = 2;
  1409. rdev->config.cik.max_gprs = 256;
  1410. rdev->config.cik.max_gs_threads = 16;
  1411. rdev->config.cik.max_hw_contexts = 8;
  1412. rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
  1413. rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
  1414. rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
  1415. rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
  1416. gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
  1417. break;
  1418. }
  1419. /* Initialize HDP */
  1420. for (i = 0, j = 0; i < 32; i++, j += 0x18) {
  1421. WREG32((0x2c14 + j), 0x00000000);
  1422. WREG32((0x2c18 + j), 0x00000000);
  1423. WREG32((0x2c1c + j), 0x00000000);
  1424. WREG32((0x2c20 + j), 0x00000000);
  1425. WREG32((0x2c24 + j), 0x00000000);
  1426. }
  1427. WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
  1428. WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
  1429. mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
  1430. mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
  1431. rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
  1432. rdev->config.cik.mem_max_burst_length_bytes = 256;
  1433. tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
  1434. rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
  1435. if (rdev->config.cik.mem_row_size_in_kb > 4)
  1436. rdev->config.cik.mem_row_size_in_kb = 4;
  1437. /* XXX use MC settings? */
  1438. rdev->config.cik.shader_engine_tile_size = 32;
  1439. rdev->config.cik.num_gpus = 1;
  1440. rdev->config.cik.multi_gpu_tile_size = 64;
  1441. /* fix up row size */
  1442. gb_addr_config &= ~ROW_SIZE_MASK;
  1443. switch (rdev->config.cik.mem_row_size_in_kb) {
  1444. case 1:
  1445. default:
  1446. gb_addr_config |= ROW_SIZE(0);
  1447. break;
  1448. case 2:
  1449. gb_addr_config |= ROW_SIZE(1);
  1450. break;
  1451. case 4:
  1452. gb_addr_config |= ROW_SIZE(2);
  1453. break;
  1454. }
  1455. /* setup tiling info dword. gb_addr_config is not adequate since it does
  1456. * not have bank info, so create a custom tiling dword.
  1457. * bits 3:0 num_pipes
  1458. * bits 7:4 num_banks
  1459. * bits 11:8 group_size
  1460. * bits 15:12 row_size
  1461. */
  1462. rdev->config.cik.tile_config = 0;
  1463. switch (rdev->config.cik.num_tile_pipes) {
  1464. case 1:
  1465. rdev->config.cik.tile_config |= (0 << 0);
  1466. break;
  1467. case 2:
  1468. rdev->config.cik.tile_config |= (1 << 0);
  1469. break;
  1470. case 4:
  1471. rdev->config.cik.tile_config |= (2 << 0);
  1472. break;
  1473. case 8:
  1474. default:
  1475. /* XXX what about 12? */
  1476. rdev->config.cik.tile_config |= (3 << 0);
  1477. break;
  1478. }
  1479. if ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT)
  1480. rdev->config.cik.tile_config |= 1 << 4;
  1481. else
  1482. rdev->config.cik.tile_config |= 0 << 4;
  1483. rdev->config.cik.tile_config |=
  1484. ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
  1485. rdev->config.cik.tile_config |=
  1486. ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
  1487. WREG32(GB_ADDR_CONFIG, gb_addr_config);
  1488. WREG32(HDP_ADDR_CONFIG, gb_addr_config);
  1489. WREG32(DMIF_ADDR_CALC, gb_addr_config);
  1490. WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
  1491. WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
  1492. WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
  1493. WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
  1494. WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
  1495. cik_tiling_mode_table_init(rdev);
  1496. cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
  1497. rdev->config.cik.max_sh_per_se,
  1498. rdev->config.cik.max_backends_per_se);
  1499. /* set HW defaults for 3D engine */
  1500. WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
  1501. WREG32(SX_DEBUG_1, 0x20);
  1502. WREG32(TA_CNTL_AUX, 0x00010000);
  1503. tmp = RREG32(SPI_CONFIG_CNTL);
  1504. tmp |= 0x03000000;
  1505. WREG32(SPI_CONFIG_CNTL, tmp);
  1506. WREG32(SQ_CONFIG, 1);
  1507. WREG32(DB_DEBUG, 0);
  1508. tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
  1509. tmp |= 0x00000400;
  1510. WREG32(DB_DEBUG2, tmp);
  1511. tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
  1512. tmp |= 0x00020200;
  1513. WREG32(DB_DEBUG3, tmp);
  1514. tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
  1515. tmp |= 0x00018208;
  1516. WREG32(CB_HW_CONTROL, tmp);
  1517. WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
  1518. WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
  1519. SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
  1520. SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
  1521. SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
  1522. WREG32(VGT_NUM_INSTANCES, 1);
  1523. WREG32(CP_PERFMON_CNTL, 0);
  1524. WREG32(SQ_CONFIG, 0);
  1525. WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
  1526. FORCE_EOV_MAX_REZ_CNT(255)));
  1527. WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
  1528. AUTO_INVLD_EN(ES_AND_GS_AUTO));
  1529. WREG32(VGT_GS_VERTEX_REUSE, 16);
  1530. WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
  1531. tmp = RREG32(HDP_MISC_CNTL);
  1532. tmp |= HDP_FLUSH_INVALIDATE_CACHE;
  1533. WREG32(HDP_MISC_CNTL, tmp);
  1534. hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
  1535. WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
  1536. WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
  1537. WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
  1538. udelay(50);
  1539. }
  1540. /*
  1541. * GPU scratch registers helpers function.
  1542. */
  1543. /**
  1544. * cik_scratch_init - setup driver info for CP scratch regs
  1545. *
  1546. * @rdev: radeon_device pointer
  1547. *
  1548. * Set up the number and offset of the CP scratch registers.
  1549. * NOTE: use of CP scratch registers is a legacy inferface and
  1550. * is not used by default on newer asics (r6xx+). On newer asics,
  1551. * memory buffers are used for fences rather than scratch regs.
  1552. */
  1553. static void cik_scratch_init(struct radeon_device *rdev)
  1554. {
  1555. int i;
  1556. rdev->scratch.num_reg = 7;
  1557. rdev->scratch.reg_base = SCRATCH_REG0;
  1558. for (i = 0; i < rdev->scratch.num_reg; i++) {
  1559. rdev->scratch.free[i] = true;
  1560. rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
  1561. }
  1562. }
  1563. /**
  1564. * cik_ring_test - basic gfx ring test
  1565. *
  1566. * @rdev: radeon_device pointer
  1567. * @ring: radeon_ring structure holding ring information
  1568. *
  1569. * Allocate a scratch register and write to it using the gfx ring (CIK).
  1570. * Provides a basic gfx ring test to verify that the ring is working.
  1571. * Used by cik_cp_gfx_resume();
  1572. * Returns 0 on success, error on failure.
  1573. */
  1574. int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
  1575. {
  1576. uint32_t scratch;
  1577. uint32_t tmp = 0;
  1578. unsigned i;
  1579. int r;
  1580. r = radeon_scratch_get(rdev, &scratch);
  1581. if (r) {
  1582. DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
  1583. return r;
  1584. }
  1585. WREG32(scratch, 0xCAFEDEAD);
  1586. r = radeon_ring_lock(rdev, ring, 3);
  1587. if (r) {
  1588. DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
  1589. radeon_scratch_free(rdev, scratch);
  1590. return r;
  1591. }
  1592. radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
  1593. radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
  1594. radeon_ring_write(ring, 0xDEADBEEF);
  1595. radeon_ring_unlock_commit(rdev, ring);
  1596. for (i = 0; i < rdev->usec_timeout; i++) {
  1597. tmp = RREG32(scratch);
  1598. if (tmp == 0xDEADBEEF)
  1599. break;
  1600. DRM_UDELAY(1);
  1601. }
  1602. if (i < rdev->usec_timeout) {
  1603. DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
  1604. } else {
  1605. DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
  1606. ring->idx, scratch, tmp);
  1607. r = -EINVAL;
  1608. }
  1609. radeon_scratch_free(rdev, scratch);
  1610. return r;
  1611. }
  1612. /**
  1613. * cik_fence_ring_emit - emit a fence on the gfx ring
  1614. *
  1615. * @rdev: radeon_device pointer
  1616. * @fence: radeon fence object
  1617. *
  1618. * Emits a fence sequnce number on the gfx ring and flushes
  1619. * GPU caches.
  1620. */
  1621. void cik_fence_ring_emit(struct radeon_device *rdev,
  1622. struct radeon_fence *fence)
  1623. {
  1624. struct radeon_ring *ring = &rdev->ring[fence->ring];
  1625. u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
  1626. /* EVENT_WRITE_EOP - flush caches, send int */
  1627. radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
  1628. radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
  1629. EOP_TC_ACTION_EN |
  1630. EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
  1631. EVENT_INDEX(5)));
  1632. radeon_ring_write(ring, addr & 0xfffffffc);
  1633. radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
  1634. radeon_ring_write(ring, fence->seq);
  1635. radeon_ring_write(ring, 0);
  1636. /* HDP flush */
  1637. /* We should be using the new WAIT_REG_MEM special op packet here
  1638. * but it causes the CP to hang
  1639. */
  1640. radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
  1641. radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
  1642. WRITE_DATA_DST_SEL(0)));
  1643. radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
  1644. radeon_ring_write(ring, 0);
  1645. radeon_ring_write(ring, 0);
  1646. }
  1647. void cik_semaphore_ring_emit(struct radeon_device *rdev,
  1648. struct radeon_ring *ring,
  1649. struct radeon_semaphore *semaphore,
  1650. bool emit_wait)
  1651. {
  1652. uint64_t addr = semaphore->gpu_addr;
  1653. unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
  1654. radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
  1655. radeon_ring_write(ring, addr & 0xffffffff);
  1656. radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
  1657. }
  1658. /*
  1659. * IB stuff
  1660. */
  1661. /**
  1662. * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
  1663. *
  1664. * @rdev: radeon_device pointer
  1665. * @ib: radeon indirect buffer object
  1666. *
  1667. * Emits an DE (drawing engine) or CE (constant engine) IB
  1668. * on the gfx ring. IBs are usually generated by userspace
  1669. * acceleration drivers and submitted to the kernel for
  1670. * sheduling on the ring. This function schedules the IB
  1671. * on the gfx ring for execution by the GPU.
  1672. */
  1673. void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
  1674. {
  1675. struct radeon_ring *ring = &rdev->ring[ib->ring];
  1676. u32 header, control = INDIRECT_BUFFER_VALID;
  1677. if (ib->is_const_ib) {
  1678. /* set switch buffer packet before const IB */
  1679. radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
  1680. radeon_ring_write(ring, 0);
  1681. header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
  1682. } else {
  1683. u32 next_rptr;
  1684. if (ring->rptr_save_reg) {
  1685. next_rptr = ring->wptr + 3 + 4;
  1686. radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
  1687. radeon_ring_write(ring, ((ring->rptr_save_reg -
  1688. PACKET3_SET_UCONFIG_REG_START) >> 2));
  1689. radeon_ring_write(ring, next_rptr);
  1690. } else if (rdev->wb.enabled) {
  1691. next_rptr = ring->wptr + 5 + 4;
  1692. radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
  1693. radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
  1694. radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
  1695. radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
  1696. radeon_ring_write(ring, next_rptr);
  1697. }
  1698. header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
  1699. }
  1700. control |= ib->length_dw |
  1701. (ib->vm ? (ib->vm->id << 24) : 0);
  1702. radeon_ring_write(ring, header);
  1703. radeon_ring_write(ring,
  1704. #ifdef __BIG_ENDIAN
  1705. (2 << 0) |
  1706. #endif
  1707. (ib->gpu_addr & 0xFFFFFFFC));
  1708. radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
  1709. radeon_ring_write(ring, control);
  1710. }
  1711. /**
  1712. * cik_ib_test - basic gfx ring IB test
  1713. *
  1714. * @rdev: radeon_device pointer
  1715. * @ring: radeon_ring structure holding ring information
  1716. *
  1717. * Allocate an IB and execute it on the gfx ring (CIK).
  1718. * Provides a basic gfx ring test to verify that IBs are working.
  1719. * Returns 0 on success, error on failure.
  1720. */
  1721. int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
  1722. {
  1723. struct radeon_ib ib;
  1724. uint32_t scratch;
  1725. uint32_t tmp = 0;
  1726. unsigned i;
  1727. int r;
  1728. r = radeon_scratch_get(rdev, &scratch);
  1729. if (r) {
  1730. DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
  1731. return r;
  1732. }
  1733. WREG32(scratch, 0xCAFEDEAD);
  1734. r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
  1735. if (r) {
  1736. DRM_ERROR("radeon: failed to get ib (%d).\n", r);
  1737. return r;
  1738. }
  1739. ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
  1740. ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
  1741. ib.ptr[2] = 0xDEADBEEF;
  1742. ib.length_dw = 3;
  1743. r = radeon_ib_schedule(rdev, &ib, NULL);
  1744. if (r) {
  1745. radeon_scratch_free(rdev, scratch);
  1746. radeon_ib_free(rdev, &ib);
  1747. DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
  1748. return r;
  1749. }
  1750. r = radeon_fence_wait(ib.fence, false);
  1751. if (r) {
  1752. DRM_ERROR("radeon: fence wait failed (%d).\n", r);
  1753. return r;
  1754. }
  1755. for (i = 0; i < rdev->usec_timeout; i++) {
  1756. tmp = RREG32(scratch);
  1757. if (tmp == 0xDEADBEEF)
  1758. break;
  1759. DRM_UDELAY(1);
  1760. }
  1761. if (i < rdev->usec_timeout) {
  1762. DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
  1763. } else {
  1764. DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
  1765. scratch, tmp);
  1766. r = -EINVAL;
  1767. }
  1768. radeon_scratch_free(rdev, scratch);
  1769. radeon_ib_free(rdev, &ib);
  1770. return r;
  1771. }
  1772. /*
  1773. * CP.
  1774. * On CIK, gfx and compute now have independant command processors.
  1775. *
  1776. * GFX
  1777. * Gfx consists of a single ring and can process both gfx jobs and
  1778. * compute jobs. The gfx CP consists of three microengines (ME):
  1779. * PFP - Pre-Fetch Parser
  1780. * ME - Micro Engine
  1781. * CE - Constant Engine
  1782. * The PFP and ME make up what is considered the Drawing Engine (DE).
  1783. * The CE is an asynchronous engine used for updating buffer desciptors
  1784. * used by the DE so that they can be loaded into cache in parallel
  1785. * while the DE is processing state update packets.
  1786. *
  1787. * Compute
  1788. * The compute CP consists of two microengines (ME):
  1789. * MEC1 - Compute MicroEngine 1
  1790. * MEC2 - Compute MicroEngine 2
  1791. * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
  1792. * The queues are exposed to userspace and are programmed directly
  1793. * by the compute runtime.
  1794. */
  1795. /**
  1796. * cik_cp_gfx_enable - enable/disable the gfx CP MEs
  1797. *
  1798. * @rdev: radeon_device pointer
  1799. * @enable: enable or disable the MEs
  1800. *
  1801. * Halts or unhalts the gfx MEs.
  1802. */
  1803. static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
  1804. {
  1805. if (enable)
  1806. WREG32(CP_ME_CNTL, 0);
  1807. else {
  1808. WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
  1809. rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
  1810. }
  1811. udelay(50);
  1812. }
  1813. /**
  1814. * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
  1815. *
  1816. * @rdev: radeon_device pointer
  1817. *
  1818. * Loads the gfx PFP, ME, and CE ucode.
  1819. * Returns 0 for success, -EINVAL if the ucode is not available.
  1820. */
  1821. static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
  1822. {
  1823. const __be32 *fw_data;
  1824. int i;
  1825. if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
  1826. return -EINVAL;
  1827. cik_cp_gfx_enable(rdev, false);
  1828. /* PFP */
  1829. fw_data = (const __be32 *)rdev->pfp_fw->data;
  1830. WREG32(CP_PFP_UCODE_ADDR, 0);
  1831. for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
  1832. WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
  1833. WREG32(CP_PFP_UCODE_ADDR, 0);
  1834. /* CE */
  1835. fw_data = (const __be32 *)rdev->ce_fw->data;
  1836. WREG32(CP_CE_UCODE_ADDR, 0);
  1837. for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
  1838. WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
  1839. WREG32(CP_CE_UCODE_ADDR, 0);
  1840. /* ME */
  1841. fw_data = (const __be32 *)rdev->me_fw->data;
  1842. WREG32(CP_ME_RAM_WADDR, 0);
  1843. for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
  1844. WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
  1845. WREG32(CP_ME_RAM_WADDR, 0);
  1846. WREG32(CP_PFP_UCODE_ADDR, 0);
  1847. WREG32(CP_CE_UCODE_ADDR, 0);
  1848. WREG32(CP_ME_RAM_WADDR, 0);
  1849. WREG32(CP_ME_RAM_RADDR, 0);
  1850. return 0;
  1851. }
  1852. /**
  1853. * cik_cp_gfx_start - start the gfx ring
  1854. *
  1855. * @rdev: radeon_device pointer
  1856. *
  1857. * Enables the ring and loads the clear state context and other
  1858. * packets required to init the ring.
  1859. * Returns 0 for success, error for failure.
  1860. */
  1861. static int cik_cp_gfx_start(struct radeon_device *rdev)
  1862. {
  1863. struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
  1864. int r, i;
  1865. /* init the CP */
  1866. WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
  1867. WREG32(CP_ENDIAN_SWAP, 0);
  1868. WREG32(CP_DEVICE_ID, 1);
  1869. cik_cp_gfx_enable(rdev, true);
  1870. r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
  1871. if (r) {
  1872. DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
  1873. return r;
  1874. }
  1875. /* init the CE partitions. CE only used for gfx on CIK */
  1876. radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
  1877. radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
  1878. radeon_ring_write(ring, 0xc000);
  1879. radeon_ring_write(ring, 0xc000);
  1880. /* setup clear context state */
  1881. radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
  1882. radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
  1883. radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
  1884. radeon_ring_write(ring, 0x80000000);
  1885. radeon_ring_write(ring, 0x80000000);
  1886. for (i = 0; i < cik_default_size; i++)
  1887. radeon_ring_write(ring, cik_default_state[i]);
  1888. radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
  1889. radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
  1890. /* set clear context state */
  1891. radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
  1892. radeon_ring_write(ring, 0);
  1893. radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
  1894. radeon_ring_write(ring, 0x00000316);
  1895. radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
  1896. radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
  1897. radeon_ring_unlock_commit(rdev, ring);
  1898. return 0;
  1899. }
  1900. /**
  1901. * cik_cp_gfx_fini - stop the gfx ring
  1902. *
  1903. * @rdev: radeon_device pointer
  1904. *
  1905. * Stop the gfx ring and tear down the driver ring
  1906. * info.
  1907. */
  1908. static void cik_cp_gfx_fini(struct radeon_device *rdev)
  1909. {
  1910. cik_cp_gfx_enable(rdev, false);
  1911. radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
  1912. }
  1913. /**
  1914. * cik_cp_gfx_resume - setup the gfx ring buffer registers
  1915. *
  1916. * @rdev: radeon_device pointer
  1917. *
  1918. * Program the location and size of the gfx ring buffer
  1919. * and test it to make sure it's working.
  1920. * Returns 0 for success, error for failure.
  1921. */
  1922. static int cik_cp_gfx_resume(struct radeon_device *rdev)
  1923. {
  1924. struct radeon_ring *ring;
  1925. u32 tmp;
  1926. u32 rb_bufsz;
  1927. u64 rb_addr;
  1928. int r;
  1929. WREG32(CP_SEM_WAIT_TIMER, 0x0);
  1930. WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
  1931. /* Set the write pointer delay */
  1932. WREG32(CP_RB_WPTR_DELAY, 0);
  1933. /* set the RB to use vmid 0 */
  1934. WREG32(CP_RB_VMID, 0);
  1935. WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
  1936. /* ring 0 - compute and gfx */
  1937. /* Set ring buffer size */
  1938. ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
  1939. rb_bufsz = drm_order(ring->ring_size / 8);
  1940. tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
  1941. #ifdef __BIG_ENDIAN
  1942. tmp |= BUF_SWAP_32BIT;
  1943. #endif
  1944. WREG32(CP_RB0_CNTL, tmp);
  1945. /* Initialize the ring buffer's read and write pointers */
  1946. WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
  1947. ring->wptr = 0;
  1948. WREG32(CP_RB0_WPTR, ring->wptr);
  1949. /* set the wb address wether it's enabled or not */
  1950. WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
  1951. WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
  1952. /* scratch register shadowing is no longer supported */
  1953. WREG32(SCRATCH_UMSK, 0);
  1954. if (!rdev->wb.enabled)
  1955. tmp |= RB_NO_UPDATE;
  1956. mdelay(1);
  1957. WREG32(CP_RB0_CNTL, tmp);
  1958. rb_addr = ring->gpu_addr >> 8;
  1959. WREG32(CP_RB0_BASE, rb_addr);
  1960. WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
  1961. ring->rptr = RREG32(CP_RB0_RPTR);
  1962. /* start the ring */
  1963. cik_cp_gfx_start(rdev);
  1964. rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
  1965. r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
  1966. if (r) {
  1967. rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
  1968. return r;
  1969. }
  1970. return 0;
  1971. }
  1972. /**
  1973. * cik_cp_compute_enable - enable/disable the compute CP MEs
  1974. *
  1975. * @rdev: radeon_device pointer
  1976. * @enable: enable or disable the MEs
  1977. *
  1978. * Halts or unhalts the compute MEs.
  1979. */
  1980. static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
  1981. {
  1982. if (enable)
  1983. WREG32(CP_MEC_CNTL, 0);
  1984. else
  1985. WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
  1986. udelay(50);
  1987. }
  1988. /**
  1989. * cik_cp_compute_load_microcode - load the compute CP ME ucode
  1990. *
  1991. * @rdev: radeon_device pointer
  1992. *
  1993. * Loads the compute MEC1&2 ucode.
  1994. * Returns 0 for success, -EINVAL if the ucode is not available.
  1995. */
  1996. static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
  1997. {
  1998. const __be32 *fw_data;
  1999. int i;
  2000. if (!rdev->mec_fw)
  2001. return -EINVAL;
  2002. cik_cp_compute_enable(rdev, false);
  2003. /* MEC1 */
  2004. fw_data = (const __be32 *)rdev->mec_fw->data;
  2005. WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
  2006. for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
  2007. WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
  2008. WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
  2009. if (rdev->family == CHIP_KAVERI) {
  2010. /* MEC2 */
  2011. fw_data = (const __be32 *)rdev->mec_fw->data;
  2012. WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
  2013. for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
  2014. WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
  2015. WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
  2016. }
  2017. return 0;
  2018. }
  2019. /**
  2020. * cik_cp_compute_start - start the compute queues
  2021. *
  2022. * @rdev: radeon_device pointer
  2023. *
  2024. * Enable the compute queues.
  2025. * Returns 0 for success, error for failure.
  2026. */
  2027. static int cik_cp_compute_start(struct radeon_device *rdev)
  2028. {
  2029. //todo
  2030. return 0;
  2031. }
  2032. /**
  2033. * cik_cp_compute_fini - stop the compute queues
  2034. *
  2035. * @rdev: radeon_device pointer
  2036. *
  2037. * Stop the compute queues and tear down the driver queue
  2038. * info.
  2039. */
  2040. static void cik_cp_compute_fini(struct radeon_device *rdev)
  2041. {
  2042. cik_cp_compute_enable(rdev, false);
  2043. //todo
  2044. }
  2045. /**
  2046. * cik_cp_compute_resume - setup the compute queue registers
  2047. *
  2048. * @rdev: radeon_device pointer
  2049. *
  2050. * Program the compute queues and test them to make sure they
  2051. * are working.
  2052. * Returns 0 for success, error for failure.
  2053. */
  2054. static int cik_cp_compute_resume(struct radeon_device *rdev)
  2055. {
  2056. int r;
  2057. //todo
  2058. r = cik_cp_compute_start(rdev);
  2059. if (r)
  2060. return r;
  2061. return 0;
  2062. }
  2063. /* XXX temporary wrappers to handle both compute and gfx */
  2064. /* XXX */
  2065. static void cik_cp_enable(struct radeon_device *rdev, bool enable)
  2066. {
  2067. cik_cp_gfx_enable(rdev, enable);
  2068. cik_cp_compute_enable(rdev, enable);
  2069. }
  2070. /* XXX */
  2071. static int cik_cp_load_microcode(struct radeon_device *rdev)
  2072. {
  2073. int r;
  2074. r = cik_cp_gfx_load_microcode(rdev);
  2075. if (r)
  2076. return r;
  2077. r = cik_cp_compute_load_microcode(rdev);
  2078. if (r)
  2079. return r;
  2080. return 0;
  2081. }
  2082. /* XXX */
  2083. static void cik_cp_fini(struct radeon_device *rdev)
  2084. {
  2085. cik_cp_gfx_fini(rdev);
  2086. cik_cp_compute_fini(rdev);
  2087. }
  2088. /* XXX */
  2089. static int cik_cp_resume(struct radeon_device *rdev)
  2090. {
  2091. int r;
  2092. /* Reset all cp blocks */
  2093. WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
  2094. RREG32(GRBM_SOFT_RESET);
  2095. mdelay(15);
  2096. WREG32(GRBM_SOFT_RESET, 0);
  2097. RREG32(GRBM_SOFT_RESET);
  2098. r = cik_cp_load_microcode(rdev);
  2099. if (r)
  2100. return r;
  2101. r = cik_cp_gfx_resume(rdev);
  2102. if (r)
  2103. return r;
  2104. r = cik_cp_compute_resume(rdev);
  2105. if (r)
  2106. return r;
  2107. return 0;
  2108. }
  2109. /*
  2110. * sDMA - System DMA
  2111. * Starting with CIK, the GPU has new asynchronous
  2112. * DMA engines. These engines are used for compute
  2113. * and gfx. There are two DMA engines (SDMA0, SDMA1)
  2114. * and each one supports 1 ring buffer used for gfx
  2115. * and 2 queues used for compute.
  2116. *
  2117. * The programming model is very similar to the CP
  2118. * (ring buffer, IBs, etc.), but sDMA has it's own
  2119. * packet format that is different from the PM4 format
  2120. * used by the CP. sDMA supports copying data, writing
  2121. * embedded data, solid fills, and a number of other
  2122. * things. It also has support for tiling/detiling of
  2123. * buffers.
  2124. */
  2125. /**
  2126. * cik_sdma_ring_ib_execute - Schedule an IB on the DMA engine
  2127. *
  2128. * @rdev: radeon_device pointer
  2129. * @ib: IB object to schedule
  2130. *
  2131. * Schedule an IB in the DMA ring (CIK).
  2132. */
  2133. void cik_sdma_ring_ib_execute(struct radeon_device *rdev,
  2134. struct radeon_ib *ib)
  2135. {
  2136. struct radeon_ring *ring = &rdev->ring[ib->ring];
  2137. u32 extra_bits = (ib->vm ? ib->vm->id : 0) & 0xf;
  2138. if (rdev->wb.enabled) {
  2139. u32 next_rptr = ring->wptr + 5;
  2140. while ((next_rptr & 7) != 4)
  2141. next_rptr++;
  2142. next_rptr += 4;
  2143. radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0));
  2144. radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
  2145. radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
  2146. radeon_ring_write(ring, 1); /* number of DWs to follow */
  2147. radeon_ring_write(ring, next_rptr);
  2148. }
  2149. /* IB packet must end on a 8 DW boundary */
  2150. while ((ring->wptr & 7) != 4)
  2151. radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
  2152. radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_INDIRECT_BUFFER, 0, extra_bits));
  2153. radeon_ring_write(ring, ib->gpu_addr & 0xffffffe0); /* base must be 32 byte aligned */
  2154. radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xffffffff);
  2155. radeon_ring_write(ring, ib->length_dw);
  2156. }
  2157. /**
  2158. * cik_sdma_fence_ring_emit - emit a fence on the DMA ring
  2159. *
  2160. * @rdev: radeon_device pointer
  2161. * @fence: radeon fence object
  2162. *
  2163. * Add a DMA fence packet to the ring to write
  2164. * the fence seq number and DMA trap packet to generate
  2165. * an interrupt if needed (CIK).
  2166. */
  2167. void cik_sdma_fence_ring_emit(struct radeon_device *rdev,
  2168. struct radeon_fence *fence)
  2169. {
  2170. struct radeon_ring *ring = &rdev->ring[fence->ring];
  2171. u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
  2172. u32 extra_bits = (SDMA_POLL_REG_MEM_EXTRA_OP(1) |
  2173. SDMA_POLL_REG_MEM_EXTRA_FUNC(3)); /* == */
  2174. u32 ref_and_mask;
  2175. if (fence->ring == R600_RING_TYPE_DMA_INDEX)
  2176. ref_and_mask = SDMA0;
  2177. else
  2178. ref_and_mask = SDMA1;
  2179. /* write the fence */
  2180. radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_FENCE, 0, 0));
  2181. radeon_ring_write(ring, addr & 0xffffffff);
  2182. radeon_ring_write(ring, upper_32_bits(addr) & 0xffffffff);
  2183. radeon_ring_write(ring, fence->seq);
  2184. /* generate an interrupt */
  2185. radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_TRAP, 0, 0));
  2186. /* flush HDP */
  2187. radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_POLL_REG_MEM, 0, extra_bits));
  2188. radeon_ring_write(ring, GPU_HDP_FLUSH_DONE);
  2189. radeon_ring_write(ring, GPU_HDP_FLUSH_REQ);
  2190. radeon_ring_write(ring, ref_and_mask); /* REFERENCE */
  2191. radeon_ring_write(ring, ref_and_mask); /* MASK */
  2192. radeon_ring_write(ring, (4 << 16) | 10); /* RETRY_COUNT, POLL_INTERVAL */
  2193. }
  2194. /**
  2195. * cik_sdma_semaphore_ring_emit - emit a semaphore on the dma ring
  2196. *
  2197. * @rdev: radeon_device pointer
  2198. * @ring: radeon_ring structure holding ring information
  2199. * @semaphore: radeon semaphore object
  2200. * @emit_wait: wait or signal semaphore
  2201. *
  2202. * Add a DMA semaphore packet to the ring wait on or signal
  2203. * other rings (CIK).
  2204. */
  2205. void cik_sdma_semaphore_ring_emit(struct radeon_device *rdev,
  2206. struct radeon_ring *ring,
  2207. struct radeon_semaphore *semaphore,
  2208. bool emit_wait)
  2209. {
  2210. u64 addr = semaphore->gpu_addr;
  2211. u32 extra_bits = emit_wait ? 0 : SDMA_SEMAPHORE_EXTRA_S;
  2212. radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SEMAPHORE, 0, extra_bits));
  2213. radeon_ring_write(ring, addr & 0xfffffff8);
  2214. radeon_ring_write(ring, upper_32_bits(addr) & 0xffffffff);
  2215. }
  2216. /**
  2217. * cik_sdma_gfx_stop - stop the gfx async dma engines
  2218. *
  2219. * @rdev: radeon_device pointer
  2220. *
  2221. * Stop the gfx async dma ring buffers (CIK).
  2222. */
  2223. static void cik_sdma_gfx_stop(struct radeon_device *rdev)
  2224. {
  2225. u32 rb_cntl, reg_offset;
  2226. int i;
  2227. radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
  2228. for (i = 0; i < 2; i++) {
  2229. if (i == 0)
  2230. reg_offset = SDMA0_REGISTER_OFFSET;
  2231. else
  2232. reg_offset = SDMA1_REGISTER_OFFSET;
  2233. rb_cntl = RREG32(SDMA0_GFX_RB_CNTL + reg_offset);
  2234. rb_cntl &= ~SDMA_RB_ENABLE;
  2235. WREG32(SDMA0_GFX_RB_CNTL + reg_offset, rb_cntl);
  2236. WREG32(SDMA0_GFX_IB_CNTL + reg_offset, 0);
  2237. }
  2238. }
  2239. /**
  2240. * cik_sdma_rlc_stop - stop the compute async dma engines
  2241. *
  2242. * @rdev: radeon_device pointer
  2243. *
  2244. * Stop the compute async dma queues (CIK).
  2245. */
  2246. static void cik_sdma_rlc_stop(struct radeon_device *rdev)
  2247. {
  2248. /* XXX todo */
  2249. }
  2250. /**
  2251. * cik_sdma_enable - stop the async dma engines
  2252. *
  2253. * @rdev: radeon_device pointer
  2254. * @enable: enable/disable the DMA MEs.
  2255. *
  2256. * Halt or unhalt the async dma engines (CIK).
  2257. */
  2258. static void cik_sdma_enable(struct radeon_device *rdev, bool enable)
  2259. {
  2260. u32 me_cntl, reg_offset;
  2261. int i;
  2262. for (i = 0; i < 2; i++) {
  2263. if (i == 0)
  2264. reg_offset = SDMA0_REGISTER_OFFSET;
  2265. else
  2266. reg_offset = SDMA1_REGISTER_OFFSET;
  2267. me_cntl = RREG32(SDMA0_ME_CNTL + reg_offset);
  2268. if (enable)
  2269. me_cntl &= ~SDMA_HALT;
  2270. else
  2271. me_cntl |= SDMA_HALT;
  2272. WREG32(SDMA0_ME_CNTL + reg_offset, me_cntl);
  2273. }
  2274. }
  2275. /**
  2276. * cik_sdma_gfx_resume - setup and start the async dma engines
  2277. *
  2278. * @rdev: radeon_device pointer
  2279. *
  2280. * Set up the gfx DMA ring buffers and enable them (CIK).
  2281. * Returns 0 for success, error for failure.
  2282. */
  2283. static int cik_sdma_gfx_resume(struct radeon_device *rdev)
  2284. {
  2285. struct radeon_ring *ring;
  2286. u32 rb_cntl, ib_cntl;
  2287. u32 rb_bufsz;
  2288. u32 reg_offset, wb_offset;
  2289. int i, r;
  2290. for (i = 0; i < 2; i++) {
  2291. if (i == 0) {
  2292. ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
  2293. reg_offset = SDMA0_REGISTER_OFFSET;
  2294. wb_offset = R600_WB_DMA_RPTR_OFFSET;
  2295. } else {
  2296. ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
  2297. reg_offset = SDMA1_REGISTER_OFFSET;
  2298. wb_offset = CAYMAN_WB_DMA1_RPTR_OFFSET;
  2299. }
  2300. WREG32(SDMA0_SEM_INCOMPLETE_TIMER_CNTL + reg_offset, 0);
  2301. WREG32(SDMA0_SEM_WAIT_FAIL_TIMER_CNTL + reg_offset, 0);
  2302. /* Set ring buffer size in dwords */
  2303. rb_bufsz = drm_order(ring->ring_size / 4);
  2304. rb_cntl = rb_bufsz << 1;
  2305. #ifdef __BIG_ENDIAN
  2306. rb_cntl |= SDMA_RB_SWAP_ENABLE | SDMA_RPTR_WRITEBACK_SWAP_ENABLE;
  2307. #endif
  2308. WREG32(SDMA0_GFX_RB_CNTL + reg_offset, rb_cntl);
  2309. /* Initialize the ring buffer's read and write pointers */
  2310. WREG32(SDMA0_GFX_RB_RPTR + reg_offset, 0);
  2311. WREG32(SDMA0_GFX_RB_WPTR + reg_offset, 0);
  2312. /* set the wb address whether it's enabled or not */
  2313. WREG32(SDMA0_GFX_RB_RPTR_ADDR_HI + reg_offset,
  2314. upper_32_bits(rdev->wb.gpu_addr + wb_offset) & 0xFFFFFFFF);
  2315. WREG32(SDMA0_GFX_RB_RPTR_ADDR_LO + reg_offset,
  2316. ((rdev->wb.gpu_addr + wb_offset) & 0xFFFFFFFC));
  2317. if (rdev->wb.enabled)
  2318. rb_cntl |= SDMA_RPTR_WRITEBACK_ENABLE;
  2319. WREG32(SDMA0_GFX_RB_BASE + reg_offset, ring->gpu_addr >> 8);
  2320. WREG32(SDMA0_GFX_RB_BASE_HI + reg_offset, ring->gpu_addr >> 40);
  2321. ring->wptr = 0;
  2322. WREG32(SDMA0_GFX_RB_WPTR + reg_offset, ring->wptr << 2);
  2323. ring->rptr = RREG32(SDMA0_GFX_RB_RPTR + reg_offset) >> 2;
  2324. /* enable DMA RB */
  2325. WREG32(SDMA0_GFX_RB_CNTL + reg_offset, rb_cntl | SDMA_RB_ENABLE);
  2326. ib_cntl = SDMA_IB_ENABLE;
  2327. #ifdef __BIG_ENDIAN
  2328. ib_cntl |= SDMA_IB_SWAP_ENABLE;
  2329. #endif
  2330. /* enable DMA IBs */
  2331. WREG32(SDMA0_GFX_IB_CNTL + reg_offset, ib_cntl);
  2332. ring->ready = true;
  2333. r = radeon_ring_test(rdev, ring->idx, ring);
  2334. if (r) {
  2335. ring->ready = false;
  2336. return r;
  2337. }
  2338. }
  2339. radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
  2340. return 0;
  2341. }
  2342. /**
  2343. * cik_sdma_rlc_resume - setup and start the async dma engines
  2344. *
  2345. * @rdev: radeon_device pointer
  2346. *
  2347. * Set up the compute DMA queues and enable them (CIK).
  2348. * Returns 0 for success, error for failure.
  2349. */
  2350. static int cik_sdma_rlc_resume(struct radeon_device *rdev)
  2351. {
  2352. /* XXX todo */
  2353. return 0;
  2354. }
  2355. /**
  2356. * cik_sdma_load_microcode - load the sDMA ME ucode
  2357. *
  2358. * @rdev: radeon_device pointer
  2359. *
  2360. * Loads the sDMA0/1 ucode.
  2361. * Returns 0 for success, -EINVAL if the ucode is not available.
  2362. */
  2363. static int cik_sdma_load_microcode(struct radeon_device *rdev)
  2364. {
  2365. const __be32 *fw_data;
  2366. int i;
  2367. if (!rdev->sdma_fw)
  2368. return -EINVAL;
  2369. /* stop the gfx rings and rlc compute queues */
  2370. cik_sdma_gfx_stop(rdev);
  2371. cik_sdma_rlc_stop(rdev);
  2372. /* halt the MEs */
  2373. cik_sdma_enable(rdev, false);
  2374. /* sdma0 */
  2375. fw_data = (const __be32 *)rdev->sdma_fw->data;
  2376. WREG32(SDMA0_UCODE_ADDR + SDMA0_REGISTER_OFFSET, 0);
  2377. for (i = 0; i < CIK_SDMA_UCODE_SIZE; i++)
  2378. WREG32(SDMA0_UCODE_DATA + SDMA0_REGISTER_OFFSET, be32_to_cpup(fw_data++));
  2379. WREG32(SDMA0_UCODE_DATA + SDMA0_REGISTER_OFFSET, CIK_SDMA_UCODE_VERSION);
  2380. /* sdma1 */
  2381. fw_data = (const __be32 *)rdev->sdma_fw->data;
  2382. WREG32(SDMA0_UCODE_ADDR + SDMA1_REGISTER_OFFSET, 0);
  2383. for (i = 0; i < CIK_SDMA_UCODE_SIZE; i++)
  2384. WREG32(SDMA0_UCODE_DATA + SDMA1_REGISTER_OFFSET, be32_to_cpup(fw_data++));
  2385. WREG32(SDMA0_UCODE_DATA + SDMA1_REGISTER_OFFSET, CIK_SDMA_UCODE_VERSION);
  2386. WREG32(SDMA0_UCODE_ADDR + SDMA0_REGISTER_OFFSET, 0);
  2387. WREG32(SDMA0_UCODE_ADDR + SDMA1_REGISTER_OFFSET, 0);
  2388. return 0;
  2389. }
  2390. /**
  2391. * cik_sdma_resume - setup and start the async dma engines
  2392. *
  2393. * @rdev: radeon_device pointer
  2394. *
  2395. * Set up the DMA engines and enable them (CIK).
  2396. * Returns 0 for success, error for failure.
  2397. */
  2398. static int cik_sdma_resume(struct radeon_device *rdev)
  2399. {
  2400. int r;
  2401. /* Reset dma */
  2402. WREG32(SRBM_SOFT_RESET, SOFT_RESET_SDMA | SOFT_RESET_SDMA1);
  2403. RREG32(SRBM_SOFT_RESET);
  2404. udelay(50);
  2405. WREG32(SRBM_SOFT_RESET, 0);
  2406. RREG32(SRBM_SOFT_RESET);
  2407. r = cik_sdma_load_microcode(rdev);
  2408. if (r)
  2409. return r;
  2410. /* unhalt the MEs */
  2411. cik_sdma_enable(rdev, true);
  2412. /* start the gfx rings and rlc compute queues */
  2413. r = cik_sdma_gfx_resume(rdev);
  2414. if (r)
  2415. return r;
  2416. r = cik_sdma_rlc_resume(rdev);
  2417. if (r)
  2418. return r;
  2419. return 0;
  2420. }
  2421. /**
  2422. * cik_sdma_fini - tear down the async dma engines
  2423. *
  2424. * @rdev: radeon_device pointer
  2425. *
  2426. * Stop the async dma engines and free the rings (CIK).
  2427. */
  2428. static void cik_sdma_fini(struct radeon_device *rdev)
  2429. {
  2430. /* stop the gfx rings and rlc compute queues */
  2431. cik_sdma_gfx_stop(rdev);
  2432. cik_sdma_rlc_stop(rdev);
  2433. /* halt the MEs */
  2434. cik_sdma_enable(rdev, false);
  2435. radeon_ring_fini(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX]);
  2436. radeon_ring_fini(rdev, &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]);
  2437. /* XXX - compute dma queue tear down */
  2438. }
  2439. /**
  2440. * cik_copy_dma - copy pages using the DMA engine
  2441. *
  2442. * @rdev: radeon_device pointer
  2443. * @src_offset: src GPU address
  2444. * @dst_offset: dst GPU address
  2445. * @num_gpu_pages: number of GPU pages to xfer
  2446. * @fence: radeon fence object
  2447. *
  2448. * Copy GPU paging using the DMA engine (CIK).
  2449. * Used by the radeon ttm implementation to move pages if
  2450. * registered as the asic copy callback.
  2451. */
  2452. int cik_copy_dma(struct radeon_device *rdev,
  2453. uint64_t src_offset, uint64_t dst_offset,
  2454. unsigned num_gpu_pages,
  2455. struct radeon_fence **fence)
  2456. {
  2457. struct radeon_semaphore *sem = NULL;
  2458. int ring_index = rdev->asic->copy.dma_ring_index;
  2459. struct radeon_ring *ring = &rdev->ring[ring_index];
  2460. u32 size_in_bytes, cur_size_in_bytes;
  2461. int i, num_loops;
  2462. int r = 0;
  2463. r = radeon_semaphore_create(rdev, &sem);
  2464. if (r) {
  2465. DRM_ERROR("radeon: moving bo (%d).\n", r);
  2466. return r;
  2467. }
  2468. size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
  2469. num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
  2470. r = radeon_ring_lock(rdev, ring, num_loops * 7 + 14);
  2471. if (r) {
  2472. DRM_ERROR("radeon: moving bo (%d).\n", r);
  2473. radeon_semaphore_free(rdev, &sem, NULL);
  2474. return r;
  2475. }
  2476. if (radeon_fence_need_sync(*fence, ring->idx)) {
  2477. radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring,
  2478. ring->idx);
  2479. radeon_fence_note_sync(*fence, ring->idx);
  2480. } else {
  2481. radeon_semaphore_free(rdev, &sem, NULL);
  2482. }
  2483. for (i = 0; i < num_loops; i++) {
  2484. cur_size_in_bytes = size_in_bytes;
  2485. if (cur_size_in_bytes > 0x1fffff)
  2486. cur_size_in_bytes = 0x1fffff;
  2487. size_in_bytes -= cur_size_in_bytes;
  2488. radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_COPY, SDMA_COPY_SUB_OPCODE_LINEAR, 0));
  2489. radeon_ring_write(ring, cur_size_in_bytes);
  2490. radeon_ring_write(ring, 0); /* src/dst endian swap */
  2491. radeon_ring_write(ring, src_offset & 0xffffffff);
  2492. radeon_ring_write(ring, upper_32_bits(src_offset) & 0xffffffff);
  2493. radeon_ring_write(ring, dst_offset & 0xfffffffc);
  2494. radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xffffffff);
  2495. src_offset += cur_size_in_bytes;
  2496. dst_offset += cur_size_in_bytes;
  2497. }
  2498. r = radeon_fence_emit(rdev, fence, ring->idx);
  2499. if (r) {
  2500. radeon_ring_unlock_undo(rdev, ring);
  2501. return r;
  2502. }
  2503. radeon_ring_unlock_commit(rdev, ring);
  2504. radeon_semaphore_free(rdev, &sem, *fence);
  2505. return r;
  2506. }
  2507. /**
  2508. * cik_sdma_ring_test - simple async dma engine test
  2509. *
  2510. * @rdev: radeon_device pointer
  2511. * @ring: radeon_ring structure holding ring information
  2512. *
  2513. * Test the DMA engine by writing using it to write an
  2514. * value to memory. (CIK).
  2515. * Returns 0 for success, error for failure.
  2516. */
  2517. int cik_sdma_ring_test(struct radeon_device *rdev,
  2518. struct radeon_ring *ring)
  2519. {
  2520. unsigned i;
  2521. int r;
  2522. void __iomem *ptr = (void *)rdev->vram_scratch.ptr;
  2523. u32 tmp;
  2524. if (!ptr) {
  2525. DRM_ERROR("invalid vram scratch pointer\n");
  2526. return -EINVAL;
  2527. }
  2528. tmp = 0xCAFEDEAD;
  2529. writel(tmp, ptr);
  2530. r = radeon_ring_lock(rdev, ring, 4);
  2531. if (r) {
  2532. DRM_ERROR("radeon: dma failed to lock ring %d (%d).\n", ring->idx, r);
  2533. return r;
  2534. }
  2535. radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0));
  2536. radeon_ring_write(ring, rdev->vram_scratch.gpu_addr & 0xfffffffc);
  2537. radeon_ring_write(ring, upper_32_bits(rdev->vram_scratch.gpu_addr) & 0xffffffff);
  2538. radeon_ring_write(ring, 1); /* number of DWs to follow */
  2539. radeon_ring_write(ring, 0xDEADBEEF);
  2540. radeon_ring_unlock_commit(rdev, ring);
  2541. for (i = 0; i < rdev->usec_timeout; i++) {
  2542. tmp = readl(ptr);
  2543. if (tmp == 0xDEADBEEF)
  2544. break;
  2545. DRM_UDELAY(1);
  2546. }
  2547. if (i < rdev->usec_timeout) {
  2548. DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
  2549. } else {
  2550. DRM_ERROR("radeon: ring %d test failed (0x%08X)\n",
  2551. ring->idx, tmp);
  2552. r = -EINVAL;
  2553. }
  2554. return r;
  2555. }
  2556. /**
  2557. * cik_sdma_ib_test - test an IB on the DMA engine
  2558. *
  2559. * @rdev: radeon_device pointer
  2560. * @ring: radeon_ring structure holding ring information
  2561. *
  2562. * Test a simple IB in the DMA ring (CIK).
  2563. * Returns 0 on success, error on failure.
  2564. */
  2565. int cik_sdma_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
  2566. {
  2567. struct radeon_ib ib;
  2568. unsigned i;
  2569. int r;
  2570. void __iomem *ptr = (void *)rdev->vram_scratch.ptr;
  2571. u32 tmp = 0;
  2572. if (!ptr) {
  2573. DRM_ERROR("invalid vram scratch pointer\n");
  2574. return -EINVAL;
  2575. }
  2576. tmp = 0xCAFEDEAD;
  2577. writel(tmp, ptr);
  2578. r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
  2579. if (r) {
  2580. DRM_ERROR("radeon: failed to get ib (%d).\n", r);
  2581. return r;
  2582. }
  2583. ib.ptr[0] = SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0);
  2584. ib.ptr[1] = rdev->vram_scratch.gpu_addr & 0xfffffffc;
  2585. ib.ptr[2] = upper_32_bits(rdev->vram_scratch.gpu_addr) & 0xffffffff;
  2586. ib.ptr[3] = 1;
  2587. ib.ptr[4] = 0xDEADBEEF;
  2588. ib.length_dw = 5;
  2589. r = radeon_ib_schedule(rdev, &ib, NULL);
  2590. if (r) {
  2591. radeon_ib_free(rdev, &ib);
  2592. DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
  2593. return r;
  2594. }
  2595. r = radeon_fence_wait(ib.fence, false);
  2596. if (r) {
  2597. DRM_ERROR("radeon: fence wait failed (%d).\n", r);
  2598. return r;
  2599. }
  2600. for (i = 0; i < rdev->usec_timeout; i++) {
  2601. tmp = readl(ptr);
  2602. if (tmp == 0xDEADBEEF)
  2603. break;
  2604. DRM_UDELAY(1);
  2605. }
  2606. if (i < rdev->usec_timeout) {
  2607. DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
  2608. } else {
  2609. DRM_ERROR("radeon: ib test failed (0x%08X)\n", tmp);
  2610. r = -EINVAL;
  2611. }
  2612. radeon_ib_free(rdev, &ib);
  2613. return r;
  2614. }
  2615. static void cik_print_gpu_status_regs(struct radeon_device *rdev)
  2616. {
  2617. dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
  2618. RREG32(GRBM_STATUS));
  2619. dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n",
  2620. RREG32(GRBM_STATUS2));
  2621. dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
  2622. RREG32(GRBM_STATUS_SE0));
  2623. dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
  2624. RREG32(GRBM_STATUS_SE1));
  2625. dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n",
  2626. RREG32(GRBM_STATUS_SE2));
  2627. dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n",
  2628. RREG32(GRBM_STATUS_SE3));
  2629. dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
  2630. RREG32(SRBM_STATUS));
  2631. dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n",
  2632. RREG32(SRBM_STATUS2));
  2633. dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n",
  2634. RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
  2635. dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n",
  2636. RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
  2637. }
  2638. /**
  2639. * cik_gpu_check_soft_reset - check which blocks are busy
  2640. *
  2641. * @rdev: radeon_device pointer
  2642. *
  2643. * Check which blocks are busy and return the relevant reset
  2644. * mask to be used by cik_gpu_soft_reset().
  2645. * Returns a mask of the blocks to be reset.
  2646. */
  2647. static u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
  2648. {
  2649. u32 reset_mask = 0;
  2650. u32 tmp;
  2651. /* GRBM_STATUS */
  2652. tmp = RREG32(GRBM_STATUS);
  2653. if (tmp & (PA_BUSY | SC_BUSY |
  2654. BCI_BUSY | SX_BUSY |
  2655. TA_BUSY | VGT_BUSY |
  2656. DB_BUSY | CB_BUSY |
  2657. GDS_BUSY | SPI_BUSY |
  2658. IA_BUSY | IA_BUSY_NO_DMA))
  2659. reset_mask |= RADEON_RESET_GFX;
  2660. if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
  2661. reset_mask |= RADEON_RESET_CP;
  2662. /* GRBM_STATUS2 */
  2663. tmp = RREG32(GRBM_STATUS2);
  2664. if (tmp & RLC_BUSY)
  2665. reset_mask |= RADEON_RESET_RLC;
  2666. /* SDMA0_STATUS_REG */
  2667. tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
  2668. if (!(tmp & SDMA_IDLE))
  2669. reset_mask |= RADEON_RESET_DMA;
  2670. /* SDMA1_STATUS_REG */
  2671. tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
  2672. if (!(tmp & SDMA_IDLE))
  2673. reset_mask |= RADEON_RESET_DMA1;
  2674. /* SRBM_STATUS2 */
  2675. tmp = RREG32(SRBM_STATUS2);
  2676. if (tmp & SDMA_BUSY)
  2677. reset_mask |= RADEON_RESET_DMA;
  2678. if (tmp & SDMA1_BUSY)
  2679. reset_mask |= RADEON_RESET_DMA1;
  2680. /* SRBM_STATUS */
  2681. tmp = RREG32(SRBM_STATUS);
  2682. if (tmp & IH_BUSY)
  2683. reset_mask |= RADEON_RESET_IH;
  2684. if (tmp & SEM_BUSY)
  2685. reset_mask |= RADEON_RESET_SEM;
  2686. if (tmp & GRBM_RQ_PENDING)
  2687. reset_mask |= RADEON_RESET_GRBM;
  2688. if (tmp & VMC_BUSY)
  2689. reset_mask |= RADEON_RESET_VMC;
  2690. if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
  2691. MCC_BUSY | MCD_BUSY))
  2692. reset_mask |= RADEON_RESET_MC;
  2693. if (evergreen_is_display_hung(rdev))
  2694. reset_mask |= RADEON_RESET_DISPLAY;
  2695. /* Skip MC reset as it's mostly likely not hung, just busy */
  2696. if (reset_mask & RADEON_RESET_MC) {
  2697. DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
  2698. reset_mask &= ~RADEON_RESET_MC;
  2699. }
  2700. return reset_mask;
  2701. }
  2702. /**
  2703. * cik_gpu_soft_reset - soft reset GPU
  2704. *
  2705. * @rdev: radeon_device pointer
  2706. * @reset_mask: mask of which blocks to reset
  2707. *
  2708. * Soft reset the blocks specified in @reset_mask.
  2709. */
  2710. static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
  2711. {
  2712. struct evergreen_mc_save save;
  2713. u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
  2714. u32 tmp;
  2715. if (reset_mask == 0)
  2716. return;
  2717. dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
  2718. cik_print_gpu_status_regs(rdev);
  2719. dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
  2720. RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
  2721. dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
  2722. RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
  2723. /* stop the rlc */
  2724. cik_rlc_stop(rdev);
  2725. /* Disable GFX parsing/prefetching */
  2726. WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
  2727. /* Disable MEC parsing/prefetching */
  2728. WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
  2729. if (reset_mask & RADEON_RESET_DMA) {
  2730. /* sdma0 */
  2731. tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
  2732. tmp |= SDMA_HALT;
  2733. WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
  2734. }
  2735. if (reset_mask & RADEON_RESET_DMA1) {
  2736. /* sdma1 */
  2737. tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
  2738. tmp |= SDMA_HALT;
  2739. WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
  2740. }
  2741. evergreen_mc_stop(rdev, &save);
  2742. if (evergreen_mc_wait_for_idle(rdev)) {
  2743. dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
  2744. }
  2745. if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
  2746. grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
  2747. if (reset_mask & RADEON_RESET_CP) {
  2748. grbm_soft_reset |= SOFT_RESET_CP;
  2749. srbm_soft_reset |= SOFT_RESET_GRBM;
  2750. }
  2751. if (reset_mask & RADEON_RESET_DMA)
  2752. srbm_soft_reset |= SOFT_RESET_SDMA;
  2753. if (reset_mask & RADEON_RESET_DMA1)
  2754. srbm_soft_reset |= SOFT_RESET_SDMA1;
  2755. if (reset_mask & RADEON_RESET_DISPLAY)
  2756. srbm_soft_reset |= SOFT_RESET_DC;
  2757. if (reset_mask & RADEON_RESET_RLC)
  2758. grbm_soft_reset |= SOFT_RESET_RLC;
  2759. if (reset_mask & RADEON_RESET_SEM)
  2760. srbm_soft_reset |= SOFT_RESET_SEM;
  2761. if (reset_mask & RADEON_RESET_IH)
  2762. srbm_soft_reset |= SOFT_RESET_IH;
  2763. if (reset_mask & RADEON_RESET_GRBM)
  2764. srbm_soft_reset |= SOFT_RESET_GRBM;
  2765. if (reset_mask & RADEON_RESET_VMC)
  2766. srbm_soft_reset |= SOFT_RESET_VMC;
  2767. if (!(rdev->flags & RADEON_IS_IGP)) {
  2768. if (reset_mask & RADEON_RESET_MC)
  2769. srbm_soft_reset |= SOFT_RESET_MC;
  2770. }
  2771. if (grbm_soft_reset) {
  2772. tmp = RREG32(GRBM_SOFT_RESET);
  2773. tmp |= grbm_soft_reset;
  2774. dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
  2775. WREG32(GRBM_SOFT_RESET, tmp);
  2776. tmp = RREG32(GRBM_SOFT_RESET);
  2777. udelay(50);
  2778. tmp &= ~grbm_soft_reset;
  2779. WREG32(GRBM_SOFT_RESET, tmp);
  2780. tmp = RREG32(GRBM_SOFT_RESET);
  2781. }
  2782. if (srbm_soft_reset) {
  2783. tmp = RREG32(SRBM_SOFT_RESET);
  2784. tmp |= srbm_soft_reset;
  2785. dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
  2786. WREG32(SRBM_SOFT_RESET, tmp);
  2787. tmp = RREG32(SRBM_SOFT_RESET);
  2788. udelay(50);
  2789. tmp &= ~srbm_soft_reset;
  2790. WREG32(SRBM_SOFT_RESET, tmp);
  2791. tmp = RREG32(SRBM_SOFT_RESET);
  2792. }
  2793. /* Wait a little for things to settle down */
  2794. udelay(50);
  2795. evergreen_mc_resume(rdev, &save);
  2796. udelay(50);
  2797. cik_print_gpu_status_regs(rdev);
  2798. }
  2799. /**
  2800. * cik_asic_reset - soft reset GPU
  2801. *
  2802. * @rdev: radeon_device pointer
  2803. *
  2804. * Look up which blocks are hung and attempt
  2805. * to reset them.
  2806. * Returns 0 for success.
  2807. */
  2808. int cik_asic_reset(struct radeon_device *rdev)
  2809. {
  2810. u32 reset_mask;
  2811. reset_mask = cik_gpu_check_soft_reset(rdev);
  2812. if (reset_mask)
  2813. r600_set_bios_scratch_engine_hung(rdev, true);
  2814. cik_gpu_soft_reset(rdev, reset_mask);
  2815. reset_mask = cik_gpu_check_soft_reset(rdev);
  2816. if (!reset_mask)
  2817. r600_set_bios_scratch_engine_hung(rdev, false);
  2818. return 0;
  2819. }
  2820. /**
  2821. * cik_gfx_is_lockup - check if the 3D engine is locked up
  2822. *
  2823. * @rdev: radeon_device pointer
  2824. * @ring: radeon_ring structure holding ring information
  2825. *
  2826. * Check if the 3D engine is locked up (CIK).
  2827. * Returns true if the engine is locked, false if not.
  2828. */
  2829. bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
  2830. {
  2831. u32 reset_mask = cik_gpu_check_soft_reset(rdev);
  2832. if (!(reset_mask & (RADEON_RESET_GFX |
  2833. RADEON_RESET_COMPUTE |
  2834. RADEON_RESET_CP))) {
  2835. radeon_ring_lockup_update(ring);
  2836. return false;
  2837. }
  2838. /* force CP activities */
  2839. radeon_ring_force_activity(rdev, ring);
  2840. return radeon_ring_test_lockup(rdev, ring);
  2841. }
  2842. /**
  2843. * cik_sdma_is_lockup - Check if the DMA engine is locked up
  2844. *
  2845. * @rdev: radeon_device pointer
  2846. * @ring: radeon_ring structure holding ring information
  2847. *
  2848. * Check if the async DMA engine is locked up (CIK).
  2849. * Returns true if the engine appears to be locked up, false if not.
  2850. */
  2851. bool cik_sdma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
  2852. {
  2853. u32 reset_mask = cik_gpu_check_soft_reset(rdev);
  2854. u32 mask;
  2855. if (ring->idx == R600_RING_TYPE_DMA_INDEX)
  2856. mask = RADEON_RESET_DMA;
  2857. else
  2858. mask = RADEON_RESET_DMA1;
  2859. if (!(reset_mask & mask)) {
  2860. radeon_ring_lockup_update(ring);
  2861. return false;
  2862. }
  2863. /* force ring activities */
  2864. radeon_ring_force_activity(rdev, ring);
  2865. return radeon_ring_test_lockup(rdev, ring);
  2866. }
  2867. /* MC */
  2868. /**
  2869. * cik_mc_program - program the GPU memory controller
  2870. *
  2871. * @rdev: radeon_device pointer
  2872. *
  2873. * Set the location of vram, gart, and AGP in the GPU's
  2874. * physical address space (CIK).
  2875. */
  2876. static void cik_mc_program(struct radeon_device *rdev)
  2877. {
  2878. struct evergreen_mc_save save;
  2879. u32 tmp;
  2880. int i, j;
  2881. /* Initialize HDP */
  2882. for (i = 0, j = 0; i < 32; i++, j += 0x18) {
  2883. WREG32((0x2c14 + j), 0x00000000);
  2884. WREG32((0x2c18 + j), 0x00000000);
  2885. WREG32((0x2c1c + j), 0x00000000);
  2886. WREG32((0x2c20 + j), 0x00000000);
  2887. WREG32((0x2c24 + j), 0x00000000);
  2888. }
  2889. WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
  2890. evergreen_mc_stop(rdev, &save);
  2891. if (radeon_mc_wait_for_idle(rdev)) {
  2892. dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
  2893. }
  2894. /* Lockout access through VGA aperture*/
  2895. WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
  2896. /* Update configuration */
  2897. WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
  2898. rdev->mc.vram_start >> 12);
  2899. WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
  2900. rdev->mc.vram_end >> 12);
  2901. WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
  2902. rdev->vram_scratch.gpu_addr >> 12);
  2903. tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
  2904. tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
  2905. WREG32(MC_VM_FB_LOCATION, tmp);
  2906. /* XXX double check these! */
  2907. WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
  2908. WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
  2909. WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
  2910. WREG32(MC_VM_AGP_BASE, 0);
  2911. WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
  2912. WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
  2913. if (radeon_mc_wait_for_idle(rdev)) {
  2914. dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
  2915. }
  2916. evergreen_mc_resume(rdev, &save);
  2917. /* we need to own VRAM, so turn off the VGA renderer here
  2918. * to stop it overwriting our objects */
  2919. rv515_vga_render_disable(rdev);
  2920. }
  2921. /**
  2922. * cik_mc_init - initialize the memory controller driver params
  2923. *
  2924. * @rdev: radeon_device pointer
  2925. *
  2926. * Look up the amount of vram, vram width, and decide how to place
  2927. * vram and gart within the GPU's physical address space (CIK).
  2928. * Returns 0 for success.
  2929. */
  2930. static int cik_mc_init(struct radeon_device *rdev)
  2931. {
  2932. u32 tmp;
  2933. int chansize, numchan;
  2934. /* Get VRAM informations */
  2935. rdev->mc.vram_is_ddr = true;
  2936. tmp = RREG32(MC_ARB_RAMCFG);
  2937. if (tmp & CHANSIZE_MASK) {
  2938. chansize = 64;
  2939. } else {
  2940. chansize = 32;
  2941. }
  2942. tmp = RREG32(MC_SHARED_CHMAP);
  2943. switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
  2944. case 0:
  2945. default:
  2946. numchan = 1;
  2947. break;
  2948. case 1:
  2949. numchan = 2;
  2950. break;
  2951. case 2:
  2952. numchan = 4;
  2953. break;
  2954. case 3:
  2955. numchan = 8;
  2956. break;
  2957. case 4:
  2958. numchan = 3;
  2959. break;
  2960. case 5:
  2961. numchan = 6;
  2962. break;
  2963. case 6:
  2964. numchan = 10;
  2965. break;
  2966. case 7:
  2967. numchan = 12;
  2968. break;
  2969. case 8:
  2970. numchan = 16;
  2971. break;
  2972. }
  2973. rdev->mc.vram_width = numchan * chansize;
  2974. /* Could aper size report 0 ? */
  2975. rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
  2976. rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
  2977. /* size in MB on si */
  2978. rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
  2979. rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
  2980. rdev->mc.visible_vram_size = rdev->mc.aper_size;
  2981. si_vram_gtt_location(rdev, &rdev->mc);
  2982. radeon_update_bandwidth_info(rdev);
  2983. return 0;
  2984. }
  2985. /*
  2986. * GART
  2987. * VMID 0 is the physical GPU addresses as used by the kernel.
  2988. * VMIDs 1-15 are used for userspace clients and are handled
  2989. * by the radeon vm/hsa code.
  2990. */
  2991. /**
  2992. * cik_pcie_gart_tlb_flush - gart tlb flush callback
  2993. *
  2994. * @rdev: radeon_device pointer
  2995. *
  2996. * Flush the TLB for the VMID 0 page table (CIK).
  2997. */
  2998. void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
  2999. {
  3000. /* flush hdp cache */
  3001. WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
  3002. /* bits 0-15 are the VM contexts0-15 */
  3003. WREG32(VM_INVALIDATE_REQUEST, 0x1);
  3004. }
  3005. /**
  3006. * cik_pcie_gart_enable - gart enable
  3007. *
  3008. * @rdev: radeon_device pointer
  3009. *
  3010. * This sets up the TLBs, programs the page tables for VMID0,
  3011. * sets up the hw for VMIDs 1-15 which are allocated on
  3012. * demand, and sets up the global locations for the LDS, GDS,
  3013. * and GPUVM for FSA64 clients (CIK).
  3014. * Returns 0 for success, errors for failure.
  3015. */
  3016. static int cik_pcie_gart_enable(struct radeon_device *rdev)
  3017. {
  3018. int r, i;
  3019. if (rdev->gart.robj == NULL) {
  3020. dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
  3021. return -EINVAL;
  3022. }
  3023. r = radeon_gart_table_vram_pin(rdev);
  3024. if (r)
  3025. return r;
  3026. radeon_gart_restore(rdev);
  3027. /* Setup TLB control */
  3028. WREG32(MC_VM_MX_L1_TLB_CNTL,
  3029. (0xA << 7) |
  3030. ENABLE_L1_TLB |
  3031. SYSTEM_ACCESS_MODE_NOT_IN_SYS |
  3032. ENABLE_ADVANCED_DRIVER_MODEL |
  3033. SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
  3034. /* Setup L2 cache */
  3035. WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
  3036. ENABLE_L2_FRAGMENT_PROCESSING |
  3037. ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
  3038. ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
  3039. EFFECTIVE_L2_QUEUE_SIZE(7) |
  3040. CONTEXT1_IDENTITY_ACCESS_MODE(1));
  3041. WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
  3042. WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
  3043. L2_CACHE_BIGK_FRAGMENT_SIZE(6));
  3044. /* setup context0 */
  3045. WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
  3046. WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
  3047. WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
  3048. WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
  3049. (u32)(rdev->dummy_page.addr >> 12));
  3050. WREG32(VM_CONTEXT0_CNTL2, 0);
  3051. WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
  3052. RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
  3053. WREG32(0x15D4, 0);
  3054. WREG32(0x15D8, 0);
  3055. WREG32(0x15DC, 0);
  3056. /* empty context1-15 */
  3057. /* FIXME start with 4G, once using 2 level pt switch to full
  3058. * vm size space
  3059. */
  3060. /* set vm size, must be a multiple of 4 */
  3061. WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
  3062. WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
  3063. for (i = 1; i < 16; i++) {
  3064. if (i < 8)
  3065. WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
  3066. rdev->gart.table_addr >> 12);
  3067. else
  3068. WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
  3069. rdev->gart.table_addr >> 12);
  3070. }
  3071. /* enable context1-15 */
  3072. WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
  3073. (u32)(rdev->dummy_page.addr >> 12));
  3074. WREG32(VM_CONTEXT1_CNTL2, 4);
  3075. WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
  3076. RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
  3077. RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
  3078. DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
  3079. DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
  3080. PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
  3081. PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
  3082. VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
  3083. VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
  3084. READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
  3085. READ_PROTECTION_FAULT_ENABLE_DEFAULT |
  3086. WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
  3087. WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
  3088. /* TC cache setup ??? */
  3089. WREG32(TC_CFG_L1_LOAD_POLICY0, 0);
  3090. WREG32(TC_CFG_L1_LOAD_POLICY1, 0);
  3091. WREG32(TC_CFG_L1_STORE_POLICY, 0);
  3092. WREG32(TC_CFG_L2_LOAD_POLICY0, 0);
  3093. WREG32(TC_CFG_L2_LOAD_POLICY1, 0);
  3094. WREG32(TC_CFG_L2_STORE_POLICY0, 0);
  3095. WREG32(TC_CFG_L2_STORE_POLICY1, 0);
  3096. WREG32(TC_CFG_L2_ATOMIC_POLICY, 0);
  3097. WREG32(TC_CFG_L1_VOLATILE, 0);
  3098. WREG32(TC_CFG_L2_VOLATILE, 0);
  3099. if (rdev->family == CHIP_KAVERI) {
  3100. u32 tmp = RREG32(CHUB_CONTROL);
  3101. tmp &= ~BYPASS_VM;
  3102. WREG32(CHUB_CONTROL, tmp);
  3103. }
  3104. /* XXX SH_MEM regs */
  3105. /* where to put LDS, scratch, GPUVM in FSA64 space */
  3106. for (i = 0; i < 16; i++) {
  3107. cik_srbm_select(rdev, 0, 0, 0, i);
  3108. /* CP and shaders */
  3109. WREG32(SH_MEM_CONFIG, 0);
  3110. WREG32(SH_MEM_APE1_BASE, 1);
  3111. WREG32(SH_MEM_APE1_LIMIT, 0);
  3112. WREG32(SH_MEM_BASES, 0);
  3113. /* SDMA GFX */
  3114. WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
  3115. WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
  3116. WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
  3117. WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
  3118. /* XXX SDMA RLC - todo */
  3119. }
  3120. cik_srbm_select(rdev, 0, 0, 0, 0);
  3121. cik_pcie_gart_tlb_flush(rdev);
  3122. DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
  3123. (unsigned)(rdev->mc.gtt_size >> 20),
  3124. (unsigned long long)rdev->gart.table_addr);
  3125. rdev->gart.ready = true;
  3126. return 0;
  3127. }
  3128. /**
  3129. * cik_pcie_gart_disable - gart disable
  3130. *
  3131. * @rdev: radeon_device pointer
  3132. *
  3133. * This disables all VM page table (CIK).
  3134. */
  3135. static void cik_pcie_gart_disable(struct radeon_device *rdev)
  3136. {
  3137. /* Disable all tables */
  3138. WREG32(VM_CONTEXT0_CNTL, 0);
  3139. WREG32(VM_CONTEXT1_CNTL, 0);
  3140. /* Setup TLB control */
  3141. WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
  3142. SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
  3143. /* Setup L2 cache */
  3144. WREG32(VM_L2_CNTL,
  3145. ENABLE_L2_FRAGMENT_PROCESSING |
  3146. ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
  3147. ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
  3148. EFFECTIVE_L2_QUEUE_SIZE(7) |
  3149. CONTEXT1_IDENTITY_ACCESS_MODE(1));
  3150. WREG32(VM_L2_CNTL2, 0);
  3151. WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
  3152. L2_CACHE_BIGK_FRAGMENT_SIZE(6));
  3153. radeon_gart_table_vram_unpin(rdev);
  3154. }
  3155. /**
  3156. * cik_pcie_gart_fini - vm fini callback
  3157. *
  3158. * @rdev: radeon_device pointer
  3159. *
  3160. * Tears down the driver GART/VM setup (CIK).
  3161. */
  3162. static void cik_pcie_gart_fini(struct radeon_device *rdev)
  3163. {
  3164. cik_pcie_gart_disable(rdev);
  3165. radeon_gart_table_vram_free(rdev);
  3166. radeon_gart_fini(rdev);
  3167. }
  3168. /* vm parser */
  3169. /**
  3170. * cik_ib_parse - vm ib_parse callback
  3171. *
  3172. * @rdev: radeon_device pointer
  3173. * @ib: indirect buffer pointer
  3174. *
  3175. * CIK uses hw IB checking so this is a nop (CIK).
  3176. */
  3177. int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
  3178. {
  3179. return 0;
  3180. }
  3181. /*
  3182. * vm
  3183. * VMID 0 is the physical GPU addresses as used by the kernel.
  3184. * VMIDs 1-15 are used for userspace clients and are handled
  3185. * by the radeon vm/hsa code.
  3186. */
  3187. /**
  3188. * cik_vm_init - cik vm init callback
  3189. *
  3190. * @rdev: radeon_device pointer
  3191. *
  3192. * Inits cik specific vm parameters (number of VMs, base of vram for
  3193. * VMIDs 1-15) (CIK).
  3194. * Returns 0 for success.
  3195. */
  3196. int cik_vm_init(struct radeon_device *rdev)
  3197. {
  3198. /* number of VMs */
  3199. rdev->vm_manager.nvm = 16;
  3200. /* base offset of vram pages */
  3201. if (rdev->flags & RADEON_IS_IGP) {
  3202. u64 tmp = RREG32(MC_VM_FB_OFFSET);
  3203. tmp <<= 22;
  3204. rdev->vm_manager.vram_base_offset = tmp;
  3205. } else
  3206. rdev->vm_manager.vram_base_offset = 0;
  3207. return 0;
  3208. }
  3209. /**
  3210. * cik_vm_fini - cik vm fini callback
  3211. *
  3212. * @rdev: radeon_device pointer
  3213. *
  3214. * Tear down any asic specific VM setup (CIK).
  3215. */
  3216. void cik_vm_fini(struct radeon_device *rdev)
  3217. {
  3218. }
  3219. /**
  3220. * cik_vm_flush - cik vm flush using the CP
  3221. *
  3222. * @rdev: radeon_device pointer
  3223. *
  3224. * Update the page table base and flush the VM TLB
  3225. * using the CP (CIK).
  3226. */
  3227. void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
  3228. {
  3229. struct radeon_ring *ring = &rdev->ring[ridx];
  3230. if (vm == NULL)
  3231. return;
  3232. radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
  3233. radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
  3234. WRITE_DATA_DST_SEL(0)));
  3235. if (vm->id < 8) {
  3236. radeon_ring_write(ring,
  3237. (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
  3238. } else {
  3239. radeon_ring_write(ring,
  3240. (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
  3241. }
  3242. radeon_ring_write(ring, 0);
  3243. radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
  3244. /* update SH_MEM_* regs */
  3245. radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
  3246. radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
  3247. WRITE_DATA_DST_SEL(0)));
  3248. radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
  3249. radeon_ring_write(ring, 0);
  3250. radeon_ring_write(ring, VMID(vm->id));
  3251. radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
  3252. radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
  3253. WRITE_DATA_DST_SEL(0)));
  3254. radeon_ring_write(ring, SH_MEM_BASES >> 2);
  3255. radeon_ring_write(ring, 0);
  3256. radeon_ring_write(ring, 0); /* SH_MEM_BASES */
  3257. radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
  3258. radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
  3259. radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
  3260. radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
  3261. radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
  3262. WRITE_DATA_DST_SEL(0)));
  3263. radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
  3264. radeon_ring_write(ring, 0);
  3265. radeon_ring_write(ring, VMID(0));
  3266. /* HDP flush */
  3267. /* We should be using the WAIT_REG_MEM packet here like in
  3268. * cik_fence_ring_emit(), but it causes the CP to hang in this
  3269. * context...
  3270. */
  3271. radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
  3272. radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
  3273. WRITE_DATA_DST_SEL(0)));
  3274. radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
  3275. radeon_ring_write(ring, 0);
  3276. radeon_ring_write(ring, 0);
  3277. /* bits 0-15 are the VM contexts0-15 */
  3278. radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
  3279. radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
  3280. WRITE_DATA_DST_SEL(0)));
  3281. radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
  3282. radeon_ring_write(ring, 0);
  3283. radeon_ring_write(ring, 1 << vm->id);
  3284. /* sync PFP to ME, otherwise we might get invalid PFP reads */
  3285. radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
  3286. radeon_ring_write(ring, 0x0);
  3287. }
  3288. /**
  3289. * cik_vm_set_page - update the page tables using sDMA
  3290. *
  3291. * @rdev: radeon_device pointer
  3292. * @ib: indirect buffer to fill with commands
  3293. * @pe: addr of the page entry
  3294. * @addr: dst addr to write into pe
  3295. * @count: number of page entries to update
  3296. * @incr: increase next addr by incr bytes
  3297. * @flags: access flags
  3298. *
  3299. * Update the page tables using CP or sDMA (CIK).
  3300. */
  3301. void cik_vm_set_page(struct radeon_device *rdev,
  3302. struct radeon_ib *ib,
  3303. uint64_t pe,
  3304. uint64_t addr, unsigned count,
  3305. uint32_t incr, uint32_t flags)
  3306. {
  3307. uint32_t r600_flags = cayman_vm_page_flags(rdev, flags);
  3308. uint64_t value;
  3309. unsigned ndw;
  3310. if (rdev->asic->vm.pt_ring_index == RADEON_RING_TYPE_GFX_INDEX) {
  3311. /* CP */
  3312. while (count) {
  3313. ndw = 2 + count * 2;
  3314. if (ndw > 0x3FFE)
  3315. ndw = 0x3FFE;
  3316. ib->ptr[ib->length_dw++] = PACKET3(PACKET3_WRITE_DATA, ndw);
  3317. ib->ptr[ib->length_dw++] = (WRITE_DATA_ENGINE_SEL(0) |
  3318. WRITE_DATA_DST_SEL(1));
  3319. ib->ptr[ib->length_dw++] = pe;
  3320. ib->ptr[ib->length_dw++] = upper_32_bits(pe);
  3321. for (; ndw > 2; ndw -= 2, --count, pe += 8) {
  3322. if (flags & RADEON_VM_PAGE_SYSTEM) {
  3323. value = radeon_vm_map_gart(rdev, addr);
  3324. value &= 0xFFFFFFFFFFFFF000ULL;
  3325. } else if (flags & RADEON_VM_PAGE_VALID) {
  3326. value = addr;
  3327. } else {
  3328. value = 0;
  3329. }
  3330. addr += incr;
  3331. value |= r600_flags;
  3332. ib->ptr[ib->length_dw++] = value;
  3333. ib->ptr[ib->length_dw++] = upper_32_bits(value);
  3334. }
  3335. }
  3336. } else {
  3337. /* DMA */
  3338. if (flags & RADEON_VM_PAGE_SYSTEM) {
  3339. while (count) {
  3340. ndw = count * 2;
  3341. if (ndw > 0xFFFFE)
  3342. ndw = 0xFFFFE;
  3343. /* for non-physically contiguous pages (system) */
  3344. ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0);
  3345. ib->ptr[ib->length_dw++] = pe;
  3346. ib->ptr[ib->length_dw++] = upper_32_bits(pe);
  3347. ib->ptr[ib->length_dw++] = ndw;
  3348. for (; ndw > 0; ndw -= 2, --count, pe += 8) {
  3349. if (flags & RADEON_VM_PAGE_SYSTEM) {
  3350. value = radeon_vm_map_gart(rdev, addr);
  3351. value &= 0xFFFFFFFFFFFFF000ULL;
  3352. } else if (flags & RADEON_VM_PAGE_VALID) {
  3353. value = addr;
  3354. } else {
  3355. value = 0;
  3356. }
  3357. addr += incr;
  3358. value |= r600_flags;
  3359. ib->ptr[ib->length_dw++] = value;
  3360. ib->ptr[ib->length_dw++] = upper_32_bits(value);
  3361. }
  3362. }
  3363. } else {
  3364. while (count) {
  3365. ndw = count;
  3366. if (ndw > 0x7FFFF)
  3367. ndw = 0x7FFFF;
  3368. if (flags & RADEON_VM_PAGE_VALID)
  3369. value = addr;
  3370. else
  3371. value = 0;
  3372. /* for physically contiguous pages (vram) */
  3373. ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_GENERATE_PTE_PDE, 0, 0);
  3374. ib->ptr[ib->length_dw++] = pe; /* dst addr */
  3375. ib->ptr[ib->length_dw++] = upper_32_bits(pe);
  3376. ib->ptr[ib->length_dw++] = r600_flags; /* mask */
  3377. ib->ptr[ib->length_dw++] = 0;
  3378. ib->ptr[ib->length_dw++] = value; /* value */
  3379. ib->ptr[ib->length_dw++] = upper_32_bits(value);
  3380. ib->ptr[ib->length_dw++] = incr; /* increment size */
  3381. ib->ptr[ib->length_dw++] = 0;
  3382. ib->ptr[ib->length_dw++] = ndw; /* number of entries */
  3383. pe += ndw * 8;
  3384. addr += ndw * incr;
  3385. count -= ndw;
  3386. }
  3387. }
  3388. while (ib->length_dw & 0x7)
  3389. ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0);
  3390. }
  3391. }
  3392. /**
  3393. * cik_dma_vm_flush - cik vm flush using sDMA
  3394. *
  3395. * @rdev: radeon_device pointer
  3396. *
  3397. * Update the page table base and flush the VM TLB
  3398. * using sDMA (CIK).
  3399. */
  3400. void cik_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
  3401. {
  3402. struct radeon_ring *ring = &rdev->ring[ridx];
  3403. u32 extra_bits = (SDMA_POLL_REG_MEM_EXTRA_OP(1) |
  3404. SDMA_POLL_REG_MEM_EXTRA_FUNC(3)); /* == */
  3405. u32 ref_and_mask;
  3406. if (vm == NULL)
  3407. return;
  3408. if (ridx == R600_RING_TYPE_DMA_INDEX)
  3409. ref_and_mask = SDMA0;
  3410. else
  3411. ref_and_mask = SDMA1;
  3412. radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
  3413. if (vm->id < 8) {
  3414. radeon_ring_write(ring, (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
  3415. } else {
  3416. radeon_ring_write(ring, (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
  3417. }
  3418. radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
  3419. /* update SH_MEM_* regs */
  3420. radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
  3421. radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
  3422. radeon_ring_write(ring, VMID(vm->id));
  3423. radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
  3424. radeon_ring_write(ring, SH_MEM_BASES >> 2);
  3425. radeon_ring_write(ring, 0);
  3426. radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
  3427. radeon_ring_write(ring, SH_MEM_CONFIG >> 2);
  3428. radeon_ring_write(ring, 0);
  3429. radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
  3430. radeon_ring_write(ring, SH_MEM_APE1_BASE >> 2);
  3431. radeon_ring_write(ring, 1);
  3432. radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
  3433. radeon_ring_write(ring, SH_MEM_APE1_LIMIT >> 2);
  3434. radeon_ring_write(ring, 0);
  3435. radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
  3436. radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
  3437. radeon_ring_write(ring, VMID(0));
  3438. /* flush HDP */
  3439. radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_POLL_REG_MEM, 0, extra_bits));
  3440. radeon_ring_write(ring, GPU_HDP_FLUSH_DONE);
  3441. radeon_ring_write(ring, GPU_HDP_FLUSH_REQ);
  3442. radeon_ring_write(ring, ref_and_mask); /* REFERENCE */
  3443. radeon_ring_write(ring, ref_and_mask); /* MASK */
  3444. radeon_ring_write(ring, (4 << 16) | 10); /* RETRY_COUNT, POLL_INTERVAL */
  3445. /* flush TLB */
  3446. radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
  3447. radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
  3448. radeon_ring_write(ring, 1 << vm->id);
  3449. }
  3450. /*
  3451. * RLC
  3452. * The RLC is a multi-purpose microengine that handles a
  3453. * variety of functions, the most important of which is
  3454. * the interrupt controller.
  3455. */
  3456. /**
  3457. * cik_rlc_stop - stop the RLC ME
  3458. *
  3459. * @rdev: radeon_device pointer
  3460. *
  3461. * Halt the RLC ME (MicroEngine) (CIK).
  3462. */
  3463. static void cik_rlc_stop(struct radeon_device *rdev)
  3464. {
  3465. int i, j, k;
  3466. u32 mask, tmp;
  3467. tmp = RREG32(CP_INT_CNTL_RING0);
  3468. tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
  3469. WREG32(CP_INT_CNTL_RING0, tmp);
  3470. RREG32(CB_CGTT_SCLK_CTRL);
  3471. RREG32(CB_CGTT_SCLK_CTRL);
  3472. RREG32(CB_CGTT_SCLK_CTRL);
  3473. RREG32(CB_CGTT_SCLK_CTRL);
  3474. tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
  3475. WREG32(RLC_CGCG_CGLS_CTRL, tmp);
  3476. WREG32(RLC_CNTL, 0);
  3477. for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
  3478. for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
  3479. cik_select_se_sh(rdev, i, j);
  3480. for (k = 0; k < rdev->usec_timeout; k++) {
  3481. if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
  3482. break;
  3483. udelay(1);
  3484. }
  3485. }
  3486. }
  3487. cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
  3488. mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
  3489. for (k = 0; k < rdev->usec_timeout; k++) {
  3490. if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
  3491. break;
  3492. udelay(1);
  3493. }
  3494. }
  3495. /**
  3496. * cik_rlc_start - start the RLC ME
  3497. *
  3498. * @rdev: radeon_device pointer
  3499. *
  3500. * Unhalt the RLC ME (MicroEngine) (CIK).
  3501. */
  3502. static void cik_rlc_start(struct radeon_device *rdev)
  3503. {
  3504. u32 tmp;
  3505. WREG32(RLC_CNTL, RLC_ENABLE);
  3506. tmp = RREG32(CP_INT_CNTL_RING0);
  3507. tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
  3508. WREG32(CP_INT_CNTL_RING0, tmp);
  3509. udelay(50);
  3510. }
  3511. /**
  3512. * cik_rlc_resume - setup the RLC hw
  3513. *
  3514. * @rdev: radeon_device pointer
  3515. *
  3516. * Initialize the RLC registers, load the ucode,
  3517. * and start the RLC (CIK).
  3518. * Returns 0 for success, -EINVAL if the ucode is not available.
  3519. */
  3520. static int cik_rlc_resume(struct radeon_device *rdev)
  3521. {
  3522. u32 i, size;
  3523. u32 clear_state_info[3];
  3524. const __be32 *fw_data;
  3525. if (!rdev->rlc_fw)
  3526. return -EINVAL;
  3527. switch (rdev->family) {
  3528. case CHIP_BONAIRE:
  3529. default:
  3530. size = BONAIRE_RLC_UCODE_SIZE;
  3531. break;
  3532. case CHIP_KAVERI:
  3533. size = KV_RLC_UCODE_SIZE;
  3534. break;
  3535. case CHIP_KABINI:
  3536. size = KB_RLC_UCODE_SIZE;
  3537. break;
  3538. }
  3539. cik_rlc_stop(rdev);
  3540. WREG32(GRBM_SOFT_RESET, SOFT_RESET_RLC);
  3541. RREG32(GRBM_SOFT_RESET);
  3542. udelay(50);
  3543. WREG32(GRBM_SOFT_RESET, 0);
  3544. RREG32(GRBM_SOFT_RESET);
  3545. udelay(50);
  3546. WREG32(RLC_LB_CNTR_INIT, 0);
  3547. WREG32(RLC_LB_CNTR_MAX, 0x00008000);
  3548. cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
  3549. WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
  3550. WREG32(RLC_LB_PARAMS, 0x00600408);
  3551. WREG32(RLC_LB_CNTL, 0x80000004);
  3552. WREG32(RLC_MC_CNTL, 0);
  3553. WREG32(RLC_UCODE_CNTL, 0);
  3554. fw_data = (const __be32 *)rdev->rlc_fw->data;
  3555. WREG32(RLC_GPM_UCODE_ADDR, 0);
  3556. for (i = 0; i < size; i++)
  3557. WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
  3558. WREG32(RLC_GPM_UCODE_ADDR, 0);
  3559. /* XXX */
  3560. clear_state_info[0] = 0;//upper_32_bits(rdev->rlc.save_restore_gpu_addr);
  3561. clear_state_info[1] = 0;//rdev->rlc.save_restore_gpu_addr;
  3562. clear_state_info[2] = 0;//cik_default_size;
  3563. WREG32(RLC_GPM_SCRATCH_ADDR, 0x3d);
  3564. for (i = 0; i < 3; i++)
  3565. WREG32(RLC_GPM_SCRATCH_DATA, clear_state_info[i]);
  3566. WREG32(RLC_DRIVER_DMA_STATUS, 0);
  3567. cik_rlc_start(rdev);
  3568. return 0;
  3569. }
  3570. /*
  3571. * Interrupts
  3572. * Starting with r6xx, interrupts are handled via a ring buffer.
  3573. * Ring buffers are areas of GPU accessible memory that the GPU
  3574. * writes interrupt vectors into and the host reads vectors out of.
  3575. * There is a rptr (read pointer) that determines where the
  3576. * host is currently reading, and a wptr (write pointer)
  3577. * which determines where the GPU has written. When the
  3578. * pointers are equal, the ring is idle. When the GPU
  3579. * writes vectors to the ring buffer, it increments the
  3580. * wptr. When there is an interrupt, the host then starts
  3581. * fetching commands and processing them until the pointers are
  3582. * equal again at which point it updates the rptr.
  3583. */
  3584. /**
  3585. * cik_enable_interrupts - Enable the interrupt ring buffer
  3586. *
  3587. * @rdev: radeon_device pointer
  3588. *
  3589. * Enable the interrupt ring buffer (CIK).
  3590. */
  3591. static void cik_enable_interrupts(struct radeon_device *rdev)
  3592. {
  3593. u32 ih_cntl = RREG32(IH_CNTL);
  3594. u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
  3595. ih_cntl |= ENABLE_INTR;
  3596. ih_rb_cntl |= IH_RB_ENABLE;
  3597. WREG32(IH_CNTL, ih_cntl);
  3598. WREG32(IH_RB_CNTL, ih_rb_cntl);
  3599. rdev->ih.enabled = true;
  3600. }
  3601. /**
  3602. * cik_disable_interrupts - Disable the interrupt ring buffer
  3603. *
  3604. * @rdev: radeon_device pointer
  3605. *
  3606. * Disable the interrupt ring buffer (CIK).
  3607. */
  3608. static void cik_disable_interrupts(struct radeon_device *rdev)
  3609. {
  3610. u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
  3611. u32 ih_cntl = RREG32(IH_CNTL);
  3612. ih_rb_cntl &= ~IH_RB_ENABLE;
  3613. ih_cntl &= ~ENABLE_INTR;
  3614. WREG32(IH_RB_CNTL, ih_rb_cntl);
  3615. WREG32(IH_CNTL, ih_cntl);
  3616. /* set rptr, wptr to 0 */
  3617. WREG32(IH_RB_RPTR, 0);
  3618. WREG32(IH_RB_WPTR, 0);
  3619. rdev->ih.enabled = false;
  3620. rdev->ih.rptr = 0;
  3621. }
  3622. /**
  3623. * cik_disable_interrupt_state - Disable all interrupt sources
  3624. *
  3625. * @rdev: radeon_device pointer
  3626. *
  3627. * Clear all interrupt enable bits used by the driver (CIK).
  3628. */
  3629. static void cik_disable_interrupt_state(struct radeon_device *rdev)
  3630. {
  3631. u32 tmp;
  3632. /* gfx ring */
  3633. WREG32(CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
  3634. /* sdma */
  3635. tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
  3636. WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
  3637. tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
  3638. WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
  3639. /* compute queues */
  3640. WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
  3641. WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
  3642. WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
  3643. WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
  3644. WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
  3645. WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
  3646. WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
  3647. WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
  3648. /* grbm */
  3649. WREG32(GRBM_INT_CNTL, 0);
  3650. /* vline/vblank, etc. */
  3651. WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
  3652. WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
  3653. if (rdev->num_crtc >= 4) {
  3654. WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
  3655. WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
  3656. }
  3657. if (rdev->num_crtc >= 6) {
  3658. WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
  3659. WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
  3660. }
  3661. /* dac hotplug */
  3662. WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
  3663. /* digital hotplug */
  3664. tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
  3665. WREG32(DC_HPD1_INT_CONTROL, tmp);
  3666. tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
  3667. WREG32(DC_HPD2_INT_CONTROL, tmp);
  3668. tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
  3669. WREG32(DC_HPD3_INT_CONTROL, tmp);
  3670. tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
  3671. WREG32(DC_HPD4_INT_CONTROL, tmp);
  3672. tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
  3673. WREG32(DC_HPD5_INT_CONTROL, tmp);
  3674. tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
  3675. WREG32(DC_HPD6_INT_CONTROL, tmp);
  3676. }
  3677. /**
  3678. * cik_irq_init - init and enable the interrupt ring
  3679. *
  3680. * @rdev: radeon_device pointer
  3681. *
  3682. * Allocate a ring buffer for the interrupt controller,
  3683. * enable the RLC, disable interrupts, enable the IH
  3684. * ring buffer and enable it (CIK).
  3685. * Called at device load and reume.
  3686. * Returns 0 for success, errors for failure.
  3687. */
  3688. static int cik_irq_init(struct radeon_device *rdev)
  3689. {
  3690. int ret = 0;
  3691. int rb_bufsz;
  3692. u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
  3693. /* allocate ring */
  3694. ret = r600_ih_ring_alloc(rdev);
  3695. if (ret)
  3696. return ret;
  3697. /* disable irqs */
  3698. cik_disable_interrupts(rdev);
  3699. /* init rlc */
  3700. ret = cik_rlc_resume(rdev);
  3701. if (ret) {
  3702. r600_ih_ring_fini(rdev);
  3703. return ret;
  3704. }
  3705. /* setup interrupt control */
  3706. /* XXX this should actually be a bus address, not an MC address. same on older asics */
  3707. WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
  3708. interrupt_cntl = RREG32(INTERRUPT_CNTL);
  3709. /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
  3710. * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
  3711. */
  3712. interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
  3713. /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
  3714. interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
  3715. WREG32(INTERRUPT_CNTL, interrupt_cntl);
  3716. WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
  3717. rb_bufsz = drm_order(rdev->ih.ring_size / 4);
  3718. ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
  3719. IH_WPTR_OVERFLOW_CLEAR |
  3720. (rb_bufsz << 1));
  3721. if (rdev->wb.enabled)
  3722. ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
  3723. /* set the writeback address whether it's enabled or not */
  3724. WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
  3725. WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
  3726. WREG32(IH_RB_CNTL, ih_rb_cntl);
  3727. /* set rptr, wptr to 0 */
  3728. WREG32(IH_RB_RPTR, 0);
  3729. WREG32(IH_RB_WPTR, 0);
  3730. /* Default settings for IH_CNTL (disabled at first) */
  3731. ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
  3732. /* RPTR_REARM only works if msi's are enabled */
  3733. if (rdev->msi_enabled)
  3734. ih_cntl |= RPTR_REARM;
  3735. WREG32(IH_CNTL, ih_cntl);
  3736. /* force the active interrupt state to all disabled */
  3737. cik_disable_interrupt_state(rdev);
  3738. pci_set_master(rdev->pdev);
  3739. /* enable irqs */
  3740. cik_enable_interrupts(rdev);
  3741. return ret;
  3742. }
  3743. /**
  3744. * cik_irq_set - enable/disable interrupt sources
  3745. *
  3746. * @rdev: radeon_device pointer
  3747. *
  3748. * Enable interrupt sources on the GPU (vblanks, hpd,
  3749. * etc.) (CIK).
  3750. * Returns 0 for success, errors for failure.
  3751. */
  3752. int cik_irq_set(struct radeon_device *rdev)
  3753. {
  3754. u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE |
  3755. PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
  3756. u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
  3757. u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
  3758. u32 grbm_int_cntl = 0;
  3759. u32 dma_cntl, dma_cntl1;
  3760. if (!rdev->irq.installed) {
  3761. WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
  3762. return -EINVAL;
  3763. }
  3764. /* don't enable anything if the ih is disabled */
  3765. if (!rdev->ih.enabled) {
  3766. cik_disable_interrupts(rdev);
  3767. /* force the active interrupt state to all disabled */
  3768. cik_disable_interrupt_state(rdev);
  3769. return 0;
  3770. }
  3771. hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
  3772. hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
  3773. hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
  3774. hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
  3775. hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
  3776. hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
  3777. dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
  3778. dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
  3779. /* enable CP interrupts on all rings */
  3780. if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
  3781. DRM_DEBUG("cik_irq_set: sw int gfx\n");
  3782. cp_int_cntl |= TIME_STAMP_INT_ENABLE;
  3783. }
  3784. /* TODO: compute queues! */
  3785. /* CP_ME[1-2]_PIPE[0-3]_INT_CNTL */
  3786. if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
  3787. DRM_DEBUG("cik_irq_set: sw int dma\n");
  3788. dma_cntl |= TRAP_ENABLE;
  3789. }
  3790. if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
  3791. DRM_DEBUG("cik_irq_set: sw int dma1\n");
  3792. dma_cntl1 |= TRAP_ENABLE;
  3793. }
  3794. if (rdev->irq.crtc_vblank_int[0] ||
  3795. atomic_read(&rdev->irq.pflip[0])) {
  3796. DRM_DEBUG("cik_irq_set: vblank 0\n");
  3797. crtc1 |= VBLANK_INTERRUPT_MASK;
  3798. }
  3799. if (rdev->irq.crtc_vblank_int[1] ||
  3800. atomic_read(&rdev->irq.pflip[1])) {
  3801. DRM_DEBUG("cik_irq_set: vblank 1\n");
  3802. crtc2 |= VBLANK_INTERRUPT_MASK;
  3803. }
  3804. if (rdev->irq.crtc_vblank_int[2] ||
  3805. atomic_read(&rdev->irq.pflip[2])) {
  3806. DRM_DEBUG("cik_irq_set: vblank 2\n");
  3807. crtc3 |= VBLANK_INTERRUPT_MASK;
  3808. }
  3809. if (rdev->irq.crtc_vblank_int[3] ||
  3810. atomic_read(&rdev->irq.pflip[3])) {
  3811. DRM_DEBUG("cik_irq_set: vblank 3\n");
  3812. crtc4 |= VBLANK_INTERRUPT_MASK;
  3813. }
  3814. if (rdev->irq.crtc_vblank_int[4] ||
  3815. atomic_read(&rdev->irq.pflip[4])) {
  3816. DRM_DEBUG("cik_irq_set: vblank 4\n");
  3817. crtc5 |= VBLANK_INTERRUPT_MASK;
  3818. }
  3819. if (rdev->irq.crtc_vblank_int[5] ||
  3820. atomic_read(&rdev->irq.pflip[5])) {
  3821. DRM_DEBUG("cik_irq_set: vblank 5\n");
  3822. crtc6 |= VBLANK_INTERRUPT_MASK;
  3823. }
  3824. if (rdev->irq.hpd[0]) {
  3825. DRM_DEBUG("cik_irq_set: hpd 1\n");
  3826. hpd1 |= DC_HPDx_INT_EN;
  3827. }
  3828. if (rdev->irq.hpd[1]) {
  3829. DRM_DEBUG("cik_irq_set: hpd 2\n");
  3830. hpd2 |= DC_HPDx_INT_EN;
  3831. }
  3832. if (rdev->irq.hpd[2]) {
  3833. DRM_DEBUG("cik_irq_set: hpd 3\n");
  3834. hpd3 |= DC_HPDx_INT_EN;
  3835. }
  3836. if (rdev->irq.hpd[3]) {
  3837. DRM_DEBUG("cik_irq_set: hpd 4\n");
  3838. hpd4 |= DC_HPDx_INT_EN;
  3839. }
  3840. if (rdev->irq.hpd[4]) {
  3841. DRM_DEBUG("cik_irq_set: hpd 5\n");
  3842. hpd5 |= DC_HPDx_INT_EN;
  3843. }
  3844. if (rdev->irq.hpd[5]) {
  3845. DRM_DEBUG("cik_irq_set: hpd 6\n");
  3846. hpd6 |= DC_HPDx_INT_EN;
  3847. }
  3848. WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
  3849. WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
  3850. WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
  3851. WREG32(GRBM_INT_CNTL, grbm_int_cntl);
  3852. WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
  3853. WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
  3854. if (rdev->num_crtc >= 4) {
  3855. WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
  3856. WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
  3857. }
  3858. if (rdev->num_crtc >= 6) {
  3859. WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
  3860. WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
  3861. }
  3862. WREG32(DC_HPD1_INT_CONTROL, hpd1);
  3863. WREG32(DC_HPD2_INT_CONTROL, hpd2);
  3864. WREG32(DC_HPD3_INT_CONTROL, hpd3);
  3865. WREG32(DC_HPD4_INT_CONTROL, hpd4);
  3866. WREG32(DC_HPD5_INT_CONTROL, hpd5);
  3867. WREG32(DC_HPD6_INT_CONTROL, hpd6);
  3868. return 0;
  3869. }
  3870. /**
  3871. * cik_irq_ack - ack interrupt sources
  3872. *
  3873. * @rdev: radeon_device pointer
  3874. *
  3875. * Ack interrupt sources on the GPU (vblanks, hpd,
  3876. * etc.) (CIK). Certain interrupts sources are sw
  3877. * generated and do not require an explicit ack.
  3878. */
  3879. static inline void cik_irq_ack(struct radeon_device *rdev)
  3880. {
  3881. u32 tmp;
  3882. rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
  3883. rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
  3884. rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
  3885. rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
  3886. rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
  3887. rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
  3888. rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
  3889. if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
  3890. WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
  3891. if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
  3892. WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
  3893. if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
  3894. WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
  3895. if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
  3896. WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
  3897. if (rdev->num_crtc >= 4) {
  3898. if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
  3899. WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
  3900. if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
  3901. WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
  3902. if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
  3903. WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
  3904. if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
  3905. WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
  3906. }
  3907. if (rdev->num_crtc >= 6) {
  3908. if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
  3909. WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
  3910. if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
  3911. WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
  3912. if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
  3913. WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
  3914. if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
  3915. WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
  3916. }
  3917. if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
  3918. tmp = RREG32(DC_HPD1_INT_CONTROL);
  3919. tmp |= DC_HPDx_INT_ACK;
  3920. WREG32(DC_HPD1_INT_CONTROL, tmp);
  3921. }
  3922. if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
  3923. tmp = RREG32(DC_HPD2_INT_CONTROL);
  3924. tmp |= DC_HPDx_INT_ACK;
  3925. WREG32(DC_HPD2_INT_CONTROL, tmp);
  3926. }
  3927. if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
  3928. tmp = RREG32(DC_HPD3_INT_CONTROL);
  3929. tmp |= DC_HPDx_INT_ACK;
  3930. WREG32(DC_HPD3_INT_CONTROL, tmp);
  3931. }
  3932. if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
  3933. tmp = RREG32(DC_HPD4_INT_CONTROL);
  3934. tmp |= DC_HPDx_INT_ACK;
  3935. WREG32(DC_HPD4_INT_CONTROL, tmp);
  3936. }
  3937. if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
  3938. tmp = RREG32(DC_HPD5_INT_CONTROL);
  3939. tmp |= DC_HPDx_INT_ACK;
  3940. WREG32(DC_HPD5_INT_CONTROL, tmp);
  3941. }
  3942. if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
  3943. tmp = RREG32(DC_HPD5_INT_CONTROL);
  3944. tmp |= DC_HPDx_INT_ACK;
  3945. WREG32(DC_HPD6_INT_CONTROL, tmp);
  3946. }
  3947. }
  3948. /**
  3949. * cik_irq_disable - disable interrupts
  3950. *
  3951. * @rdev: radeon_device pointer
  3952. *
  3953. * Disable interrupts on the hw (CIK).
  3954. */
  3955. static void cik_irq_disable(struct radeon_device *rdev)
  3956. {
  3957. cik_disable_interrupts(rdev);
  3958. /* Wait and acknowledge irq */
  3959. mdelay(1);
  3960. cik_irq_ack(rdev);
  3961. cik_disable_interrupt_state(rdev);
  3962. }
  3963. /**
  3964. * cik_irq_disable - disable interrupts for suspend
  3965. *
  3966. * @rdev: radeon_device pointer
  3967. *
  3968. * Disable interrupts and stop the RLC (CIK).
  3969. * Used for suspend.
  3970. */
  3971. static void cik_irq_suspend(struct radeon_device *rdev)
  3972. {
  3973. cik_irq_disable(rdev);
  3974. cik_rlc_stop(rdev);
  3975. }
  3976. /**
  3977. * cik_irq_fini - tear down interrupt support
  3978. *
  3979. * @rdev: radeon_device pointer
  3980. *
  3981. * Disable interrupts on the hw and free the IH ring
  3982. * buffer (CIK).
  3983. * Used for driver unload.
  3984. */
  3985. static void cik_irq_fini(struct radeon_device *rdev)
  3986. {
  3987. cik_irq_suspend(rdev);
  3988. r600_ih_ring_fini(rdev);
  3989. }
  3990. /**
  3991. * cik_get_ih_wptr - get the IH ring buffer wptr
  3992. *
  3993. * @rdev: radeon_device pointer
  3994. *
  3995. * Get the IH ring buffer wptr from either the register
  3996. * or the writeback memory buffer (CIK). Also check for
  3997. * ring buffer overflow and deal with it.
  3998. * Used by cik_irq_process().
  3999. * Returns the value of the wptr.
  4000. */
  4001. static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
  4002. {
  4003. u32 wptr, tmp;
  4004. if (rdev->wb.enabled)
  4005. wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
  4006. else
  4007. wptr = RREG32(IH_RB_WPTR);
  4008. if (wptr & RB_OVERFLOW) {
  4009. /* When a ring buffer overflow happen start parsing interrupt
  4010. * from the last not overwritten vector (wptr + 16). Hopefully
  4011. * this should allow us to catchup.
  4012. */
  4013. dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
  4014. wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
  4015. rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
  4016. tmp = RREG32(IH_RB_CNTL);
  4017. tmp |= IH_WPTR_OVERFLOW_CLEAR;
  4018. WREG32(IH_RB_CNTL, tmp);
  4019. }
  4020. return (wptr & rdev->ih.ptr_mask);
  4021. }
  4022. /* CIK IV Ring
  4023. * Each IV ring entry is 128 bits:
  4024. * [7:0] - interrupt source id
  4025. * [31:8] - reserved
  4026. * [59:32] - interrupt source data
  4027. * [63:60] - reserved
  4028. * [71:64] - RINGID
  4029. * CP:
  4030. * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
  4031. * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
  4032. * - for gfx, hw shader state (0=PS...5=LS, 6=CS)
  4033. * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
  4034. * PIPE_ID - ME0 0=3D
  4035. * - ME1&2 compute dispatcher (4 pipes each)
  4036. * SDMA:
  4037. * INSTANCE_ID [1:0], QUEUE_ID[1:0]
  4038. * INSTANCE_ID - 0 = sdma0, 1 = sdma1
  4039. * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
  4040. * [79:72] - VMID
  4041. * [95:80] - PASID
  4042. * [127:96] - reserved
  4043. */
  4044. /**
  4045. * cik_irq_process - interrupt handler
  4046. *
  4047. * @rdev: radeon_device pointer
  4048. *
  4049. * Interrupt hander (CIK). Walk the IH ring,
  4050. * ack interrupts and schedule work to handle
  4051. * interrupt events.
  4052. * Returns irq process return code.
  4053. */
  4054. int cik_irq_process(struct radeon_device *rdev)
  4055. {
  4056. u32 wptr;
  4057. u32 rptr;
  4058. u32 src_id, src_data, ring_id;
  4059. u8 me_id, pipe_id, queue_id;
  4060. u32 ring_index;
  4061. bool queue_hotplug = false;
  4062. bool queue_reset = false;
  4063. if (!rdev->ih.enabled || rdev->shutdown)
  4064. return IRQ_NONE;
  4065. wptr = cik_get_ih_wptr(rdev);
  4066. restart_ih:
  4067. /* is somebody else already processing irqs? */
  4068. if (atomic_xchg(&rdev->ih.lock, 1))
  4069. return IRQ_NONE;
  4070. rptr = rdev->ih.rptr;
  4071. DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
  4072. /* Order reading of wptr vs. reading of IH ring data */
  4073. rmb();
  4074. /* display interrupts */
  4075. cik_irq_ack(rdev);
  4076. while (rptr != wptr) {
  4077. /* wptr/rptr are in bytes! */
  4078. ring_index = rptr / 4;
  4079. src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
  4080. src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
  4081. ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
  4082. switch (src_id) {
  4083. case 1: /* D1 vblank/vline */
  4084. switch (src_data) {
  4085. case 0: /* D1 vblank */
  4086. if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) {
  4087. if (rdev->irq.crtc_vblank_int[0]) {
  4088. drm_handle_vblank(rdev->ddev, 0);
  4089. rdev->pm.vblank_sync = true;
  4090. wake_up(&rdev->irq.vblank_queue);
  4091. }
  4092. if (atomic_read(&rdev->irq.pflip[0]))
  4093. radeon_crtc_handle_flip(rdev, 0);
  4094. rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
  4095. DRM_DEBUG("IH: D1 vblank\n");
  4096. }
  4097. break;
  4098. case 1: /* D1 vline */
  4099. if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) {
  4100. rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
  4101. DRM_DEBUG("IH: D1 vline\n");
  4102. }
  4103. break;
  4104. default:
  4105. DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
  4106. break;
  4107. }
  4108. break;
  4109. case 2: /* D2 vblank/vline */
  4110. switch (src_data) {
  4111. case 0: /* D2 vblank */
  4112. if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
  4113. if (rdev->irq.crtc_vblank_int[1]) {
  4114. drm_handle_vblank(rdev->ddev, 1);
  4115. rdev->pm.vblank_sync = true;
  4116. wake_up(&rdev->irq.vblank_queue);
  4117. }
  4118. if (atomic_read(&rdev->irq.pflip[1]))
  4119. radeon_crtc_handle_flip(rdev, 1);
  4120. rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
  4121. DRM_DEBUG("IH: D2 vblank\n");
  4122. }
  4123. break;
  4124. case 1: /* D2 vline */
  4125. if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
  4126. rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
  4127. DRM_DEBUG("IH: D2 vline\n");
  4128. }
  4129. break;
  4130. default:
  4131. DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
  4132. break;
  4133. }
  4134. break;
  4135. case 3: /* D3 vblank/vline */
  4136. switch (src_data) {
  4137. case 0: /* D3 vblank */
  4138. if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
  4139. if (rdev->irq.crtc_vblank_int[2]) {
  4140. drm_handle_vblank(rdev->ddev, 2);
  4141. rdev->pm.vblank_sync = true;
  4142. wake_up(&rdev->irq.vblank_queue);
  4143. }
  4144. if (atomic_read(&rdev->irq.pflip[2]))
  4145. radeon_crtc_handle_flip(rdev, 2);
  4146. rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
  4147. DRM_DEBUG("IH: D3 vblank\n");
  4148. }
  4149. break;
  4150. case 1: /* D3 vline */
  4151. if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
  4152. rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
  4153. DRM_DEBUG("IH: D3 vline\n");
  4154. }
  4155. break;
  4156. default:
  4157. DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
  4158. break;
  4159. }
  4160. break;
  4161. case 4: /* D4 vblank/vline */
  4162. switch (src_data) {
  4163. case 0: /* D4 vblank */
  4164. if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
  4165. if (rdev->irq.crtc_vblank_int[3]) {
  4166. drm_handle_vblank(rdev->ddev, 3);
  4167. rdev->pm.vblank_sync = true;
  4168. wake_up(&rdev->irq.vblank_queue);
  4169. }
  4170. if (atomic_read(&rdev->irq.pflip[3]))
  4171. radeon_crtc_handle_flip(rdev, 3);
  4172. rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
  4173. DRM_DEBUG("IH: D4 vblank\n");
  4174. }
  4175. break;
  4176. case 1: /* D4 vline */
  4177. if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
  4178. rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
  4179. DRM_DEBUG("IH: D4 vline\n");
  4180. }
  4181. break;
  4182. default:
  4183. DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
  4184. break;
  4185. }
  4186. break;
  4187. case 5: /* D5 vblank/vline */
  4188. switch (src_data) {
  4189. case 0: /* D5 vblank */
  4190. if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
  4191. if (rdev->irq.crtc_vblank_int[4]) {
  4192. drm_handle_vblank(rdev->ddev, 4);
  4193. rdev->pm.vblank_sync = true;
  4194. wake_up(&rdev->irq.vblank_queue);
  4195. }
  4196. if (atomic_read(&rdev->irq.pflip[4]))
  4197. radeon_crtc_handle_flip(rdev, 4);
  4198. rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
  4199. DRM_DEBUG("IH: D5 vblank\n");
  4200. }
  4201. break;
  4202. case 1: /* D5 vline */
  4203. if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
  4204. rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
  4205. DRM_DEBUG("IH: D5 vline\n");
  4206. }
  4207. break;
  4208. default:
  4209. DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
  4210. break;
  4211. }
  4212. break;
  4213. case 6: /* D6 vblank/vline */
  4214. switch (src_data) {
  4215. case 0: /* D6 vblank */
  4216. if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
  4217. if (rdev->irq.crtc_vblank_int[5]) {
  4218. drm_handle_vblank(rdev->ddev, 5);
  4219. rdev->pm.vblank_sync = true;
  4220. wake_up(&rdev->irq.vblank_queue);
  4221. }
  4222. if (atomic_read(&rdev->irq.pflip[5]))
  4223. radeon_crtc_handle_flip(rdev, 5);
  4224. rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
  4225. DRM_DEBUG("IH: D6 vblank\n");
  4226. }
  4227. break;
  4228. case 1: /* D6 vline */
  4229. if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
  4230. rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
  4231. DRM_DEBUG("IH: D6 vline\n");
  4232. }
  4233. break;
  4234. default:
  4235. DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
  4236. break;
  4237. }
  4238. break;
  4239. case 42: /* HPD hotplug */
  4240. switch (src_data) {
  4241. case 0:
  4242. if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
  4243. rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
  4244. queue_hotplug = true;
  4245. DRM_DEBUG("IH: HPD1\n");
  4246. }
  4247. break;
  4248. case 1:
  4249. if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
  4250. rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
  4251. queue_hotplug = true;
  4252. DRM_DEBUG("IH: HPD2\n");
  4253. }
  4254. break;
  4255. case 2:
  4256. if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
  4257. rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
  4258. queue_hotplug = true;
  4259. DRM_DEBUG("IH: HPD3\n");
  4260. }
  4261. break;
  4262. case 3:
  4263. if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
  4264. rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
  4265. queue_hotplug = true;
  4266. DRM_DEBUG("IH: HPD4\n");
  4267. }
  4268. break;
  4269. case 4:
  4270. if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
  4271. rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
  4272. queue_hotplug = true;
  4273. DRM_DEBUG("IH: HPD5\n");
  4274. }
  4275. break;
  4276. case 5:
  4277. if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
  4278. rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
  4279. queue_hotplug = true;
  4280. DRM_DEBUG("IH: HPD6\n");
  4281. }
  4282. break;
  4283. default:
  4284. DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
  4285. break;
  4286. }
  4287. break;
  4288. case 146:
  4289. case 147:
  4290. dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
  4291. dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
  4292. RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
  4293. dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
  4294. RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
  4295. /* reset addr and status */
  4296. WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
  4297. break;
  4298. case 176: /* GFX RB CP_INT */
  4299. case 177: /* GFX IB CP_INT */
  4300. radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
  4301. break;
  4302. case 181: /* CP EOP event */
  4303. DRM_DEBUG("IH: CP EOP\n");
  4304. /* XXX check the bitfield order! */
  4305. me_id = (ring_id & 0x60) >> 5;
  4306. pipe_id = (ring_id & 0x18) >> 3;
  4307. queue_id = (ring_id & 0x7) >> 0;
  4308. switch (me_id) {
  4309. case 0:
  4310. radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
  4311. break;
  4312. case 1:
  4313. /* XXX compute */
  4314. break;
  4315. case 2:
  4316. /* XXX compute */
  4317. break;
  4318. }
  4319. break;
  4320. case 184: /* CP Privileged reg access */
  4321. DRM_ERROR("Illegal register access in command stream\n");
  4322. /* XXX check the bitfield order! */
  4323. me_id = (ring_id & 0x60) >> 5;
  4324. pipe_id = (ring_id & 0x18) >> 3;
  4325. queue_id = (ring_id & 0x7) >> 0;
  4326. switch (me_id) {
  4327. case 0:
  4328. /* This results in a full GPU reset, but all we need to do is soft
  4329. * reset the CP for gfx
  4330. */
  4331. queue_reset = true;
  4332. break;
  4333. case 1:
  4334. /* XXX compute */
  4335. break;
  4336. case 2:
  4337. /* XXX compute */
  4338. break;
  4339. }
  4340. break;
  4341. case 185: /* CP Privileged inst */
  4342. DRM_ERROR("Illegal instruction in command stream\n");
  4343. /* XXX check the bitfield order! */
  4344. me_id = (ring_id & 0x60) >> 5;
  4345. pipe_id = (ring_id & 0x18) >> 3;
  4346. queue_id = (ring_id & 0x7) >> 0;
  4347. switch (me_id) {
  4348. case 0:
  4349. /* This results in a full GPU reset, but all we need to do is soft
  4350. * reset the CP for gfx
  4351. */
  4352. queue_reset = true;
  4353. break;
  4354. case 1:
  4355. /* XXX compute */
  4356. break;
  4357. case 2:
  4358. /* XXX compute */
  4359. break;
  4360. }
  4361. break;
  4362. case 224: /* SDMA trap event */
  4363. /* XXX check the bitfield order! */
  4364. me_id = (ring_id & 0x3) >> 0;
  4365. queue_id = (ring_id & 0xc) >> 2;
  4366. DRM_DEBUG("IH: SDMA trap\n");
  4367. switch (me_id) {
  4368. case 0:
  4369. switch (queue_id) {
  4370. case 0:
  4371. radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
  4372. break;
  4373. case 1:
  4374. /* XXX compute */
  4375. break;
  4376. case 2:
  4377. /* XXX compute */
  4378. break;
  4379. }
  4380. break;
  4381. case 1:
  4382. switch (queue_id) {
  4383. case 0:
  4384. radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
  4385. break;
  4386. case 1:
  4387. /* XXX compute */
  4388. break;
  4389. case 2:
  4390. /* XXX compute */
  4391. break;
  4392. }
  4393. break;
  4394. }
  4395. break;
  4396. case 241: /* SDMA Privileged inst */
  4397. case 247: /* SDMA Privileged inst */
  4398. DRM_ERROR("Illegal instruction in SDMA command stream\n");
  4399. /* XXX check the bitfield order! */
  4400. me_id = (ring_id & 0x3) >> 0;
  4401. queue_id = (ring_id & 0xc) >> 2;
  4402. switch (me_id) {
  4403. case 0:
  4404. switch (queue_id) {
  4405. case 0:
  4406. queue_reset = true;
  4407. break;
  4408. case 1:
  4409. /* XXX compute */
  4410. queue_reset = true;
  4411. break;
  4412. case 2:
  4413. /* XXX compute */
  4414. queue_reset = true;
  4415. break;
  4416. }
  4417. break;
  4418. case 1:
  4419. switch (queue_id) {
  4420. case 0:
  4421. queue_reset = true;
  4422. break;
  4423. case 1:
  4424. /* XXX compute */
  4425. queue_reset = true;
  4426. break;
  4427. case 2:
  4428. /* XXX compute */
  4429. queue_reset = true;
  4430. break;
  4431. }
  4432. break;
  4433. }
  4434. break;
  4435. case 233: /* GUI IDLE */
  4436. DRM_DEBUG("IH: GUI idle\n");
  4437. break;
  4438. default:
  4439. DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
  4440. break;
  4441. }
  4442. /* wptr/rptr are in bytes! */
  4443. rptr += 16;
  4444. rptr &= rdev->ih.ptr_mask;
  4445. }
  4446. if (queue_hotplug)
  4447. schedule_work(&rdev->hotplug_work);
  4448. if (queue_reset)
  4449. schedule_work(&rdev->reset_work);
  4450. rdev->ih.rptr = rptr;
  4451. WREG32(IH_RB_RPTR, rdev->ih.rptr);
  4452. atomic_set(&rdev->ih.lock, 0);
  4453. /* make sure wptr hasn't changed while processing */
  4454. wptr = cik_get_ih_wptr(rdev);
  4455. if (wptr != rptr)
  4456. goto restart_ih;
  4457. return IRQ_HANDLED;
  4458. }
  4459. /*
  4460. * startup/shutdown callbacks
  4461. */
  4462. /**
  4463. * cik_startup - program the asic to a functional state
  4464. *
  4465. * @rdev: radeon_device pointer
  4466. *
  4467. * Programs the asic to a functional state (CIK).
  4468. * Called by cik_init() and cik_resume().
  4469. * Returns 0 for success, error for failure.
  4470. */
  4471. static int cik_startup(struct radeon_device *rdev)
  4472. {
  4473. struct radeon_ring *ring;
  4474. int r;
  4475. if (rdev->flags & RADEON_IS_IGP) {
  4476. if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
  4477. !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
  4478. r = cik_init_microcode(rdev);
  4479. if (r) {
  4480. DRM_ERROR("Failed to load firmware!\n");
  4481. return r;
  4482. }
  4483. }
  4484. } else {
  4485. if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
  4486. !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
  4487. !rdev->mc_fw) {
  4488. r = cik_init_microcode(rdev);
  4489. if (r) {
  4490. DRM_ERROR("Failed to load firmware!\n");
  4491. return r;
  4492. }
  4493. }
  4494. r = ci_mc_load_microcode(rdev);
  4495. if (r) {
  4496. DRM_ERROR("Failed to load MC firmware!\n");
  4497. return r;
  4498. }
  4499. }
  4500. r = r600_vram_scratch_init(rdev);
  4501. if (r)
  4502. return r;
  4503. cik_mc_program(rdev);
  4504. r = cik_pcie_gart_enable(rdev);
  4505. if (r)
  4506. return r;
  4507. cik_gpu_init(rdev);
  4508. /* allocate rlc buffers */
  4509. r = si_rlc_init(rdev);
  4510. if (r) {
  4511. DRM_ERROR("Failed to init rlc BOs!\n");
  4512. return r;
  4513. }
  4514. /* allocate wb buffer */
  4515. r = radeon_wb_init(rdev);
  4516. if (r)
  4517. return r;
  4518. r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
  4519. if (r) {
  4520. dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
  4521. return r;
  4522. }
  4523. r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
  4524. if (r) {
  4525. dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
  4526. return r;
  4527. }
  4528. r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
  4529. if (r) {
  4530. dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
  4531. return r;
  4532. }
  4533. r = cik_uvd_resume(rdev);
  4534. if (!r) {
  4535. r = radeon_fence_driver_start_ring(rdev,
  4536. R600_RING_TYPE_UVD_INDEX);
  4537. if (r)
  4538. dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
  4539. }
  4540. if (r)
  4541. rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
  4542. /* Enable IRQ */
  4543. if (!rdev->irq.installed) {
  4544. r = radeon_irq_kms_init(rdev);
  4545. if (r)
  4546. return r;
  4547. }
  4548. r = cik_irq_init(rdev);
  4549. if (r) {
  4550. DRM_ERROR("radeon: IH init failed (%d).\n", r);
  4551. radeon_irq_kms_fini(rdev);
  4552. return r;
  4553. }
  4554. cik_irq_set(rdev);
  4555. ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
  4556. r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
  4557. CP_RB0_RPTR, CP_RB0_WPTR,
  4558. 0, 0xfffff, RADEON_CP_PACKET2);
  4559. if (r)
  4560. return r;
  4561. ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
  4562. r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
  4563. SDMA0_GFX_RB_RPTR + SDMA0_REGISTER_OFFSET,
  4564. SDMA0_GFX_RB_WPTR + SDMA0_REGISTER_OFFSET,
  4565. 2, 0xfffffffc, SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
  4566. if (r)
  4567. return r;
  4568. ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
  4569. r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
  4570. SDMA0_GFX_RB_RPTR + SDMA1_REGISTER_OFFSET,
  4571. SDMA0_GFX_RB_WPTR + SDMA1_REGISTER_OFFSET,
  4572. 2, 0xfffffffc, SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
  4573. if (r)
  4574. return r;
  4575. r = cik_cp_resume(rdev);
  4576. if (r)
  4577. return r;
  4578. r = cik_sdma_resume(rdev);
  4579. if (r)
  4580. return r;
  4581. ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
  4582. if (ring->ring_size) {
  4583. r = radeon_ring_init(rdev, ring, ring->ring_size,
  4584. R600_WB_UVD_RPTR_OFFSET,
  4585. UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
  4586. 0, 0xfffff, RADEON_CP_PACKET2);
  4587. if (!r)
  4588. r = r600_uvd_init(rdev);
  4589. if (r)
  4590. DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
  4591. }
  4592. r = radeon_ib_pool_init(rdev);
  4593. if (r) {
  4594. dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
  4595. return r;
  4596. }
  4597. r = radeon_vm_manager_init(rdev);
  4598. if (r) {
  4599. dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
  4600. return r;
  4601. }
  4602. return 0;
  4603. }
  4604. /**
  4605. * cik_resume - resume the asic to a functional state
  4606. *
  4607. * @rdev: radeon_device pointer
  4608. *
  4609. * Programs the asic to a functional state (CIK).
  4610. * Called at resume.
  4611. * Returns 0 for success, error for failure.
  4612. */
  4613. int cik_resume(struct radeon_device *rdev)
  4614. {
  4615. int r;
  4616. /* post card */
  4617. atom_asic_init(rdev->mode_info.atom_context);
  4618. rdev->accel_working = true;
  4619. r = cik_startup(rdev);
  4620. if (r) {
  4621. DRM_ERROR("cik startup failed on resume\n");
  4622. rdev->accel_working = false;
  4623. return r;
  4624. }
  4625. return r;
  4626. }
  4627. /**
  4628. * cik_suspend - suspend the asic
  4629. *
  4630. * @rdev: radeon_device pointer
  4631. *
  4632. * Bring the chip into a state suitable for suspend (CIK).
  4633. * Called at suspend.
  4634. * Returns 0 for success.
  4635. */
  4636. int cik_suspend(struct radeon_device *rdev)
  4637. {
  4638. radeon_vm_manager_fini(rdev);
  4639. cik_cp_enable(rdev, false);
  4640. cik_sdma_enable(rdev, false);
  4641. r600_uvd_rbc_stop(rdev);
  4642. radeon_uvd_suspend(rdev);
  4643. cik_irq_suspend(rdev);
  4644. radeon_wb_disable(rdev);
  4645. cik_pcie_gart_disable(rdev);
  4646. return 0;
  4647. }
  4648. /* Plan is to move initialization in that function and use
  4649. * helper function so that radeon_device_init pretty much
  4650. * do nothing more than calling asic specific function. This
  4651. * should also allow to remove a bunch of callback function
  4652. * like vram_info.
  4653. */
  4654. /**
  4655. * cik_init - asic specific driver and hw init
  4656. *
  4657. * @rdev: radeon_device pointer
  4658. *
  4659. * Setup asic specific driver variables and program the hw
  4660. * to a functional state (CIK).
  4661. * Called at driver startup.
  4662. * Returns 0 for success, errors for failure.
  4663. */
  4664. int cik_init(struct radeon_device *rdev)
  4665. {
  4666. struct radeon_ring *ring;
  4667. int r;
  4668. /* Read BIOS */
  4669. if (!radeon_get_bios(rdev)) {
  4670. if (ASIC_IS_AVIVO(rdev))
  4671. return -EINVAL;
  4672. }
  4673. /* Must be an ATOMBIOS */
  4674. if (!rdev->is_atom_bios) {
  4675. dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
  4676. return -EINVAL;
  4677. }
  4678. r = radeon_atombios_init(rdev);
  4679. if (r)
  4680. return r;
  4681. /* Post card if necessary */
  4682. if (!radeon_card_posted(rdev)) {
  4683. if (!rdev->bios) {
  4684. dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
  4685. return -EINVAL;
  4686. }
  4687. DRM_INFO("GPU not posted. posting now...\n");
  4688. atom_asic_init(rdev->mode_info.atom_context);
  4689. }
  4690. /* Initialize scratch registers */
  4691. cik_scratch_init(rdev);
  4692. /* Initialize surface registers */
  4693. radeon_surface_init(rdev);
  4694. /* Initialize clocks */
  4695. radeon_get_clock_info(rdev->ddev);
  4696. /* Fence driver */
  4697. r = radeon_fence_driver_init(rdev);
  4698. if (r)
  4699. return r;
  4700. /* initialize memory controller */
  4701. r = cik_mc_init(rdev);
  4702. if (r)
  4703. return r;
  4704. /* Memory manager */
  4705. r = radeon_bo_init(rdev);
  4706. if (r)
  4707. return r;
  4708. ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
  4709. ring->ring_obj = NULL;
  4710. r600_ring_init(rdev, ring, 1024 * 1024);
  4711. ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
  4712. ring->ring_obj = NULL;
  4713. r600_ring_init(rdev, ring, 256 * 1024);
  4714. ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
  4715. ring->ring_obj = NULL;
  4716. r600_ring_init(rdev, ring, 256 * 1024);
  4717. r = radeon_uvd_init(rdev);
  4718. if (!r) {
  4719. ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
  4720. ring->ring_obj = NULL;
  4721. r600_ring_init(rdev, ring, 4096);
  4722. }
  4723. rdev->ih.ring_obj = NULL;
  4724. r600_ih_ring_init(rdev, 64 * 1024);
  4725. r = r600_pcie_gart_init(rdev);
  4726. if (r)
  4727. return r;
  4728. rdev->accel_working = true;
  4729. r = cik_startup(rdev);
  4730. if (r) {
  4731. dev_err(rdev->dev, "disabling GPU acceleration\n");
  4732. cik_cp_fini(rdev);
  4733. cik_sdma_fini(rdev);
  4734. cik_irq_fini(rdev);
  4735. si_rlc_fini(rdev);
  4736. radeon_wb_fini(rdev);
  4737. radeon_ib_pool_fini(rdev);
  4738. radeon_vm_manager_fini(rdev);
  4739. radeon_irq_kms_fini(rdev);
  4740. cik_pcie_gart_fini(rdev);
  4741. rdev->accel_working = false;
  4742. }
  4743. /* Don't start up if the MC ucode is missing.
  4744. * The default clocks and voltages before the MC ucode
  4745. * is loaded are not suffient for advanced operations.
  4746. */
  4747. if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
  4748. DRM_ERROR("radeon: MC ucode required for NI+.\n");
  4749. return -EINVAL;
  4750. }
  4751. return 0;
  4752. }
  4753. /**
  4754. * cik_fini - asic specific driver and hw fini
  4755. *
  4756. * @rdev: radeon_device pointer
  4757. *
  4758. * Tear down the asic specific driver variables and program the hw
  4759. * to an idle state (CIK).
  4760. * Called at driver unload.
  4761. */
  4762. void cik_fini(struct radeon_device *rdev)
  4763. {
  4764. cik_cp_fini(rdev);
  4765. cik_sdma_fini(rdev);
  4766. cik_irq_fini(rdev);
  4767. si_rlc_fini(rdev);
  4768. radeon_wb_fini(rdev);
  4769. radeon_vm_manager_fini(rdev);
  4770. radeon_ib_pool_fini(rdev);
  4771. radeon_irq_kms_fini(rdev);
  4772. radeon_uvd_fini(rdev);
  4773. cik_pcie_gart_fini(rdev);
  4774. r600_vram_scratch_fini(rdev);
  4775. radeon_gem_fini(rdev);
  4776. radeon_fence_driver_fini(rdev);
  4777. radeon_bo_fini(rdev);
  4778. radeon_atombios_fini(rdev);
  4779. kfree(rdev->bios);
  4780. rdev->bios = NULL;
  4781. }
  4782. /* display watermark setup */
  4783. /**
  4784. * dce8_line_buffer_adjust - Set up the line buffer
  4785. *
  4786. * @rdev: radeon_device pointer
  4787. * @radeon_crtc: the selected display controller
  4788. * @mode: the current display mode on the selected display
  4789. * controller
  4790. *
  4791. * Setup up the line buffer allocation for
  4792. * the selected display controller (CIK).
  4793. * Returns the line buffer size in pixels.
  4794. */
  4795. static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
  4796. struct radeon_crtc *radeon_crtc,
  4797. struct drm_display_mode *mode)
  4798. {
  4799. u32 tmp;
  4800. /*
  4801. * Line Buffer Setup
  4802. * There are 6 line buffers, one for each display controllers.
  4803. * There are 3 partitions per LB. Select the number of partitions
  4804. * to enable based on the display width. For display widths larger
  4805. * than 4096, you need use to use 2 display controllers and combine
  4806. * them using the stereo blender.
  4807. */
  4808. if (radeon_crtc->base.enabled && mode) {
  4809. if (mode->crtc_hdisplay < 1920)
  4810. tmp = 1;
  4811. else if (mode->crtc_hdisplay < 2560)
  4812. tmp = 2;
  4813. else if (mode->crtc_hdisplay < 4096)
  4814. tmp = 0;
  4815. else {
  4816. DRM_DEBUG_KMS("Mode too big for LB!\n");
  4817. tmp = 0;
  4818. }
  4819. } else
  4820. tmp = 1;
  4821. WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
  4822. LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
  4823. if (radeon_crtc->base.enabled && mode) {
  4824. switch (tmp) {
  4825. case 0:
  4826. default:
  4827. return 4096 * 2;
  4828. case 1:
  4829. return 1920 * 2;
  4830. case 2:
  4831. return 2560 * 2;
  4832. }
  4833. }
  4834. /* controller not enabled, so no lb used */
  4835. return 0;
  4836. }
  4837. /**
  4838. * cik_get_number_of_dram_channels - get the number of dram channels
  4839. *
  4840. * @rdev: radeon_device pointer
  4841. *
  4842. * Look up the number of video ram channels (CIK).
  4843. * Used for display watermark bandwidth calculations
  4844. * Returns the number of dram channels
  4845. */
  4846. static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
  4847. {
  4848. u32 tmp = RREG32(MC_SHARED_CHMAP);
  4849. switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
  4850. case 0:
  4851. default:
  4852. return 1;
  4853. case 1:
  4854. return 2;
  4855. case 2:
  4856. return 4;
  4857. case 3:
  4858. return 8;
  4859. case 4:
  4860. return 3;
  4861. case 5:
  4862. return 6;
  4863. case 6:
  4864. return 10;
  4865. case 7:
  4866. return 12;
  4867. case 8:
  4868. return 16;
  4869. }
  4870. }
  4871. struct dce8_wm_params {
  4872. u32 dram_channels; /* number of dram channels */
  4873. u32 yclk; /* bandwidth per dram data pin in kHz */
  4874. u32 sclk; /* engine clock in kHz */
  4875. u32 disp_clk; /* display clock in kHz */
  4876. u32 src_width; /* viewport width */
  4877. u32 active_time; /* active display time in ns */
  4878. u32 blank_time; /* blank time in ns */
  4879. bool interlaced; /* mode is interlaced */
  4880. fixed20_12 vsc; /* vertical scale ratio */
  4881. u32 num_heads; /* number of active crtcs */
  4882. u32 bytes_per_pixel; /* bytes per pixel display + overlay */
  4883. u32 lb_size; /* line buffer allocated to pipe */
  4884. u32 vtaps; /* vertical scaler taps */
  4885. };
  4886. /**
  4887. * dce8_dram_bandwidth - get the dram bandwidth
  4888. *
  4889. * @wm: watermark calculation data
  4890. *
  4891. * Calculate the raw dram bandwidth (CIK).
  4892. * Used for display watermark bandwidth calculations
  4893. * Returns the dram bandwidth in MBytes/s
  4894. */
  4895. static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
  4896. {
  4897. /* Calculate raw DRAM Bandwidth */
  4898. fixed20_12 dram_efficiency; /* 0.7 */
  4899. fixed20_12 yclk, dram_channels, bandwidth;
  4900. fixed20_12 a;
  4901. a.full = dfixed_const(1000);
  4902. yclk.full = dfixed_const(wm->yclk);
  4903. yclk.full = dfixed_div(yclk, a);
  4904. dram_channels.full = dfixed_const(wm->dram_channels * 4);
  4905. a.full = dfixed_const(10);
  4906. dram_efficiency.full = dfixed_const(7);
  4907. dram_efficiency.full = dfixed_div(dram_efficiency, a);
  4908. bandwidth.full = dfixed_mul(dram_channels, yclk);
  4909. bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
  4910. return dfixed_trunc(bandwidth);
  4911. }
  4912. /**
  4913. * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
  4914. *
  4915. * @wm: watermark calculation data
  4916. *
  4917. * Calculate the dram bandwidth used for display (CIK).
  4918. * Used for display watermark bandwidth calculations
  4919. * Returns the dram bandwidth for display in MBytes/s
  4920. */
  4921. static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
  4922. {
  4923. /* Calculate DRAM Bandwidth and the part allocated to display. */
  4924. fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
  4925. fixed20_12 yclk, dram_channels, bandwidth;
  4926. fixed20_12 a;
  4927. a.full = dfixed_const(1000);
  4928. yclk.full = dfixed_const(wm->yclk);
  4929. yclk.full = dfixed_div(yclk, a);
  4930. dram_channels.full = dfixed_const(wm->dram_channels * 4);
  4931. a.full = dfixed_const(10);
  4932. disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
  4933. disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
  4934. bandwidth.full = dfixed_mul(dram_channels, yclk);
  4935. bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
  4936. return dfixed_trunc(bandwidth);
  4937. }
  4938. /**
  4939. * dce8_data_return_bandwidth - get the data return bandwidth
  4940. *
  4941. * @wm: watermark calculation data
  4942. *
  4943. * Calculate the data return bandwidth used for display (CIK).
  4944. * Used for display watermark bandwidth calculations
  4945. * Returns the data return bandwidth in MBytes/s
  4946. */
  4947. static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
  4948. {
  4949. /* Calculate the display Data return Bandwidth */
  4950. fixed20_12 return_efficiency; /* 0.8 */
  4951. fixed20_12 sclk, bandwidth;
  4952. fixed20_12 a;
  4953. a.full = dfixed_const(1000);
  4954. sclk.full = dfixed_const(wm->sclk);
  4955. sclk.full = dfixed_div(sclk, a);
  4956. a.full = dfixed_const(10);
  4957. return_efficiency.full = dfixed_const(8);
  4958. return_efficiency.full = dfixed_div(return_efficiency, a);
  4959. a.full = dfixed_const(32);
  4960. bandwidth.full = dfixed_mul(a, sclk);
  4961. bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
  4962. return dfixed_trunc(bandwidth);
  4963. }
  4964. /**
  4965. * dce8_dmif_request_bandwidth - get the dmif bandwidth
  4966. *
  4967. * @wm: watermark calculation data
  4968. *
  4969. * Calculate the dmif bandwidth used for display (CIK).
  4970. * Used for display watermark bandwidth calculations
  4971. * Returns the dmif bandwidth in MBytes/s
  4972. */
  4973. static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
  4974. {
  4975. /* Calculate the DMIF Request Bandwidth */
  4976. fixed20_12 disp_clk_request_efficiency; /* 0.8 */
  4977. fixed20_12 disp_clk, bandwidth;
  4978. fixed20_12 a, b;
  4979. a.full = dfixed_const(1000);
  4980. disp_clk.full = dfixed_const(wm->disp_clk);
  4981. disp_clk.full = dfixed_div(disp_clk, a);
  4982. a.full = dfixed_const(32);
  4983. b.full = dfixed_mul(a, disp_clk);
  4984. a.full = dfixed_const(10);
  4985. disp_clk_request_efficiency.full = dfixed_const(8);
  4986. disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
  4987. bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
  4988. return dfixed_trunc(bandwidth);
  4989. }
  4990. /**
  4991. * dce8_available_bandwidth - get the min available bandwidth
  4992. *
  4993. * @wm: watermark calculation data
  4994. *
  4995. * Calculate the min available bandwidth used for display (CIK).
  4996. * Used for display watermark bandwidth calculations
  4997. * Returns the min available bandwidth in MBytes/s
  4998. */
  4999. static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
  5000. {
  5001. /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
  5002. u32 dram_bandwidth = dce8_dram_bandwidth(wm);
  5003. u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
  5004. u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
  5005. return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
  5006. }
  5007. /**
  5008. * dce8_average_bandwidth - get the average available bandwidth
  5009. *
  5010. * @wm: watermark calculation data
  5011. *
  5012. * Calculate the average available bandwidth used for display (CIK).
  5013. * Used for display watermark bandwidth calculations
  5014. * Returns the average available bandwidth in MBytes/s
  5015. */
  5016. static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
  5017. {
  5018. /* Calculate the display mode Average Bandwidth
  5019. * DisplayMode should contain the source and destination dimensions,
  5020. * timing, etc.
  5021. */
  5022. fixed20_12 bpp;
  5023. fixed20_12 line_time;
  5024. fixed20_12 src_width;
  5025. fixed20_12 bandwidth;
  5026. fixed20_12 a;
  5027. a.full = dfixed_const(1000);
  5028. line_time.full = dfixed_const(wm->active_time + wm->blank_time);
  5029. line_time.full = dfixed_div(line_time, a);
  5030. bpp.full = dfixed_const(wm->bytes_per_pixel);
  5031. src_width.full = dfixed_const(wm->src_width);
  5032. bandwidth.full = dfixed_mul(src_width, bpp);
  5033. bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
  5034. bandwidth.full = dfixed_div(bandwidth, line_time);
  5035. return dfixed_trunc(bandwidth);
  5036. }
  5037. /**
  5038. * dce8_latency_watermark - get the latency watermark
  5039. *
  5040. * @wm: watermark calculation data
  5041. *
  5042. * Calculate the latency watermark (CIK).
  5043. * Used for display watermark bandwidth calculations
  5044. * Returns the latency watermark in ns
  5045. */
  5046. static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
  5047. {
  5048. /* First calculate the latency in ns */
  5049. u32 mc_latency = 2000; /* 2000 ns. */
  5050. u32 available_bandwidth = dce8_available_bandwidth(wm);
  5051. u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
  5052. u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
  5053. u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
  5054. u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
  5055. (wm->num_heads * cursor_line_pair_return_time);
  5056. u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
  5057. u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
  5058. u32 tmp, dmif_size = 12288;
  5059. fixed20_12 a, b, c;
  5060. if (wm->num_heads == 0)
  5061. return 0;
  5062. a.full = dfixed_const(2);
  5063. b.full = dfixed_const(1);
  5064. if ((wm->vsc.full > a.full) ||
  5065. ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
  5066. (wm->vtaps >= 5) ||
  5067. ((wm->vsc.full >= a.full) && wm->interlaced))
  5068. max_src_lines_per_dst_line = 4;
  5069. else
  5070. max_src_lines_per_dst_line = 2;
  5071. a.full = dfixed_const(available_bandwidth);
  5072. b.full = dfixed_const(wm->num_heads);
  5073. a.full = dfixed_div(a, b);
  5074. b.full = dfixed_const(mc_latency + 512);
  5075. c.full = dfixed_const(wm->disp_clk);
  5076. b.full = dfixed_div(b, c);
  5077. c.full = dfixed_const(dmif_size);
  5078. b.full = dfixed_div(c, b);
  5079. tmp = min(dfixed_trunc(a), dfixed_trunc(b));
  5080. b.full = dfixed_const(1000);
  5081. c.full = dfixed_const(wm->disp_clk);
  5082. b.full = dfixed_div(c, b);
  5083. c.full = dfixed_const(wm->bytes_per_pixel);
  5084. b.full = dfixed_mul(b, c);
  5085. lb_fill_bw = min(tmp, dfixed_trunc(b));
  5086. a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
  5087. b.full = dfixed_const(1000);
  5088. c.full = dfixed_const(lb_fill_bw);
  5089. b.full = dfixed_div(c, b);
  5090. a.full = dfixed_div(a, b);
  5091. line_fill_time = dfixed_trunc(a);
  5092. if (line_fill_time < wm->active_time)
  5093. return latency;
  5094. else
  5095. return latency + (line_fill_time - wm->active_time);
  5096. }
  5097. /**
  5098. * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
  5099. * average and available dram bandwidth
  5100. *
  5101. * @wm: watermark calculation data
  5102. *
  5103. * Check if the display average bandwidth fits in the display
  5104. * dram bandwidth (CIK).
  5105. * Used for display watermark bandwidth calculations
  5106. * Returns true if the display fits, false if not.
  5107. */
  5108. static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
  5109. {
  5110. if (dce8_average_bandwidth(wm) <=
  5111. (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
  5112. return true;
  5113. else
  5114. return false;
  5115. }
  5116. /**
  5117. * dce8_average_bandwidth_vs_available_bandwidth - check
  5118. * average and available bandwidth
  5119. *
  5120. * @wm: watermark calculation data
  5121. *
  5122. * Check if the display average bandwidth fits in the display
  5123. * available bandwidth (CIK).
  5124. * Used for display watermark bandwidth calculations
  5125. * Returns true if the display fits, false if not.
  5126. */
  5127. static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
  5128. {
  5129. if (dce8_average_bandwidth(wm) <=
  5130. (dce8_available_bandwidth(wm) / wm->num_heads))
  5131. return true;
  5132. else
  5133. return false;
  5134. }
  5135. /**
  5136. * dce8_check_latency_hiding - check latency hiding
  5137. *
  5138. * @wm: watermark calculation data
  5139. *
  5140. * Check latency hiding (CIK).
  5141. * Used for display watermark bandwidth calculations
  5142. * Returns true if the display fits, false if not.
  5143. */
  5144. static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
  5145. {
  5146. u32 lb_partitions = wm->lb_size / wm->src_width;
  5147. u32 line_time = wm->active_time + wm->blank_time;
  5148. u32 latency_tolerant_lines;
  5149. u32 latency_hiding;
  5150. fixed20_12 a;
  5151. a.full = dfixed_const(1);
  5152. if (wm->vsc.full > a.full)
  5153. latency_tolerant_lines = 1;
  5154. else {
  5155. if (lb_partitions <= (wm->vtaps + 1))
  5156. latency_tolerant_lines = 1;
  5157. else
  5158. latency_tolerant_lines = 2;
  5159. }
  5160. latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
  5161. if (dce8_latency_watermark(wm) <= latency_hiding)
  5162. return true;
  5163. else
  5164. return false;
  5165. }
  5166. /**
  5167. * dce8_program_watermarks - program display watermarks
  5168. *
  5169. * @rdev: radeon_device pointer
  5170. * @radeon_crtc: the selected display controller
  5171. * @lb_size: line buffer size
  5172. * @num_heads: number of display controllers in use
  5173. *
  5174. * Calculate and program the display watermarks for the
  5175. * selected display controller (CIK).
  5176. */
  5177. static void dce8_program_watermarks(struct radeon_device *rdev,
  5178. struct radeon_crtc *radeon_crtc,
  5179. u32 lb_size, u32 num_heads)
  5180. {
  5181. struct drm_display_mode *mode = &radeon_crtc->base.mode;
  5182. struct dce8_wm_params wm;
  5183. u32 pixel_period;
  5184. u32 line_time = 0;
  5185. u32 latency_watermark_a = 0, latency_watermark_b = 0;
  5186. u32 tmp, wm_mask;
  5187. if (radeon_crtc->base.enabled && num_heads && mode) {
  5188. pixel_period = 1000000 / (u32)mode->clock;
  5189. line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
  5190. wm.yclk = rdev->pm.current_mclk * 10;
  5191. wm.sclk = rdev->pm.current_sclk * 10;
  5192. wm.disp_clk = mode->clock;
  5193. wm.src_width = mode->crtc_hdisplay;
  5194. wm.active_time = mode->crtc_hdisplay * pixel_period;
  5195. wm.blank_time = line_time - wm.active_time;
  5196. wm.interlaced = false;
  5197. if (mode->flags & DRM_MODE_FLAG_INTERLACE)
  5198. wm.interlaced = true;
  5199. wm.vsc = radeon_crtc->vsc;
  5200. wm.vtaps = 1;
  5201. if (radeon_crtc->rmx_type != RMX_OFF)
  5202. wm.vtaps = 2;
  5203. wm.bytes_per_pixel = 4; /* XXX: get this from fb config */
  5204. wm.lb_size = lb_size;
  5205. wm.dram_channels = cik_get_number_of_dram_channels(rdev);
  5206. wm.num_heads = num_heads;
  5207. /* set for high clocks */
  5208. latency_watermark_a = min(dce8_latency_watermark(&wm), (u32)65535);
  5209. /* set for low clocks */
  5210. /* wm.yclk = low clk; wm.sclk = low clk */
  5211. latency_watermark_b = min(dce8_latency_watermark(&wm), (u32)65535);
  5212. /* possibly force display priority to high */
  5213. /* should really do this at mode validation time... */
  5214. if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm) ||
  5215. !dce8_average_bandwidth_vs_available_bandwidth(&wm) ||
  5216. !dce8_check_latency_hiding(&wm) ||
  5217. (rdev->disp_priority == 2)) {
  5218. DRM_DEBUG_KMS("force priority to high\n");
  5219. }
  5220. }
  5221. /* select wm A */
  5222. wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
  5223. tmp = wm_mask;
  5224. tmp &= ~LATENCY_WATERMARK_MASK(3);
  5225. tmp |= LATENCY_WATERMARK_MASK(1);
  5226. WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
  5227. WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
  5228. (LATENCY_LOW_WATERMARK(latency_watermark_a) |
  5229. LATENCY_HIGH_WATERMARK(line_time)));
  5230. /* select wm B */
  5231. tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
  5232. tmp &= ~LATENCY_WATERMARK_MASK(3);
  5233. tmp |= LATENCY_WATERMARK_MASK(2);
  5234. WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
  5235. WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
  5236. (LATENCY_LOW_WATERMARK(latency_watermark_b) |
  5237. LATENCY_HIGH_WATERMARK(line_time)));
  5238. /* restore original selection */
  5239. WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
  5240. }
  5241. /**
  5242. * dce8_bandwidth_update - program display watermarks
  5243. *
  5244. * @rdev: radeon_device pointer
  5245. *
  5246. * Calculate and program the display watermarks and line
  5247. * buffer allocation (CIK).
  5248. */
  5249. void dce8_bandwidth_update(struct radeon_device *rdev)
  5250. {
  5251. struct drm_display_mode *mode = NULL;
  5252. u32 num_heads = 0, lb_size;
  5253. int i;
  5254. radeon_update_display_priority(rdev);
  5255. for (i = 0; i < rdev->num_crtc; i++) {
  5256. if (rdev->mode_info.crtcs[i]->base.enabled)
  5257. num_heads++;
  5258. }
  5259. for (i = 0; i < rdev->num_crtc; i++) {
  5260. mode = &rdev->mode_info.crtcs[i]->base.mode;
  5261. lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
  5262. dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
  5263. }
  5264. }
  5265. /**
  5266. * cik_get_gpu_clock_counter - return GPU clock counter snapshot
  5267. *
  5268. * @rdev: radeon_device pointer
  5269. *
  5270. * Fetches a GPU clock counter snapshot (SI).
  5271. * Returns the 64 bit clock counter snapshot.
  5272. */
  5273. uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
  5274. {
  5275. uint64_t clock;
  5276. mutex_lock(&rdev->gpu_clock_mutex);
  5277. WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
  5278. clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
  5279. ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
  5280. mutex_unlock(&rdev->gpu_clock_mutex);
  5281. return clock;
  5282. }
  5283. static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
  5284. u32 cntl_reg, u32 status_reg)
  5285. {
  5286. int r, i;
  5287. struct atom_clock_dividers dividers;
  5288. uint32_t tmp;
  5289. r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
  5290. clock, false, &dividers);
  5291. if (r)
  5292. return r;
  5293. tmp = RREG32_SMC(cntl_reg);
  5294. tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
  5295. tmp |= dividers.post_divider;
  5296. WREG32_SMC(cntl_reg, tmp);
  5297. for (i = 0; i < 100; i++) {
  5298. if (RREG32_SMC(status_reg) & DCLK_STATUS)
  5299. break;
  5300. mdelay(10);
  5301. }
  5302. if (i == 100)
  5303. return -ETIMEDOUT;
  5304. return 0;
  5305. }
  5306. int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
  5307. {
  5308. int r = 0;
  5309. r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
  5310. if (r)
  5311. return r;
  5312. r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
  5313. return r;
  5314. }
  5315. int cik_uvd_resume(struct radeon_device *rdev)
  5316. {
  5317. uint64_t addr;
  5318. uint32_t size;
  5319. int r;
  5320. r = radeon_uvd_resume(rdev);
  5321. if (r)
  5322. return r;
  5323. /* programm the VCPU memory controller bits 0-27 */
  5324. addr = rdev->uvd.gpu_addr >> 3;
  5325. size = RADEON_GPU_PAGE_ALIGN(rdev->uvd_fw->size + 4) >> 3;
  5326. WREG32(UVD_VCPU_CACHE_OFFSET0, addr);
  5327. WREG32(UVD_VCPU_CACHE_SIZE0, size);
  5328. addr += size;
  5329. size = RADEON_UVD_STACK_SIZE >> 3;
  5330. WREG32(UVD_VCPU_CACHE_OFFSET1, addr);
  5331. WREG32(UVD_VCPU_CACHE_SIZE1, size);
  5332. addr += size;
  5333. size = RADEON_UVD_HEAP_SIZE >> 3;
  5334. WREG32(UVD_VCPU_CACHE_OFFSET2, addr);
  5335. WREG32(UVD_VCPU_CACHE_SIZE2, size);
  5336. /* bits 28-31 */
  5337. addr = (rdev->uvd.gpu_addr >> 28) & 0xF;
  5338. WREG32(UVD_LMI_ADDR_EXT, (addr << 12) | (addr << 0));
  5339. /* bits 32-39 */
  5340. addr = (rdev->uvd.gpu_addr >> 32) & 0xFF;
  5341. WREG32(UVD_LMI_EXT40_ADDR, addr | (0x9 << 16) | (0x1 << 31));
  5342. return 0;
  5343. }