i915_gem.c 132 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124412541264127412841294130413141324133413441354136413741384139414041414142414341444145414641474148414941504151415241534154415541564157415841594160416141624163416441654166416741684169417041714172417341744175417641774178417941804181418241834184418541864187418841894190419141924193419441954196419741984199420042014202420342044205420642074208420942104211421242134214421542164217421842194220422142224223422442254226422742284229423042314232423342344235423642374238423942404241424242434244424542464247424842494250425142524253425442554256425742584259426042614262426342644265426642674268426942704271427242734274427542764277427842794280428142824283428442854286428742884289429042914292429342944295429642974298429943004301430243034304430543064307430843094310431143124313431443154316431743184319432043214322432343244325432643274328432943304331433243334334433543364337433843394340434143424343434443454346434743484349435043514352435343544355435643574358435943604361436243634364436543664367436843694370437143724373437443754376437743784379438043814382438343844385438643874388438943904391439243934394439543964397439843994400440144024403440444054406440744084409441044114412441344144415441644174418441944204421442244234424442544264427442844294430443144324433443444354436443744384439444044414442444344444445444644474448444944504451445244534454445544564457445844594460446144624463446444654466446744684469447044714472447344744475447644774478447944804481448244834484448544864487448844894490449144924493449444954496449744984499450045014502450345044505450645074508450945104511451245134514451545164517451845194520452145224523452445254526452745284529453045314532453345344535453645374538453945404541454245434544454545464547454845494550455145524553455445554556455745584559456045614562456345644565456645674568456945704571457245734574457545764577457845794580458145824583458445854586458745884589459045914592459345944595459645974598459946004601460246034604460546064607460846094610461146124613461446154616461746184619462046214622462346244625462646274628462946304631463246334634463546364637463846394640464146424643464446454646464746484649465046514652465346544655465646574658465946604661466246634664466546664667466846694670467146724673467446754676467746784679468046814682468346844685468646874688468946904691469246934694469546964697469846994700470147024703470447054706470747084709471047114712471347144715471647174718471947204721472247234724472547264727472847294730473147324733473447354736473747384739474047414742474347444745474647474748474947504751475247534754475547564757475847594760476147624763476447654766476747684769477047714772477347744775477647774778477947804781478247834784478547864787478847894790479147924793479447954796479747984799480048014802480348044805480648074808480948104811481248134814481548164817481848194820482148224823482448254826482748284829483048314832483348344835483648374838483948404841484248434844484548464847484848494850485148524853485448554856485748584859486048614862486348644865486648674868486948704871487248734874487548764877487848794880488148824883488448854886488748884889489048914892489348944895489648974898489949004901490249034904490549064907490849094910491149124913491449154916491749184919492049214922492349244925492649274928492949304931493249334934493549364937493849394940494149424943494449454946494749484949495049514952495349544955495649574958495949604961496249634964496549664967496849694970497149724973497449754976497749784979498049814982498349844985498649874988498949904991499249934994499549964997499849995000500150025003500450055006500750085009501050115012501350145015501650175018501950205021502250235024502550265027
  1. /*
  2. * Copyright © 2008 Intel Corporation
  3. *
  4. * Permission is hereby granted, free of charge, to any person obtaining a
  5. * copy of this software and associated documentation files (the "Software"),
  6. * to deal in the Software without restriction, including without limitation
  7. * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8. * and/or sell copies of the Software, and to permit persons to whom the
  9. * Software is furnished to do so, subject to the following conditions:
  10. *
  11. * The above copyright notice and this permission notice (including the next
  12. * paragraph) shall be included in all copies or substantial portions of the
  13. * Software.
  14. *
  15. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17. * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  18. * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19. * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20. * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21. * IN THE SOFTWARE.
  22. *
  23. * Authors:
  24. * Eric Anholt <eric@anholt.net>
  25. *
  26. */
  27. #include "drmP.h"
  28. #include "drm.h"
  29. #include "i915_drm.h"
  30. #include "i915_drv.h"
  31. #include "i915_trace.h"
  32. #include "intel_drv.h"
  33. #include <linux/slab.h>
  34. #include <linux/swap.h>
  35. #include <linux/pci.h>
  36. static uint32_t i915_gem_get_gtt_alignment(struct drm_gem_object *obj);
  37. static int i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj);
  38. static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
  39. static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
  40. static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj,
  41. int write);
  42. static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
  43. uint64_t offset,
  44. uint64_t size);
  45. static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj);
  46. static int i915_gem_object_wait_rendering(struct drm_gem_object *obj);
  47. static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
  48. unsigned alignment);
  49. static void i915_gem_clear_fence_reg(struct drm_gem_object *obj);
  50. static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
  51. struct drm_i915_gem_pwrite *args,
  52. struct drm_file *file_priv);
  53. static void i915_gem_free_object_tail(struct drm_gem_object *obj);
  54. static LIST_HEAD(shrink_list);
  55. static DEFINE_SPINLOCK(shrink_list_lock);
  56. static inline bool
  57. i915_gem_object_is_inactive(struct drm_i915_gem_object *obj_priv)
  58. {
  59. return obj_priv->gtt_space &&
  60. !obj_priv->active &&
  61. obj_priv->pin_count == 0;
  62. }
  63. int i915_gem_do_init(struct drm_device *dev, unsigned long start,
  64. unsigned long end)
  65. {
  66. drm_i915_private_t *dev_priv = dev->dev_private;
  67. if (start >= end ||
  68. (start & (PAGE_SIZE - 1)) != 0 ||
  69. (end & (PAGE_SIZE - 1)) != 0) {
  70. return -EINVAL;
  71. }
  72. drm_mm_init(&dev_priv->mm.gtt_space, start,
  73. end - start);
  74. dev->gtt_total = (uint32_t) (end - start);
  75. return 0;
  76. }
  77. int
  78. i915_gem_init_ioctl(struct drm_device *dev, void *data,
  79. struct drm_file *file_priv)
  80. {
  81. struct drm_i915_gem_init *args = data;
  82. int ret;
  83. mutex_lock(&dev->struct_mutex);
  84. ret = i915_gem_do_init(dev, args->gtt_start, args->gtt_end);
  85. mutex_unlock(&dev->struct_mutex);
  86. return ret;
  87. }
  88. int
  89. i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
  90. struct drm_file *file_priv)
  91. {
  92. struct drm_i915_gem_get_aperture *args = data;
  93. if (!(dev->driver->driver_features & DRIVER_GEM))
  94. return -ENODEV;
  95. args->aper_size = dev->gtt_total;
  96. args->aper_available_size = (args->aper_size -
  97. atomic_read(&dev->pin_memory));
  98. return 0;
  99. }
  100. /**
  101. * Creates a new mm object and returns a handle to it.
  102. */
  103. int
  104. i915_gem_create_ioctl(struct drm_device *dev, void *data,
  105. struct drm_file *file_priv)
  106. {
  107. struct drm_i915_gem_create *args = data;
  108. struct drm_gem_object *obj;
  109. int ret;
  110. u32 handle;
  111. args->size = roundup(args->size, PAGE_SIZE);
  112. /* Allocate the new object */
  113. obj = i915_gem_alloc_object(dev, args->size);
  114. if (obj == NULL)
  115. return -ENOMEM;
  116. ret = drm_gem_handle_create(file_priv, obj, &handle);
  117. if (ret) {
  118. drm_gem_object_unreference_unlocked(obj);
  119. return ret;
  120. }
  121. /* Sink the floating reference from kref_init(handlecount) */
  122. drm_gem_object_handle_unreference_unlocked(obj);
  123. args->handle = handle;
  124. return 0;
  125. }
  126. static inline int
  127. fast_shmem_read(struct page **pages,
  128. loff_t page_base, int page_offset,
  129. char __user *data,
  130. int length)
  131. {
  132. char __iomem *vaddr;
  133. int unwritten;
  134. vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
  135. if (vaddr == NULL)
  136. return -ENOMEM;
  137. unwritten = __copy_to_user_inatomic(data, vaddr + page_offset, length);
  138. kunmap_atomic(vaddr, KM_USER0);
  139. if (unwritten)
  140. return -EFAULT;
  141. return 0;
  142. }
  143. static int i915_gem_object_needs_bit17_swizzle(struct drm_gem_object *obj)
  144. {
  145. drm_i915_private_t *dev_priv = obj->dev->dev_private;
  146. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  147. return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
  148. obj_priv->tiling_mode != I915_TILING_NONE;
  149. }
  150. static inline void
  151. slow_shmem_copy(struct page *dst_page,
  152. int dst_offset,
  153. struct page *src_page,
  154. int src_offset,
  155. int length)
  156. {
  157. char *dst_vaddr, *src_vaddr;
  158. dst_vaddr = kmap(dst_page);
  159. src_vaddr = kmap(src_page);
  160. memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);
  161. kunmap(src_page);
  162. kunmap(dst_page);
  163. }
  164. static inline void
  165. slow_shmem_bit17_copy(struct page *gpu_page,
  166. int gpu_offset,
  167. struct page *cpu_page,
  168. int cpu_offset,
  169. int length,
  170. int is_read)
  171. {
  172. char *gpu_vaddr, *cpu_vaddr;
  173. /* Use the unswizzled path if this page isn't affected. */
  174. if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
  175. if (is_read)
  176. return slow_shmem_copy(cpu_page, cpu_offset,
  177. gpu_page, gpu_offset, length);
  178. else
  179. return slow_shmem_copy(gpu_page, gpu_offset,
  180. cpu_page, cpu_offset, length);
  181. }
  182. gpu_vaddr = kmap(gpu_page);
  183. cpu_vaddr = kmap(cpu_page);
  184. /* Copy the data, XORing A6 with A17 (1). The user already knows he's
  185. * XORing with the other bits (A9 for Y, A9 and A10 for X)
  186. */
  187. while (length > 0) {
  188. int cacheline_end = ALIGN(gpu_offset + 1, 64);
  189. int this_length = min(cacheline_end - gpu_offset, length);
  190. int swizzled_gpu_offset = gpu_offset ^ 64;
  191. if (is_read) {
  192. memcpy(cpu_vaddr + cpu_offset,
  193. gpu_vaddr + swizzled_gpu_offset,
  194. this_length);
  195. } else {
  196. memcpy(gpu_vaddr + swizzled_gpu_offset,
  197. cpu_vaddr + cpu_offset,
  198. this_length);
  199. }
  200. cpu_offset += this_length;
  201. gpu_offset += this_length;
  202. length -= this_length;
  203. }
  204. kunmap(cpu_page);
  205. kunmap(gpu_page);
  206. }
  207. /**
  208. * This is the fast shmem pread path, which attempts to copy_from_user directly
  209. * from the backing pages of the object to the user's address space. On a
  210. * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
  211. */
  212. static int
  213. i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj,
  214. struct drm_i915_gem_pread *args,
  215. struct drm_file *file_priv)
  216. {
  217. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  218. ssize_t remain;
  219. loff_t offset, page_base;
  220. char __user *user_data;
  221. int page_offset, page_length;
  222. int ret;
  223. user_data = (char __user *) (uintptr_t) args->data_ptr;
  224. remain = args->size;
  225. mutex_lock(&dev->struct_mutex);
  226. ret = i915_gem_object_get_pages(obj, 0);
  227. if (ret != 0)
  228. goto fail_unlock;
  229. ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
  230. args->size);
  231. if (ret != 0)
  232. goto fail_put_pages;
  233. obj_priv = to_intel_bo(obj);
  234. offset = args->offset;
  235. while (remain > 0) {
  236. /* Operation in this page
  237. *
  238. * page_base = page offset within aperture
  239. * page_offset = offset within page
  240. * page_length = bytes to copy for this page
  241. */
  242. page_base = (offset & ~(PAGE_SIZE-1));
  243. page_offset = offset & (PAGE_SIZE-1);
  244. page_length = remain;
  245. if ((page_offset + remain) > PAGE_SIZE)
  246. page_length = PAGE_SIZE - page_offset;
  247. ret = fast_shmem_read(obj_priv->pages,
  248. page_base, page_offset,
  249. user_data, page_length);
  250. if (ret)
  251. goto fail_put_pages;
  252. remain -= page_length;
  253. user_data += page_length;
  254. offset += page_length;
  255. }
  256. fail_put_pages:
  257. i915_gem_object_put_pages(obj);
  258. fail_unlock:
  259. mutex_unlock(&dev->struct_mutex);
  260. return ret;
  261. }
  262. static int
  263. i915_gem_object_get_pages_or_evict(struct drm_gem_object *obj)
  264. {
  265. int ret;
  266. ret = i915_gem_object_get_pages(obj, __GFP_NORETRY | __GFP_NOWARN);
  267. /* If we've insufficient memory to map in the pages, attempt
  268. * to make some space by throwing out some old buffers.
  269. */
  270. if (ret == -ENOMEM) {
  271. struct drm_device *dev = obj->dev;
  272. ret = i915_gem_evict_something(dev, obj->size,
  273. i915_gem_get_gtt_alignment(obj));
  274. if (ret)
  275. return ret;
  276. ret = i915_gem_object_get_pages(obj, 0);
  277. }
  278. return ret;
  279. }
  280. /**
  281. * This is the fallback shmem pread path, which allocates temporary storage
  282. * in kernel space to copy_to_user into outside of the struct_mutex, so we
  283. * can copy out of the object's backing pages while holding the struct mutex
  284. * and not take page faults.
  285. */
  286. static int
  287. i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
  288. struct drm_i915_gem_pread *args,
  289. struct drm_file *file_priv)
  290. {
  291. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  292. struct mm_struct *mm = current->mm;
  293. struct page **user_pages;
  294. ssize_t remain;
  295. loff_t offset, pinned_pages, i;
  296. loff_t first_data_page, last_data_page, num_pages;
  297. int shmem_page_index, shmem_page_offset;
  298. int data_page_index, data_page_offset;
  299. int page_length;
  300. int ret;
  301. uint64_t data_ptr = args->data_ptr;
  302. int do_bit17_swizzling;
  303. remain = args->size;
  304. /* Pin the user pages containing the data. We can't fault while
  305. * holding the struct mutex, yet we want to hold it while
  306. * dereferencing the user data.
  307. */
  308. first_data_page = data_ptr / PAGE_SIZE;
  309. last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
  310. num_pages = last_data_page - first_data_page + 1;
  311. user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
  312. if (user_pages == NULL)
  313. return -ENOMEM;
  314. down_read(&mm->mmap_sem);
  315. pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
  316. num_pages, 1, 0, user_pages, NULL);
  317. up_read(&mm->mmap_sem);
  318. if (pinned_pages < num_pages) {
  319. ret = -EFAULT;
  320. goto fail_put_user_pages;
  321. }
  322. do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
  323. mutex_lock(&dev->struct_mutex);
  324. ret = i915_gem_object_get_pages_or_evict(obj);
  325. if (ret)
  326. goto fail_unlock;
  327. ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
  328. args->size);
  329. if (ret != 0)
  330. goto fail_put_pages;
  331. obj_priv = to_intel_bo(obj);
  332. offset = args->offset;
  333. while (remain > 0) {
  334. /* Operation in this page
  335. *
  336. * shmem_page_index = page number within shmem file
  337. * shmem_page_offset = offset within page in shmem file
  338. * data_page_index = page number in get_user_pages return
  339. * data_page_offset = offset with data_page_index page.
  340. * page_length = bytes to copy for this page
  341. */
  342. shmem_page_index = offset / PAGE_SIZE;
  343. shmem_page_offset = offset & ~PAGE_MASK;
  344. data_page_index = data_ptr / PAGE_SIZE - first_data_page;
  345. data_page_offset = data_ptr & ~PAGE_MASK;
  346. page_length = remain;
  347. if ((shmem_page_offset + page_length) > PAGE_SIZE)
  348. page_length = PAGE_SIZE - shmem_page_offset;
  349. if ((data_page_offset + page_length) > PAGE_SIZE)
  350. page_length = PAGE_SIZE - data_page_offset;
  351. if (do_bit17_swizzling) {
  352. slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
  353. shmem_page_offset,
  354. user_pages[data_page_index],
  355. data_page_offset,
  356. page_length,
  357. 1);
  358. } else {
  359. slow_shmem_copy(user_pages[data_page_index],
  360. data_page_offset,
  361. obj_priv->pages[shmem_page_index],
  362. shmem_page_offset,
  363. page_length);
  364. }
  365. remain -= page_length;
  366. data_ptr += page_length;
  367. offset += page_length;
  368. }
  369. fail_put_pages:
  370. i915_gem_object_put_pages(obj);
  371. fail_unlock:
  372. mutex_unlock(&dev->struct_mutex);
  373. fail_put_user_pages:
  374. for (i = 0; i < pinned_pages; i++) {
  375. SetPageDirty(user_pages[i]);
  376. page_cache_release(user_pages[i]);
  377. }
  378. drm_free_large(user_pages);
  379. return ret;
  380. }
  381. /**
  382. * Reads data from the object referenced by handle.
  383. *
  384. * On error, the contents of *data are undefined.
  385. */
  386. int
  387. i915_gem_pread_ioctl(struct drm_device *dev, void *data,
  388. struct drm_file *file_priv)
  389. {
  390. struct drm_i915_gem_pread *args = data;
  391. struct drm_gem_object *obj;
  392. struct drm_i915_gem_object *obj_priv;
  393. int ret;
  394. obj = drm_gem_object_lookup(dev, file_priv, args->handle);
  395. if (obj == NULL)
  396. return -ENOENT;
  397. obj_priv = to_intel_bo(obj);
  398. /* Bounds check source.
  399. *
  400. * XXX: This could use review for overflow issues...
  401. */
  402. if (args->offset > obj->size || args->size > obj->size ||
  403. args->offset + args->size > obj->size) {
  404. drm_gem_object_unreference_unlocked(obj);
  405. return -EINVAL;
  406. }
  407. if (i915_gem_object_needs_bit17_swizzle(obj)) {
  408. ret = i915_gem_shmem_pread_slow(dev, obj, args, file_priv);
  409. } else {
  410. ret = i915_gem_shmem_pread_fast(dev, obj, args, file_priv);
  411. if (ret != 0)
  412. ret = i915_gem_shmem_pread_slow(dev, obj, args,
  413. file_priv);
  414. }
  415. drm_gem_object_unreference_unlocked(obj);
  416. return ret;
  417. }
  418. /* This is the fast write path which cannot handle
  419. * page faults in the source data
  420. */
  421. static inline int
  422. fast_user_write(struct io_mapping *mapping,
  423. loff_t page_base, int page_offset,
  424. char __user *user_data,
  425. int length)
  426. {
  427. char *vaddr_atomic;
  428. unsigned long unwritten;
  429. vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base, KM_USER0);
  430. unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
  431. user_data, length);
  432. io_mapping_unmap_atomic(vaddr_atomic, KM_USER0);
  433. if (unwritten)
  434. return -EFAULT;
  435. return 0;
  436. }
  437. /* Here's the write path which can sleep for
  438. * page faults
  439. */
  440. static inline void
  441. slow_kernel_write(struct io_mapping *mapping,
  442. loff_t gtt_base, int gtt_offset,
  443. struct page *user_page, int user_offset,
  444. int length)
  445. {
  446. char __iomem *dst_vaddr;
  447. char *src_vaddr;
  448. dst_vaddr = io_mapping_map_wc(mapping, gtt_base);
  449. src_vaddr = kmap(user_page);
  450. memcpy_toio(dst_vaddr + gtt_offset,
  451. src_vaddr + user_offset,
  452. length);
  453. kunmap(user_page);
  454. io_mapping_unmap(dst_vaddr);
  455. }
  456. static inline int
  457. fast_shmem_write(struct page **pages,
  458. loff_t page_base, int page_offset,
  459. char __user *data,
  460. int length)
  461. {
  462. char __iomem *vaddr;
  463. unsigned long unwritten;
  464. vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
  465. if (vaddr == NULL)
  466. return -ENOMEM;
  467. unwritten = __copy_from_user_inatomic(vaddr + page_offset, data, length);
  468. kunmap_atomic(vaddr, KM_USER0);
  469. if (unwritten)
  470. return -EFAULT;
  471. return 0;
  472. }
  473. /**
  474. * This is the fast pwrite path, where we copy the data directly from the
  475. * user into the GTT, uncached.
  476. */
  477. static int
  478. i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
  479. struct drm_i915_gem_pwrite *args,
  480. struct drm_file *file_priv)
  481. {
  482. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  483. drm_i915_private_t *dev_priv = dev->dev_private;
  484. ssize_t remain;
  485. loff_t offset, page_base;
  486. char __user *user_data;
  487. int page_offset, page_length;
  488. int ret;
  489. user_data = (char __user *) (uintptr_t) args->data_ptr;
  490. remain = args->size;
  491. if (!access_ok(VERIFY_READ, user_data, remain))
  492. return -EFAULT;
  493. mutex_lock(&dev->struct_mutex);
  494. ret = i915_gem_object_pin(obj, 0);
  495. if (ret) {
  496. mutex_unlock(&dev->struct_mutex);
  497. return ret;
  498. }
  499. ret = i915_gem_object_set_to_gtt_domain(obj, 1);
  500. if (ret)
  501. goto fail;
  502. obj_priv = to_intel_bo(obj);
  503. offset = obj_priv->gtt_offset + args->offset;
  504. while (remain > 0) {
  505. /* Operation in this page
  506. *
  507. * page_base = page offset within aperture
  508. * page_offset = offset within page
  509. * page_length = bytes to copy for this page
  510. */
  511. page_base = (offset & ~(PAGE_SIZE-1));
  512. page_offset = offset & (PAGE_SIZE-1);
  513. page_length = remain;
  514. if ((page_offset + remain) > PAGE_SIZE)
  515. page_length = PAGE_SIZE - page_offset;
  516. ret = fast_user_write (dev_priv->mm.gtt_mapping, page_base,
  517. page_offset, user_data, page_length);
  518. /* If we get a fault while copying data, then (presumably) our
  519. * source page isn't available. Return the error and we'll
  520. * retry in the slow path.
  521. */
  522. if (ret)
  523. goto fail;
  524. remain -= page_length;
  525. user_data += page_length;
  526. offset += page_length;
  527. }
  528. fail:
  529. i915_gem_object_unpin(obj);
  530. mutex_unlock(&dev->struct_mutex);
  531. return ret;
  532. }
  533. /**
  534. * This is the fallback GTT pwrite path, which uses get_user_pages to pin
  535. * the memory and maps it using kmap_atomic for copying.
  536. *
  537. * This code resulted in x11perf -rgb10text consuming about 10% more CPU
  538. * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
  539. */
  540. static int
  541. i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
  542. struct drm_i915_gem_pwrite *args,
  543. struct drm_file *file_priv)
  544. {
  545. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  546. drm_i915_private_t *dev_priv = dev->dev_private;
  547. ssize_t remain;
  548. loff_t gtt_page_base, offset;
  549. loff_t first_data_page, last_data_page, num_pages;
  550. loff_t pinned_pages, i;
  551. struct page **user_pages;
  552. struct mm_struct *mm = current->mm;
  553. int gtt_page_offset, data_page_offset, data_page_index, page_length;
  554. int ret;
  555. uint64_t data_ptr = args->data_ptr;
  556. remain = args->size;
  557. /* Pin the user pages containing the data. We can't fault while
  558. * holding the struct mutex, and all of the pwrite implementations
  559. * want to hold it while dereferencing the user data.
  560. */
  561. first_data_page = data_ptr / PAGE_SIZE;
  562. last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
  563. num_pages = last_data_page - first_data_page + 1;
  564. user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
  565. if (user_pages == NULL)
  566. return -ENOMEM;
  567. down_read(&mm->mmap_sem);
  568. pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
  569. num_pages, 0, 0, user_pages, NULL);
  570. up_read(&mm->mmap_sem);
  571. if (pinned_pages < num_pages) {
  572. ret = -EFAULT;
  573. goto out_unpin_pages;
  574. }
  575. mutex_lock(&dev->struct_mutex);
  576. ret = i915_gem_object_pin(obj, 0);
  577. if (ret)
  578. goto out_unlock;
  579. ret = i915_gem_object_set_to_gtt_domain(obj, 1);
  580. if (ret)
  581. goto out_unpin_object;
  582. obj_priv = to_intel_bo(obj);
  583. offset = obj_priv->gtt_offset + args->offset;
  584. while (remain > 0) {
  585. /* Operation in this page
  586. *
  587. * gtt_page_base = page offset within aperture
  588. * gtt_page_offset = offset within page in aperture
  589. * data_page_index = page number in get_user_pages return
  590. * data_page_offset = offset with data_page_index page.
  591. * page_length = bytes to copy for this page
  592. */
  593. gtt_page_base = offset & PAGE_MASK;
  594. gtt_page_offset = offset & ~PAGE_MASK;
  595. data_page_index = data_ptr / PAGE_SIZE - first_data_page;
  596. data_page_offset = data_ptr & ~PAGE_MASK;
  597. page_length = remain;
  598. if ((gtt_page_offset + page_length) > PAGE_SIZE)
  599. page_length = PAGE_SIZE - gtt_page_offset;
  600. if ((data_page_offset + page_length) > PAGE_SIZE)
  601. page_length = PAGE_SIZE - data_page_offset;
  602. slow_kernel_write(dev_priv->mm.gtt_mapping,
  603. gtt_page_base, gtt_page_offset,
  604. user_pages[data_page_index],
  605. data_page_offset,
  606. page_length);
  607. remain -= page_length;
  608. offset += page_length;
  609. data_ptr += page_length;
  610. }
  611. out_unpin_object:
  612. i915_gem_object_unpin(obj);
  613. out_unlock:
  614. mutex_unlock(&dev->struct_mutex);
  615. out_unpin_pages:
  616. for (i = 0; i < pinned_pages; i++)
  617. page_cache_release(user_pages[i]);
  618. drm_free_large(user_pages);
  619. return ret;
  620. }
  621. /**
  622. * This is the fast shmem pwrite path, which attempts to directly
  623. * copy_from_user into the kmapped pages backing the object.
  624. */
  625. static int
  626. i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
  627. struct drm_i915_gem_pwrite *args,
  628. struct drm_file *file_priv)
  629. {
  630. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  631. ssize_t remain;
  632. loff_t offset, page_base;
  633. char __user *user_data;
  634. int page_offset, page_length;
  635. int ret;
  636. user_data = (char __user *) (uintptr_t) args->data_ptr;
  637. remain = args->size;
  638. mutex_lock(&dev->struct_mutex);
  639. ret = i915_gem_object_get_pages(obj, 0);
  640. if (ret != 0)
  641. goto fail_unlock;
  642. ret = i915_gem_object_set_to_cpu_domain(obj, 1);
  643. if (ret != 0)
  644. goto fail_put_pages;
  645. obj_priv = to_intel_bo(obj);
  646. offset = args->offset;
  647. obj_priv->dirty = 1;
  648. while (remain > 0) {
  649. /* Operation in this page
  650. *
  651. * page_base = page offset within aperture
  652. * page_offset = offset within page
  653. * page_length = bytes to copy for this page
  654. */
  655. page_base = (offset & ~(PAGE_SIZE-1));
  656. page_offset = offset & (PAGE_SIZE-1);
  657. page_length = remain;
  658. if ((page_offset + remain) > PAGE_SIZE)
  659. page_length = PAGE_SIZE - page_offset;
  660. ret = fast_shmem_write(obj_priv->pages,
  661. page_base, page_offset,
  662. user_data, page_length);
  663. if (ret)
  664. goto fail_put_pages;
  665. remain -= page_length;
  666. user_data += page_length;
  667. offset += page_length;
  668. }
  669. fail_put_pages:
  670. i915_gem_object_put_pages(obj);
  671. fail_unlock:
  672. mutex_unlock(&dev->struct_mutex);
  673. return ret;
  674. }
  675. /**
  676. * This is the fallback shmem pwrite path, which uses get_user_pages to pin
  677. * the memory and maps it using kmap_atomic for copying.
  678. *
  679. * This avoids taking mmap_sem for faulting on the user's address while the
  680. * struct_mutex is held.
  681. */
  682. static int
  683. i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
  684. struct drm_i915_gem_pwrite *args,
  685. struct drm_file *file_priv)
  686. {
  687. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  688. struct mm_struct *mm = current->mm;
  689. struct page **user_pages;
  690. ssize_t remain;
  691. loff_t offset, pinned_pages, i;
  692. loff_t first_data_page, last_data_page, num_pages;
  693. int shmem_page_index, shmem_page_offset;
  694. int data_page_index, data_page_offset;
  695. int page_length;
  696. int ret;
  697. uint64_t data_ptr = args->data_ptr;
  698. int do_bit17_swizzling;
  699. remain = args->size;
  700. /* Pin the user pages containing the data. We can't fault while
  701. * holding the struct mutex, and all of the pwrite implementations
  702. * want to hold it while dereferencing the user data.
  703. */
  704. first_data_page = data_ptr / PAGE_SIZE;
  705. last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
  706. num_pages = last_data_page - first_data_page + 1;
  707. user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
  708. if (user_pages == NULL)
  709. return -ENOMEM;
  710. down_read(&mm->mmap_sem);
  711. pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
  712. num_pages, 0, 0, user_pages, NULL);
  713. up_read(&mm->mmap_sem);
  714. if (pinned_pages < num_pages) {
  715. ret = -EFAULT;
  716. goto fail_put_user_pages;
  717. }
  718. do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
  719. mutex_lock(&dev->struct_mutex);
  720. ret = i915_gem_object_get_pages_or_evict(obj);
  721. if (ret)
  722. goto fail_unlock;
  723. ret = i915_gem_object_set_to_cpu_domain(obj, 1);
  724. if (ret != 0)
  725. goto fail_put_pages;
  726. obj_priv = to_intel_bo(obj);
  727. offset = args->offset;
  728. obj_priv->dirty = 1;
  729. while (remain > 0) {
  730. /* Operation in this page
  731. *
  732. * shmem_page_index = page number within shmem file
  733. * shmem_page_offset = offset within page in shmem file
  734. * data_page_index = page number in get_user_pages return
  735. * data_page_offset = offset with data_page_index page.
  736. * page_length = bytes to copy for this page
  737. */
  738. shmem_page_index = offset / PAGE_SIZE;
  739. shmem_page_offset = offset & ~PAGE_MASK;
  740. data_page_index = data_ptr / PAGE_SIZE - first_data_page;
  741. data_page_offset = data_ptr & ~PAGE_MASK;
  742. page_length = remain;
  743. if ((shmem_page_offset + page_length) > PAGE_SIZE)
  744. page_length = PAGE_SIZE - shmem_page_offset;
  745. if ((data_page_offset + page_length) > PAGE_SIZE)
  746. page_length = PAGE_SIZE - data_page_offset;
  747. if (do_bit17_swizzling) {
  748. slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
  749. shmem_page_offset,
  750. user_pages[data_page_index],
  751. data_page_offset,
  752. page_length,
  753. 0);
  754. } else {
  755. slow_shmem_copy(obj_priv->pages[shmem_page_index],
  756. shmem_page_offset,
  757. user_pages[data_page_index],
  758. data_page_offset,
  759. page_length);
  760. }
  761. remain -= page_length;
  762. data_ptr += page_length;
  763. offset += page_length;
  764. }
  765. fail_put_pages:
  766. i915_gem_object_put_pages(obj);
  767. fail_unlock:
  768. mutex_unlock(&dev->struct_mutex);
  769. fail_put_user_pages:
  770. for (i = 0; i < pinned_pages; i++)
  771. page_cache_release(user_pages[i]);
  772. drm_free_large(user_pages);
  773. return ret;
  774. }
  775. /**
  776. * Writes data to the object referenced by handle.
  777. *
  778. * On error, the contents of the buffer that were to be modified are undefined.
  779. */
  780. int
  781. i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
  782. struct drm_file *file_priv)
  783. {
  784. struct drm_i915_gem_pwrite *args = data;
  785. struct drm_gem_object *obj;
  786. struct drm_i915_gem_object *obj_priv;
  787. int ret = 0;
  788. obj = drm_gem_object_lookup(dev, file_priv, args->handle);
  789. if (obj == NULL)
  790. return -ENOENT;
  791. obj_priv = to_intel_bo(obj);
  792. /* Bounds check destination.
  793. *
  794. * XXX: This could use review for overflow issues...
  795. */
  796. if (args->offset > obj->size || args->size > obj->size ||
  797. args->offset + args->size > obj->size) {
  798. drm_gem_object_unreference_unlocked(obj);
  799. return -EINVAL;
  800. }
  801. /* We can only do the GTT pwrite on untiled buffers, as otherwise
  802. * it would end up going through the fenced access, and we'll get
  803. * different detiling behavior between reading and writing.
  804. * pread/pwrite currently are reading and writing from the CPU
  805. * perspective, requiring manual detiling by the client.
  806. */
  807. if (obj_priv->phys_obj)
  808. ret = i915_gem_phys_pwrite(dev, obj, args, file_priv);
  809. else if (obj_priv->tiling_mode == I915_TILING_NONE &&
  810. dev->gtt_total != 0 &&
  811. obj->write_domain != I915_GEM_DOMAIN_CPU) {
  812. ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file_priv);
  813. if (ret == -EFAULT) {
  814. ret = i915_gem_gtt_pwrite_slow(dev, obj, args,
  815. file_priv);
  816. }
  817. } else if (i915_gem_object_needs_bit17_swizzle(obj)) {
  818. ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file_priv);
  819. } else {
  820. ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file_priv);
  821. if (ret == -EFAULT) {
  822. ret = i915_gem_shmem_pwrite_slow(dev, obj, args,
  823. file_priv);
  824. }
  825. }
  826. #if WATCH_PWRITE
  827. if (ret)
  828. DRM_INFO("pwrite failed %d\n", ret);
  829. #endif
  830. drm_gem_object_unreference_unlocked(obj);
  831. return ret;
  832. }
  833. /**
  834. * Called when user space prepares to use an object with the CPU, either
  835. * through the mmap ioctl's mapping or a GTT mapping.
  836. */
  837. int
  838. i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
  839. struct drm_file *file_priv)
  840. {
  841. struct drm_i915_private *dev_priv = dev->dev_private;
  842. struct drm_i915_gem_set_domain *args = data;
  843. struct drm_gem_object *obj;
  844. struct drm_i915_gem_object *obj_priv;
  845. uint32_t read_domains = args->read_domains;
  846. uint32_t write_domain = args->write_domain;
  847. int ret;
  848. if (!(dev->driver->driver_features & DRIVER_GEM))
  849. return -ENODEV;
  850. /* Only handle setting domains to types used by the CPU. */
  851. if (write_domain & I915_GEM_GPU_DOMAINS)
  852. return -EINVAL;
  853. if (read_domains & I915_GEM_GPU_DOMAINS)
  854. return -EINVAL;
  855. /* Having something in the write domain implies it's in the read
  856. * domain, and only that read domain. Enforce that in the request.
  857. */
  858. if (write_domain != 0 && read_domains != write_domain)
  859. return -EINVAL;
  860. obj = drm_gem_object_lookup(dev, file_priv, args->handle);
  861. if (obj == NULL)
  862. return -ENOENT;
  863. obj_priv = to_intel_bo(obj);
  864. mutex_lock(&dev->struct_mutex);
  865. intel_mark_busy(dev, obj);
  866. #if WATCH_BUF
  867. DRM_INFO("set_domain_ioctl %p(%zd), %08x %08x\n",
  868. obj, obj->size, read_domains, write_domain);
  869. #endif
  870. if (read_domains & I915_GEM_DOMAIN_GTT) {
  871. ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
  872. /* Update the LRU on the fence for the CPU access that's
  873. * about to occur.
  874. */
  875. if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
  876. struct drm_i915_fence_reg *reg =
  877. &dev_priv->fence_regs[obj_priv->fence_reg];
  878. list_move_tail(&reg->lru_list,
  879. &dev_priv->mm.fence_list);
  880. }
  881. /* Silently promote "you're not bound, there was nothing to do"
  882. * to success, since the client was just asking us to
  883. * make sure everything was done.
  884. */
  885. if (ret == -EINVAL)
  886. ret = 0;
  887. } else {
  888. ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
  889. }
  890. /* Maintain LRU order of "inactive" objects */
  891. if (ret == 0 && i915_gem_object_is_inactive(obj_priv))
  892. list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
  893. drm_gem_object_unreference(obj);
  894. mutex_unlock(&dev->struct_mutex);
  895. return ret;
  896. }
  897. /**
  898. * Called when user space has done writes to this buffer
  899. */
  900. int
  901. i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
  902. struct drm_file *file_priv)
  903. {
  904. struct drm_i915_gem_sw_finish *args = data;
  905. struct drm_gem_object *obj;
  906. struct drm_i915_gem_object *obj_priv;
  907. int ret = 0;
  908. if (!(dev->driver->driver_features & DRIVER_GEM))
  909. return -ENODEV;
  910. mutex_lock(&dev->struct_mutex);
  911. obj = drm_gem_object_lookup(dev, file_priv, args->handle);
  912. if (obj == NULL) {
  913. mutex_unlock(&dev->struct_mutex);
  914. return -ENOENT;
  915. }
  916. #if WATCH_BUF
  917. DRM_INFO("%s: sw_finish %d (%p %zd)\n",
  918. __func__, args->handle, obj, obj->size);
  919. #endif
  920. obj_priv = to_intel_bo(obj);
  921. /* Pinned buffers may be scanout, so flush the cache */
  922. if (obj_priv->pin_count)
  923. i915_gem_object_flush_cpu_write_domain(obj);
  924. drm_gem_object_unreference(obj);
  925. mutex_unlock(&dev->struct_mutex);
  926. return ret;
  927. }
  928. /**
  929. * Maps the contents of an object, returning the address it is mapped
  930. * into.
  931. *
  932. * While the mapping holds a reference on the contents of the object, it doesn't
  933. * imply a ref on the object itself.
  934. */
  935. int
  936. i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
  937. struct drm_file *file_priv)
  938. {
  939. struct drm_i915_gem_mmap *args = data;
  940. struct drm_gem_object *obj;
  941. loff_t offset;
  942. unsigned long addr;
  943. if (!(dev->driver->driver_features & DRIVER_GEM))
  944. return -ENODEV;
  945. obj = drm_gem_object_lookup(dev, file_priv, args->handle);
  946. if (obj == NULL)
  947. return -ENOENT;
  948. offset = args->offset;
  949. down_write(&current->mm->mmap_sem);
  950. addr = do_mmap(obj->filp, 0, args->size,
  951. PROT_READ | PROT_WRITE, MAP_SHARED,
  952. args->offset);
  953. up_write(&current->mm->mmap_sem);
  954. drm_gem_object_unreference_unlocked(obj);
  955. if (IS_ERR((void *)addr))
  956. return addr;
  957. args->addr_ptr = (uint64_t) addr;
  958. return 0;
  959. }
  960. /**
  961. * i915_gem_fault - fault a page into the GTT
  962. * vma: VMA in question
  963. * vmf: fault info
  964. *
  965. * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
  966. * from userspace. The fault handler takes care of binding the object to
  967. * the GTT (if needed), allocating and programming a fence register (again,
  968. * only if needed based on whether the old reg is still valid or the object
  969. * is tiled) and inserting a new PTE into the faulting process.
  970. *
  971. * Note that the faulting process may involve evicting existing objects
  972. * from the GTT and/or fence registers to make room. So performance may
  973. * suffer if the GTT working set is large or there are few fence registers
  974. * left.
  975. */
  976. int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
  977. {
  978. struct drm_gem_object *obj = vma->vm_private_data;
  979. struct drm_device *dev = obj->dev;
  980. drm_i915_private_t *dev_priv = dev->dev_private;
  981. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  982. pgoff_t page_offset;
  983. unsigned long pfn;
  984. int ret = 0;
  985. bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
  986. /* We don't use vmf->pgoff since that has the fake offset */
  987. page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
  988. PAGE_SHIFT;
  989. /* Now bind it into the GTT if needed */
  990. mutex_lock(&dev->struct_mutex);
  991. if (!obj_priv->gtt_space) {
  992. ret = i915_gem_object_bind_to_gtt(obj, 0);
  993. if (ret)
  994. goto unlock;
  995. ret = i915_gem_object_set_to_gtt_domain(obj, write);
  996. if (ret)
  997. goto unlock;
  998. }
  999. /* Need a new fence register? */
  1000. if (obj_priv->tiling_mode != I915_TILING_NONE) {
  1001. ret = i915_gem_object_get_fence_reg(obj);
  1002. if (ret)
  1003. goto unlock;
  1004. }
  1005. if (i915_gem_object_is_inactive(obj_priv))
  1006. list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
  1007. pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) +
  1008. page_offset;
  1009. /* Finally, remap it using the new GTT offset */
  1010. ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
  1011. unlock:
  1012. mutex_unlock(&dev->struct_mutex);
  1013. switch (ret) {
  1014. case 0:
  1015. case -ERESTARTSYS:
  1016. return VM_FAULT_NOPAGE;
  1017. case -ENOMEM:
  1018. case -EAGAIN:
  1019. return VM_FAULT_OOM;
  1020. default:
  1021. return VM_FAULT_SIGBUS;
  1022. }
  1023. }
  1024. /**
  1025. * i915_gem_create_mmap_offset - create a fake mmap offset for an object
  1026. * @obj: obj in question
  1027. *
  1028. * GEM memory mapping works by handing back to userspace a fake mmap offset
  1029. * it can use in a subsequent mmap(2) call. The DRM core code then looks
  1030. * up the object based on the offset and sets up the various memory mapping
  1031. * structures.
  1032. *
  1033. * This routine allocates and attaches a fake offset for @obj.
  1034. */
  1035. static int
  1036. i915_gem_create_mmap_offset(struct drm_gem_object *obj)
  1037. {
  1038. struct drm_device *dev = obj->dev;
  1039. struct drm_gem_mm *mm = dev->mm_private;
  1040. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  1041. struct drm_map_list *list;
  1042. struct drm_local_map *map;
  1043. int ret = 0;
  1044. /* Set the object up for mmap'ing */
  1045. list = &obj->map_list;
  1046. list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL);
  1047. if (!list->map)
  1048. return -ENOMEM;
  1049. map = list->map;
  1050. map->type = _DRM_GEM;
  1051. map->size = obj->size;
  1052. map->handle = obj;
  1053. /* Get a DRM GEM mmap offset allocated... */
  1054. list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
  1055. obj->size / PAGE_SIZE, 0, 0);
  1056. if (!list->file_offset_node) {
  1057. DRM_ERROR("failed to allocate offset for bo %d\n", obj->name);
  1058. ret = -ENOMEM;
  1059. goto out_free_list;
  1060. }
  1061. list->file_offset_node = drm_mm_get_block(list->file_offset_node,
  1062. obj->size / PAGE_SIZE, 0);
  1063. if (!list->file_offset_node) {
  1064. ret = -ENOMEM;
  1065. goto out_free_list;
  1066. }
  1067. list->hash.key = list->file_offset_node->start;
  1068. if (drm_ht_insert_item(&mm->offset_hash, &list->hash)) {
  1069. DRM_ERROR("failed to add to map hash\n");
  1070. ret = -ENOMEM;
  1071. goto out_free_mm;
  1072. }
  1073. /* By now we should be all set, any drm_mmap request on the offset
  1074. * below will get to our mmap & fault handler */
  1075. obj_priv->mmap_offset = ((uint64_t) list->hash.key) << PAGE_SHIFT;
  1076. return 0;
  1077. out_free_mm:
  1078. drm_mm_put_block(list->file_offset_node);
  1079. out_free_list:
  1080. kfree(list->map);
  1081. return ret;
  1082. }
  1083. /**
  1084. * i915_gem_release_mmap - remove physical page mappings
  1085. * @obj: obj in question
  1086. *
  1087. * Preserve the reservation of the mmapping with the DRM core code, but
  1088. * relinquish ownership of the pages back to the system.
  1089. *
  1090. * It is vital that we remove the page mapping if we have mapped a tiled
  1091. * object through the GTT and then lose the fence register due to
  1092. * resource pressure. Similarly if the object has been moved out of the
  1093. * aperture, than pages mapped into userspace must be revoked. Removing the
  1094. * mapping will then trigger a page fault on the next user access, allowing
  1095. * fixup by i915_gem_fault().
  1096. */
  1097. void
  1098. i915_gem_release_mmap(struct drm_gem_object *obj)
  1099. {
  1100. struct drm_device *dev = obj->dev;
  1101. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  1102. if (dev->dev_mapping)
  1103. unmap_mapping_range(dev->dev_mapping,
  1104. obj_priv->mmap_offset, obj->size, 1);
  1105. }
  1106. static void
  1107. i915_gem_free_mmap_offset(struct drm_gem_object *obj)
  1108. {
  1109. struct drm_device *dev = obj->dev;
  1110. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  1111. struct drm_gem_mm *mm = dev->mm_private;
  1112. struct drm_map_list *list;
  1113. list = &obj->map_list;
  1114. drm_ht_remove_item(&mm->offset_hash, &list->hash);
  1115. if (list->file_offset_node) {
  1116. drm_mm_put_block(list->file_offset_node);
  1117. list->file_offset_node = NULL;
  1118. }
  1119. if (list->map) {
  1120. kfree(list->map);
  1121. list->map = NULL;
  1122. }
  1123. obj_priv->mmap_offset = 0;
  1124. }
  1125. /**
  1126. * i915_gem_get_gtt_alignment - return required GTT alignment for an object
  1127. * @obj: object to check
  1128. *
  1129. * Return the required GTT alignment for an object, taking into account
  1130. * potential fence register mapping if needed.
  1131. */
  1132. static uint32_t
  1133. i915_gem_get_gtt_alignment(struct drm_gem_object *obj)
  1134. {
  1135. struct drm_device *dev = obj->dev;
  1136. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  1137. int start, i;
  1138. /*
  1139. * Minimum alignment is 4k (GTT page size), but might be greater
  1140. * if a fence register is needed for the object.
  1141. */
  1142. if (IS_I965G(dev) || obj_priv->tiling_mode == I915_TILING_NONE)
  1143. return 4096;
  1144. /*
  1145. * Previous chips need to be aligned to the size of the smallest
  1146. * fence register that can contain the object.
  1147. */
  1148. if (IS_I9XX(dev))
  1149. start = 1024*1024;
  1150. else
  1151. start = 512*1024;
  1152. for (i = start; i < obj->size; i <<= 1)
  1153. ;
  1154. return i;
  1155. }
  1156. /**
  1157. * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
  1158. * @dev: DRM device
  1159. * @data: GTT mapping ioctl data
  1160. * @file_priv: GEM object info
  1161. *
  1162. * Simply returns the fake offset to userspace so it can mmap it.
  1163. * The mmap call will end up in drm_gem_mmap(), which will set things
  1164. * up so we can get faults in the handler above.
  1165. *
  1166. * The fault handler will take care of binding the object into the GTT
  1167. * (since it may have been evicted to make room for something), allocating
  1168. * a fence register, and mapping the appropriate aperture address into
  1169. * userspace.
  1170. */
  1171. int
  1172. i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
  1173. struct drm_file *file_priv)
  1174. {
  1175. struct drm_i915_gem_mmap_gtt *args = data;
  1176. struct drm_gem_object *obj;
  1177. struct drm_i915_gem_object *obj_priv;
  1178. int ret;
  1179. if (!(dev->driver->driver_features & DRIVER_GEM))
  1180. return -ENODEV;
  1181. obj = drm_gem_object_lookup(dev, file_priv, args->handle);
  1182. if (obj == NULL)
  1183. return -ENOENT;
  1184. mutex_lock(&dev->struct_mutex);
  1185. obj_priv = to_intel_bo(obj);
  1186. if (obj_priv->madv != I915_MADV_WILLNEED) {
  1187. DRM_ERROR("Attempting to mmap a purgeable buffer\n");
  1188. drm_gem_object_unreference(obj);
  1189. mutex_unlock(&dev->struct_mutex);
  1190. return -EINVAL;
  1191. }
  1192. if (!obj_priv->mmap_offset) {
  1193. ret = i915_gem_create_mmap_offset(obj);
  1194. if (ret) {
  1195. drm_gem_object_unreference(obj);
  1196. mutex_unlock(&dev->struct_mutex);
  1197. return ret;
  1198. }
  1199. }
  1200. args->offset = obj_priv->mmap_offset;
  1201. /*
  1202. * Pull it into the GTT so that we have a page list (makes the
  1203. * initial fault faster and any subsequent flushing possible).
  1204. */
  1205. if (!obj_priv->agp_mem) {
  1206. ret = i915_gem_object_bind_to_gtt(obj, 0);
  1207. if (ret) {
  1208. drm_gem_object_unreference(obj);
  1209. mutex_unlock(&dev->struct_mutex);
  1210. return ret;
  1211. }
  1212. }
  1213. drm_gem_object_unreference(obj);
  1214. mutex_unlock(&dev->struct_mutex);
  1215. return 0;
  1216. }
  1217. void
  1218. i915_gem_object_put_pages(struct drm_gem_object *obj)
  1219. {
  1220. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  1221. int page_count = obj->size / PAGE_SIZE;
  1222. int i;
  1223. BUG_ON(obj_priv->pages_refcount == 0);
  1224. BUG_ON(obj_priv->madv == __I915_MADV_PURGED);
  1225. if (--obj_priv->pages_refcount != 0)
  1226. return;
  1227. if (obj_priv->tiling_mode != I915_TILING_NONE)
  1228. i915_gem_object_save_bit_17_swizzle(obj);
  1229. if (obj_priv->madv == I915_MADV_DONTNEED)
  1230. obj_priv->dirty = 0;
  1231. for (i = 0; i < page_count; i++) {
  1232. if (obj_priv->dirty)
  1233. set_page_dirty(obj_priv->pages[i]);
  1234. if (obj_priv->madv == I915_MADV_WILLNEED)
  1235. mark_page_accessed(obj_priv->pages[i]);
  1236. page_cache_release(obj_priv->pages[i]);
  1237. }
  1238. obj_priv->dirty = 0;
  1239. drm_free_large(obj_priv->pages);
  1240. obj_priv->pages = NULL;
  1241. }
  1242. static void
  1243. i915_gem_object_move_to_active(struct drm_gem_object *obj, uint32_t seqno,
  1244. struct intel_ring_buffer *ring)
  1245. {
  1246. struct drm_device *dev = obj->dev;
  1247. drm_i915_private_t *dev_priv = dev->dev_private;
  1248. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  1249. BUG_ON(ring == NULL);
  1250. obj_priv->ring = ring;
  1251. /* Add a reference if we're newly entering the active list. */
  1252. if (!obj_priv->active) {
  1253. drm_gem_object_reference(obj);
  1254. obj_priv->active = 1;
  1255. }
  1256. /* Move from whatever list we were on to the tail of execution. */
  1257. spin_lock(&dev_priv->mm.active_list_lock);
  1258. list_move_tail(&obj_priv->list, &ring->active_list);
  1259. spin_unlock(&dev_priv->mm.active_list_lock);
  1260. obj_priv->last_rendering_seqno = seqno;
  1261. }
  1262. static void
  1263. i915_gem_object_move_to_flushing(struct drm_gem_object *obj)
  1264. {
  1265. struct drm_device *dev = obj->dev;
  1266. drm_i915_private_t *dev_priv = dev->dev_private;
  1267. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  1268. BUG_ON(!obj_priv->active);
  1269. list_move_tail(&obj_priv->list, &dev_priv->mm.flushing_list);
  1270. obj_priv->last_rendering_seqno = 0;
  1271. }
  1272. /* Immediately discard the backing storage */
  1273. static void
  1274. i915_gem_object_truncate(struct drm_gem_object *obj)
  1275. {
  1276. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  1277. struct inode *inode;
  1278. /* Our goal here is to return as much of the memory as
  1279. * is possible back to the system as we are called from OOM.
  1280. * To do this we must instruct the shmfs to drop all of its
  1281. * backing pages, *now*. Here we mirror the actions taken
  1282. * when by shmem_delete_inode() to release the backing store.
  1283. */
  1284. inode = obj->filp->f_path.dentry->d_inode;
  1285. truncate_inode_pages(inode->i_mapping, 0);
  1286. if (inode->i_op->truncate_range)
  1287. inode->i_op->truncate_range(inode, 0, (loff_t)-1);
  1288. obj_priv->madv = __I915_MADV_PURGED;
  1289. }
  1290. static inline int
  1291. i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj_priv)
  1292. {
  1293. return obj_priv->madv == I915_MADV_DONTNEED;
  1294. }
  1295. static void
  1296. i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
  1297. {
  1298. struct drm_device *dev = obj->dev;
  1299. drm_i915_private_t *dev_priv = dev->dev_private;
  1300. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  1301. i915_verify_inactive(dev, __FILE__, __LINE__);
  1302. if (obj_priv->pin_count != 0)
  1303. list_del_init(&obj_priv->list);
  1304. else
  1305. list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
  1306. BUG_ON(!list_empty(&obj_priv->gpu_write_list));
  1307. obj_priv->last_rendering_seqno = 0;
  1308. obj_priv->ring = NULL;
  1309. if (obj_priv->active) {
  1310. obj_priv->active = 0;
  1311. drm_gem_object_unreference(obj);
  1312. }
  1313. i915_verify_inactive(dev, __FILE__, __LINE__);
  1314. }
  1315. static void
  1316. i915_gem_process_flushing_list(struct drm_device *dev,
  1317. uint32_t flush_domains, uint32_t seqno,
  1318. struct intel_ring_buffer *ring)
  1319. {
  1320. drm_i915_private_t *dev_priv = dev->dev_private;
  1321. struct drm_i915_gem_object *obj_priv, *next;
  1322. list_for_each_entry_safe(obj_priv, next,
  1323. &dev_priv->mm.gpu_write_list,
  1324. gpu_write_list) {
  1325. struct drm_gem_object *obj = &obj_priv->base;
  1326. if ((obj->write_domain & flush_domains) ==
  1327. obj->write_domain &&
  1328. obj_priv->ring->ring_flag == ring->ring_flag) {
  1329. uint32_t old_write_domain = obj->write_domain;
  1330. obj->write_domain = 0;
  1331. list_del_init(&obj_priv->gpu_write_list);
  1332. i915_gem_object_move_to_active(obj, seqno, ring);
  1333. /* update the fence lru list */
  1334. if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
  1335. struct drm_i915_fence_reg *reg =
  1336. &dev_priv->fence_regs[obj_priv->fence_reg];
  1337. list_move_tail(&reg->lru_list,
  1338. &dev_priv->mm.fence_list);
  1339. }
  1340. trace_i915_gem_object_change_domain(obj,
  1341. obj->read_domains,
  1342. old_write_domain);
  1343. }
  1344. }
  1345. }
  1346. uint32_t
  1347. i915_add_request(struct drm_device *dev, struct drm_file *file_priv,
  1348. uint32_t flush_domains, struct intel_ring_buffer *ring)
  1349. {
  1350. drm_i915_private_t *dev_priv = dev->dev_private;
  1351. struct drm_i915_file_private *i915_file_priv = NULL;
  1352. struct drm_i915_gem_request *request;
  1353. uint32_t seqno;
  1354. int was_empty;
  1355. if (file_priv != NULL)
  1356. i915_file_priv = file_priv->driver_priv;
  1357. request = kzalloc(sizeof(*request), GFP_KERNEL);
  1358. if (request == NULL)
  1359. return 0;
  1360. seqno = ring->add_request(dev, ring, file_priv, flush_domains);
  1361. request->seqno = seqno;
  1362. request->ring = ring;
  1363. request->emitted_jiffies = jiffies;
  1364. was_empty = list_empty(&ring->request_list);
  1365. list_add_tail(&request->list, &ring->request_list);
  1366. if (i915_file_priv) {
  1367. list_add_tail(&request->client_list,
  1368. &i915_file_priv->mm.request_list);
  1369. } else {
  1370. INIT_LIST_HEAD(&request->client_list);
  1371. }
  1372. /* Associate any objects on the flushing list matching the write
  1373. * domain we're flushing with our flush.
  1374. */
  1375. if (flush_domains != 0)
  1376. i915_gem_process_flushing_list(dev, flush_domains, seqno, ring);
  1377. if (!dev_priv->mm.suspended) {
  1378. mod_timer(&dev_priv->hangcheck_timer, jiffies + DRM_I915_HANGCHECK_PERIOD);
  1379. if (was_empty)
  1380. queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
  1381. }
  1382. return seqno;
  1383. }
  1384. /**
  1385. * Command execution barrier
  1386. *
  1387. * Ensures that all commands in the ring are finished
  1388. * before signalling the CPU
  1389. */
  1390. static uint32_t
  1391. i915_retire_commands(struct drm_device *dev, struct intel_ring_buffer *ring)
  1392. {
  1393. uint32_t flush_domains = 0;
  1394. /* The sampler always gets flushed on i965 (sigh) */
  1395. if (IS_I965G(dev))
  1396. flush_domains |= I915_GEM_DOMAIN_SAMPLER;
  1397. ring->flush(dev, ring,
  1398. I915_GEM_DOMAIN_COMMAND, flush_domains);
  1399. return flush_domains;
  1400. }
  1401. /**
  1402. * Moves buffers associated only with the given active seqno from the active
  1403. * to inactive list, potentially freeing them.
  1404. */
  1405. static void
  1406. i915_gem_retire_request(struct drm_device *dev,
  1407. struct drm_i915_gem_request *request)
  1408. {
  1409. drm_i915_private_t *dev_priv = dev->dev_private;
  1410. trace_i915_gem_request_retire(dev, request->seqno);
  1411. /* Move any buffers on the active list that are no longer referenced
  1412. * by the ringbuffer to the flushing/inactive lists as appropriate.
  1413. */
  1414. spin_lock(&dev_priv->mm.active_list_lock);
  1415. while (!list_empty(&request->ring->active_list)) {
  1416. struct drm_gem_object *obj;
  1417. struct drm_i915_gem_object *obj_priv;
  1418. obj_priv = list_first_entry(&request->ring->active_list,
  1419. struct drm_i915_gem_object,
  1420. list);
  1421. obj = &obj_priv->base;
  1422. /* If the seqno being retired doesn't match the oldest in the
  1423. * list, then the oldest in the list must still be newer than
  1424. * this seqno.
  1425. */
  1426. if (obj_priv->last_rendering_seqno != request->seqno)
  1427. goto out;
  1428. #if WATCH_LRU
  1429. DRM_INFO("%s: retire %d moves to inactive list %p\n",
  1430. __func__, request->seqno, obj);
  1431. #endif
  1432. if (obj->write_domain != 0)
  1433. i915_gem_object_move_to_flushing(obj);
  1434. else {
  1435. /* Take a reference on the object so it won't be
  1436. * freed while the spinlock is held. The list
  1437. * protection for this spinlock is safe when breaking
  1438. * the lock like this since the next thing we do
  1439. * is just get the head of the list again.
  1440. */
  1441. drm_gem_object_reference(obj);
  1442. i915_gem_object_move_to_inactive(obj);
  1443. spin_unlock(&dev_priv->mm.active_list_lock);
  1444. drm_gem_object_unreference(obj);
  1445. spin_lock(&dev_priv->mm.active_list_lock);
  1446. }
  1447. }
  1448. out:
  1449. spin_unlock(&dev_priv->mm.active_list_lock);
  1450. }
  1451. /**
  1452. * Returns true if seq1 is later than seq2.
  1453. */
  1454. bool
  1455. i915_seqno_passed(uint32_t seq1, uint32_t seq2)
  1456. {
  1457. return (int32_t)(seq1 - seq2) >= 0;
  1458. }
  1459. uint32_t
  1460. i915_get_gem_seqno(struct drm_device *dev,
  1461. struct intel_ring_buffer *ring)
  1462. {
  1463. return ring->get_gem_seqno(dev, ring);
  1464. }
  1465. /**
  1466. * This function clears the request list as sequence numbers are passed.
  1467. */
  1468. static void
  1469. i915_gem_retire_requests_ring(struct drm_device *dev,
  1470. struct intel_ring_buffer *ring)
  1471. {
  1472. drm_i915_private_t *dev_priv = dev->dev_private;
  1473. uint32_t seqno;
  1474. if (!ring->status_page.page_addr
  1475. || list_empty(&ring->request_list))
  1476. return;
  1477. seqno = i915_get_gem_seqno(dev, ring);
  1478. while (!list_empty(&ring->request_list)) {
  1479. struct drm_i915_gem_request *request;
  1480. uint32_t retiring_seqno;
  1481. request = list_first_entry(&ring->request_list,
  1482. struct drm_i915_gem_request,
  1483. list);
  1484. retiring_seqno = request->seqno;
  1485. if (i915_seqno_passed(seqno, retiring_seqno) ||
  1486. atomic_read(&dev_priv->mm.wedged)) {
  1487. i915_gem_retire_request(dev, request);
  1488. list_del(&request->list);
  1489. list_del(&request->client_list);
  1490. kfree(request);
  1491. } else
  1492. break;
  1493. }
  1494. if (unlikely (dev_priv->trace_irq_seqno &&
  1495. i915_seqno_passed(dev_priv->trace_irq_seqno, seqno))) {
  1496. ring->user_irq_put(dev, ring);
  1497. dev_priv->trace_irq_seqno = 0;
  1498. }
  1499. }
  1500. void
  1501. i915_gem_retire_requests(struct drm_device *dev)
  1502. {
  1503. drm_i915_private_t *dev_priv = dev->dev_private;
  1504. if (!list_empty(&dev_priv->mm.deferred_free_list)) {
  1505. struct drm_i915_gem_object *obj_priv, *tmp;
  1506. /* We must be careful that during unbind() we do not
  1507. * accidentally infinitely recurse into retire requests.
  1508. * Currently:
  1509. * retire -> free -> unbind -> wait -> retire_ring
  1510. */
  1511. list_for_each_entry_safe(obj_priv, tmp,
  1512. &dev_priv->mm.deferred_free_list,
  1513. list)
  1514. i915_gem_free_object_tail(&obj_priv->base);
  1515. }
  1516. i915_gem_retire_requests_ring(dev, &dev_priv->render_ring);
  1517. if (HAS_BSD(dev))
  1518. i915_gem_retire_requests_ring(dev, &dev_priv->bsd_ring);
  1519. }
  1520. void
  1521. i915_gem_retire_work_handler(struct work_struct *work)
  1522. {
  1523. drm_i915_private_t *dev_priv;
  1524. struct drm_device *dev;
  1525. dev_priv = container_of(work, drm_i915_private_t,
  1526. mm.retire_work.work);
  1527. dev = dev_priv->dev;
  1528. mutex_lock(&dev->struct_mutex);
  1529. i915_gem_retire_requests(dev);
  1530. if (!dev_priv->mm.suspended &&
  1531. (!list_empty(&dev_priv->render_ring.request_list) ||
  1532. (HAS_BSD(dev) &&
  1533. !list_empty(&dev_priv->bsd_ring.request_list))))
  1534. queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
  1535. mutex_unlock(&dev->struct_mutex);
  1536. }
  1537. int
  1538. i915_do_wait_request(struct drm_device *dev, uint32_t seqno,
  1539. int interruptible, struct intel_ring_buffer *ring)
  1540. {
  1541. drm_i915_private_t *dev_priv = dev->dev_private;
  1542. u32 ier;
  1543. int ret = 0;
  1544. BUG_ON(seqno == 0);
  1545. if (atomic_read(&dev_priv->mm.wedged))
  1546. return -EIO;
  1547. if (!i915_seqno_passed(ring->get_gem_seqno(dev, ring), seqno)) {
  1548. if (HAS_PCH_SPLIT(dev))
  1549. ier = I915_READ(DEIER) | I915_READ(GTIER);
  1550. else
  1551. ier = I915_READ(IER);
  1552. if (!ier) {
  1553. DRM_ERROR("something (likely vbetool) disabled "
  1554. "interrupts, re-enabling\n");
  1555. i915_driver_irq_preinstall(dev);
  1556. i915_driver_irq_postinstall(dev);
  1557. }
  1558. trace_i915_gem_request_wait_begin(dev, seqno);
  1559. ring->waiting_gem_seqno = seqno;
  1560. ring->user_irq_get(dev, ring);
  1561. if (interruptible)
  1562. ret = wait_event_interruptible(ring->irq_queue,
  1563. i915_seqno_passed(
  1564. ring->get_gem_seqno(dev, ring), seqno)
  1565. || atomic_read(&dev_priv->mm.wedged));
  1566. else
  1567. wait_event(ring->irq_queue,
  1568. i915_seqno_passed(
  1569. ring->get_gem_seqno(dev, ring), seqno)
  1570. || atomic_read(&dev_priv->mm.wedged));
  1571. ring->user_irq_put(dev, ring);
  1572. ring->waiting_gem_seqno = 0;
  1573. trace_i915_gem_request_wait_end(dev, seqno);
  1574. }
  1575. if (atomic_read(&dev_priv->mm.wedged))
  1576. ret = -EIO;
  1577. if (ret && ret != -ERESTARTSYS)
  1578. DRM_ERROR("%s returns %d (awaiting %d at %d)\n",
  1579. __func__, ret, seqno, ring->get_gem_seqno(dev, ring));
  1580. /* Directly dispatch request retiring. While we have the work queue
  1581. * to handle this, the waiter on a request often wants an associated
  1582. * buffer to have made it to the inactive list, and we would need
  1583. * a separate wait queue to handle that.
  1584. */
  1585. if (ret == 0)
  1586. i915_gem_retire_requests_ring(dev, ring);
  1587. return ret;
  1588. }
  1589. /**
  1590. * Waits for a sequence number to be signaled, and cleans up the
  1591. * request and object lists appropriately for that event.
  1592. */
  1593. static int
  1594. i915_wait_request(struct drm_device *dev, uint32_t seqno,
  1595. struct intel_ring_buffer *ring)
  1596. {
  1597. return i915_do_wait_request(dev, seqno, 1, ring);
  1598. }
  1599. static void
  1600. i915_gem_flush(struct drm_device *dev,
  1601. uint32_t invalidate_domains,
  1602. uint32_t flush_domains)
  1603. {
  1604. drm_i915_private_t *dev_priv = dev->dev_private;
  1605. if (flush_domains & I915_GEM_DOMAIN_CPU)
  1606. drm_agp_chipset_flush(dev);
  1607. dev_priv->render_ring.flush(dev, &dev_priv->render_ring,
  1608. invalidate_domains,
  1609. flush_domains);
  1610. if (HAS_BSD(dev))
  1611. dev_priv->bsd_ring.flush(dev, &dev_priv->bsd_ring,
  1612. invalidate_domains,
  1613. flush_domains);
  1614. }
  1615. /**
  1616. * Ensures that all rendering to the object has completed and the object is
  1617. * safe to unbind from the GTT or access from the CPU.
  1618. */
  1619. static int
  1620. i915_gem_object_wait_rendering(struct drm_gem_object *obj)
  1621. {
  1622. struct drm_device *dev = obj->dev;
  1623. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  1624. int ret;
  1625. /* This function only exists to support waiting for existing rendering,
  1626. * not for emitting required flushes.
  1627. */
  1628. BUG_ON((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0);
  1629. /* If there is rendering queued on the buffer being evicted, wait for
  1630. * it.
  1631. */
  1632. if (obj_priv->active) {
  1633. #if WATCH_BUF
  1634. DRM_INFO("%s: object %p wait for seqno %08x\n",
  1635. __func__, obj, obj_priv->last_rendering_seqno);
  1636. #endif
  1637. ret = i915_wait_request(dev,
  1638. obj_priv->last_rendering_seqno, obj_priv->ring);
  1639. if (ret != 0)
  1640. return ret;
  1641. }
  1642. return 0;
  1643. }
  1644. /**
  1645. * Unbinds an object from the GTT aperture.
  1646. */
  1647. int
  1648. i915_gem_object_unbind(struct drm_gem_object *obj)
  1649. {
  1650. struct drm_device *dev = obj->dev;
  1651. drm_i915_private_t *dev_priv = dev->dev_private;
  1652. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  1653. int ret = 0;
  1654. #if WATCH_BUF
  1655. DRM_INFO("%s:%d %p\n", __func__, __LINE__, obj);
  1656. DRM_INFO("gtt_space %p\n", obj_priv->gtt_space);
  1657. #endif
  1658. if (obj_priv->gtt_space == NULL)
  1659. return 0;
  1660. if (obj_priv->pin_count != 0) {
  1661. DRM_ERROR("Attempting to unbind pinned buffer\n");
  1662. return -EINVAL;
  1663. }
  1664. /* blow away mappings if mapped through GTT */
  1665. i915_gem_release_mmap(obj);
  1666. /* Move the object to the CPU domain to ensure that
  1667. * any possible CPU writes while it's not in the GTT
  1668. * are flushed when we go to remap it. This will
  1669. * also ensure that all pending GPU writes are finished
  1670. * before we unbind.
  1671. */
  1672. ret = i915_gem_object_set_to_cpu_domain(obj, 1);
  1673. if (ret == -ERESTARTSYS)
  1674. return ret;
  1675. /* Continue on if we fail due to EIO, the GPU is hung so we
  1676. * should be safe and we need to cleanup or else we might
  1677. * cause memory corruption through use-after-free.
  1678. */
  1679. /* release the fence reg _after_ flushing */
  1680. if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
  1681. i915_gem_clear_fence_reg(obj);
  1682. if (obj_priv->agp_mem != NULL) {
  1683. drm_unbind_agp(obj_priv->agp_mem);
  1684. drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE);
  1685. obj_priv->agp_mem = NULL;
  1686. }
  1687. i915_gem_object_put_pages(obj);
  1688. BUG_ON(obj_priv->pages_refcount);
  1689. if (obj_priv->gtt_space) {
  1690. atomic_dec(&dev->gtt_count);
  1691. atomic_sub(obj->size, &dev->gtt_memory);
  1692. drm_mm_put_block(obj_priv->gtt_space);
  1693. obj_priv->gtt_space = NULL;
  1694. }
  1695. /* Remove ourselves from the LRU list if present. */
  1696. spin_lock(&dev_priv->mm.active_list_lock);
  1697. if (!list_empty(&obj_priv->list))
  1698. list_del_init(&obj_priv->list);
  1699. spin_unlock(&dev_priv->mm.active_list_lock);
  1700. if (i915_gem_object_is_purgeable(obj_priv))
  1701. i915_gem_object_truncate(obj);
  1702. trace_i915_gem_object_unbind(obj);
  1703. return ret;
  1704. }
  1705. int
  1706. i915_gpu_idle(struct drm_device *dev)
  1707. {
  1708. drm_i915_private_t *dev_priv = dev->dev_private;
  1709. bool lists_empty;
  1710. uint32_t seqno1, seqno2;
  1711. int ret;
  1712. spin_lock(&dev_priv->mm.active_list_lock);
  1713. lists_empty = (list_empty(&dev_priv->mm.flushing_list) &&
  1714. list_empty(&dev_priv->render_ring.active_list) &&
  1715. (!HAS_BSD(dev) ||
  1716. list_empty(&dev_priv->bsd_ring.active_list)));
  1717. spin_unlock(&dev_priv->mm.active_list_lock);
  1718. if (lists_empty)
  1719. return 0;
  1720. /* Flush everything onto the inactive list. */
  1721. i915_gem_flush(dev, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
  1722. seqno1 = i915_add_request(dev, NULL, I915_GEM_GPU_DOMAINS,
  1723. &dev_priv->render_ring);
  1724. if (seqno1 == 0)
  1725. return -ENOMEM;
  1726. ret = i915_wait_request(dev, seqno1, &dev_priv->render_ring);
  1727. if (HAS_BSD(dev)) {
  1728. seqno2 = i915_add_request(dev, NULL, I915_GEM_GPU_DOMAINS,
  1729. &dev_priv->bsd_ring);
  1730. if (seqno2 == 0)
  1731. return -ENOMEM;
  1732. ret = i915_wait_request(dev, seqno2, &dev_priv->bsd_ring);
  1733. if (ret)
  1734. return ret;
  1735. }
  1736. return ret;
  1737. }
  1738. int
  1739. i915_gem_object_get_pages(struct drm_gem_object *obj,
  1740. gfp_t gfpmask)
  1741. {
  1742. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  1743. int page_count, i;
  1744. struct address_space *mapping;
  1745. struct inode *inode;
  1746. struct page *page;
  1747. BUG_ON(obj_priv->pages_refcount
  1748. == DRM_I915_GEM_OBJECT_MAX_PAGES_REFCOUNT);
  1749. if (obj_priv->pages_refcount++ != 0)
  1750. return 0;
  1751. /* Get the list of pages out of our struct file. They'll be pinned
  1752. * at this point until we release them.
  1753. */
  1754. page_count = obj->size / PAGE_SIZE;
  1755. BUG_ON(obj_priv->pages != NULL);
  1756. obj_priv->pages = drm_calloc_large(page_count, sizeof(struct page *));
  1757. if (obj_priv->pages == NULL) {
  1758. obj_priv->pages_refcount--;
  1759. return -ENOMEM;
  1760. }
  1761. inode = obj->filp->f_path.dentry->d_inode;
  1762. mapping = inode->i_mapping;
  1763. for (i = 0; i < page_count; i++) {
  1764. page = read_cache_page_gfp(mapping, i,
  1765. GFP_HIGHUSER |
  1766. __GFP_COLD |
  1767. __GFP_RECLAIMABLE |
  1768. gfpmask);
  1769. if (IS_ERR(page))
  1770. goto err_pages;
  1771. obj_priv->pages[i] = page;
  1772. }
  1773. if (obj_priv->tiling_mode != I915_TILING_NONE)
  1774. i915_gem_object_do_bit_17_swizzle(obj);
  1775. return 0;
  1776. err_pages:
  1777. while (i--)
  1778. page_cache_release(obj_priv->pages[i]);
  1779. drm_free_large(obj_priv->pages);
  1780. obj_priv->pages = NULL;
  1781. obj_priv->pages_refcount--;
  1782. return PTR_ERR(page);
  1783. }
  1784. static void sandybridge_write_fence_reg(struct drm_i915_fence_reg *reg)
  1785. {
  1786. struct drm_gem_object *obj = reg->obj;
  1787. struct drm_device *dev = obj->dev;
  1788. drm_i915_private_t *dev_priv = dev->dev_private;
  1789. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  1790. int regnum = obj_priv->fence_reg;
  1791. uint64_t val;
  1792. val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
  1793. 0xfffff000) << 32;
  1794. val |= obj_priv->gtt_offset & 0xfffff000;
  1795. val |= (uint64_t)((obj_priv->stride / 128) - 1) <<
  1796. SANDYBRIDGE_FENCE_PITCH_SHIFT;
  1797. if (obj_priv->tiling_mode == I915_TILING_Y)
  1798. val |= 1 << I965_FENCE_TILING_Y_SHIFT;
  1799. val |= I965_FENCE_REG_VALID;
  1800. I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (regnum * 8), val);
  1801. }
  1802. static void i965_write_fence_reg(struct drm_i915_fence_reg *reg)
  1803. {
  1804. struct drm_gem_object *obj = reg->obj;
  1805. struct drm_device *dev = obj->dev;
  1806. drm_i915_private_t *dev_priv = dev->dev_private;
  1807. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  1808. int regnum = obj_priv->fence_reg;
  1809. uint64_t val;
  1810. val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
  1811. 0xfffff000) << 32;
  1812. val |= obj_priv->gtt_offset & 0xfffff000;
  1813. val |= ((obj_priv->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
  1814. if (obj_priv->tiling_mode == I915_TILING_Y)
  1815. val |= 1 << I965_FENCE_TILING_Y_SHIFT;
  1816. val |= I965_FENCE_REG_VALID;
  1817. I915_WRITE64(FENCE_REG_965_0 + (regnum * 8), val);
  1818. }
  1819. static void i915_write_fence_reg(struct drm_i915_fence_reg *reg)
  1820. {
  1821. struct drm_gem_object *obj = reg->obj;
  1822. struct drm_device *dev = obj->dev;
  1823. drm_i915_private_t *dev_priv = dev->dev_private;
  1824. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  1825. int regnum = obj_priv->fence_reg;
  1826. int tile_width;
  1827. uint32_t fence_reg, val;
  1828. uint32_t pitch_val;
  1829. if ((obj_priv->gtt_offset & ~I915_FENCE_START_MASK) ||
  1830. (obj_priv->gtt_offset & (obj->size - 1))) {
  1831. WARN(1, "%s: object 0x%08x not 1M or size (0x%zx) aligned\n",
  1832. __func__, obj_priv->gtt_offset, obj->size);
  1833. return;
  1834. }
  1835. if (obj_priv->tiling_mode == I915_TILING_Y &&
  1836. HAS_128_BYTE_Y_TILING(dev))
  1837. tile_width = 128;
  1838. else
  1839. tile_width = 512;
  1840. /* Note: pitch better be a power of two tile widths */
  1841. pitch_val = obj_priv->stride / tile_width;
  1842. pitch_val = ffs(pitch_val) - 1;
  1843. if (obj_priv->tiling_mode == I915_TILING_Y &&
  1844. HAS_128_BYTE_Y_TILING(dev))
  1845. WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
  1846. else
  1847. WARN_ON(pitch_val > I915_FENCE_MAX_PITCH_VAL);
  1848. val = obj_priv->gtt_offset;
  1849. if (obj_priv->tiling_mode == I915_TILING_Y)
  1850. val |= 1 << I830_FENCE_TILING_Y_SHIFT;
  1851. val |= I915_FENCE_SIZE_BITS(obj->size);
  1852. val |= pitch_val << I830_FENCE_PITCH_SHIFT;
  1853. val |= I830_FENCE_REG_VALID;
  1854. if (regnum < 8)
  1855. fence_reg = FENCE_REG_830_0 + (regnum * 4);
  1856. else
  1857. fence_reg = FENCE_REG_945_8 + ((regnum - 8) * 4);
  1858. I915_WRITE(fence_reg, val);
  1859. }
  1860. static void i830_write_fence_reg(struct drm_i915_fence_reg *reg)
  1861. {
  1862. struct drm_gem_object *obj = reg->obj;
  1863. struct drm_device *dev = obj->dev;
  1864. drm_i915_private_t *dev_priv = dev->dev_private;
  1865. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  1866. int regnum = obj_priv->fence_reg;
  1867. uint32_t val;
  1868. uint32_t pitch_val;
  1869. uint32_t fence_size_bits;
  1870. if ((obj_priv->gtt_offset & ~I830_FENCE_START_MASK) ||
  1871. (obj_priv->gtt_offset & (obj->size - 1))) {
  1872. WARN(1, "%s: object 0x%08x not 512K or size aligned\n",
  1873. __func__, obj_priv->gtt_offset);
  1874. return;
  1875. }
  1876. pitch_val = obj_priv->stride / 128;
  1877. pitch_val = ffs(pitch_val) - 1;
  1878. WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
  1879. val = obj_priv->gtt_offset;
  1880. if (obj_priv->tiling_mode == I915_TILING_Y)
  1881. val |= 1 << I830_FENCE_TILING_Y_SHIFT;
  1882. fence_size_bits = I830_FENCE_SIZE_BITS(obj->size);
  1883. WARN_ON(fence_size_bits & ~0x00000f00);
  1884. val |= fence_size_bits;
  1885. val |= pitch_val << I830_FENCE_PITCH_SHIFT;
  1886. val |= I830_FENCE_REG_VALID;
  1887. I915_WRITE(FENCE_REG_830_0 + (regnum * 4), val);
  1888. }
  1889. static int i915_find_fence_reg(struct drm_device *dev)
  1890. {
  1891. struct drm_i915_fence_reg *reg = NULL;
  1892. struct drm_i915_gem_object *obj_priv = NULL;
  1893. struct drm_i915_private *dev_priv = dev->dev_private;
  1894. struct drm_gem_object *obj = NULL;
  1895. int i, avail, ret;
  1896. /* First try to find a free reg */
  1897. avail = 0;
  1898. for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
  1899. reg = &dev_priv->fence_regs[i];
  1900. if (!reg->obj)
  1901. return i;
  1902. obj_priv = to_intel_bo(reg->obj);
  1903. if (!obj_priv->pin_count)
  1904. avail++;
  1905. }
  1906. if (avail == 0)
  1907. return -ENOSPC;
  1908. /* None available, try to steal one or wait for a user to finish */
  1909. i = I915_FENCE_REG_NONE;
  1910. list_for_each_entry(reg, &dev_priv->mm.fence_list,
  1911. lru_list) {
  1912. obj = reg->obj;
  1913. obj_priv = to_intel_bo(obj);
  1914. if (obj_priv->pin_count)
  1915. continue;
  1916. /* found one! */
  1917. i = obj_priv->fence_reg;
  1918. break;
  1919. }
  1920. BUG_ON(i == I915_FENCE_REG_NONE);
  1921. /* We only have a reference on obj from the active list. put_fence_reg
  1922. * might drop that one, causing a use-after-free in it. So hold a
  1923. * private reference to obj like the other callers of put_fence_reg
  1924. * (set_tiling ioctl) do. */
  1925. drm_gem_object_reference(obj);
  1926. ret = i915_gem_object_put_fence_reg(obj);
  1927. drm_gem_object_unreference(obj);
  1928. if (ret != 0)
  1929. return ret;
  1930. return i;
  1931. }
  1932. /**
  1933. * i915_gem_object_get_fence_reg - set up a fence reg for an object
  1934. * @obj: object to map through a fence reg
  1935. *
  1936. * When mapping objects through the GTT, userspace wants to be able to write
  1937. * to them without having to worry about swizzling if the object is tiled.
  1938. *
  1939. * This function walks the fence regs looking for a free one for @obj,
  1940. * stealing one if it can't find any.
  1941. *
  1942. * It then sets up the reg based on the object's properties: address, pitch
  1943. * and tiling format.
  1944. */
  1945. int
  1946. i915_gem_object_get_fence_reg(struct drm_gem_object *obj)
  1947. {
  1948. struct drm_device *dev = obj->dev;
  1949. struct drm_i915_private *dev_priv = dev->dev_private;
  1950. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  1951. struct drm_i915_fence_reg *reg = NULL;
  1952. int ret;
  1953. /* Just update our place in the LRU if our fence is getting used. */
  1954. if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
  1955. reg = &dev_priv->fence_regs[obj_priv->fence_reg];
  1956. list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
  1957. return 0;
  1958. }
  1959. switch (obj_priv->tiling_mode) {
  1960. case I915_TILING_NONE:
  1961. WARN(1, "allocating a fence for non-tiled object?\n");
  1962. break;
  1963. case I915_TILING_X:
  1964. if (!obj_priv->stride)
  1965. return -EINVAL;
  1966. WARN((obj_priv->stride & (512 - 1)),
  1967. "object 0x%08x is X tiled but has non-512B pitch\n",
  1968. obj_priv->gtt_offset);
  1969. break;
  1970. case I915_TILING_Y:
  1971. if (!obj_priv->stride)
  1972. return -EINVAL;
  1973. WARN((obj_priv->stride & (128 - 1)),
  1974. "object 0x%08x is Y tiled but has non-128B pitch\n",
  1975. obj_priv->gtt_offset);
  1976. break;
  1977. }
  1978. ret = i915_find_fence_reg(dev);
  1979. if (ret < 0)
  1980. return ret;
  1981. obj_priv->fence_reg = ret;
  1982. reg = &dev_priv->fence_regs[obj_priv->fence_reg];
  1983. list_add_tail(&reg->lru_list, &dev_priv->mm.fence_list);
  1984. reg->obj = obj;
  1985. if (IS_GEN6(dev))
  1986. sandybridge_write_fence_reg(reg);
  1987. else if (IS_I965G(dev))
  1988. i965_write_fence_reg(reg);
  1989. else if (IS_I9XX(dev))
  1990. i915_write_fence_reg(reg);
  1991. else
  1992. i830_write_fence_reg(reg);
  1993. trace_i915_gem_object_get_fence(obj, obj_priv->fence_reg,
  1994. obj_priv->tiling_mode);
  1995. return 0;
  1996. }
  1997. /**
  1998. * i915_gem_clear_fence_reg - clear out fence register info
  1999. * @obj: object to clear
  2000. *
  2001. * Zeroes out the fence register itself and clears out the associated
  2002. * data structures in dev_priv and obj_priv.
  2003. */
  2004. static void
  2005. i915_gem_clear_fence_reg(struct drm_gem_object *obj)
  2006. {
  2007. struct drm_device *dev = obj->dev;
  2008. drm_i915_private_t *dev_priv = dev->dev_private;
  2009. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  2010. struct drm_i915_fence_reg *reg =
  2011. &dev_priv->fence_regs[obj_priv->fence_reg];
  2012. if (IS_GEN6(dev)) {
  2013. I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 +
  2014. (obj_priv->fence_reg * 8), 0);
  2015. } else if (IS_I965G(dev)) {
  2016. I915_WRITE64(FENCE_REG_965_0 + (obj_priv->fence_reg * 8), 0);
  2017. } else {
  2018. uint32_t fence_reg;
  2019. if (obj_priv->fence_reg < 8)
  2020. fence_reg = FENCE_REG_830_0 + obj_priv->fence_reg * 4;
  2021. else
  2022. fence_reg = FENCE_REG_945_8 + (obj_priv->fence_reg -
  2023. 8) * 4;
  2024. I915_WRITE(fence_reg, 0);
  2025. }
  2026. reg->obj = NULL;
  2027. obj_priv->fence_reg = I915_FENCE_REG_NONE;
  2028. list_del_init(&reg->lru_list);
  2029. }
  2030. /**
  2031. * i915_gem_object_put_fence_reg - waits on outstanding fenced access
  2032. * to the buffer to finish, and then resets the fence register.
  2033. * @obj: tiled object holding a fence register.
  2034. *
  2035. * Zeroes out the fence register itself and clears out the associated
  2036. * data structures in dev_priv and obj_priv.
  2037. */
  2038. int
  2039. i915_gem_object_put_fence_reg(struct drm_gem_object *obj)
  2040. {
  2041. struct drm_device *dev = obj->dev;
  2042. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  2043. if (obj_priv->fence_reg == I915_FENCE_REG_NONE)
  2044. return 0;
  2045. /* If we've changed tiling, GTT-mappings of the object
  2046. * need to re-fault to ensure that the correct fence register
  2047. * setup is in place.
  2048. */
  2049. i915_gem_release_mmap(obj);
  2050. /* On the i915, GPU access to tiled buffers is via a fence,
  2051. * therefore we must wait for any outstanding access to complete
  2052. * before clearing the fence.
  2053. */
  2054. if (!IS_I965G(dev)) {
  2055. int ret;
  2056. ret = i915_gem_object_flush_gpu_write_domain(obj);
  2057. if (ret != 0)
  2058. return ret;
  2059. ret = i915_gem_object_wait_rendering(obj);
  2060. if (ret != 0)
  2061. return ret;
  2062. }
  2063. i915_gem_object_flush_gtt_write_domain(obj);
  2064. i915_gem_clear_fence_reg (obj);
  2065. return 0;
  2066. }
  2067. /**
  2068. * Finds free space in the GTT aperture and binds the object there.
  2069. */
  2070. static int
  2071. i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
  2072. {
  2073. struct drm_device *dev = obj->dev;
  2074. drm_i915_private_t *dev_priv = dev->dev_private;
  2075. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  2076. struct drm_mm_node *free_space;
  2077. gfp_t gfpmask = __GFP_NORETRY | __GFP_NOWARN;
  2078. int ret;
  2079. if (obj_priv->madv != I915_MADV_WILLNEED) {
  2080. DRM_ERROR("Attempting to bind a purgeable object\n");
  2081. return -EINVAL;
  2082. }
  2083. if (alignment == 0)
  2084. alignment = i915_gem_get_gtt_alignment(obj);
  2085. if (alignment & (i915_gem_get_gtt_alignment(obj) - 1)) {
  2086. DRM_ERROR("Invalid object alignment requested %u\n", alignment);
  2087. return -EINVAL;
  2088. }
  2089. /* If the object is bigger than the entire aperture, reject it early
  2090. * before evicting everything in a vain attempt to find space.
  2091. */
  2092. if (obj->size > dev->gtt_total) {
  2093. DRM_ERROR("Attempting to bind an object larger than the aperture\n");
  2094. return -E2BIG;
  2095. }
  2096. search_free:
  2097. free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
  2098. obj->size, alignment, 0);
  2099. if (free_space != NULL) {
  2100. obj_priv->gtt_space = drm_mm_get_block(free_space, obj->size,
  2101. alignment);
  2102. if (obj_priv->gtt_space != NULL)
  2103. obj_priv->gtt_offset = obj_priv->gtt_space->start;
  2104. }
  2105. if (obj_priv->gtt_space == NULL) {
  2106. /* If the gtt is empty and we're still having trouble
  2107. * fitting our object in, we're out of memory.
  2108. */
  2109. #if WATCH_LRU
  2110. DRM_INFO("%s: GTT full, evicting something\n", __func__);
  2111. #endif
  2112. ret = i915_gem_evict_something(dev, obj->size, alignment);
  2113. if (ret)
  2114. return ret;
  2115. goto search_free;
  2116. }
  2117. #if WATCH_BUF
  2118. DRM_INFO("Binding object of size %zd at 0x%08x\n",
  2119. obj->size, obj_priv->gtt_offset);
  2120. #endif
  2121. ret = i915_gem_object_get_pages(obj, gfpmask);
  2122. if (ret) {
  2123. drm_mm_put_block(obj_priv->gtt_space);
  2124. obj_priv->gtt_space = NULL;
  2125. if (ret == -ENOMEM) {
  2126. /* first try to clear up some space from the GTT */
  2127. ret = i915_gem_evict_something(dev, obj->size,
  2128. alignment);
  2129. if (ret) {
  2130. /* now try to shrink everyone else */
  2131. if (gfpmask) {
  2132. gfpmask = 0;
  2133. goto search_free;
  2134. }
  2135. return ret;
  2136. }
  2137. goto search_free;
  2138. }
  2139. return ret;
  2140. }
  2141. /* Create an AGP memory structure pointing at our pages, and bind it
  2142. * into the GTT.
  2143. */
  2144. obj_priv->agp_mem = drm_agp_bind_pages(dev,
  2145. obj_priv->pages,
  2146. obj->size >> PAGE_SHIFT,
  2147. obj_priv->gtt_offset,
  2148. obj_priv->agp_type);
  2149. if (obj_priv->agp_mem == NULL) {
  2150. i915_gem_object_put_pages(obj);
  2151. drm_mm_put_block(obj_priv->gtt_space);
  2152. obj_priv->gtt_space = NULL;
  2153. ret = i915_gem_evict_something(dev, obj->size, alignment);
  2154. if (ret)
  2155. return ret;
  2156. goto search_free;
  2157. }
  2158. atomic_inc(&dev->gtt_count);
  2159. atomic_add(obj->size, &dev->gtt_memory);
  2160. /* keep track of bounds object by adding it to the inactive list */
  2161. list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
  2162. /* Assert that the object is not currently in any GPU domain. As it
  2163. * wasn't in the GTT, there shouldn't be any way it could have been in
  2164. * a GPU cache
  2165. */
  2166. BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS);
  2167. BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);
  2168. trace_i915_gem_object_bind(obj, obj_priv->gtt_offset);
  2169. return 0;
  2170. }
  2171. void
  2172. i915_gem_clflush_object(struct drm_gem_object *obj)
  2173. {
  2174. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  2175. /* If we don't have a page list set up, then we're not pinned
  2176. * to GPU, and we can ignore the cache flush because it'll happen
  2177. * again at bind time.
  2178. */
  2179. if (obj_priv->pages == NULL)
  2180. return;
  2181. trace_i915_gem_object_clflush(obj);
  2182. drm_clflush_pages(obj_priv->pages, obj->size / PAGE_SIZE);
  2183. }
  2184. /** Flushes any GPU write domain for the object if it's dirty. */
  2185. static int
  2186. i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj)
  2187. {
  2188. struct drm_device *dev = obj->dev;
  2189. uint32_t old_write_domain;
  2190. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  2191. if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
  2192. return 0;
  2193. /* Queue the GPU write cache flushing we need. */
  2194. old_write_domain = obj->write_domain;
  2195. i915_gem_flush(dev, 0, obj->write_domain);
  2196. if (i915_add_request(dev, NULL, obj->write_domain, obj_priv->ring) == 0)
  2197. return -ENOMEM;
  2198. trace_i915_gem_object_change_domain(obj,
  2199. obj->read_domains,
  2200. old_write_domain);
  2201. return 0;
  2202. }
  2203. /** Flushes the GTT write domain for the object if it's dirty. */
  2204. static void
  2205. i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj)
  2206. {
  2207. uint32_t old_write_domain;
  2208. if (obj->write_domain != I915_GEM_DOMAIN_GTT)
  2209. return;
  2210. /* No actual flushing is required for the GTT write domain. Writes
  2211. * to it immediately go to main memory as far as we know, so there's
  2212. * no chipset flush. It also doesn't land in render cache.
  2213. */
  2214. old_write_domain = obj->write_domain;
  2215. obj->write_domain = 0;
  2216. trace_i915_gem_object_change_domain(obj,
  2217. obj->read_domains,
  2218. old_write_domain);
  2219. }
  2220. /** Flushes the CPU write domain for the object if it's dirty. */
  2221. static void
  2222. i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj)
  2223. {
  2224. struct drm_device *dev = obj->dev;
  2225. uint32_t old_write_domain;
  2226. if (obj->write_domain != I915_GEM_DOMAIN_CPU)
  2227. return;
  2228. i915_gem_clflush_object(obj);
  2229. drm_agp_chipset_flush(dev);
  2230. old_write_domain = obj->write_domain;
  2231. obj->write_domain = 0;
  2232. trace_i915_gem_object_change_domain(obj,
  2233. obj->read_domains,
  2234. old_write_domain);
  2235. }
  2236. int
  2237. i915_gem_object_flush_write_domain(struct drm_gem_object *obj)
  2238. {
  2239. int ret = 0;
  2240. switch (obj->write_domain) {
  2241. case I915_GEM_DOMAIN_GTT:
  2242. i915_gem_object_flush_gtt_write_domain(obj);
  2243. break;
  2244. case I915_GEM_DOMAIN_CPU:
  2245. i915_gem_object_flush_cpu_write_domain(obj);
  2246. break;
  2247. default:
  2248. ret = i915_gem_object_flush_gpu_write_domain(obj);
  2249. break;
  2250. }
  2251. return ret;
  2252. }
  2253. /**
  2254. * Moves a single object to the GTT read, and possibly write domain.
  2255. *
  2256. * This function returns when the move is complete, including waiting on
  2257. * flushes to occur.
  2258. */
  2259. int
  2260. i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
  2261. {
  2262. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  2263. uint32_t old_write_domain, old_read_domains;
  2264. int ret;
  2265. /* Not valid to be called on unbound objects. */
  2266. if (obj_priv->gtt_space == NULL)
  2267. return -EINVAL;
  2268. ret = i915_gem_object_flush_gpu_write_domain(obj);
  2269. if (ret != 0)
  2270. return ret;
  2271. /* Wait on any GPU rendering and flushing to occur. */
  2272. ret = i915_gem_object_wait_rendering(obj);
  2273. if (ret != 0)
  2274. return ret;
  2275. old_write_domain = obj->write_domain;
  2276. old_read_domains = obj->read_domains;
  2277. /* If we're writing through the GTT domain, then CPU and GPU caches
  2278. * will need to be invalidated at next use.
  2279. */
  2280. if (write)
  2281. obj->read_domains &= I915_GEM_DOMAIN_GTT;
  2282. i915_gem_object_flush_cpu_write_domain(obj);
  2283. /* It should now be out of any other write domains, and we can update
  2284. * the domain values for our changes.
  2285. */
  2286. BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
  2287. obj->read_domains |= I915_GEM_DOMAIN_GTT;
  2288. if (write) {
  2289. obj->write_domain = I915_GEM_DOMAIN_GTT;
  2290. obj_priv->dirty = 1;
  2291. }
  2292. trace_i915_gem_object_change_domain(obj,
  2293. old_read_domains,
  2294. old_write_domain);
  2295. return 0;
  2296. }
  2297. /*
  2298. * Prepare buffer for display plane. Use uninterruptible for possible flush
  2299. * wait, as in modesetting process we're not supposed to be interrupted.
  2300. */
  2301. int
  2302. i915_gem_object_set_to_display_plane(struct drm_gem_object *obj)
  2303. {
  2304. struct drm_device *dev = obj->dev;
  2305. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  2306. uint32_t old_write_domain, old_read_domains;
  2307. int ret;
  2308. /* Not valid to be called on unbound objects. */
  2309. if (obj_priv->gtt_space == NULL)
  2310. return -EINVAL;
  2311. ret = i915_gem_object_flush_gpu_write_domain(obj);
  2312. if (ret)
  2313. return ret;
  2314. /* Wait on any GPU rendering and flushing to occur. */
  2315. if (obj_priv->active) {
  2316. #if WATCH_BUF
  2317. DRM_INFO("%s: object %p wait for seqno %08x\n",
  2318. __func__, obj, obj_priv->last_rendering_seqno);
  2319. #endif
  2320. ret = i915_do_wait_request(dev,
  2321. obj_priv->last_rendering_seqno,
  2322. 0,
  2323. obj_priv->ring);
  2324. if (ret != 0)
  2325. return ret;
  2326. }
  2327. i915_gem_object_flush_cpu_write_domain(obj);
  2328. old_write_domain = obj->write_domain;
  2329. old_read_domains = obj->read_domains;
  2330. /* It should now be out of any other write domains, and we can update
  2331. * the domain values for our changes.
  2332. */
  2333. BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
  2334. obj->read_domains = I915_GEM_DOMAIN_GTT;
  2335. obj->write_domain = I915_GEM_DOMAIN_GTT;
  2336. obj_priv->dirty = 1;
  2337. trace_i915_gem_object_change_domain(obj,
  2338. old_read_domains,
  2339. old_write_domain);
  2340. return 0;
  2341. }
  2342. /**
  2343. * Moves a single object to the CPU read, and possibly write domain.
  2344. *
  2345. * This function returns when the move is complete, including waiting on
  2346. * flushes to occur.
  2347. */
  2348. static int
  2349. i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
  2350. {
  2351. uint32_t old_write_domain, old_read_domains;
  2352. int ret;
  2353. ret = i915_gem_object_flush_gpu_write_domain(obj);
  2354. if (ret)
  2355. return ret;
  2356. /* Wait on any GPU rendering and flushing to occur. */
  2357. ret = i915_gem_object_wait_rendering(obj);
  2358. if (ret != 0)
  2359. return ret;
  2360. i915_gem_object_flush_gtt_write_domain(obj);
  2361. /* If we have a partially-valid cache of the object in the CPU,
  2362. * finish invalidating it and free the per-page flags.
  2363. */
  2364. i915_gem_object_set_to_full_cpu_read_domain(obj);
  2365. old_write_domain = obj->write_domain;
  2366. old_read_domains = obj->read_domains;
  2367. /* Flush the CPU cache if it's still invalid. */
  2368. if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
  2369. i915_gem_clflush_object(obj);
  2370. obj->read_domains |= I915_GEM_DOMAIN_CPU;
  2371. }
  2372. /* It should now be out of any other write domains, and we can update
  2373. * the domain values for our changes.
  2374. */
  2375. BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
  2376. /* If we're writing through the CPU, then the GPU read domains will
  2377. * need to be invalidated at next use.
  2378. */
  2379. if (write) {
  2380. obj->read_domains &= I915_GEM_DOMAIN_CPU;
  2381. obj->write_domain = I915_GEM_DOMAIN_CPU;
  2382. }
  2383. trace_i915_gem_object_change_domain(obj,
  2384. old_read_domains,
  2385. old_write_domain);
  2386. return 0;
  2387. }
  2388. /*
  2389. * Set the next domain for the specified object. This
  2390. * may not actually perform the necessary flushing/invaliding though,
  2391. * as that may want to be batched with other set_domain operations
  2392. *
  2393. * This is (we hope) the only really tricky part of gem. The goal
  2394. * is fairly simple -- track which caches hold bits of the object
  2395. * and make sure they remain coherent. A few concrete examples may
  2396. * help to explain how it works. For shorthand, we use the notation
  2397. * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
  2398. * a pair of read and write domain masks.
  2399. *
  2400. * Case 1: the batch buffer
  2401. *
  2402. * 1. Allocated
  2403. * 2. Written by CPU
  2404. * 3. Mapped to GTT
  2405. * 4. Read by GPU
  2406. * 5. Unmapped from GTT
  2407. * 6. Freed
  2408. *
  2409. * Let's take these a step at a time
  2410. *
  2411. * 1. Allocated
  2412. * Pages allocated from the kernel may still have
  2413. * cache contents, so we set them to (CPU, CPU) always.
  2414. * 2. Written by CPU (using pwrite)
  2415. * The pwrite function calls set_domain (CPU, CPU) and
  2416. * this function does nothing (as nothing changes)
  2417. * 3. Mapped by GTT
  2418. * This function asserts that the object is not
  2419. * currently in any GPU-based read or write domains
  2420. * 4. Read by GPU
  2421. * i915_gem_execbuffer calls set_domain (COMMAND, 0).
  2422. * As write_domain is zero, this function adds in the
  2423. * current read domains (CPU+COMMAND, 0).
  2424. * flush_domains is set to CPU.
  2425. * invalidate_domains is set to COMMAND
  2426. * clflush is run to get data out of the CPU caches
  2427. * then i915_dev_set_domain calls i915_gem_flush to
  2428. * emit an MI_FLUSH and drm_agp_chipset_flush
  2429. * 5. Unmapped from GTT
  2430. * i915_gem_object_unbind calls set_domain (CPU, CPU)
  2431. * flush_domains and invalidate_domains end up both zero
  2432. * so no flushing/invalidating happens
  2433. * 6. Freed
  2434. * yay, done
  2435. *
  2436. * Case 2: The shared render buffer
  2437. *
  2438. * 1. Allocated
  2439. * 2. Mapped to GTT
  2440. * 3. Read/written by GPU
  2441. * 4. set_domain to (CPU,CPU)
  2442. * 5. Read/written by CPU
  2443. * 6. Read/written by GPU
  2444. *
  2445. * 1. Allocated
  2446. * Same as last example, (CPU, CPU)
  2447. * 2. Mapped to GTT
  2448. * Nothing changes (assertions find that it is not in the GPU)
  2449. * 3. Read/written by GPU
  2450. * execbuffer calls set_domain (RENDER, RENDER)
  2451. * flush_domains gets CPU
  2452. * invalidate_domains gets GPU
  2453. * clflush (obj)
  2454. * MI_FLUSH and drm_agp_chipset_flush
  2455. * 4. set_domain (CPU, CPU)
  2456. * flush_domains gets GPU
  2457. * invalidate_domains gets CPU
  2458. * wait_rendering (obj) to make sure all drawing is complete.
  2459. * This will include an MI_FLUSH to get the data from GPU
  2460. * to memory
  2461. * clflush (obj) to invalidate the CPU cache
  2462. * Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
  2463. * 5. Read/written by CPU
  2464. * cache lines are loaded and dirtied
  2465. * 6. Read written by GPU
  2466. * Same as last GPU access
  2467. *
  2468. * Case 3: The constant buffer
  2469. *
  2470. * 1. Allocated
  2471. * 2. Written by CPU
  2472. * 3. Read by GPU
  2473. * 4. Updated (written) by CPU again
  2474. * 5. Read by GPU
  2475. *
  2476. * 1. Allocated
  2477. * (CPU, CPU)
  2478. * 2. Written by CPU
  2479. * (CPU, CPU)
  2480. * 3. Read by GPU
  2481. * (CPU+RENDER, 0)
  2482. * flush_domains = CPU
  2483. * invalidate_domains = RENDER
  2484. * clflush (obj)
  2485. * MI_FLUSH
  2486. * drm_agp_chipset_flush
  2487. * 4. Updated (written) by CPU again
  2488. * (CPU, CPU)
  2489. * flush_domains = 0 (no previous write domain)
  2490. * invalidate_domains = 0 (no new read domains)
  2491. * 5. Read by GPU
  2492. * (CPU+RENDER, 0)
  2493. * flush_domains = CPU
  2494. * invalidate_domains = RENDER
  2495. * clflush (obj)
  2496. * MI_FLUSH
  2497. * drm_agp_chipset_flush
  2498. */
  2499. static void
  2500. i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj)
  2501. {
  2502. struct drm_device *dev = obj->dev;
  2503. drm_i915_private_t *dev_priv = dev->dev_private;
  2504. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  2505. uint32_t invalidate_domains = 0;
  2506. uint32_t flush_domains = 0;
  2507. uint32_t old_read_domains;
  2508. BUG_ON(obj->pending_read_domains & I915_GEM_DOMAIN_CPU);
  2509. BUG_ON(obj->pending_write_domain == I915_GEM_DOMAIN_CPU);
  2510. intel_mark_busy(dev, obj);
  2511. #if WATCH_BUF
  2512. DRM_INFO("%s: object %p read %08x -> %08x write %08x -> %08x\n",
  2513. __func__, obj,
  2514. obj->read_domains, obj->pending_read_domains,
  2515. obj->write_domain, obj->pending_write_domain);
  2516. #endif
  2517. /*
  2518. * If the object isn't moving to a new write domain,
  2519. * let the object stay in multiple read domains
  2520. */
  2521. if (obj->pending_write_domain == 0)
  2522. obj->pending_read_domains |= obj->read_domains;
  2523. else
  2524. obj_priv->dirty = 1;
  2525. /*
  2526. * Flush the current write domain if
  2527. * the new read domains don't match. Invalidate
  2528. * any read domains which differ from the old
  2529. * write domain
  2530. */
  2531. if (obj->write_domain &&
  2532. obj->write_domain != obj->pending_read_domains) {
  2533. flush_domains |= obj->write_domain;
  2534. invalidate_domains |=
  2535. obj->pending_read_domains & ~obj->write_domain;
  2536. }
  2537. /*
  2538. * Invalidate any read caches which may have
  2539. * stale data. That is, any new read domains.
  2540. */
  2541. invalidate_domains |= obj->pending_read_domains & ~obj->read_domains;
  2542. if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU) {
  2543. #if WATCH_BUF
  2544. DRM_INFO("%s: CPU domain flush %08x invalidate %08x\n",
  2545. __func__, flush_domains, invalidate_domains);
  2546. #endif
  2547. i915_gem_clflush_object(obj);
  2548. }
  2549. old_read_domains = obj->read_domains;
  2550. /* The actual obj->write_domain will be updated with
  2551. * pending_write_domain after we emit the accumulated flush for all
  2552. * of our domain changes in execbuffers (which clears objects'
  2553. * write_domains). So if we have a current write domain that we
  2554. * aren't changing, set pending_write_domain to that.
  2555. */
  2556. if (flush_domains == 0 && obj->pending_write_domain == 0)
  2557. obj->pending_write_domain = obj->write_domain;
  2558. obj->read_domains = obj->pending_read_domains;
  2559. if (flush_domains & I915_GEM_GPU_DOMAINS) {
  2560. if (obj_priv->ring == &dev_priv->render_ring)
  2561. dev_priv->flush_rings |= FLUSH_RENDER_RING;
  2562. else if (obj_priv->ring == &dev_priv->bsd_ring)
  2563. dev_priv->flush_rings |= FLUSH_BSD_RING;
  2564. }
  2565. dev->invalidate_domains |= invalidate_domains;
  2566. dev->flush_domains |= flush_domains;
  2567. #if WATCH_BUF
  2568. DRM_INFO("%s: read %08x write %08x invalidate %08x flush %08x\n",
  2569. __func__,
  2570. obj->read_domains, obj->write_domain,
  2571. dev->invalidate_domains, dev->flush_domains);
  2572. #endif
  2573. trace_i915_gem_object_change_domain(obj,
  2574. old_read_domains,
  2575. obj->write_domain);
  2576. }
  2577. /**
  2578. * Moves the object from a partially CPU read to a full one.
  2579. *
  2580. * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
  2581. * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
  2582. */
  2583. static void
  2584. i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj)
  2585. {
  2586. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  2587. if (!obj_priv->page_cpu_valid)
  2588. return;
  2589. /* If we're partially in the CPU read domain, finish moving it in.
  2590. */
  2591. if (obj->read_domains & I915_GEM_DOMAIN_CPU) {
  2592. int i;
  2593. for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) {
  2594. if (obj_priv->page_cpu_valid[i])
  2595. continue;
  2596. drm_clflush_pages(obj_priv->pages + i, 1);
  2597. }
  2598. }
  2599. /* Free the page_cpu_valid mappings which are now stale, whether
  2600. * or not we've got I915_GEM_DOMAIN_CPU.
  2601. */
  2602. kfree(obj_priv->page_cpu_valid);
  2603. obj_priv->page_cpu_valid = NULL;
  2604. }
  2605. /**
  2606. * Set the CPU read domain on a range of the object.
  2607. *
  2608. * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
  2609. * not entirely valid. The page_cpu_valid member of the object flags which
  2610. * pages have been flushed, and will be respected by
  2611. * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
  2612. * of the whole object.
  2613. *
  2614. * This function returns when the move is complete, including waiting on
  2615. * flushes to occur.
  2616. */
  2617. static int
  2618. i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
  2619. uint64_t offset, uint64_t size)
  2620. {
  2621. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  2622. uint32_t old_read_domains;
  2623. int i, ret;
  2624. if (offset == 0 && size == obj->size)
  2625. return i915_gem_object_set_to_cpu_domain(obj, 0);
  2626. ret = i915_gem_object_flush_gpu_write_domain(obj);
  2627. if (ret)
  2628. return ret;
  2629. /* Wait on any GPU rendering and flushing to occur. */
  2630. ret = i915_gem_object_wait_rendering(obj);
  2631. if (ret != 0)
  2632. return ret;
  2633. i915_gem_object_flush_gtt_write_domain(obj);
  2634. /* If we're already fully in the CPU read domain, we're done. */
  2635. if (obj_priv->page_cpu_valid == NULL &&
  2636. (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0)
  2637. return 0;
  2638. /* Otherwise, create/clear the per-page CPU read domain flag if we're
  2639. * newly adding I915_GEM_DOMAIN_CPU
  2640. */
  2641. if (obj_priv->page_cpu_valid == NULL) {
  2642. obj_priv->page_cpu_valid = kzalloc(obj->size / PAGE_SIZE,
  2643. GFP_KERNEL);
  2644. if (obj_priv->page_cpu_valid == NULL)
  2645. return -ENOMEM;
  2646. } else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0)
  2647. memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE);
  2648. /* Flush the cache on any pages that are still invalid from the CPU's
  2649. * perspective.
  2650. */
  2651. for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
  2652. i++) {
  2653. if (obj_priv->page_cpu_valid[i])
  2654. continue;
  2655. drm_clflush_pages(obj_priv->pages + i, 1);
  2656. obj_priv->page_cpu_valid[i] = 1;
  2657. }
  2658. /* It should now be out of any other write domains, and we can update
  2659. * the domain values for our changes.
  2660. */
  2661. BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
  2662. old_read_domains = obj->read_domains;
  2663. obj->read_domains |= I915_GEM_DOMAIN_CPU;
  2664. trace_i915_gem_object_change_domain(obj,
  2665. old_read_domains,
  2666. obj->write_domain);
  2667. return 0;
  2668. }
  2669. /**
  2670. * Pin an object to the GTT and evaluate the relocations landing in it.
  2671. */
  2672. static int
  2673. i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
  2674. struct drm_file *file_priv,
  2675. struct drm_i915_gem_exec_object2 *entry,
  2676. struct drm_i915_gem_relocation_entry *relocs)
  2677. {
  2678. struct drm_device *dev = obj->dev;
  2679. drm_i915_private_t *dev_priv = dev->dev_private;
  2680. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  2681. int i, ret;
  2682. void __iomem *reloc_page;
  2683. bool need_fence;
  2684. need_fence = entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
  2685. obj_priv->tiling_mode != I915_TILING_NONE;
  2686. /* Check fence reg constraints and rebind if necessary */
  2687. if (need_fence &&
  2688. !i915_gem_object_fence_offset_ok(obj,
  2689. obj_priv->tiling_mode)) {
  2690. ret = i915_gem_object_unbind(obj);
  2691. if (ret)
  2692. return ret;
  2693. }
  2694. /* Choose the GTT offset for our buffer and put it there. */
  2695. ret = i915_gem_object_pin(obj, (uint32_t) entry->alignment);
  2696. if (ret)
  2697. return ret;
  2698. /*
  2699. * Pre-965 chips need a fence register set up in order to
  2700. * properly handle blits to/from tiled surfaces.
  2701. */
  2702. if (need_fence) {
  2703. ret = i915_gem_object_get_fence_reg(obj);
  2704. if (ret != 0) {
  2705. i915_gem_object_unpin(obj);
  2706. return ret;
  2707. }
  2708. }
  2709. entry->offset = obj_priv->gtt_offset;
  2710. /* Apply the relocations, using the GTT aperture to avoid cache
  2711. * flushing requirements.
  2712. */
  2713. for (i = 0; i < entry->relocation_count; i++) {
  2714. struct drm_i915_gem_relocation_entry *reloc= &relocs[i];
  2715. struct drm_gem_object *target_obj;
  2716. struct drm_i915_gem_object *target_obj_priv;
  2717. uint32_t reloc_val, reloc_offset;
  2718. uint32_t __iomem *reloc_entry;
  2719. target_obj = drm_gem_object_lookup(obj->dev, file_priv,
  2720. reloc->target_handle);
  2721. if (target_obj == NULL) {
  2722. i915_gem_object_unpin(obj);
  2723. return -ENOENT;
  2724. }
  2725. target_obj_priv = to_intel_bo(target_obj);
  2726. #if WATCH_RELOC
  2727. DRM_INFO("%s: obj %p offset %08x target %d "
  2728. "read %08x write %08x gtt %08x "
  2729. "presumed %08x delta %08x\n",
  2730. __func__,
  2731. obj,
  2732. (int) reloc->offset,
  2733. (int) reloc->target_handle,
  2734. (int) reloc->read_domains,
  2735. (int) reloc->write_domain,
  2736. (int) target_obj_priv->gtt_offset,
  2737. (int) reloc->presumed_offset,
  2738. reloc->delta);
  2739. #endif
  2740. /* The target buffer should have appeared before us in the
  2741. * exec_object list, so it should have a GTT space bound by now.
  2742. */
  2743. if (target_obj_priv->gtt_space == NULL) {
  2744. DRM_ERROR("No GTT space found for object %d\n",
  2745. reloc->target_handle);
  2746. drm_gem_object_unreference(target_obj);
  2747. i915_gem_object_unpin(obj);
  2748. return -EINVAL;
  2749. }
  2750. /* Validate that the target is in a valid r/w GPU domain */
  2751. if (reloc->write_domain & (reloc->write_domain - 1)) {
  2752. DRM_ERROR("reloc with multiple write domains: "
  2753. "obj %p target %d offset %d "
  2754. "read %08x write %08x",
  2755. obj, reloc->target_handle,
  2756. (int) reloc->offset,
  2757. reloc->read_domains,
  2758. reloc->write_domain);
  2759. return -EINVAL;
  2760. }
  2761. if (reloc->write_domain & I915_GEM_DOMAIN_CPU ||
  2762. reloc->read_domains & I915_GEM_DOMAIN_CPU) {
  2763. DRM_ERROR("reloc with read/write CPU domains: "
  2764. "obj %p target %d offset %d "
  2765. "read %08x write %08x",
  2766. obj, reloc->target_handle,
  2767. (int) reloc->offset,
  2768. reloc->read_domains,
  2769. reloc->write_domain);
  2770. drm_gem_object_unreference(target_obj);
  2771. i915_gem_object_unpin(obj);
  2772. return -EINVAL;
  2773. }
  2774. if (reloc->write_domain && target_obj->pending_write_domain &&
  2775. reloc->write_domain != target_obj->pending_write_domain) {
  2776. DRM_ERROR("Write domain conflict: "
  2777. "obj %p target %d offset %d "
  2778. "new %08x old %08x\n",
  2779. obj, reloc->target_handle,
  2780. (int) reloc->offset,
  2781. reloc->write_domain,
  2782. target_obj->pending_write_domain);
  2783. drm_gem_object_unreference(target_obj);
  2784. i915_gem_object_unpin(obj);
  2785. return -EINVAL;
  2786. }
  2787. target_obj->pending_read_domains |= reloc->read_domains;
  2788. target_obj->pending_write_domain |= reloc->write_domain;
  2789. /* If the relocation already has the right value in it, no
  2790. * more work needs to be done.
  2791. */
  2792. if (target_obj_priv->gtt_offset == reloc->presumed_offset) {
  2793. drm_gem_object_unreference(target_obj);
  2794. continue;
  2795. }
  2796. /* Check that the relocation address is valid... */
  2797. if (reloc->offset > obj->size - 4) {
  2798. DRM_ERROR("Relocation beyond object bounds: "
  2799. "obj %p target %d offset %d size %d.\n",
  2800. obj, reloc->target_handle,
  2801. (int) reloc->offset, (int) obj->size);
  2802. drm_gem_object_unreference(target_obj);
  2803. i915_gem_object_unpin(obj);
  2804. return -EINVAL;
  2805. }
  2806. if (reloc->offset & 3) {
  2807. DRM_ERROR("Relocation not 4-byte aligned: "
  2808. "obj %p target %d offset %d.\n",
  2809. obj, reloc->target_handle,
  2810. (int) reloc->offset);
  2811. drm_gem_object_unreference(target_obj);
  2812. i915_gem_object_unpin(obj);
  2813. return -EINVAL;
  2814. }
  2815. /* and points to somewhere within the target object. */
  2816. if (reloc->delta >= target_obj->size) {
  2817. DRM_ERROR("Relocation beyond target object bounds: "
  2818. "obj %p target %d delta %d size %d.\n",
  2819. obj, reloc->target_handle,
  2820. (int) reloc->delta, (int) target_obj->size);
  2821. drm_gem_object_unreference(target_obj);
  2822. i915_gem_object_unpin(obj);
  2823. return -EINVAL;
  2824. }
  2825. ret = i915_gem_object_set_to_gtt_domain(obj, 1);
  2826. if (ret != 0) {
  2827. drm_gem_object_unreference(target_obj);
  2828. i915_gem_object_unpin(obj);
  2829. return -EINVAL;
  2830. }
  2831. /* Map the page containing the relocation we're going to
  2832. * perform.
  2833. */
  2834. reloc_offset = obj_priv->gtt_offset + reloc->offset;
  2835. reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
  2836. (reloc_offset &
  2837. ~(PAGE_SIZE - 1)),
  2838. KM_USER0);
  2839. reloc_entry = (uint32_t __iomem *)(reloc_page +
  2840. (reloc_offset & (PAGE_SIZE - 1)));
  2841. reloc_val = target_obj_priv->gtt_offset + reloc->delta;
  2842. #if WATCH_BUF
  2843. DRM_INFO("Applied relocation: %p@0x%08x %08x -> %08x\n",
  2844. obj, (unsigned int) reloc->offset,
  2845. readl(reloc_entry), reloc_val);
  2846. #endif
  2847. writel(reloc_val, reloc_entry);
  2848. io_mapping_unmap_atomic(reloc_page, KM_USER0);
  2849. /* The updated presumed offset for this entry will be
  2850. * copied back out to the user.
  2851. */
  2852. reloc->presumed_offset = target_obj_priv->gtt_offset;
  2853. drm_gem_object_unreference(target_obj);
  2854. }
  2855. #if WATCH_BUF
  2856. if (0)
  2857. i915_gem_dump_object(obj, 128, __func__, ~0);
  2858. #endif
  2859. return 0;
  2860. }
  2861. /* Throttle our rendering by waiting until the ring has completed our requests
  2862. * emitted over 20 msec ago.
  2863. *
  2864. * Note that if we were to use the current jiffies each time around the loop,
  2865. * we wouldn't escape the function with any frames outstanding if the time to
  2866. * render a frame was over 20ms.
  2867. *
  2868. * This should get us reasonable parallelism between CPU and GPU but also
  2869. * relatively low latency when blocking on a particular request to finish.
  2870. */
  2871. static int
  2872. i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file_priv)
  2873. {
  2874. struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
  2875. int ret = 0;
  2876. unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
  2877. mutex_lock(&dev->struct_mutex);
  2878. while (!list_empty(&i915_file_priv->mm.request_list)) {
  2879. struct drm_i915_gem_request *request;
  2880. request = list_first_entry(&i915_file_priv->mm.request_list,
  2881. struct drm_i915_gem_request,
  2882. client_list);
  2883. if (time_after_eq(request->emitted_jiffies, recent_enough))
  2884. break;
  2885. ret = i915_wait_request(dev, request->seqno, request->ring);
  2886. if (ret != 0)
  2887. break;
  2888. }
  2889. mutex_unlock(&dev->struct_mutex);
  2890. return ret;
  2891. }
  2892. static int
  2893. i915_gem_get_relocs_from_user(struct drm_i915_gem_exec_object2 *exec_list,
  2894. uint32_t buffer_count,
  2895. struct drm_i915_gem_relocation_entry **relocs)
  2896. {
  2897. uint32_t reloc_count = 0, reloc_index = 0, i;
  2898. int ret;
  2899. *relocs = NULL;
  2900. for (i = 0; i < buffer_count; i++) {
  2901. if (reloc_count + exec_list[i].relocation_count < reloc_count)
  2902. return -EINVAL;
  2903. reloc_count += exec_list[i].relocation_count;
  2904. }
  2905. *relocs = drm_calloc_large(reloc_count, sizeof(**relocs));
  2906. if (*relocs == NULL) {
  2907. DRM_ERROR("failed to alloc relocs, count %d\n", reloc_count);
  2908. return -ENOMEM;
  2909. }
  2910. for (i = 0; i < buffer_count; i++) {
  2911. struct drm_i915_gem_relocation_entry __user *user_relocs;
  2912. user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
  2913. ret = copy_from_user(&(*relocs)[reloc_index],
  2914. user_relocs,
  2915. exec_list[i].relocation_count *
  2916. sizeof(**relocs));
  2917. if (ret != 0) {
  2918. drm_free_large(*relocs);
  2919. *relocs = NULL;
  2920. return -EFAULT;
  2921. }
  2922. reloc_index += exec_list[i].relocation_count;
  2923. }
  2924. return 0;
  2925. }
  2926. static int
  2927. i915_gem_put_relocs_to_user(struct drm_i915_gem_exec_object2 *exec_list,
  2928. uint32_t buffer_count,
  2929. struct drm_i915_gem_relocation_entry *relocs)
  2930. {
  2931. uint32_t reloc_count = 0, i;
  2932. int ret = 0;
  2933. if (relocs == NULL)
  2934. return 0;
  2935. for (i = 0; i < buffer_count; i++) {
  2936. struct drm_i915_gem_relocation_entry __user *user_relocs;
  2937. int unwritten;
  2938. user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
  2939. unwritten = copy_to_user(user_relocs,
  2940. &relocs[reloc_count],
  2941. exec_list[i].relocation_count *
  2942. sizeof(*relocs));
  2943. if (unwritten) {
  2944. ret = -EFAULT;
  2945. goto err;
  2946. }
  2947. reloc_count += exec_list[i].relocation_count;
  2948. }
  2949. err:
  2950. drm_free_large(relocs);
  2951. return ret;
  2952. }
  2953. static int
  2954. i915_gem_check_execbuffer (struct drm_i915_gem_execbuffer2 *exec,
  2955. uint64_t exec_offset)
  2956. {
  2957. uint32_t exec_start, exec_len;
  2958. exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
  2959. exec_len = (uint32_t) exec->batch_len;
  2960. if ((exec_start | exec_len) & 0x7)
  2961. return -EINVAL;
  2962. if (!exec_start)
  2963. return -EINVAL;
  2964. return 0;
  2965. }
  2966. static int
  2967. i915_gem_wait_for_pending_flip(struct drm_device *dev,
  2968. struct drm_gem_object **object_list,
  2969. int count)
  2970. {
  2971. drm_i915_private_t *dev_priv = dev->dev_private;
  2972. struct drm_i915_gem_object *obj_priv;
  2973. DEFINE_WAIT(wait);
  2974. int i, ret = 0;
  2975. for (;;) {
  2976. prepare_to_wait(&dev_priv->pending_flip_queue,
  2977. &wait, TASK_INTERRUPTIBLE);
  2978. for (i = 0; i < count; i++) {
  2979. obj_priv = to_intel_bo(object_list[i]);
  2980. if (atomic_read(&obj_priv->pending_flip) > 0)
  2981. break;
  2982. }
  2983. if (i == count)
  2984. break;
  2985. if (!signal_pending(current)) {
  2986. mutex_unlock(&dev->struct_mutex);
  2987. schedule();
  2988. mutex_lock(&dev->struct_mutex);
  2989. continue;
  2990. }
  2991. ret = -ERESTARTSYS;
  2992. break;
  2993. }
  2994. finish_wait(&dev_priv->pending_flip_queue, &wait);
  2995. return ret;
  2996. }
  2997. int
  2998. i915_gem_do_execbuffer(struct drm_device *dev, void *data,
  2999. struct drm_file *file_priv,
  3000. struct drm_i915_gem_execbuffer2 *args,
  3001. struct drm_i915_gem_exec_object2 *exec_list)
  3002. {
  3003. drm_i915_private_t *dev_priv = dev->dev_private;
  3004. struct drm_gem_object **object_list = NULL;
  3005. struct drm_gem_object *batch_obj;
  3006. struct drm_i915_gem_object *obj_priv;
  3007. struct drm_clip_rect *cliprects = NULL;
  3008. struct drm_i915_gem_relocation_entry *relocs = NULL;
  3009. int ret = 0, ret2, i, pinned = 0;
  3010. uint64_t exec_offset;
  3011. uint32_t seqno, flush_domains, reloc_index;
  3012. int pin_tries, flips;
  3013. struct intel_ring_buffer *ring = NULL;
  3014. #if WATCH_EXEC
  3015. DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
  3016. (int) args->buffers_ptr, args->buffer_count, args->batch_len);
  3017. #endif
  3018. if (args->flags & I915_EXEC_BSD) {
  3019. if (!HAS_BSD(dev)) {
  3020. DRM_ERROR("execbuf with wrong flag\n");
  3021. return -EINVAL;
  3022. }
  3023. ring = &dev_priv->bsd_ring;
  3024. } else {
  3025. ring = &dev_priv->render_ring;
  3026. }
  3027. if (args->buffer_count < 1) {
  3028. DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
  3029. return -EINVAL;
  3030. }
  3031. object_list = drm_malloc_ab(sizeof(*object_list), args->buffer_count);
  3032. if (object_list == NULL) {
  3033. DRM_ERROR("Failed to allocate object list for %d buffers\n",
  3034. args->buffer_count);
  3035. ret = -ENOMEM;
  3036. goto pre_mutex_err;
  3037. }
  3038. if (args->num_cliprects != 0) {
  3039. cliprects = kcalloc(args->num_cliprects, sizeof(*cliprects),
  3040. GFP_KERNEL);
  3041. if (cliprects == NULL) {
  3042. ret = -ENOMEM;
  3043. goto pre_mutex_err;
  3044. }
  3045. ret = copy_from_user(cliprects,
  3046. (struct drm_clip_rect __user *)
  3047. (uintptr_t) args->cliprects_ptr,
  3048. sizeof(*cliprects) * args->num_cliprects);
  3049. if (ret != 0) {
  3050. DRM_ERROR("copy %d cliprects failed: %d\n",
  3051. args->num_cliprects, ret);
  3052. ret = -EFAULT;
  3053. goto pre_mutex_err;
  3054. }
  3055. }
  3056. ret = i915_gem_get_relocs_from_user(exec_list, args->buffer_count,
  3057. &relocs);
  3058. if (ret != 0)
  3059. goto pre_mutex_err;
  3060. mutex_lock(&dev->struct_mutex);
  3061. i915_verify_inactive(dev, __FILE__, __LINE__);
  3062. if (atomic_read(&dev_priv->mm.wedged)) {
  3063. mutex_unlock(&dev->struct_mutex);
  3064. ret = -EIO;
  3065. goto pre_mutex_err;
  3066. }
  3067. if (dev_priv->mm.suspended) {
  3068. mutex_unlock(&dev->struct_mutex);
  3069. ret = -EBUSY;
  3070. goto pre_mutex_err;
  3071. }
  3072. /* Look up object handles */
  3073. flips = 0;
  3074. for (i = 0; i < args->buffer_count; i++) {
  3075. object_list[i] = drm_gem_object_lookup(dev, file_priv,
  3076. exec_list[i].handle);
  3077. if (object_list[i] == NULL) {
  3078. DRM_ERROR("Invalid object handle %d at index %d\n",
  3079. exec_list[i].handle, i);
  3080. /* prevent error path from reading uninitialized data */
  3081. args->buffer_count = i + 1;
  3082. ret = -ENOENT;
  3083. goto err;
  3084. }
  3085. obj_priv = to_intel_bo(object_list[i]);
  3086. if (obj_priv->in_execbuffer) {
  3087. DRM_ERROR("Object %p appears more than once in object list\n",
  3088. object_list[i]);
  3089. /* prevent error path from reading uninitialized data */
  3090. args->buffer_count = i + 1;
  3091. ret = -EINVAL;
  3092. goto err;
  3093. }
  3094. obj_priv->in_execbuffer = true;
  3095. flips += atomic_read(&obj_priv->pending_flip);
  3096. }
  3097. if (flips > 0) {
  3098. ret = i915_gem_wait_for_pending_flip(dev, object_list,
  3099. args->buffer_count);
  3100. if (ret)
  3101. goto err;
  3102. }
  3103. /* Pin and relocate */
  3104. for (pin_tries = 0; ; pin_tries++) {
  3105. ret = 0;
  3106. reloc_index = 0;
  3107. for (i = 0; i < args->buffer_count; i++) {
  3108. object_list[i]->pending_read_domains = 0;
  3109. object_list[i]->pending_write_domain = 0;
  3110. ret = i915_gem_object_pin_and_relocate(object_list[i],
  3111. file_priv,
  3112. &exec_list[i],
  3113. &relocs[reloc_index]);
  3114. if (ret)
  3115. break;
  3116. pinned = i + 1;
  3117. reloc_index += exec_list[i].relocation_count;
  3118. }
  3119. /* success */
  3120. if (ret == 0)
  3121. break;
  3122. /* error other than GTT full, or we've already tried again */
  3123. if (ret != -ENOSPC || pin_tries >= 1) {
  3124. if (ret != -ERESTARTSYS) {
  3125. unsigned long long total_size = 0;
  3126. int num_fences = 0;
  3127. for (i = 0; i < args->buffer_count; i++) {
  3128. obj_priv = to_intel_bo(object_list[i]);
  3129. total_size += object_list[i]->size;
  3130. num_fences +=
  3131. exec_list[i].flags & EXEC_OBJECT_NEEDS_FENCE &&
  3132. obj_priv->tiling_mode != I915_TILING_NONE;
  3133. }
  3134. DRM_ERROR("Failed to pin buffer %d of %d, total %llu bytes, %d fences: %d\n",
  3135. pinned+1, args->buffer_count,
  3136. total_size, num_fences,
  3137. ret);
  3138. DRM_ERROR("%d objects [%d pinned], "
  3139. "%d object bytes [%d pinned], "
  3140. "%d/%d gtt bytes\n",
  3141. atomic_read(&dev->object_count),
  3142. atomic_read(&dev->pin_count),
  3143. atomic_read(&dev->object_memory),
  3144. atomic_read(&dev->pin_memory),
  3145. atomic_read(&dev->gtt_memory),
  3146. dev->gtt_total);
  3147. }
  3148. goto err;
  3149. }
  3150. /* unpin all of our buffers */
  3151. for (i = 0; i < pinned; i++)
  3152. i915_gem_object_unpin(object_list[i]);
  3153. pinned = 0;
  3154. /* evict everyone we can from the aperture */
  3155. ret = i915_gem_evict_everything(dev);
  3156. if (ret && ret != -ENOSPC)
  3157. goto err;
  3158. }
  3159. /* Set the pending read domains for the batch buffer to COMMAND */
  3160. batch_obj = object_list[args->buffer_count-1];
  3161. if (batch_obj->pending_write_domain) {
  3162. DRM_ERROR("Attempting to use self-modifying batch buffer\n");
  3163. ret = -EINVAL;
  3164. goto err;
  3165. }
  3166. batch_obj->pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
  3167. /* Sanity check the batch buffer, prior to moving objects */
  3168. exec_offset = exec_list[args->buffer_count - 1].offset;
  3169. ret = i915_gem_check_execbuffer (args, exec_offset);
  3170. if (ret != 0) {
  3171. DRM_ERROR("execbuf with invalid offset/length\n");
  3172. goto err;
  3173. }
  3174. i915_verify_inactive(dev, __FILE__, __LINE__);
  3175. /* Zero the global flush/invalidate flags. These
  3176. * will be modified as new domains are computed
  3177. * for each object
  3178. */
  3179. dev->invalidate_domains = 0;
  3180. dev->flush_domains = 0;
  3181. dev_priv->flush_rings = 0;
  3182. for (i = 0; i < args->buffer_count; i++) {
  3183. struct drm_gem_object *obj = object_list[i];
  3184. /* Compute new gpu domains and update invalidate/flush */
  3185. i915_gem_object_set_to_gpu_domain(obj);
  3186. }
  3187. i915_verify_inactive(dev, __FILE__, __LINE__);
  3188. if (dev->invalidate_domains | dev->flush_domains) {
  3189. #if WATCH_EXEC
  3190. DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
  3191. __func__,
  3192. dev->invalidate_domains,
  3193. dev->flush_domains);
  3194. #endif
  3195. i915_gem_flush(dev,
  3196. dev->invalidate_domains,
  3197. dev->flush_domains);
  3198. if (dev_priv->flush_rings & FLUSH_RENDER_RING)
  3199. (void)i915_add_request(dev, file_priv,
  3200. dev->flush_domains,
  3201. &dev_priv->render_ring);
  3202. if (dev_priv->flush_rings & FLUSH_BSD_RING)
  3203. (void)i915_add_request(dev, file_priv,
  3204. dev->flush_domains,
  3205. &dev_priv->bsd_ring);
  3206. }
  3207. for (i = 0; i < args->buffer_count; i++) {
  3208. struct drm_gem_object *obj = object_list[i];
  3209. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  3210. uint32_t old_write_domain = obj->write_domain;
  3211. obj->write_domain = obj->pending_write_domain;
  3212. if (obj->write_domain)
  3213. list_move_tail(&obj_priv->gpu_write_list,
  3214. &dev_priv->mm.gpu_write_list);
  3215. else
  3216. list_del_init(&obj_priv->gpu_write_list);
  3217. trace_i915_gem_object_change_domain(obj,
  3218. obj->read_domains,
  3219. old_write_domain);
  3220. }
  3221. i915_verify_inactive(dev, __FILE__, __LINE__);
  3222. #if WATCH_COHERENCY
  3223. for (i = 0; i < args->buffer_count; i++) {
  3224. i915_gem_object_check_coherency(object_list[i],
  3225. exec_list[i].handle);
  3226. }
  3227. #endif
  3228. #if WATCH_EXEC
  3229. i915_gem_dump_object(batch_obj,
  3230. args->batch_len,
  3231. __func__,
  3232. ~0);
  3233. #endif
  3234. /* Exec the batchbuffer */
  3235. ret = ring->dispatch_gem_execbuffer(dev, ring, args,
  3236. cliprects, exec_offset);
  3237. if (ret) {
  3238. DRM_ERROR("dispatch failed %d\n", ret);
  3239. goto err;
  3240. }
  3241. /*
  3242. * Ensure that the commands in the batch buffer are
  3243. * finished before the interrupt fires
  3244. */
  3245. flush_domains = i915_retire_commands(dev, ring);
  3246. i915_verify_inactive(dev, __FILE__, __LINE__);
  3247. /*
  3248. * Get a seqno representing the execution of the current buffer,
  3249. * which we can wait on. We would like to mitigate these interrupts,
  3250. * likely by only creating seqnos occasionally (so that we have
  3251. * *some* interrupts representing completion of buffers that we can
  3252. * wait on when trying to clear up gtt space).
  3253. */
  3254. seqno = i915_add_request(dev, file_priv, flush_domains, ring);
  3255. BUG_ON(seqno == 0);
  3256. for (i = 0; i < args->buffer_count; i++) {
  3257. struct drm_gem_object *obj = object_list[i];
  3258. obj_priv = to_intel_bo(obj);
  3259. i915_gem_object_move_to_active(obj, seqno, ring);
  3260. #if WATCH_LRU
  3261. DRM_INFO("%s: move to exec list %p\n", __func__, obj);
  3262. #endif
  3263. }
  3264. #if WATCH_LRU
  3265. i915_dump_lru(dev, __func__);
  3266. #endif
  3267. i915_verify_inactive(dev, __FILE__, __LINE__);
  3268. err:
  3269. for (i = 0; i < pinned; i++)
  3270. i915_gem_object_unpin(object_list[i]);
  3271. for (i = 0; i < args->buffer_count; i++) {
  3272. if (object_list[i]) {
  3273. obj_priv = to_intel_bo(object_list[i]);
  3274. obj_priv->in_execbuffer = false;
  3275. }
  3276. drm_gem_object_unreference(object_list[i]);
  3277. }
  3278. mutex_unlock(&dev->struct_mutex);
  3279. pre_mutex_err:
  3280. /* Copy the updated relocations out regardless of current error
  3281. * state. Failure to update the relocs would mean that the next
  3282. * time userland calls execbuf, it would do so with presumed offset
  3283. * state that didn't match the actual object state.
  3284. */
  3285. ret2 = i915_gem_put_relocs_to_user(exec_list, args->buffer_count,
  3286. relocs);
  3287. if (ret2 != 0) {
  3288. DRM_ERROR("Failed to copy relocations back out: %d\n", ret2);
  3289. if (ret == 0)
  3290. ret = ret2;
  3291. }
  3292. drm_free_large(object_list);
  3293. kfree(cliprects);
  3294. return ret;
  3295. }
  3296. /*
  3297. * Legacy execbuffer just creates an exec2 list from the original exec object
  3298. * list array and passes it to the real function.
  3299. */
  3300. int
  3301. i915_gem_execbuffer(struct drm_device *dev, void *data,
  3302. struct drm_file *file_priv)
  3303. {
  3304. struct drm_i915_gem_execbuffer *args = data;
  3305. struct drm_i915_gem_execbuffer2 exec2;
  3306. struct drm_i915_gem_exec_object *exec_list = NULL;
  3307. struct drm_i915_gem_exec_object2 *exec2_list = NULL;
  3308. int ret, i;
  3309. #if WATCH_EXEC
  3310. DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
  3311. (int) args->buffers_ptr, args->buffer_count, args->batch_len);
  3312. #endif
  3313. if (args->buffer_count < 1) {
  3314. DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
  3315. return -EINVAL;
  3316. }
  3317. /* Copy in the exec list from userland */
  3318. exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
  3319. exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
  3320. if (exec_list == NULL || exec2_list == NULL) {
  3321. DRM_ERROR("Failed to allocate exec list for %d buffers\n",
  3322. args->buffer_count);
  3323. drm_free_large(exec_list);
  3324. drm_free_large(exec2_list);
  3325. return -ENOMEM;
  3326. }
  3327. ret = copy_from_user(exec_list,
  3328. (struct drm_i915_relocation_entry __user *)
  3329. (uintptr_t) args->buffers_ptr,
  3330. sizeof(*exec_list) * args->buffer_count);
  3331. if (ret != 0) {
  3332. DRM_ERROR("copy %d exec entries failed %d\n",
  3333. args->buffer_count, ret);
  3334. drm_free_large(exec_list);
  3335. drm_free_large(exec2_list);
  3336. return -EFAULT;
  3337. }
  3338. for (i = 0; i < args->buffer_count; i++) {
  3339. exec2_list[i].handle = exec_list[i].handle;
  3340. exec2_list[i].relocation_count = exec_list[i].relocation_count;
  3341. exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
  3342. exec2_list[i].alignment = exec_list[i].alignment;
  3343. exec2_list[i].offset = exec_list[i].offset;
  3344. if (!IS_I965G(dev))
  3345. exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
  3346. else
  3347. exec2_list[i].flags = 0;
  3348. }
  3349. exec2.buffers_ptr = args->buffers_ptr;
  3350. exec2.buffer_count = args->buffer_count;
  3351. exec2.batch_start_offset = args->batch_start_offset;
  3352. exec2.batch_len = args->batch_len;
  3353. exec2.DR1 = args->DR1;
  3354. exec2.DR4 = args->DR4;
  3355. exec2.num_cliprects = args->num_cliprects;
  3356. exec2.cliprects_ptr = args->cliprects_ptr;
  3357. exec2.flags = I915_EXEC_RENDER;
  3358. ret = i915_gem_do_execbuffer(dev, data, file_priv, &exec2, exec2_list);
  3359. if (!ret) {
  3360. /* Copy the new buffer offsets back to the user's exec list. */
  3361. for (i = 0; i < args->buffer_count; i++)
  3362. exec_list[i].offset = exec2_list[i].offset;
  3363. /* ... and back out to userspace */
  3364. ret = copy_to_user((struct drm_i915_relocation_entry __user *)
  3365. (uintptr_t) args->buffers_ptr,
  3366. exec_list,
  3367. sizeof(*exec_list) * args->buffer_count);
  3368. if (ret) {
  3369. ret = -EFAULT;
  3370. DRM_ERROR("failed to copy %d exec entries "
  3371. "back to user (%d)\n",
  3372. args->buffer_count, ret);
  3373. }
  3374. }
  3375. drm_free_large(exec_list);
  3376. drm_free_large(exec2_list);
  3377. return ret;
  3378. }
  3379. int
  3380. i915_gem_execbuffer2(struct drm_device *dev, void *data,
  3381. struct drm_file *file_priv)
  3382. {
  3383. struct drm_i915_gem_execbuffer2 *args = data;
  3384. struct drm_i915_gem_exec_object2 *exec2_list = NULL;
  3385. int ret;
  3386. #if WATCH_EXEC
  3387. DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
  3388. (int) args->buffers_ptr, args->buffer_count, args->batch_len);
  3389. #endif
  3390. if (args->buffer_count < 1) {
  3391. DRM_ERROR("execbuf2 with %d buffers\n", args->buffer_count);
  3392. return -EINVAL;
  3393. }
  3394. exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
  3395. if (exec2_list == NULL) {
  3396. DRM_ERROR("Failed to allocate exec list for %d buffers\n",
  3397. args->buffer_count);
  3398. return -ENOMEM;
  3399. }
  3400. ret = copy_from_user(exec2_list,
  3401. (struct drm_i915_relocation_entry __user *)
  3402. (uintptr_t) args->buffers_ptr,
  3403. sizeof(*exec2_list) * args->buffer_count);
  3404. if (ret != 0) {
  3405. DRM_ERROR("copy %d exec entries failed %d\n",
  3406. args->buffer_count, ret);
  3407. drm_free_large(exec2_list);
  3408. return -EFAULT;
  3409. }
  3410. ret = i915_gem_do_execbuffer(dev, data, file_priv, args, exec2_list);
  3411. if (!ret) {
  3412. /* Copy the new buffer offsets back to the user's exec list. */
  3413. ret = copy_to_user((struct drm_i915_relocation_entry __user *)
  3414. (uintptr_t) args->buffers_ptr,
  3415. exec2_list,
  3416. sizeof(*exec2_list) * args->buffer_count);
  3417. if (ret) {
  3418. ret = -EFAULT;
  3419. DRM_ERROR("failed to copy %d exec entries "
  3420. "back to user (%d)\n",
  3421. args->buffer_count, ret);
  3422. }
  3423. }
  3424. drm_free_large(exec2_list);
  3425. return ret;
  3426. }
  3427. int
  3428. i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment)
  3429. {
  3430. struct drm_device *dev = obj->dev;
  3431. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  3432. int ret;
  3433. BUG_ON(obj_priv->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
  3434. i915_verify_inactive(dev, __FILE__, __LINE__);
  3435. if (obj_priv->gtt_space != NULL) {
  3436. if (alignment == 0)
  3437. alignment = i915_gem_get_gtt_alignment(obj);
  3438. if (obj_priv->gtt_offset & (alignment - 1)) {
  3439. WARN(obj_priv->pin_count,
  3440. "bo is already pinned with incorrect alignment:"
  3441. " offset=%x, req.alignment=%x\n",
  3442. obj_priv->gtt_offset, alignment);
  3443. ret = i915_gem_object_unbind(obj);
  3444. if (ret)
  3445. return ret;
  3446. }
  3447. }
  3448. if (obj_priv->gtt_space == NULL) {
  3449. ret = i915_gem_object_bind_to_gtt(obj, alignment);
  3450. if (ret)
  3451. return ret;
  3452. }
  3453. obj_priv->pin_count++;
  3454. /* If the object is not active and not pending a flush,
  3455. * remove it from the inactive list
  3456. */
  3457. if (obj_priv->pin_count == 1) {
  3458. atomic_inc(&dev->pin_count);
  3459. atomic_add(obj->size, &dev->pin_memory);
  3460. if (!obj_priv->active &&
  3461. (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
  3462. list_del_init(&obj_priv->list);
  3463. }
  3464. i915_verify_inactive(dev, __FILE__, __LINE__);
  3465. return 0;
  3466. }
  3467. void
  3468. i915_gem_object_unpin(struct drm_gem_object *obj)
  3469. {
  3470. struct drm_device *dev = obj->dev;
  3471. drm_i915_private_t *dev_priv = dev->dev_private;
  3472. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  3473. i915_verify_inactive(dev, __FILE__, __LINE__);
  3474. obj_priv->pin_count--;
  3475. BUG_ON(obj_priv->pin_count < 0);
  3476. BUG_ON(obj_priv->gtt_space == NULL);
  3477. /* If the object is no longer pinned, and is
  3478. * neither active nor being flushed, then stick it on
  3479. * the inactive list
  3480. */
  3481. if (obj_priv->pin_count == 0) {
  3482. if (!obj_priv->active &&
  3483. (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
  3484. list_move_tail(&obj_priv->list,
  3485. &dev_priv->mm.inactive_list);
  3486. atomic_dec(&dev->pin_count);
  3487. atomic_sub(obj->size, &dev->pin_memory);
  3488. }
  3489. i915_verify_inactive(dev, __FILE__, __LINE__);
  3490. }
  3491. int
  3492. i915_gem_pin_ioctl(struct drm_device *dev, void *data,
  3493. struct drm_file *file_priv)
  3494. {
  3495. struct drm_i915_gem_pin *args = data;
  3496. struct drm_gem_object *obj;
  3497. struct drm_i915_gem_object *obj_priv;
  3498. int ret;
  3499. mutex_lock(&dev->struct_mutex);
  3500. obj = drm_gem_object_lookup(dev, file_priv, args->handle);
  3501. if (obj == NULL) {
  3502. DRM_ERROR("Bad handle in i915_gem_pin_ioctl(): %d\n",
  3503. args->handle);
  3504. mutex_unlock(&dev->struct_mutex);
  3505. return -ENOENT;
  3506. }
  3507. obj_priv = to_intel_bo(obj);
  3508. if (obj_priv->madv != I915_MADV_WILLNEED) {
  3509. DRM_ERROR("Attempting to pin a purgeable buffer\n");
  3510. drm_gem_object_unreference(obj);
  3511. mutex_unlock(&dev->struct_mutex);
  3512. return -EINVAL;
  3513. }
  3514. if (obj_priv->pin_filp != NULL && obj_priv->pin_filp != file_priv) {
  3515. DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
  3516. args->handle);
  3517. drm_gem_object_unreference(obj);
  3518. mutex_unlock(&dev->struct_mutex);
  3519. return -EINVAL;
  3520. }
  3521. obj_priv->user_pin_count++;
  3522. obj_priv->pin_filp = file_priv;
  3523. if (obj_priv->user_pin_count == 1) {
  3524. ret = i915_gem_object_pin(obj, args->alignment);
  3525. if (ret != 0) {
  3526. drm_gem_object_unreference(obj);
  3527. mutex_unlock(&dev->struct_mutex);
  3528. return ret;
  3529. }
  3530. }
  3531. /* XXX - flush the CPU caches for pinned objects
  3532. * as the X server doesn't manage domains yet
  3533. */
  3534. i915_gem_object_flush_cpu_write_domain(obj);
  3535. args->offset = obj_priv->gtt_offset;
  3536. drm_gem_object_unreference(obj);
  3537. mutex_unlock(&dev->struct_mutex);
  3538. return 0;
  3539. }
  3540. int
  3541. i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
  3542. struct drm_file *file_priv)
  3543. {
  3544. struct drm_i915_gem_pin *args = data;
  3545. struct drm_gem_object *obj;
  3546. struct drm_i915_gem_object *obj_priv;
  3547. mutex_lock(&dev->struct_mutex);
  3548. obj = drm_gem_object_lookup(dev, file_priv, args->handle);
  3549. if (obj == NULL) {
  3550. DRM_ERROR("Bad handle in i915_gem_unpin_ioctl(): %d\n",
  3551. args->handle);
  3552. mutex_unlock(&dev->struct_mutex);
  3553. return -ENOENT;
  3554. }
  3555. obj_priv = to_intel_bo(obj);
  3556. if (obj_priv->pin_filp != file_priv) {
  3557. DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
  3558. args->handle);
  3559. drm_gem_object_unreference(obj);
  3560. mutex_unlock(&dev->struct_mutex);
  3561. return -EINVAL;
  3562. }
  3563. obj_priv->user_pin_count--;
  3564. if (obj_priv->user_pin_count == 0) {
  3565. obj_priv->pin_filp = NULL;
  3566. i915_gem_object_unpin(obj);
  3567. }
  3568. drm_gem_object_unreference(obj);
  3569. mutex_unlock(&dev->struct_mutex);
  3570. return 0;
  3571. }
  3572. int
  3573. i915_gem_busy_ioctl(struct drm_device *dev, void *data,
  3574. struct drm_file *file_priv)
  3575. {
  3576. struct drm_i915_gem_busy *args = data;
  3577. struct drm_gem_object *obj;
  3578. struct drm_i915_gem_object *obj_priv;
  3579. obj = drm_gem_object_lookup(dev, file_priv, args->handle);
  3580. if (obj == NULL) {
  3581. DRM_ERROR("Bad handle in i915_gem_busy_ioctl(): %d\n",
  3582. args->handle);
  3583. return -ENOENT;
  3584. }
  3585. mutex_lock(&dev->struct_mutex);
  3586. /* Count all active objects as busy, even if they are currently not used
  3587. * by the gpu. Users of this interface expect objects to eventually
  3588. * become non-busy without any further actions, therefore emit any
  3589. * necessary flushes here.
  3590. */
  3591. obj_priv = to_intel_bo(obj);
  3592. args->busy = obj_priv->active;
  3593. if (args->busy) {
  3594. /* Unconditionally flush objects, even when the gpu still uses this
  3595. * object. Userspace calling this function indicates that it wants to
  3596. * use this buffer rather sooner than later, so issuing the required
  3597. * flush earlier is beneficial.
  3598. */
  3599. if (obj->write_domain) {
  3600. i915_gem_flush(dev, 0, obj->write_domain);
  3601. (void)i915_add_request(dev, file_priv, obj->write_domain, obj_priv->ring);
  3602. }
  3603. /* Update the active list for the hardware's current position.
  3604. * Otherwise this only updates on a delayed timer or when irqs
  3605. * are actually unmasked, and our working set ends up being
  3606. * larger than required.
  3607. */
  3608. i915_gem_retire_requests_ring(dev, obj_priv->ring);
  3609. args->busy = obj_priv->active;
  3610. }
  3611. drm_gem_object_unreference(obj);
  3612. mutex_unlock(&dev->struct_mutex);
  3613. return 0;
  3614. }
  3615. int
  3616. i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
  3617. struct drm_file *file_priv)
  3618. {
  3619. return i915_gem_ring_throttle(dev, file_priv);
  3620. }
  3621. int
  3622. i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
  3623. struct drm_file *file_priv)
  3624. {
  3625. struct drm_i915_gem_madvise *args = data;
  3626. struct drm_gem_object *obj;
  3627. struct drm_i915_gem_object *obj_priv;
  3628. switch (args->madv) {
  3629. case I915_MADV_DONTNEED:
  3630. case I915_MADV_WILLNEED:
  3631. break;
  3632. default:
  3633. return -EINVAL;
  3634. }
  3635. obj = drm_gem_object_lookup(dev, file_priv, args->handle);
  3636. if (obj == NULL) {
  3637. DRM_ERROR("Bad handle in i915_gem_madvise_ioctl(): %d\n",
  3638. args->handle);
  3639. return -ENOENT;
  3640. }
  3641. mutex_lock(&dev->struct_mutex);
  3642. obj_priv = to_intel_bo(obj);
  3643. if (obj_priv->pin_count) {
  3644. drm_gem_object_unreference(obj);
  3645. mutex_unlock(&dev->struct_mutex);
  3646. DRM_ERROR("Attempted i915_gem_madvise_ioctl() on a pinned object\n");
  3647. return -EINVAL;
  3648. }
  3649. if (obj_priv->madv != __I915_MADV_PURGED)
  3650. obj_priv->madv = args->madv;
  3651. /* if the object is no longer bound, discard its backing storage */
  3652. if (i915_gem_object_is_purgeable(obj_priv) &&
  3653. obj_priv->gtt_space == NULL)
  3654. i915_gem_object_truncate(obj);
  3655. args->retained = obj_priv->madv != __I915_MADV_PURGED;
  3656. drm_gem_object_unreference(obj);
  3657. mutex_unlock(&dev->struct_mutex);
  3658. return 0;
  3659. }
  3660. struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev,
  3661. size_t size)
  3662. {
  3663. struct drm_i915_gem_object *obj;
  3664. obj = kzalloc(sizeof(*obj), GFP_KERNEL);
  3665. if (obj == NULL)
  3666. return NULL;
  3667. if (drm_gem_object_init(dev, &obj->base, size) != 0) {
  3668. kfree(obj);
  3669. return NULL;
  3670. }
  3671. obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  3672. obj->base.read_domains = I915_GEM_DOMAIN_CPU;
  3673. obj->agp_type = AGP_USER_MEMORY;
  3674. obj->base.driver_private = NULL;
  3675. obj->fence_reg = I915_FENCE_REG_NONE;
  3676. INIT_LIST_HEAD(&obj->list);
  3677. INIT_LIST_HEAD(&obj->gpu_write_list);
  3678. obj->madv = I915_MADV_WILLNEED;
  3679. trace_i915_gem_object_create(&obj->base);
  3680. return &obj->base;
  3681. }
  3682. int i915_gem_init_object(struct drm_gem_object *obj)
  3683. {
  3684. BUG();
  3685. return 0;
  3686. }
  3687. static void i915_gem_free_object_tail(struct drm_gem_object *obj)
  3688. {
  3689. struct drm_device *dev = obj->dev;
  3690. drm_i915_private_t *dev_priv = dev->dev_private;
  3691. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  3692. int ret;
  3693. ret = i915_gem_object_unbind(obj);
  3694. if (ret == -ERESTARTSYS) {
  3695. list_move(&obj_priv->list,
  3696. &dev_priv->mm.deferred_free_list);
  3697. return;
  3698. }
  3699. if (obj_priv->mmap_offset)
  3700. i915_gem_free_mmap_offset(obj);
  3701. drm_gem_object_release(obj);
  3702. kfree(obj_priv->page_cpu_valid);
  3703. kfree(obj_priv->bit_17);
  3704. kfree(obj_priv);
  3705. }
  3706. void i915_gem_free_object(struct drm_gem_object *obj)
  3707. {
  3708. struct drm_device *dev = obj->dev;
  3709. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  3710. trace_i915_gem_object_destroy(obj);
  3711. while (obj_priv->pin_count > 0)
  3712. i915_gem_object_unpin(obj);
  3713. if (obj_priv->phys_obj)
  3714. i915_gem_detach_phys_object(dev, obj);
  3715. i915_gem_free_object_tail(obj);
  3716. }
  3717. int
  3718. i915_gem_idle(struct drm_device *dev)
  3719. {
  3720. drm_i915_private_t *dev_priv = dev->dev_private;
  3721. int ret;
  3722. mutex_lock(&dev->struct_mutex);
  3723. if (dev_priv->mm.suspended ||
  3724. (dev_priv->render_ring.gem_object == NULL) ||
  3725. (HAS_BSD(dev) &&
  3726. dev_priv->bsd_ring.gem_object == NULL)) {
  3727. mutex_unlock(&dev->struct_mutex);
  3728. return 0;
  3729. }
  3730. ret = i915_gpu_idle(dev);
  3731. if (ret) {
  3732. mutex_unlock(&dev->struct_mutex);
  3733. return ret;
  3734. }
  3735. /* Under UMS, be paranoid and evict. */
  3736. if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
  3737. ret = i915_gem_evict_inactive(dev);
  3738. if (ret) {
  3739. mutex_unlock(&dev->struct_mutex);
  3740. return ret;
  3741. }
  3742. }
  3743. /* Hack! Don't let anybody do execbuf while we don't control the chip.
  3744. * We need to replace this with a semaphore, or something.
  3745. * And not confound mm.suspended!
  3746. */
  3747. dev_priv->mm.suspended = 1;
  3748. del_timer(&dev_priv->hangcheck_timer);
  3749. i915_kernel_lost_context(dev);
  3750. i915_gem_cleanup_ringbuffer(dev);
  3751. mutex_unlock(&dev->struct_mutex);
  3752. /* Cancel the retire work handler, which should be idle now. */
  3753. cancel_delayed_work_sync(&dev_priv->mm.retire_work);
  3754. return 0;
  3755. }
  3756. /*
  3757. * 965+ support PIPE_CONTROL commands, which provide finer grained control
  3758. * over cache flushing.
  3759. */
  3760. static int
  3761. i915_gem_init_pipe_control(struct drm_device *dev)
  3762. {
  3763. drm_i915_private_t *dev_priv = dev->dev_private;
  3764. struct drm_gem_object *obj;
  3765. struct drm_i915_gem_object *obj_priv;
  3766. int ret;
  3767. obj = i915_gem_alloc_object(dev, 4096);
  3768. if (obj == NULL) {
  3769. DRM_ERROR("Failed to allocate seqno page\n");
  3770. ret = -ENOMEM;
  3771. goto err;
  3772. }
  3773. obj_priv = to_intel_bo(obj);
  3774. obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
  3775. ret = i915_gem_object_pin(obj, 4096);
  3776. if (ret)
  3777. goto err_unref;
  3778. dev_priv->seqno_gfx_addr = obj_priv->gtt_offset;
  3779. dev_priv->seqno_page = kmap(obj_priv->pages[0]);
  3780. if (dev_priv->seqno_page == NULL)
  3781. goto err_unpin;
  3782. dev_priv->seqno_obj = obj;
  3783. memset(dev_priv->seqno_page, 0, PAGE_SIZE);
  3784. return 0;
  3785. err_unpin:
  3786. i915_gem_object_unpin(obj);
  3787. err_unref:
  3788. drm_gem_object_unreference(obj);
  3789. err:
  3790. return ret;
  3791. }
  3792. static void
  3793. i915_gem_cleanup_pipe_control(struct drm_device *dev)
  3794. {
  3795. drm_i915_private_t *dev_priv = dev->dev_private;
  3796. struct drm_gem_object *obj;
  3797. struct drm_i915_gem_object *obj_priv;
  3798. obj = dev_priv->seqno_obj;
  3799. obj_priv = to_intel_bo(obj);
  3800. kunmap(obj_priv->pages[0]);
  3801. i915_gem_object_unpin(obj);
  3802. drm_gem_object_unreference(obj);
  3803. dev_priv->seqno_obj = NULL;
  3804. dev_priv->seqno_page = NULL;
  3805. }
  3806. int
  3807. i915_gem_init_ringbuffer(struct drm_device *dev)
  3808. {
  3809. drm_i915_private_t *dev_priv = dev->dev_private;
  3810. int ret;
  3811. dev_priv->render_ring = render_ring;
  3812. if (!I915_NEED_GFX_HWS(dev)) {
  3813. dev_priv->render_ring.status_page.page_addr
  3814. = dev_priv->status_page_dmah->vaddr;
  3815. memset(dev_priv->render_ring.status_page.page_addr,
  3816. 0, PAGE_SIZE);
  3817. }
  3818. if (HAS_PIPE_CONTROL(dev)) {
  3819. ret = i915_gem_init_pipe_control(dev);
  3820. if (ret)
  3821. return ret;
  3822. }
  3823. ret = intel_init_ring_buffer(dev, &dev_priv->render_ring);
  3824. if (ret)
  3825. goto cleanup_pipe_control;
  3826. if (HAS_BSD(dev)) {
  3827. dev_priv->bsd_ring = bsd_ring;
  3828. ret = intel_init_ring_buffer(dev, &dev_priv->bsd_ring);
  3829. if (ret)
  3830. goto cleanup_render_ring;
  3831. }
  3832. dev_priv->next_seqno = 1;
  3833. return 0;
  3834. cleanup_render_ring:
  3835. intel_cleanup_ring_buffer(dev, &dev_priv->render_ring);
  3836. cleanup_pipe_control:
  3837. if (HAS_PIPE_CONTROL(dev))
  3838. i915_gem_cleanup_pipe_control(dev);
  3839. return ret;
  3840. }
  3841. void
  3842. i915_gem_cleanup_ringbuffer(struct drm_device *dev)
  3843. {
  3844. drm_i915_private_t *dev_priv = dev->dev_private;
  3845. intel_cleanup_ring_buffer(dev, &dev_priv->render_ring);
  3846. if (HAS_BSD(dev))
  3847. intel_cleanup_ring_buffer(dev, &dev_priv->bsd_ring);
  3848. if (HAS_PIPE_CONTROL(dev))
  3849. i915_gem_cleanup_pipe_control(dev);
  3850. }
  3851. int
  3852. i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
  3853. struct drm_file *file_priv)
  3854. {
  3855. drm_i915_private_t *dev_priv = dev->dev_private;
  3856. int ret;
  3857. if (drm_core_check_feature(dev, DRIVER_MODESET))
  3858. return 0;
  3859. if (atomic_read(&dev_priv->mm.wedged)) {
  3860. DRM_ERROR("Reenabling wedged hardware, good luck\n");
  3861. atomic_set(&dev_priv->mm.wedged, 0);
  3862. }
  3863. mutex_lock(&dev->struct_mutex);
  3864. dev_priv->mm.suspended = 0;
  3865. ret = i915_gem_init_ringbuffer(dev);
  3866. if (ret != 0) {
  3867. mutex_unlock(&dev->struct_mutex);
  3868. return ret;
  3869. }
  3870. spin_lock(&dev_priv->mm.active_list_lock);
  3871. BUG_ON(!list_empty(&dev_priv->render_ring.active_list));
  3872. BUG_ON(HAS_BSD(dev) && !list_empty(&dev_priv->bsd_ring.active_list));
  3873. spin_unlock(&dev_priv->mm.active_list_lock);
  3874. BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
  3875. BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
  3876. BUG_ON(!list_empty(&dev_priv->render_ring.request_list));
  3877. BUG_ON(HAS_BSD(dev) && !list_empty(&dev_priv->bsd_ring.request_list));
  3878. mutex_unlock(&dev->struct_mutex);
  3879. ret = drm_irq_install(dev);
  3880. if (ret)
  3881. goto cleanup_ringbuffer;
  3882. return 0;
  3883. cleanup_ringbuffer:
  3884. mutex_lock(&dev->struct_mutex);
  3885. i915_gem_cleanup_ringbuffer(dev);
  3886. dev_priv->mm.suspended = 1;
  3887. mutex_unlock(&dev->struct_mutex);
  3888. return ret;
  3889. }
  3890. int
  3891. i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
  3892. struct drm_file *file_priv)
  3893. {
  3894. if (drm_core_check_feature(dev, DRIVER_MODESET))
  3895. return 0;
  3896. drm_irq_uninstall(dev);
  3897. return i915_gem_idle(dev);
  3898. }
  3899. void
  3900. i915_gem_lastclose(struct drm_device *dev)
  3901. {
  3902. int ret;
  3903. if (drm_core_check_feature(dev, DRIVER_MODESET))
  3904. return;
  3905. ret = i915_gem_idle(dev);
  3906. if (ret)
  3907. DRM_ERROR("failed to idle hardware: %d\n", ret);
  3908. }
  3909. void
  3910. i915_gem_load(struct drm_device *dev)
  3911. {
  3912. int i;
  3913. drm_i915_private_t *dev_priv = dev->dev_private;
  3914. spin_lock_init(&dev_priv->mm.active_list_lock);
  3915. INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
  3916. INIT_LIST_HEAD(&dev_priv->mm.gpu_write_list);
  3917. INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
  3918. INIT_LIST_HEAD(&dev_priv->mm.fence_list);
  3919. INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
  3920. INIT_LIST_HEAD(&dev_priv->render_ring.active_list);
  3921. INIT_LIST_HEAD(&dev_priv->render_ring.request_list);
  3922. if (HAS_BSD(dev)) {
  3923. INIT_LIST_HEAD(&dev_priv->bsd_ring.active_list);
  3924. INIT_LIST_HEAD(&dev_priv->bsd_ring.request_list);
  3925. }
  3926. for (i = 0; i < 16; i++)
  3927. INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
  3928. INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
  3929. i915_gem_retire_work_handler);
  3930. spin_lock(&shrink_list_lock);
  3931. list_add(&dev_priv->mm.shrink_list, &shrink_list);
  3932. spin_unlock(&shrink_list_lock);
  3933. /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
  3934. if (IS_GEN3(dev)) {
  3935. u32 tmp = I915_READ(MI_ARB_STATE);
  3936. if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) {
  3937. /* arb state is a masked write, so set bit + bit in mask */
  3938. tmp = MI_ARB_C3_LP_WRITE_ENABLE | (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT);
  3939. I915_WRITE(MI_ARB_STATE, tmp);
  3940. }
  3941. }
  3942. /* Old X drivers will take 0-2 for front, back, depth buffers */
  3943. if (!drm_core_check_feature(dev, DRIVER_MODESET))
  3944. dev_priv->fence_reg_start = 3;
  3945. if (IS_I965G(dev) || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
  3946. dev_priv->num_fence_regs = 16;
  3947. else
  3948. dev_priv->num_fence_regs = 8;
  3949. /* Initialize fence registers to zero */
  3950. if (IS_I965G(dev)) {
  3951. for (i = 0; i < 16; i++)
  3952. I915_WRITE64(FENCE_REG_965_0 + (i * 8), 0);
  3953. } else {
  3954. for (i = 0; i < 8; i++)
  3955. I915_WRITE(FENCE_REG_830_0 + (i * 4), 0);
  3956. if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
  3957. for (i = 0; i < 8; i++)
  3958. I915_WRITE(FENCE_REG_945_8 + (i * 4), 0);
  3959. }
  3960. i915_gem_detect_bit_6_swizzle(dev);
  3961. init_waitqueue_head(&dev_priv->pending_flip_queue);
  3962. }
  3963. /*
  3964. * Create a physically contiguous memory object for this object
  3965. * e.g. for cursor + overlay regs
  3966. */
  3967. int i915_gem_init_phys_object(struct drm_device *dev,
  3968. int id, int size, int align)
  3969. {
  3970. drm_i915_private_t *dev_priv = dev->dev_private;
  3971. struct drm_i915_gem_phys_object *phys_obj;
  3972. int ret;
  3973. if (dev_priv->mm.phys_objs[id - 1] || !size)
  3974. return 0;
  3975. phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
  3976. if (!phys_obj)
  3977. return -ENOMEM;
  3978. phys_obj->id = id;
  3979. phys_obj->handle = drm_pci_alloc(dev, size, align);
  3980. if (!phys_obj->handle) {
  3981. ret = -ENOMEM;
  3982. goto kfree_obj;
  3983. }
  3984. #ifdef CONFIG_X86
  3985. set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
  3986. #endif
  3987. dev_priv->mm.phys_objs[id - 1] = phys_obj;
  3988. return 0;
  3989. kfree_obj:
  3990. kfree(phys_obj);
  3991. return ret;
  3992. }
  3993. void i915_gem_free_phys_object(struct drm_device *dev, int id)
  3994. {
  3995. drm_i915_private_t *dev_priv = dev->dev_private;
  3996. struct drm_i915_gem_phys_object *phys_obj;
  3997. if (!dev_priv->mm.phys_objs[id - 1])
  3998. return;
  3999. phys_obj = dev_priv->mm.phys_objs[id - 1];
  4000. if (phys_obj->cur_obj) {
  4001. i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
  4002. }
  4003. #ifdef CONFIG_X86
  4004. set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
  4005. #endif
  4006. drm_pci_free(dev, phys_obj->handle);
  4007. kfree(phys_obj);
  4008. dev_priv->mm.phys_objs[id - 1] = NULL;
  4009. }
  4010. void i915_gem_free_all_phys_object(struct drm_device *dev)
  4011. {
  4012. int i;
  4013. for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
  4014. i915_gem_free_phys_object(dev, i);
  4015. }
  4016. void i915_gem_detach_phys_object(struct drm_device *dev,
  4017. struct drm_gem_object *obj)
  4018. {
  4019. struct drm_i915_gem_object *obj_priv;
  4020. int i;
  4021. int ret;
  4022. int page_count;
  4023. obj_priv = to_intel_bo(obj);
  4024. if (!obj_priv->phys_obj)
  4025. return;
  4026. ret = i915_gem_object_get_pages(obj, 0);
  4027. if (ret)
  4028. goto out;
  4029. page_count = obj->size / PAGE_SIZE;
  4030. for (i = 0; i < page_count; i++) {
  4031. char *dst = kmap_atomic(obj_priv->pages[i], KM_USER0);
  4032. char *src = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
  4033. memcpy(dst, src, PAGE_SIZE);
  4034. kunmap_atomic(dst, KM_USER0);
  4035. }
  4036. drm_clflush_pages(obj_priv->pages, page_count);
  4037. drm_agp_chipset_flush(dev);
  4038. i915_gem_object_put_pages(obj);
  4039. out:
  4040. obj_priv->phys_obj->cur_obj = NULL;
  4041. obj_priv->phys_obj = NULL;
  4042. }
  4043. int
  4044. i915_gem_attach_phys_object(struct drm_device *dev,
  4045. struct drm_gem_object *obj,
  4046. int id,
  4047. int align)
  4048. {
  4049. drm_i915_private_t *dev_priv = dev->dev_private;
  4050. struct drm_i915_gem_object *obj_priv;
  4051. int ret = 0;
  4052. int page_count;
  4053. int i;
  4054. if (id > I915_MAX_PHYS_OBJECT)
  4055. return -EINVAL;
  4056. obj_priv = to_intel_bo(obj);
  4057. if (obj_priv->phys_obj) {
  4058. if (obj_priv->phys_obj->id == id)
  4059. return 0;
  4060. i915_gem_detach_phys_object(dev, obj);
  4061. }
  4062. /* create a new object */
  4063. if (!dev_priv->mm.phys_objs[id - 1]) {
  4064. ret = i915_gem_init_phys_object(dev, id,
  4065. obj->size, align);
  4066. if (ret) {
  4067. DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size);
  4068. goto out;
  4069. }
  4070. }
  4071. /* bind to the object */
  4072. obj_priv->phys_obj = dev_priv->mm.phys_objs[id - 1];
  4073. obj_priv->phys_obj->cur_obj = obj;
  4074. ret = i915_gem_object_get_pages(obj, 0);
  4075. if (ret) {
  4076. DRM_ERROR("failed to get page list\n");
  4077. goto out;
  4078. }
  4079. page_count = obj->size / PAGE_SIZE;
  4080. for (i = 0; i < page_count; i++) {
  4081. char *src = kmap_atomic(obj_priv->pages[i], KM_USER0);
  4082. char *dst = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
  4083. memcpy(dst, src, PAGE_SIZE);
  4084. kunmap_atomic(src, KM_USER0);
  4085. }
  4086. i915_gem_object_put_pages(obj);
  4087. return 0;
  4088. out:
  4089. return ret;
  4090. }
  4091. static int
  4092. i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
  4093. struct drm_i915_gem_pwrite *args,
  4094. struct drm_file *file_priv)
  4095. {
  4096. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  4097. void *obj_addr;
  4098. int ret;
  4099. char __user *user_data;
  4100. user_data = (char __user *) (uintptr_t) args->data_ptr;
  4101. obj_addr = obj_priv->phys_obj->handle->vaddr + args->offset;
  4102. DRM_DEBUG_DRIVER("obj_addr %p, %lld\n", obj_addr, args->size);
  4103. ret = copy_from_user(obj_addr, user_data, args->size);
  4104. if (ret)
  4105. return -EFAULT;
  4106. drm_agp_chipset_flush(dev);
  4107. return 0;
  4108. }
  4109. void i915_gem_release(struct drm_device * dev, struct drm_file *file_priv)
  4110. {
  4111. struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
  4112. /* Clean up our request list when the client is going away, so that
  4113. * later retire_requests won't dereference our soon-to-be-gone
  4114. * file_priv.
  4115. */
  4116. mutex_lock(&dev->struct_mutex);
  4117. while (!list_empty(&i915_file_priv->mm.request_list))
  4118. list_del_init(i915_file_priv->mm.request_list.next);
  4119. mutex_unlock(&dev->struct_mutex);
  4120. }
  4121. static int
  4122. i915_gpu_is_active(struct drm_device *dev)
  4123. {
  4124. drm_i915_private_t *dev_priv = dev->dev_private;
  4125. int lists_empty;
  4126. spin_lock(&dev_priv->mm.active_list_lock);
  4127. lists_empty = list_empty(&dev_priv->mm.flushing_list) &&
  4128. list_empty(&dev_priv->render_ring.active_list);
  4129. if (HAS_BSD(dev))
  4130. lists_empty &= list_empty(&dev_priv->bsd_ring.active_list);
  4131. spin_unlock(&dev_priv->mm.active_list_lock);
  4132. return !lists_empty;
  4133. }
  4134. static int
  4135. i915_gem_shrink(struct shrinker *shrink, int nr_to_scan, gfp_t gfp_mask)
  4136. {
  4137. drm_i915_private_t *dev_priv, *next_dev;
  4138. struct drm_i915_gem_object *obj_priv, *next_obj;
  4139. int cnt = 0;
  4140. int would_deadlock = 1;
  4141. /* "fast-path" to count number of available objects */
  4142. if (nr_to_scan == 0) {
  4143. spin_lock(&shrink_list_lock);
  4144. list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
  4145. struct drm_device *dev = dev_priv->dev;
  4146. if (mutex_trylock(&dev->struct_mutex)) {
  4147. list_for_each_entry(obj_priv,
  4148. &dev_priv->mm.inactive_list,
  4149. list)
  4150. cnt++;
  4151. mutex_unlock(&dev->struct_mutex);
  4152. }
  4153. }
  4154. spin_unlock(&shrink_list_lock);
  4155. return (cnt / 100) * sysctl_vfs_cache_pressure;
  4156. }
  4157. spin_lock(&shrink_list_lock);
  4158. rescan:
  4159. /* first scan for clean buffers */
  4160. list_for_each_entry_safe(dev_priv, next_dev,
  4161. &shrink_list, mm.shrink_list) {
  4162. struct drm_device *dev = dev_priv->dev;
  4163. if (! mutex_trylock(&dev->struct_mutex))
  4164. continue;
  4165. spin_unlock(&shrink_list_lock);
  4166. i915_gem_retire_requests(dev);
  4167. list_for_each_entry_safe(obj_priv, next_obj,
  4168. &dev_priv->mm.inactive_list,
  4169. list) {
  4170. if (i915_gem_object_is_purgeable(obj_priv)) {
  4171. i915_gem_object_unbind(&obj_priv->base);
  4172. if (--nr_to_scan <= 0)
  4173. break;
  4174. }
  4175. }
  4176. spin_lock(&shrink_list_lock);
  4177. mutex_unlock(&dev->struct_mutex);
  4178. would_deadlock = 0;
  4179. if (nr_to_scan <= 0)
  4180. break;
  4181. }
  4182. /* second pass, evict/count anything still on the inactive list */
  4183. list_for_each_entry_safe(dev_priv, next_dev,
  4184. &shrink_list, mm.shrink_list) {
  4185. struct drm_device *dev = dev_priv->dev;
  4186. if (! mutex_trylock(&dev->struct_mutex))
  4187. continue;
  4188. spin_unlock(&shrink_list_lock);
  4189. list_for_each_entry_safe(obj_priv, next_obj,
  4190. &dev_priv->mm.inactive_list,
  4191. list) {
  4192. if (nr_to_scan > 0) {
  4193. i915_gem_object_unbind(&obj_priv->base);
  4194. nr_to_scan--;
  4195. } else
  4196. cnt++;
  4197. }
  4198. spin_lock(&shrink_list_lock);
  4199. mutex_unlock(&dev->struct_mutex);
  4200. would_deadlock = 0;
  4201. }
  4202. if (nr_to_scan) {
  4203. int active = 0;
  4204. /*
  4205. * We are desperate for pages, so as a last resort, wait
  4206. * for the GPU to finish and discard whatever we can.
  4207. * This has a dramatic impact to reduce the number of
  4208. * OOM-killer events whilst running the GPU aggressively.
  4209. */
  4210. list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
  4211. struct drm_device *dev = dev_priv->dev;
  4212. if (!mutex_trylock(&dev->struct_mutex))
  4213. continue;
  4214. spin_unlock(&shrink_list_lock);
  4215. if (i915_gpu_is_active(dev)) {
  4216. i915_gpu_idle(dev);
  4217. active++;
  4218. }
  4219. spin_lock(&shrink_list_lock);
  4220. mutex_unlock(&dev->struct_mutex);
  4221. }
  4222. if (active)
  4223. goto rescan;
  4224. }
  4225. spin_unlock(&shrink_list_lock);
  4226. if (would_deadlock)
  4227. return -1;
  4228. else if (cnt > 0)
  4229. return (cnt / 100) * sysctl_vfs_cache_pressure;
  4230. else
  4231. return 0;
  4232. }
  4233. static struct shrinker shrinker = {
  4234. .shrink = i915_gem_shrink,
  4235. .seeks = DEFAULT_SEEKS,
  4236. };
  4237. __init void
  4238. i915_gem_shrinker_init(void)
  4239. {
  4240. register_shrinker(&shrinker);
  4241. }
  4242. __exit void
  4243. i915_gem_shrinker_exit(void)
  4244. {
  4245. unregister_shrinker(&shrinker);
  4246. }