i915_gem.c 122 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124412541264127412841294130413141324133413441354136413741384139414041414142414341444145414641474148414941504151415241534154415541564157415841594160416141624163416441654166416741684169417041714172417341744175417641774178417941804181418241834184418541864187418841894190419141924193419441954196419741984199420042014202420342044205420642074208420942104211421242134214421542164217421842194220422142224223422442254226422742284229423042314232423342344235423642374238423942404241424242434244424542464247424842494250425142524253425442554256425742584259426042614262426342644265426642674268426942704271427242734274427542764277427842794280428142824283428442854286428742884289429042914292429342944295429642974298429943004301430243034304430543064307430843094310431143124313431443154316431743184319432043214322432343244325432643274328432943304331433243334334433543364337433843394340434143424343434443454346434743484349435043514352435343544355435643574358435943604361436243634364436543664367436843694370437143724373437443754376437743784379438043814382438343844385438643874388438943904391439243934394439543964397439843994400440144024403440444054406440744084409441044114412441344144415441644174418441944204421442244234424442544264427442844294430443144324433443444354436443744384439444044414442444344444445444644474448444944504451445244534454445544564457445844594460446144624463446444654466446744684469447044714472447344744475447644774478447944804481448244834484448544864487448844894490449144924493449444954496449744984499450045014502450345044505450645074508450945104511451245134514451545164517451845194520452145224523452445254526452745284529453045314532453345344535453645374538453945404541454245434544454545464547454845494550455145524553455445554556455745584559456045614562456345644565456645674568456945704571457245734574457545764577457845794580458145824583458445854586458745884589459045914592459345944595459645974598459946004601460246034604460546064607460846094610461146124613461446154616461746184619462046214622462346244625462646274628462946304631463246334634463546364637463846394640464146424643464446454646464746484649465046514652465346544655465646574658465946604661466246634664466546664667466846694670467146724673467446754676467746784679468046814682468346844685468646874688468946904691469246934694469546964697469846994700470147024703470447054706470747084709471047114712471347144715471647174718471947204721472247234724472547264727472847294730473147324733473447354736473747384739474047414742474347444745474647474748474947504751475247534754475547564757475847594760476147624763476447654766476747684769477047714772477347744775477647774778477947804781478247834784478547864787478847894790479147924793479447954796479747984799480048014802480348044805480648074808480948104811481248134814481548164817481848194820482148224823482448254826482748284829483048314832483348344835483648374838483948404841484248434844484548464847484848494850485148524853485448554856485748584859486048614862486348644865486648674868486948704871487248734874487548764877487848794880
  1. /*
  2. * Copyright © 2008 Intel Corporation
  3. *
  4. * Permission is hereby granted, free of charge, to any person obtaining a
  5. * copy of this software and associated documentation files (the "Software"),
  6. * to deal in the Software without restriction, including without limitation
  7. * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8. * and/or sell copies of the Software, and to permit persons to whom the
  9. * Software is furnished to do so, subject to the following conditions:
  10. *
  11. * The above copyright notice and this permission notice (including the next
  12. * paragraph) shall be included in all copies or substantial portions of the
  13. * Software.
  14. *
  15. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17. * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  18. * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19. * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20. * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21. * IN THE SOFTWARE.
  22. *
  23. * Authors:
  24. * Eric Anholt <eric@anholt.net>
  25. *
  26. */
  27. #include <drm/drmP.h>
  28. #include <drm/i915_drm.h>
  29. #include "i915_drv.h"
  30. #include "i915_trace.h"
  31. #include "intel_drv.h"
  32. #include <linux/shmem_fs.h>
  33. #include <linux/slab.h>
  34. #include <linux/swap.h>
  35. #include <linux/pci.h>
  36. #include <linux/dma-buf.h>
  37. static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
  38. static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj,
  39. bool force);
  40. static __must_check int
  41. i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
  42. struct i915_address_space *vm,
  43. unsigned alignment,
  44. bool map_and_fenceable,
  45. bool nonblocking);
  46. static int i915_gem_phys_pwrite(struct drm_device *dev,
  47. struct drm_i915_gem_object *obj,
  48. struct drm_i915_gem_pwrite *args,
  49. struct drm_file *file);
  50. static void i915_gem_write_fence(struct drm_device *dev, int reg,
  51. struct drm_i915_gem_object *obj);
  52. static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
  53. struct drm_i915_fence_reg *fence,
  54. bool enable);
  55. static int i915_gem_inactive_shrink(struct shrinker *shrinker,
  56. struct shrink_control *sc);
  57. static long i915_gem_purge(struct drm_i915_private *dev_priv, long target);
  58. static void i915_gem_shrink_all(struct drm_i915_private *dev_priv);
  59. static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
  60. static bool cpu_cache_is_coherent(struct drm_device *dev,
  61. enum i915_cache_level level)
  62. {
  63. return HAS_LLC(dev) || level != I915_CACHE_NONE;
  64. }
  65. static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
  66. {
  67. if (!cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
  68. return true;
  69. return obj->pin_display;
  70. }
  71. static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
  72. {
  73. if (obj->tiling_mode)
  74. i915_gem_release_mmap(obj);
  75. /* As we do not have an associated fence register, we will force
  76. * a tiling change if we ever need to acquire one.
  77. */
  78. obj->fence_dirty = false;
  79. obj->fence_reg = I915_FENCE_REG_NONE;
  80. }
  81. /* some bookkeeping */
  82. static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
  83. size_t size)
  84. {
  85. spin_lock(&dev_priv->mm.object_stat_lock);
  86. dev_priv->mm.object_count++;
  87. dev_priv->mm.object_memory += size;
  88. spin_unlock(&dev_priv->mm.object_stat_lock);
  89. }
  90. static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
  91. size_t size)
  92. {
  93. spin_lock(&dev_priv->mm.object_stat_lock);
  94. dev_priv->mm.object_count--;
  95. dev_priv->mm.object_memory -= size;
  96. spin_unlock(&dev_priv->mm.object_stat_lock);
  97. }
  98. static int
  99. i915_gem_wait_for_error(struct i915_gpu_error *error)
  100. {
  101. int ret;
  102. #define EXIT_COND (!i915_reset_in_progress(error) || \
  103. i915_terminally_wedged(error))
  104. if (EXIT_COND)
  105. return 0;
  106. /*
  107. * Only wait 10 seconds for the gpu reset to complete to avoid hanging
  108. * userspace. If it takes that long something really bad is going on and
  109. * we should simply try to bail out and fail as gracefully as possible.
  110. */
  111. ret = wait_event_interruptible_timeout(error->reset_queue,
  112. EXIT_COND,
  113. 10*HZ);
  114. if (ret == 0) {
  115. DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
  116. return -EIO;
  117. } else if (ret < 0) {
  118. return ret;
  119. }
  120. #undef EXIT_COND
  121. return 0;
  122. }
  123. int i915_mutex_lock_interruptible(struct drm_device *dev)
  124. {
  125. struct drm_i915_private *dev_priv = dev->dev_private;
  126. int ret;
  127. ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
  128. if (ret)
  129. return ret;
  130. ret = mutex_lock_interruptible(&dev->struct_mutex);
  131. if (ret)
  132. return ret;
  133. WARN_ON(i915_verify_lists(dev));
  134. return 0;
  135. }
  136. static inline bool
  137. i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
  138. {
  139. return i915_gem_obj_bound_any(obj) && !obj->active;
  140. }
  141. int
  142. i915_gem_init_ioctl(struct drm_device *dev, void *data,
  143. struct drm_file *file)
  144. {
  145. struct drm_i915_private *dev_priv = dev->dev_private;
  146. struct drm_i915_gem_init *args = data;
  147. if (drm_core_check_feature(dev, DRIVER_MODESET))
  148. return -ENODEV;
  149. if (args->gtt_start >= args->gtt_end ||
  150. (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
  151. return -EINVAL;
  152. /* GEM with user mode setting was never supported on ilk and later. */
  153. if (INTEL_INFO(dev)->gen >= 5)
  154. return -ENODEV;
  155. mutex_lock(&dev->struct_mutex);
  156. i915_gem_setup_global_gtt(dev, args->gtt_start, args->gtt_end,
  157. args->gtt_end);
  158. dev_priv->gtt.mappable_end = args->gtt_end;
  159. mutex_unlock(&dev->struct_mutex);
  160. return 0;
  161. }
  162. int
  163. i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
  164. struct drm_file *file)
  165. {
  166. struct drm_i915_private *dev_priv = dev->dev_private;
  167. struct drm_i915_gem_get_aperture *args = data;
  168. struct drm_i915_gem_object *obj;
  169. size_t pinned;
  170. pinned = 0;
  171. mutex_lock(&dev->struct_mutex);
  172. list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
  173. if (obj->pin_count)
  174. pinned += i915_gem_obj_ggtt_size(obj);
  175. mutex_unlock(&dev->struct_mutex);
  176. args->aper_size = dev_priv->gtt.base.total;
  177. args->aper_available_size = args->aper_size - pinned;
  178. return 0;
  179. }
  180. void *i915_gem_object_alloc(struct drm_device *dev)
  181. {
  182. struct drm_i915_private *dev_priv = dev->dev_private;
  183. return kmem_cache_alloc(dev_priv->slab, GFP_KERNEL | __GFP_ZERO);
  184. }
  185. void i915_gem_object_free(struct drm_i915_gem_object *obj)
  186. {
  187. struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  188. kmem_cache_free(dev_priv->slab, obj);
  189. }
  190. static int
  191. i915_gem_create(struct drm_file *file,
  192. struct drm_device *dev,
  193. uint64_t size,
  194. uint32_t *handle_p)
  195. {
  196. struct drm_i915_gem_object *obj;
  197. int ret;
  198. u32 handle;
  199. size = roundup(size, PAGE_SIZE);
  200. if (size == 0)
  201. return -EINVAL;
  202. /* Allocate the new object */
  203. obj = i915_gem_alloc_object(dev, size);
  204. if (obj == NULL)
  205. return -ENOMEM;
  206. ret = drm_gem_handle_create(file, &obj->base, &handle);
  207. /* drop reference from allocate - handle holds it now */
  208. drm_gem_object_unreference_unlocked(&obj->base);
  209. if (ret)
  210. return ret;
  211. *handle_p = handle;
  212. return 0;
  213. }
  214. int
  215. i915_gem_dumb_create(struct drm_file *file,
  216. struct drm_device *dev,
  217. struct drm_mode_create_dumb *args)
  218. {
  219. /* have to work out size/pitch and return them */
  220. args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
  221. args->size = args->pitch * args->height;
  222. return i915_gem_create(file, dev,
  223. args->size, &args->handle);
  224. }
  225. int i915_gem_dumb_destroy(struct drm_file *file,
  226. struct drm_device *dev,
  227. uint32_t handle)
  228. {
  229. return drm_gem_handle_delete(file, handle);
  230. }
  231. /**
  232. * Creates a new mm object and returns a handle to it.
  233. */
  234. int
  235. i915_gem_create_ioctl(struct drm_device *dev, void *data,
  236. struct drm_file *file)
  237. {
  238. struct drm_i915_gem_create *args = data;
  239. return i915_gem_create(file, dev,
  240. args->size, &args->handle);
  241. }
  242. static inline int
  243. __copy_to_user_swizzled(char __user *cpu_vaddr,
  244. const char *gpu_vaddr, int gpu_offset,
  245. int length)
  246. {
  247. int ret, cpu_offset = 0;
  248. while (length > 0) {
  249. int cacheline_end = ALIGN(gpu_offset + 1, 64);
  250. int this_length = min(cacheline_end - gpu_offset, length);
  251. int swizzled_gpu_offset = gpu_offset ^ 64;
  252. ret = __copy_to_user(cpu_vaddr + cpu_offset,
  253. gpu_vaddr + swizzled_gpu_offset,
  254. this_length);
  255. if (ret)
  256. return ret + length;
  257. cpu_offset += this_length;
  258. gpu_offset += this_length;
  259. length -= this_length;
  260. }
  261. return 0;
  262. }
  263. static inline int
  264. __copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
  265. const char __user *cpu_vaddr,
  266. int length)
  267. {
  268. int ret, cpu_offset = 0;
  269. while (length > 0) {
  270. int cacheline_end = ALIGN(gpu_offset + 1, 64);
  271. int this_length = min(cacheline_end - gpu_offset, length);
  272. int swizzled_gpu_offset = gpu_offset ^ 64;
  273. ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
  274. cpu_vaddr + cpu_offset,
  275. this_length);
  276. if (ret)
  277. return ret + length;
  278. cpu_offset += this_length;
  279. gpu_offset += this_length;
  280. length -= this_length;
  281. }
  282. return 0;
  283. }
  284. /* Per-page copy function for the shmem pread fastpath.
  285. * Flushes invalid cachelines before reading the target if
  286. * needs_clflush is set. */
  287. static int
  288. shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
  289. char __user *user_data,
  290. bool page_do_bit17_swizzling, bool needs_clflush)
  291. {
  292. char *vaddr;
  293. int ret;
  294. if (unlikely(page_do_bit17_swizzling))
  295. return -EINVAL;
  296. vaddr = kmap_atomic(page);
  297. if (needs_clflush)
  298. drm_clflush_virt_range(vaddr + shmem_page_offset,
  299. page_length);
  300. ret = __copy_to_user_inatomic(user_data,
  301. vaddr + shmem_page_offset,
  302. page_length);
  303. kunmap_atomic(vaddr);
  304. return ret ? -EFAULT : 0;
  305. }
  306. static void
  307. shmem_clflush_swizzled_range(char *addr, unsigned long length,
  308. bool swizzled)
  309. {
  310. if (unlikely(swizzled)) {
  311. unsigned long start = (unsigned long) addr;
  312. unsigned long end = (unsigned long) addr + length;
  313. /* For swizzling simply ensure that we always flush both
  314. * channels. Lame, but simple and it works. Swizzled
  315. * pwrite/pread is far from a hotpath - current userspace
  316. * doesn't use it at all. */
  317. start = round_down(start, 128);
  318. end = round_up(end, 128);
  319. drm_clflush_virt_range((void *)start, end - start);
  320. } else {
  321. drm_clflush_virt_range(addr, length);
  322. }
  323. }
  324. /* Only difference to the fast-path function is that this can handle bit17
  325. * and uses non-atomic copy and kmap functions. */
  326. static int
  327. shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
  328. char __user *user_data,
  329. bool page_do_bit17_swizzling, bool needs_clflush)
  330. {
  331. char *vaddr;
  332. int ret;
  333. vaddr = kmap(page);
  334. if (needs_clflush)
  335. shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
  336. page_length,
  337. page_do_bit17_swizzling);
  338. if (page_do_bit17_swizzling)
  339. ret = __copy_to_user_swizzled(user_data,
  340. vaddr, shmem_page_offset,
  341. page_length);
  342. else
  343. ret = __copy_to_user(user_data,
  344. vaddr + shmem_page_offset,
  345. page_length);
  346. kunmap(page);
  347. return ret ? - EFAULT : 0;
  348. }
  349. static int
  350. i915_gem_shmem_pread(struct drm_device *dev,
  351. struct drm_i915_gem_object *obj,
  352. struct drm_i915_gem_pread *args,
  353. struct drm_file *file)
  354. {
  355. char __user *user_data;
  356. ssize_t remain;
  357. loff_t offset;
  358. int shmem_page_offset, page_length, ret = 0;
  359. int obj_do_bit17_swizzling, page_do_bit17_swizzling;
  360. int prefaulted = 0;
  361. int needs_clflush = 0;
  362. struct sg_page_iter sg_iter;
  363. user_data = to_user_ptr(args->data_ptr);
  364. remain = args->size;
  365. obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
  366. if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
  367. /* If we're not in the cpu read domain, set ourself into the gtt
  368. * read domain and manually flush cachelines (if required). This
  369. * optimizes for the case when the gpu will dirty the data
  370. * anyway again before the next pread happens. */
  371. needs_clflush = !cpu_cache_is_coherent(dev, obj->cache_level);
  372. if (i915_gem_obj_bound_any(obj)) {
  373. ret = i915_gem_object_set_to_gtt_domain(obj, false);
  374. if (ret)
  375. return ret;
  376. }
  377. }
  378. ret = i915_gem_object_get_pages(obj);
  379. if (ret)
  380. return ret;
  381. i915_gem_object_pin_pages(obj);
  382. offset = args->offset;
  383. for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
  384. offset >> PAGE_SHIFT) {
  385. struct page *page = sg_page_iter_page(&sg_iter);
  386. if (remain <= 0)
  387. break;
  388. /* Operation in this page
  389. *
  390. * shmem_page_offset = offset within page in shmem file
  391. * page_length = bytes to copy for this page
  392. */
  393. shmem_page_offset = offset_in_page(offset);
  394. page_length = remain;
  395. if ((shmem_page_offset + page_length) > PAGE_SIZE)
  396. page_length = PAGE_SIZE - shmem_page_offset;
  397. page_do_bit17_swizzling = obj_do_bit17_swizzling &&
  398. (page_to_phys(page) & (1 << 17)) != 0;
  399. ret = shmem_pread_fast(page, shmem_page_offset, page_length,
  400. user_data, page_do_bit17_swizzling,
  401. needs_clflush);
  402. if (ret == 0)
  403. goto next_page;
  404. mutex_unlock(&dev->struct_mutex);
  405. if (likely(!i915_prefault_disable) && !prefaulted) {
  406. ret = fault_in_multipages_writeable(user_data, remain);
  407. /* Userspace is tricking us, but we've already clobbered
  408. * its pages with the prefault and promised to write the
  409. * data up to the first fault. Hence ignore any errors
  410. * and just continue. */
  411. (void)ret;
  412. prefaulted = 1;
  413. }
  414. ret = shmem_pread_slow(page, shmem_page_offset, page_length,
  415. user_data, page_do_bit17_swizzling,
  416. needs_clflush);
  417. mutex_lock(&dev->struct_mutex);
  418. next_page:
  419. mark_page_accessed(page);
  420. if (ret)
  421. goto out;
  422. remain -= page_length;
  423. user_data += page_length;
  424. offset += page_length;
  425. }
  426. out:
  427. i915_gem_object_unpin_pages(obj);
  428. return ret;
  429. }
  430. /**
  431. * Reads data from the object referenced by handle.
  432. *
  433. * On error, the contents of *data are undefined.
  434. */
  435. int
  436. i915_gem_pread_ioctl(struct drm_device *dev, void *data,
  437. struct drm_file *file)
  438. {
  439. struct drm_i915_gem_pread *args = data;
  440. struct drm_i915_gem_object *obj;
  441. int ret = 0;
  442. if (args->size == 0)
  443. return 0;
  444. if (!access_ok(VERIFY_WRITE,
  445. to_user_ptr(args->data_ptr),
  446. args->size))
  447. return -EFAULT;
  448. ret = i915_mutex_lock_interruptible(dev);
  449. if (ret)
  450. return ret;
  451. obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  452. if (&obj->base == NULL) {
  453. ret = -ENOENT;
  454. goto unlock;
  455. }
  456. /* Bounds check source. */
  457. if (args->offset > obj->base.size ||
  458. args->size > obj->base.size - args->offset) {
  459. ret = -EINVAL;
  460. goto out;
  461. }
  462. /* prime objects have no backing filp to GEM pread/pwrite
  463. * pages from.
  464. */
  465. if (!obj->base.filp) {
  466. ret = -EINVAL;
  467. goto out;
  468. }
  469. trace_i915_gem_object_pread(obj, args->offset, args->size);
  470. ret = i915_gem_shmem_pread(dev, obj, args, file);
  471. out:
  472. drm_gem_object_unreference(&obj->base);
  473. unlock:
  474. mutex_unlock(&dev->struct_mutex);
  475. return ret;
  476. }
  477. /* This is the fast write path which cannot handle
  478. * page faults in the source data
  479. */
  480. static inline int
  481. fast_user_write(struct io_mapping *mapping,
  482. loff_t page_base, int page_offset,
  483. char __user *user_data,
  484. int length)
  485. {
  486. void __iomem *vaddr_atomic;
  487. void *vaddr;
  488. unsigned long unwritten;
  489. vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
  490. /* We can use the cpu mem copy function because this is X86. */
  491. vaddr = (void __force*)vaddr_atomic + page_offset;
  492. unwritten = __copy_from_user_inatomic_nocache(vaddr,
  493. user_data, length);
  494. io_mapping_unmap_atomic(vaddr_atomic);
  495. return unwritten;
  496. }
  497. /**
  498. * This is the fast pwrite path, where we copy the data directly from the
  499. * user into the GTT, uncached.
  500. */
  501. static int
  502. i915_gem_gtt_pwrite_fast(struct drm_device *dev,
  503. struct drm_i915_gem_object *obj,
  504. struct drm_i915_gem_pwrite *args,
  505. struct drm_file *file)
  506. {
  507. drm_i915_private_t *dev_priv = dev->dev_private;
  508. ssize_t remain;
  509. loff_t offset, page_base;
  510. char __user *user_data;
  511. int page_offset, page_length, ret;
  512. ret = i915_gem_obj_ggtt_pin(obj, 0, true, true);
  513. if (ret)
  514. goto out;
  515. ret = i915_gem_object_set_to_gtt_domain(obj, true);
  516. if (ret)
  517. goto out_unpin;
  518. ret = i915_gem_object_put_fence(obj);
  519. if (ret)
  520. goto out_unpin;
  521. user_data = to_user_ptr(args->data_ptr);
  522. remain = args->size;
  523. offset = i915_gem_obj_ggtt_offset(obj) + args->offset;
  524. while (remain > 0) {
  525. /* Operation in this page
  526. *
  527. * page_base = page offset within aperture
  528. * page_offset = offset within page
  529. * page_length = bytes to copy for this page
  530. */
  531. page_base = offset & PAGE_MASK;
  532. page_offset = offset_in_page(offset);
  533. page_length = remain;
  534. if ((page_offset + remain) > PAGE_SIZE)
  535. page_length = PAGE_SIZE - page_offset;
  536. /* If we get a fault while copying data, then (presumably) our
  537. * source page isn't available. Return the error and we'll
  538. * retry in the slow path.
  539. */
  540. if (fast_user_write(dev_priv->gtt.mappable, page_base,
  541. page_offset, user_data, page_length)) {
  542. ret = -EFAULT;
  543. goto out_unpin;
  544. }
  545. remain -= page_length;
  546. user_data += page_length;
  547. offset += page_length;
  548. }
  549. out_unpin:
  550. i915_gem_object_unpin(obj);
  551. out:
  552. return ret;
  553. }
  554. /* Per-page copy function for the shmem pwrite fastpath.
  555. * Flushes invalid cachelines before writing to the target if
  556. * needs_clflush_before is set and flushes out any written cachelines after
  557. * writing if needs_clflush is set. */
  558. static int
  559. shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
  560. char __user *user_data,
  561. bool page_do_bit17_swizzling,
  562. bool needs_clflush_before,
  563. bool needs_clflush_after)
  564. {
  565. char *vaddr;
  566. int ret;
  567. if (unlikely(page_do_bit17_swizzling))
  568. return -EINVAL;
  569. vaddr = kmap_atomic(page);
  570. if (needs_clflush_before)
  571. drm_clflush_virt_range(vaddr + shmem_page_offset,
  572. page_length);
  573. ret = __copy_from_user_inatomic_nocache(vaddr + shmem_page_offset,
  574. user_data,
  575. page_length);
  576. if (needs_clflush_after)
  577. drm_clflush_virt_range(vaddr + shmem_page_offset,
  578. page_length);
  579. kunmap_atomic(vaddr);
  580. return ret ? -EFAULT : 0;
  581. }
  582. /* Only difference to the fast-path function is that this can handle bit17
  583. * and uses non-atomic copy and kmap functions. */
  584. static int
  585. shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
  586. char __user *user_data,
  587. bool page_do_bit17_swizzling,
  588. bool needs_clflush_before,
  589. bool needs_clflush_after)
  590. {
  591. char *vaddr;
  592. int ret;
  593. vaddr = kmap(page);
  594. if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
  595. shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
  596. page_length,
  597. page_do_bit17_swizzling);
  598. if (page_do_bit17_swizzling)
  599. ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
  600. user_data,
  601. page_length);
  602. else
  603. ret = __copy_from_user(vaddr + shmem_page_offset,
  604. user_data,
  605. page_length);
  606. if (needs_clflush_after)
  607. shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
  608. page_length,
  609. page_do_bit17_swizzling);
  610. kunmap(page);
  611. return ret ? -EFAULT : 0;
  612. }
  613. static int
  614. i915_gem_shmem_pwrite(struct drm_device *dev,
  615. struct drm_i915_gem_object *obj,
  616. struct drm_i915_gem_pwrite *args,
  617. struct drm_file *file)
  618. {
  619. ssize_t remain;
  620. loff_t offset;
  621. char __user *user_data;
  622. int shmem_page_offset, page_length, ret = 0;
  623. int obj_do_bit17_swizzling, page_do_bit17_swizzling;
  624. int hit_slowpath = 0;
  625. int needs_clflush_after = 0;
  626. int needs_clflush_before = 0;
  627. struct sg_page_iter sg_iter;
  628. user_data = to_user_ptr(args->data_ptr);
  629. remain = args->size;
  630. obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
  631. if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
  632. /* If we're not in the cpu write domain, set ourself into the gtt
  633. * write domain and manually flush cachelines (if required). This
  634. * optimizes for the case when the gpu will use the data
  635. * right away and we therefore have to clflush anyway. */
  636. needs_clflush_after = cpu_write_needs_clflush(obj);
  637. if (i915_gem_obj_bound_any(obj)) {
  638. ret = i915_gem_object_set_to_gtt_domain(obj, true);
  639. if (ret)
  640. return ret;
  641. }
  642. }
  643. /* Same trick applies to invalidate partially written cachelines read
  644. * before writing. */
  645. if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
  646. needs_clflush_before =
  647. !cpu_cache_is_coherent(dev, obj->cache_level);
  648. ret = i915_gem_object_get_pages(obj);
  649. if (ret)
  650. return ret;
  651. i915_gem_object_pin_pages(obj);
  652. offset = args->offset;
  653. obj->dirty = 1;
  654. for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
  655. offset >> PAGE_SHIFT) {
  656. struct page *page = sg_page_iter_page(&sg_iter);
  657. int partial_cacheline_write;
  658. if (remain <= 0)
  659. break;
  660. /* Operation in this page
  661. *
  662. * shmem_page_offset = offset within page in shmem file
  663. * page_length = bytes to copy for this page
  664. */
  665. shmem_page_offset = offset_in_page(offset);
  666. page_length = remain;
  667. if ((shmem_page_offset + page_length) > PAGE_SIZE)
  668. page_length = PAGE_SIZE - shmem_page_offset;
  669. /* If we don't overwrite a cacheline completely we need to be
  670. * careful to have up-to-date data by first clflushing. Don't
  671. * overcomplicate things and flush the entire patch. */
  672. partial_cacheline_write = needs_clflush_before &&
  673. ((shmem_page_offset | page_length)
  674. & (boot_cpu_data.x86_clflush_size - 1));
  675. page_do_bit17_swizzling = obj_do_bit17_swizzling &&
  676. (page_to_phys(page) & (1 << 17)) != 0;
  677. ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
  678. user_data, page_do_bit17_swizzling,
  679. partial_cacheline_write,
  680. needs_clflush_after);
  681. if (ret == 0)
  682. goto next_page;
  683. hit_slowpath = 1;
  684. mutex_unlock(&dev->struct_mutex);
  685. ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
  686. user_data, page_do_bit17_swizzling,
  687. partial_cacheline_write,
  688. needs_clflush_after);
  689. mutex_lock(&dev->struct_mutex);
  690. next_page:
  691. set_page_dirty(page);
  692. mark_page_accessed(page);
  693. if (ret)
  694. goto out;
  695. remain -= page_length;
  696. user_data += page_length;
  697. offset += page_length;
  698. }
  699. out:
  700. i915_gem_object_unpin_pages(obj);
  701. if (hit_slowpath) {
  702. /*
  703. * Fixup: Flush cpu caches in case we didn't flush the dirty
  704. * cachelines in-line while writing and the object moved
  705. * out of the cpu write domain while we've dropped the lock.
  706. */
  707. if (!needs_clflush_after &&
  708. obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
  709. if (i915_gem_clflush_object(obj, obj->pin_display))
  710. i915_gem_chipset_flush(dev);
  711. }
  712. }
  713. if (needs_clflush_after)
  714. i915_gem_chipset_flush(dev);
  715. return ret;
  716. }
  717. /**
  718. * Writes data to the object referenced by handle.
  719. *
  720. * On error, the contents of the buffer that were to be modified are undefined.
  721. */
  722. int
  723. i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
  724. struct drm_file *file)
  725. {
  726. struct drm_i915_gem_pwrite *args = data;
  727. struct drm_i915_gem_object *obj;
  728. int ret;
  729. if (args->size == 0)
  730. return 0;
  731. if (!access_ok(VERIFY_READ,
  732. to_user_ptr(args->data_ptr),
  733. args->size))
  734. return -EFAULT;
  735. if (likely(!i915_prefault_disable)) {
  736. ret = fault_in_multipages_readable(to_user_ptr(args->data_ptr),
  737. args->size);
  738. if (ret)
  739. return -EFAULT;
  740. }
  741. ret = i915_mutex_lock_interruptible(dev);
  742. if (ret)
  743. return ret;
  744. obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  745. if (&obj->base == NULL) {
  746. ret = -ENOENT;
  747. goto unlock;
  748. }
  749. /* Bounds check destination. */
  750. if (args->offset > obj->base.size ||
  751. args->size > obj->base.size - args->offset) {
  752. ret = -EINVAL;
  753. goto out;
  754. }
  755. /* prime objects have no backing filp to GEM pread/pwrite
  756. * pages from.
  757. */
  758. if (!obj->base.filp) {
  759. ret = -EINVAL;
  760. goto out;
  761. }
  762. trace_i915_gem_object_pwrite(obj, args->offset, args->size);
  763. ret = -EFAULT;
  764. /* We can only do the GTT pwrite on untiled buffers, as otherwise
  765. * it would end up going through the fenced access, and we'll get
  766. * different detiling behavior between reading and writing.
  767. * pread/pwrite currently are reading and writing from the CPU
  768. * perspective, requiring manual detiling by the client.
  769. */
  770. if (obj->phys_obj) {
  771. ret = i915_gem_phys_pwrite(dev, obj, args, file);
  772. goto out;
  773. }
  774. if (obj->tiling_mode == I915_TILING_NONE &&
  775. obj->base.write_domain != I915_GEM_DOMAIN_CPU &&
  776. cpu_write_needs_clflush(obj)) {
  777. ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
  778. /* Note that the gtt paths might fail with non-page-backed user
  779. * pointers (e.g. gtt mappings when moving data between
  780. * textures). Fallback to the shmem path in that case. */
  781. }
  782. if (ret == -EFAULT || ret == -ENOSPC)
  783. ret = i915_gem_shmem_pwrite(dev, obj, args, file);
  784. out:
  785. drm_gem_object_unreference(&obj->base);
  786. unlock:
  787. mutex_unlock(&dev->struct_mutex);
  788. return ret;
  789. }
  790. int
  791. i915_gem_check_wedge(struct i915_gpu_error *error,
  792. bool interruptible)
  793. {
  794. if (i915_reset_in_progress(error)) {
  795. /* Non-interruptible callers can't handle -EAGAIN, hence return
  796. * -EIO unconditionally for these. */
  797. if (!interruptible)
  798. return -EIO;
  799. /* Recovery complete, but the reset failed ... */
  800. if (i915_terminally_wedged(error))
  801. return -EIO;
  802. return -EAGAIN;
  803. }
  804. return 0;
  805. }
  806. /*
  807. * Compare seqno against outstanding lazy request. Emit a request if they are
  808. * equal.
  809. */
  810. static int
  811. i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno)
  812. {
  813. int ret;
  814. BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex));
  815. ret = 0;
  816. if (seqno == ring->outstanding_lazy_request)
  817. ret = i915_add_request(ring, NULL);
  818. return ret;
  819. }
  820. /**
  821. * __wait_seqno - wait until execution of seqno has finished
  822. * @ring: the ring expected to report seqno
  823. * @seqno: duh!
  824. * @reset_counter: reset sequence associated with the given seqno
  825. * @interruptible: do an interruptible wait (normally yes)
  826. * @timeout: in - how long to wait (NULL forever); out - how much time remaining
  827. *
  828. * Note: It is of utmost importance that the passed in seqno and reset_counter
  829. * values have been read by the caller in an smp safe manner. Where read-side
  830. * locks are involved, it is sufficient to read the reset_counter before
  831. * unlocking the lock that protects the seqno. For lockless tricks, the
  832. * reset_counter _must_ be read before, and an appropriate smp_rmb must be
  833. * inserted.
  834. *
  835. * Returns 0 if the seqno was found within the alloted time. Else returns the
  836. * errno with remaining time filled in timeout argument.
  837. */
  838. static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
  839. unsigned reset_counter,
  840. bool interruptible, struct timespec *timeout)
  841. {
  842. drm_i915_private_t *dev_priv = ring->dev->dev_private;
  843. struct timespec before, now, wait_time={1,0};
  844. unsigned long timeout_jiffies;
  845. long end;
  846. bool wait_forever = true;
  847. int ret;
  848. if (i915_seqno_passed(ring->get_seqno(ring, true), seqno))
  849. return 0;
  850. trace_i915_gem_request_wait_begin(ring, seqno);
  851. if (timeout != NULL) {
  852. wait_time = *timeout;
  853. wait_forever = false;
  854. }
  855. timeout_jiffies = timespec_to_jiffies_timeout(&wait_time);
  856. if (WARN_ON(!ring->irq_get(ring)))
  857. return -ENODEV;
  858. /* Record current time in case interrupted by signal, or wedged * */
  859. getrawmonotonic(&before);
  860. #define EXIT_COND \
  861. (i915_seqno_passed(ring->get_seqno(ring, false), seqno) || \
  862. i915_reset_in_progress(&dev_priv->gpu_error) || \
  863. reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter))
  864. do {
  865. if (interruptible)
  866. end = wait_event_interruptible_timeout(ring->irq_queue,
  867. EXIT_COND,
  868. timeout_jiffies);
  869. else
  870. end = wait_event_timeout(ring->irq_queue, EXIT_COND,
  871. timeout_jiffies);
  872. /* We need to check whether any gpu reset happened in between
  873. * the caller grabbing the seqno and now ... */
  874. if (reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter))
  875. end = -EAGAIN;
  876. /* ... but upgrade the -EGAIN to an -EIO if the gpu is truely
  877. * gone. */
  878. ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
  879. if (ret)
  880. end = ret;
  881. } while (end == 0 && wait_forever);
  882. getrawmonotonic(&now);
  883. ring->irq_put(ring);
  884. trace_i915_gem_request_wait_end(ring, seqno);
  885. #undef EXIT_COND
  886. if (timeout) {
  887. struct timespec sleep_time = timespec_sub(now, before);
  888. *timeout = timespec_sub(*timeout, sleep_time);
  889. if (!timespec_valid(timeout)) /* i.e. negative time remains */
  890. set_normalized_timespec(timeout, 0, 0);
  891. }
  892. switch (end) {
  893. case -EIO:
  894. case -EAGAIN: /* Wedged */
  895. case -ERESTARTSYS: /* Signal */
  896. return (int)end;
  897. case 0: /* Timeout */
  898. return -ETIME;
  899. default: /* Completed */
  900. WARN_ON(end < 0); /* We're not aware of other errors */
  901. return 0;
  902. }
  903. }
  904. /**
  905. * Waits for a sequence number to be signaled, and cleans up the
  906. * request and object lists appropriately for that event.
  907. */
  908. int
  909. i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno)
  910. {
  911. struct drm_device *dev = ring->dev;
  912. struct drm_i915_private *dev_priv = dev->dev_private;
  913. bool interruptible = dev_priv->mm.interruptible;
  914. int ret;
  915. BUG_ON(!mutex_is_locked(&dev->struct_mutex));
  916. BUG_ON(seqno == 0);
  917. ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
  918. if (ret)
  919. return ret;
  920. ret = i915_gem_check_olr(ring, seqno);
  921. if (ret)
  922. return ret;
  923. return __wait_seqno(ring, seqno,
  924. atomic_read(&dev_priv->gpu_error.reset_counter),
  925. interruptible, NULL);
  926. }
  927. static int
  928. i915_gem_object_wait_rendering__tail(struct drm_i915_gem_object *obj,
  929. struct intel_ring_buffer *ring)
  930. {
  931. i915_gem_retire_requests_ring(ring);
  932. /* Manually manage the write flush as we may have not yet
  933. * retired the buffer.
  934. *
  935. * Note that the last_write_seqno is always the earlier of
  936. * the two (read/write) seqno, so if we haved successfully waited,
  937. * we know we have passed the last write.
  938. */
  939. obj->last_write_seqno = 0;
  940. obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
  941. return 0;
  942. }
  943. /**
  944. * Ensures that all rendering to the object has completed and the object is
  945. * safe to unbind from the GTT or access from the CPU.
  946. */
  947. static __must_check int
  948. i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
  949. bool readonly)
  950. {
  951. struct intel_ring_buffer *ring = obj->ring;
  952. u32 seqno;
  953. int ret;
  954. seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
  955. if (seqno == 0)
  956. return 0;
  957. ret = i915_wait_seqno(ring, seqno);
  958. if (ret)
  959. return ret;
  960. return i915_gem_object_wait_rendering__tail(obj, ring);
  961. }
  962. /* A nonblocking variant of the above wait. This is a highly dangerous routine
  963. * as the object state may change during this call.
  964. */
  965. static __must_check int
  966. i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
  967. bool readonly)
  968. {
  969. struct drm_device *dev = obj->base.dev;
  970. struct drm_i915_private *dev_priv = dev->dev_private;
  971. struct intel_ring_buffer *ring = obj->ring;
  972. unsigned reset_counter;
  973. u32 seqno;
  974. int ret;
  975. BUG_ON(!mutex_is_locked(&dev->struct_mutex));
  976. BUG_ON(!dev_priv->mm.interruptible);
  977. seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
  978. if (seqno == 0)
  979. return 0;
  980. ret = i915_gem_check_wedge(&dev_priv->gpu_error, true);
  981. if (ret)
  982. return ret;
  983. ret = i915_gem_check_olr(ring, seqno);
  984. if (ret)
  985. return ret;
  986. reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
  987. mutex_unlock(&dev->struct_mutex);
  988. ret = __wait_seqno(ring, seqno, reset_counter, true, NULL);
  989. mutex_lock(&dev->struct_mutex);
  990. if (ret)
  991. return ret;
  992. return i915_gem_object_wait_rendering__tail(obj, ring);
  993. }
  994. /**
  995. * Called when user space prepares to use an object with the CPU, either
  996. * through the mmap ioctl's mapping or a GTT mapping.
  997. */
  998. int
  999. i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
  1000. struct drm_file *file)
  1001. {
  1002. struct drm_i915_gem_set_domain *args = data;
  1003. struct drm_i915_gem_object *obj;
  1004. uint32_t read_domains = args->read_domains;
  1005. uint32_t write_domain = args->write_domain;
  1006. int ret;
  1007. /* Only handle setting domains to types used by the CPU. */
  1008. if (write_domain & I915_GEM_GPU_DOMAINS)
  1009. return -EINVAL;
  1010. if (read_domains & I915_GEM_GPU_DOMAINS)
  1011. return -EINVAL;
  1012. /* Having something in the write domain implies it's in the read
  1013. * domain, and only that read domain. Enforce that in the request.
  1014. */
  1015. if (write_domain != 0 && read_domains != write_domain)
  1016. return -EINVAL;
  1017. ret = i915_mutex_lock_interruptible(dev);
  1018. if (ret)
  1019. return ret;
  1020. obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  1021. if (&obj->base == NULL) {
  1022. ret = -ENOENT;
  1023. goto unlock;
  1024. }
  1025. /* Try to flush the object off the GPU without holding the lock.
  1026. * We will repeat the flush holding the lock in the normal manner
  1027. * to catch cases where we are gazumped.
  1028. */
  1029. ret = i915_gem_object_wait_rendering__nonblocking(obj, !write_domain);
  1030. if (ret)
  1031. goto unref;
  1032. if (read_domains & I915_GEM_DOMAIN_GTT) {
  1033. ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
  1034. /* Silently promote "you're not bound, there was nothing to do"
  1035. * to success, since the client was just asking us to
  1036. * make sure everything was done.
  1037. */
  1038. if (ret == -EINVAL)
  1039. ret = 0;
  1040. } else {
  1041. ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
  1042. }
  1043. unref:
  1044. drm_gem_object_unreference(&obj->base);
  1045. unlock:
  1046. mutex_unlock(&dev->struct_mutex);
  1047. return ret;
  1048. }
  1049. /**
  1050. * Called when user space has done writes to this buffer
  1051. */
  1052. int
  1053. i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
  1054. struct drm_file *file)
  1055. {
  1056. struct drm_i915_gem_sw_finish *args = data;
  1057. struct drm_i915_gem_object *obj;
  1058. int ret = 0;
  1059. ret = i915_mutex_lock_interruptible(dev);
  1060. if (ret)
  1061. return ret;
  1062. obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  1063. if (&obj->base == NULL) {
  1064. ret = -ENOENT;
  1065. goto unlock;
  1066. }
  1067. /* Pinned buffers may be scanout, so flush the cache */
  1068. if (obj->pin_display)
  1069. i915_gem_object_flush_cpu_write_domain(obj, true);
  1070. drm_gem_object_unreference(&obj->base);
  1071. unlock:
  1072. mutex_unlock(&dev->struct_mutex);
  1073. return ret;
  1074. }
  1075. /**
  1076. * Maps the contents of an object, returning the address it is mapped
  1077. * into.
  1078. *
  1079. * While the mapping holds a reference on the contents of the object, it doesn't
  1080. * imply a ref on the object itself.
  1081. */
  1082. int
  1083. i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
  1084. struct drm_file *file)
  1085. {
  1086. struct drm_i915_gem_mmap *args = data;
  1087. struct drm_gem_object *obj;
  1088. unsigned long addr;
  1089. obj = drm_gem_object_lookup(dev, file, args->handle);
  1090. if (obj == NULL)
  1091. return -ENOENT;
  1092. /* prime objects have no backing filp to GEM mmap
  1093. * pages from.
  1094. */
  1095. if (!obj->filp) {
  1096. drm_gem_object_unreference_unlocked(obj);
  1097. return -EINVAL;
  1098. }
  1099. addr = vm_mmap(obj->filp, 0, args->size,
  1100. PROT_READ | PROT_WRITE, MAP_SHARED,
  1101. args->offset);
  1102. drm_gem_object_unreference_unlocked(obj);
  1103. if (IS_ERR((void *)addr))
  1104. return addr;
  1105. args->addr_ptr = (uint64_t) addr;
  1106. return 0;
  1107. }
  1108. /**
  1109. * i915_gem_fault - fault a page into the GTT
  1110. * vma: VMA in question
  1111. * vmf: fault info
  1112. *
  1113. * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
  1114. * from userspace. The fault handler takes care of binding the object to
  1115. * the GTT (if needed), allocating and programming a fence register (again,
  1116. * only if needed based on whether the old reg is still valid or the object
  1117. * is tiled) and inserting a new PTE into the faulting process.
  1118. *
  1119. * Note that the faulting process may involve evicting existing objects
  1120. * from the GTT and/or fence registers to make room. So performance may
  1121. * suffer if the GTT working set is large or there are few fence registers
  1122. * left.
  1123. */
  1124. int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
  1125. {
  1126. struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
  1127. struct drm_device *dev = obj->base.dev;
  1128. drm_i915_private_t *dev_priv = dev->dev_private;
  1129. pgoff_t page_offset;
  1130. unsigned long pfn;
  1131. int ret = 0;
  1132. bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
  1133. /* We don't use vmf->pgoff since that has the fake offset */
  1134. page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
  1135. PAGE_SHIFT;
  1136. ret = i915_mutex_lock_interruptible(dev);
  1137. if (ret)
  1138. goto out;
  1139. trace_i915_gem_object_fault(obj, page_offset, true, write);
  1140. /* Access to snoopable pages through the GTT is incoherent. */
  1141. if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev)) {
  1142. ret = -EINVAL;
  1143. goto unlock;
  1144. }
  1145. /* Now bind it into the GTT if needed */
  1146. ret = i915_gem_obj_ggtt_pin(obj, 0, true, false);
  1147. if (ret)
  1148. goto unlock;
  1149. ret = i915_gem_object_set_to_gtt_domain(obj, write);
  1150. if (ret)
  1151. goto unpin;
  1152. ret = i915_gem_object_get_fence(obj);
  1153. if (ret)
  1154. goto unpin;
  1155. obj->fault_mappable = true;
  1156. pfn = dev_priv->gtt.mappable_base + i915_gem_obj_ggtt_offset(obj);
  1157. pfn >>= PAGE_SHIFT;
  1158. pfn += page_offset;
  1159. /* Finally, remap it using the new GTT offset */
  1160. ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
  1161. unpin:
  1162. i915_gem_object_unpin(obj);
  1163. unlock:
  1164. mutex_unlock(&dev->struct_mutex);
  1165. out:
  1166. switch (ret) {
  1167. case -EIO:
  1168. /* If this -EIO is due to a gpu hang, give the reset code a
  1169. * chance to clean up the mess. Otherwise return the proper
  1170. * SIGBUS. */
  1171. if (i915_terminally_wedged(&dev_priv->gpu_error))
  1172. return VM_FAULT_SIGBUS;
  1173. case -EAGAIN:
  1174. /* Give the error handler a chance to run and move the
  1175. * objects off the GPU active list. Next time we service the
  1176. * fault, we should be able to transition the page into the
  1177. * GTT without touching the GPU (and so avoid further
  1178. * EIO/EGAIN). If the GPU is wedged, then there is no issue
  1179. * with coherency, just lost writes.
  1180. */
  1181. set_need_resched();
  1182. case 0:
  1183. case -ERESTARTSYS:
  1184. case -EINTR:
  1185. case -EBUSY:
  1186. /*
  1187. * EBUSY is ok: this just means that another thread
  1188. * already did the job.
  1189. */
  1190. return VM_FAULT_NOPAGE;
  1191. case -ENOMEM:
  1192. return VM_FAULT_OOM;
  1193. case -ENOSPC:
  1194. return VM_FAULT_SIGBUS;
  1195. default:
  1196. WARN_ONCE(ret, "unhandled error in i915_gem_fault: %i\n", ret);
  1197. return VM_FAULT_SIGBUS;
  1198. }
  1199. }
  1200. /**
  1201. * i915_gem_release_mmap - remove physical page mappings
  1202. * @obj: obj in question
  1203. *
  1204. * Preserve the reservation of the mmapping with the DRM core code, but
  1205. * relinquish ownership of the pages back to the system.
  1206. *
  1207. * It is vital that we remove the page mapping if we have mapped a tiled
  1208. * object through the GTT and then lose the fence register due to
  1209. * resource pressure. Similarly if the object has been moved out of the
  1210. * aperture, than pages mapped into userspace must be revoked. Removing the
  1211. * mapping will then trigger a page fault on the next user access, allowing
  1212. * fixup by i915_gem_fault().
  1213. */
  1214. void
  1215. i915_gem_release_mmap(struct drm_i915_gem_object *obj)
  1216. {
  1217. if (!obj->fault_mappable)
  1218. return;
  1219. if (obj->base.dev->dev_mapping)
  1220. unmap_mapping_range(obj->base.dev->dev_mapping,
  1221. (loff_t)obj->base.map_list.hash.key<<PAGE_SHIFT,
  1222. obj->base.size, 1);
  1223. obj->fault_mappable = false;
  1224. }
  1225. uint32_t
  1226. i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
  1227. {
  1228. uint32_t gtt_size;
  1229. if (INTEL_INFO(dev)->gen >= 4 ||
  1230. tiling_mode == I915_TILING_NONE)
  1231. return size;
  1232. /* Previous chips need a power-of-two fence region when tiling */
  1233. if (INTEL_INFO(dev)->gen == 3)
  1234. gtt_size = 1024*1024;
  1235. else
  1236. gtt_size = 512*1024;
  1237. while (gtt_size < size)
  1238. gtt_size <<= 1;
  1239. return gtt_size;
  1240. }
  1241. /**
  1242. * i915_gem_get_gtt_alignment - return required GTT alignment for an object
  1243. * @obj: object to check
  1244. *
  1245. * Return the required GTT alignment for an object, taking into account
  1246. * potential fence register mapping.
  1247. */
  1248. uint32_t
  1249. i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size,
  1250. int tiling_mode, bool fenced)
  1251. {
  1252. /*
  1253. * Minimum alignment is 4k (GTT page size), but might be greater
  1254. * if a fence register is needed for the object.
  1255. */
  1256. if (INTEL_INFO(dev)->gen >= 4 || (!fenced && IS_G33(dev)) ||
  1257. tiling_mode == I915_TILING_NONE)
  1258. return 4096;
  1259. /*
  1260. * Previous chips need to be aligned to the size of the smallest
  1261. * fence register that can contain the object.
  1262. */
  1263. return i915_gem_get_gtt_size(dev, size, tiling_mode);
  1264. }
  1265. static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj)
  1266. {
  1267. struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  1268. int ret;
  1269. if (obj->base.map_list.map)
  1270. return 0;
  1271. dev_priv->mm.shrinker_no_lock_stealing = true;
  1272. ret = drm_gem_create_mmap_offset(&obj->base);
  1273. if (ret != -ENOSPC)
  1274. goto out;
  1275. /* Badly fragmented mmap space? The only way we can recover
  1276. * space is by destroying unwanted objects. We can't randomly release
  1277. * mmap_offsets as userspace expects them to be persistent for the
  1278. * lifetime of the objects. The closest we can is to release the
  1279. * offsets on purgeable objects by truncating it and marking it purged,
  1280. * which prevents userspace from ever using that object again.
  1281. */
  1282. i915_gem_purge(dev_priv, obj->base.size >> PAGE_SHIFT);
  1283. ret = drm_gem_create_mmap_offset(&obj->base);
  1284. if (ret != -ENOSPC)
  1285. goto out;
  1286. i915_gem_shrink_all(dev_priv);
  1287. ret = drm_gem_create_mmap_offset(&obj->base);
  1288. out:
  1289. dev_priv->mm.shrinker_no_lock_stealing = false;
  1290. return ret;
  1291. }
  1292. static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj)
  1293. {
  1294. if (!obj->base.map_list.map)
  1295. return;
  1296. drm_gem_free_mmap_offset(&obj->base);
  1297. }
  1298. int
  1299. i915_gem_mmap_gtt(struct drm_file *file,
  1300. struct drm_device *dev,
  1301. uint32_t handle,
  1302. uint64_t *offset)
  1303. {
  1304. struct drm_i915_private *dev_priv = dev->dev_private;
  1305. struct drm_i915_gem_object *obj;
  1306. int ret;
  1307. ret = i915_mutex_lock_interruptible(dev);
  1308. if (ret)
  1309. return ret;
  1310. obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
  1311. if (&obj->base == NULL) {
  1312. ret = -ENOENT;
  1313. goto unlock;
  1314. }
  1315. if (obj->base.size > dev_priv->gtt.mappable_end) {
  1316. ret = -E2BIG;
  1317. goto out;
  1318. }
  1319. if (obj->madv != I915_MADV_WILLNEED) {
  1320. DRM_ERROR("Attempting to mmap a purgeable buffer\n");
  1321. ret = -EINVAL;
  1322. goto out;
  1323. }
  1324. ret = i915_gem_object_create_mmap_offset(obj);
  1325. if (ret)
  1326. goto out;
  1327. *offset = (u64)obj->base.map_list.hash.key << PAGE_SHIFT;
  1328. out:
  1329. drm_gem_object_unreference(&obj->base);
  1330. unlock:
  1331. mutex_unlock(&dev->struct_mutex);
  1332. return ret;
  1333. }
  1334. /**
  1335. * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
  1336. * @dev: DRM device
  1337. * @data: GTT mapping ioctl data
  1338. * @file: GEM object info
  1339. *
  1340. * Simply returns the fake offset to userspace so it can mmap it.
  1341. * The mmap call will end up in drm_gem_mmap(), which will set things
  1342. * up so we can get faults in the handler above.
  1343. *
  1344. * The fault handler will take care of binding the object into the GTT
  1345. * (since it may have been evicted to make room for something), allocating
  1346. * a fence register, and mapping the appropriate aperture address into
  1347. * userspace.
  1348. */
  1349. int
  1350. i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
  1351. struct drm_file *file)
  1352. {
  1353. struct drm_i915_gem_mmap_gtt *args = data;
  1354. return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
  1355. }
  1356. /* Immediately discard the backing storage */
  1357. static void
  1358. i915_gem_object_truncate(struct drm_i915_gem_object *obj)
  1359. {
  1360. struct inode *inode;
  1361. i915_gem_object_free_mmap_offset(obj);
  1362. if (obj->base.filp == NULL)
  1363. return;
  1364. /* Our goal here is to return as much of the memory as
  1365. * is possible back to the system as we are called from OOM.
  1366. * To do this we must instruct the shmfs to drop all of its
  1367. * backing pages, *now*.
  1368. */
  1369. inode = file_inode(obj->base.filp);
  1370. shmem_truncate_range(inode, 0, (loff_t)-1);
  1371. obj->madv = __I915_MADV_PURGED;
  1372. }
  1373. static inline int
  1374. i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
  1375. {
  1376. return obj->madv == I915_MADV_DONTNEED;
  1377. }
  1378. static void
  1379. i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
  1380. {
  1381. struct sg_page_iter sg_iter;
  1382. int ret;
  1383. BUG_ON(obj->madv == __I915_MADV_PURGED);
  1384. ret = i915_gem_object_set_to_cpu_domain(obj, true);
  1385. if (ret) {
  1386. /* In the event of a disaster, abandon all caches and
  1387. * hope for the best.
  1388. */
  1389. WARN_ON(ret != -EIO);
  1390. i915_gem_clflush_object(obj, true);
  1391. obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  1392. }
  1393. if (i915_gem_object_needs_bit17_swizzle(obj))
  1394. i915_gem_object_save_bit_17_swizzle(obj);
  1395. if (obj->madv == I915_MADV_DONTNEED)
  1396. obj->dirty = 0;
  1397. for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) {
  1398. struct page *page = sg_page_iter_page(&sg_iter);
  1399. if (obj->dirty)
  1400. set_page_dirty(page);
  1401. if (obj->madv == I915_MADV_WILLNEED)
  1402. mark_page_accessed(page);
  1403. page_cache_release(page);
  1404. }
  1405. obj->dirty = 0;
  1406. sg_free_table(obj->pages);
  1407. kfree(obj->pages);
  1408. }
  1409. int
  1410. i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
  1411. {
  1412. const struct drm_i915_gem_object_ops *ops = obj->ops;
  1413. if (obj->pages == NULL)
  1414. return 0;
  1415. if (obj->pages_pin_count)
  1416. return -EBUSY;
  1417. BUG_ON(i915_gem_obj_bound_any(obj));
  1418. /* ->put_pages might need to allocate memory for the bit17 swizzle
  1419. * array, hence protect them from being reaped by removing them from gtt
  1420. * lists early. */
  1421. list_del(&obj->global_list);
  1422. ops->put_pages(obj);
  1423. obj->pages = NULL;
  1424. if (i915_gem_object_is_purgeable(obj))
  1425. i915_gem_object_truncate(obj);
  1426. return 0;
  1427. }
  1428. static long
  1429. __i915_gem_shrink(struct drm_i915_private *dev_priv, long target,
  1430. bool purgeable_only)
  1431. {
  1432. struct drm_i915_gem_object *obj, *next;
  1433. long count = 0;
  1434. list_for_each_entry_safe(obj, next,
  1435. &dev_priv->mm.unbound_list,
  1436. global_list) {
  1437. if ((i915_gem_object_is_purgeable(obj) || !purgeable_only) &&
  1438. i915_gem_object_put_pages(obj) == 0) {
  1439. count += obj->base.size >> PAGE_SHIFT;
  1440. if (count >= target)
  1441. return count;
  1442. }
  1443. }
  1444. list_for_each_entry_safe(obj, next, &dev_priv->mm.bound_list,
  1445. global_list) {
  1446. struct i915_vma *vma, *v;
  1447. if (!i915_gem_object_is_purgeable(obj) && purgeable_only)
  1448. continue;
  1449. list_for_each_entry_safe(vma, v, &obj->vma_list, vma_link)
  1450. if (i915_vma_unbind(vma))
  1451. break;
  1452. if (!i915_gem_object_put_pages(obj)) {
  1453. count += obj->base.size >> PAGE_SHIFT;
  1454. if (count >= target)
  1455. return count;
  1456. }
  1457. }
  1458. return count;
  1459. }
  1460. static long
  1461. i915_gem_purge(struct drm_i915_private *dev_priv, long target)
  1462. {
  1463. return __i915_gem_shrink(dev_priv, target, true);
  1464. }
  1465. static void
  1466. i915_gem_shrink_all(struct drm_i915_private *dev_priv)
  1467. {
  1468. struct drm_i915_gem_object *obj, *next;
  1469. i915_gem_evict_everything(dev_priv->dev);
  1470. list_for_each_entry_safe(obj, next, &dev_priv->mm.unbound_list,
  1471. global_list)
  1472. i915_gem_object_put_pages(obj);
  1473. }
  1474. static int
  1475. i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
  1476. {
  1477. struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  1478. int page_count, i;
  1479. struct address_space *mapping;
  1480. struct sg_table *st;
  1481. struct scatterlist *sg;
  1482. struct sg_page_iter sg_iter;
  1483. struct page *page;
  1484. unsigned long last_pfn = 0; /* suppress gcc warning */
  1485. gfp_t gfp;
  1486. /* Assert that the object is not currently in any GPU domain. As it
  1487. * wasn't in the GTT, there shouldn't be any way it could have been in
  1488. * a GPU cache
  1489. */
  1490. BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
  1491. BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
  1492. st = kmalloc(sizeof(*st), GFP_KERNEL);
  1493. if (st == NULL)
  1494. return -ENOMEM;
  1495. page_count = obj->base.size / PAGE_SIZE;
  1496. if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
  1497. sg_free_table(st);
  1498. kfree(st);
  1499. return -ENOMEM;
  1500. }
  1501. /* Get the list of pages out of our struct file. They'll be pinned
  1502. * at this point until we release them.
  1503. *
  1504. * Fail silently without starting the shrinker
  1505. */
  1506. mapping = file_inode(obj->base.filp)->i_mapping;
  1507. gfp = mapping_gfp_mask(mapping);
  1508. gfp |= __GFP_NORETRY | __GFP_NOWARN | __GFP_NO_KSWAPD;
  1509. gfp &= ~(__GFP_IO | __GFP_WAIT);
  1510. sg = st->sgl;
  1511. st->nents = 0;
  1512. for (i = 0; i < page_count; i++) {
  1513. page = shmem_read_mapping_page_gfp(mapping, i, gfp);
  1514. if (IS_ERR(page)) {
  1515. i915_gem_purge(dev_priv, page_count);
  1516. page = shmem_read_mapping_page_gfp(mapping, i, gfp);
  1517. }
  1518. if (IS_ERR(page)) {
  1519. /* We've tried hard to allocate the memory by reaping
  1520. * our own buffer, now let the real VM do its job and
  1521. * go down in flames if truly OOM.
  1522. */
  1523. gfp &= ~(__GFP_NORETRY | __GFP_NOWARN | __GFP_NO_KSWAPD);
  1524. gfp |= __GFP_IO | __GFP_WAIT;
  1525. i915_gem_shrink_all(dev_priv);
  1526. page = shmem_read_mapping_page_gfp(mapping, i, gfp);
  1527. if (IS_ERR(page))
  1528. goto err_pages;
  1529. gfp |= __GFP_NORETRY | __GFP_NOWARN | __GFP_NO_KSWAPD;
  1530. gfp &= ~(__GFP_IO | __GFP_WAIT);
  1531. }
  1532. #ifdef CONFIG_SWIOTLB
  1533. if (swiotlb_nr_tbl()) {
  1534. st->nents++;
  1535. sg_set_page(sg, page, PAGE_SIZE, 0);
  1536. sg = sg_next(sg);
  1537. continue;
  1538. }
  1539. #endif
  1540. if (!i || page_to_pfn(page) != last_pfn + 1) {
  1541. if (i)
  1542. sg = sg_next(sg);
  1543. st->nents++;
  1544. sg_set_page(sg, page, PAGE_SIZE, 0);
  1545. } else {
  1546. sg->length += PAGE_SIZE;
  1547. }
  1548. last_pfn = page_to_pfn(page);
  1549. }
  1550. #ifdef CONFIG_SWIOTLB
  1551. if (!swiotlb_nr_tbl())
  1552. #endif
  1553. sg_mark_end(sg);
  1554. obj->pages = st;
  1555. if (i915_gem_object_needs_bit17_swizzle(obj))
  1556. i915_gem_object_do_bit_17_swizzle(obj);
  1557. return 0;
  1558. err_pages:
  1559. sg_mark_end(sg);
  1560. for_each_sg_page(st->sgl, &sg_iter, st->nents, 0)
  1561. page_cache_release(sg_page_iter_page(&sg_iter));
  1562. sg_free_table(st);
  1563. kfree(st);
  1564. return PTR_ERR(page);
  1565. }
  1566. /* Ensure that the associated pages are gathered from the backing storage
  1567. * and pinned into our object. i915_gem_object_get_pages() may be called
  1568. * multiple times before they are released by a single call to
  1569. * i915_gem_object_put_pages() - once the pages are no longer referenced
  1570. * either as a result of memory pressure (reaping pages under the shrinker)
  1571. * or as the object is itself released.
  1572. */
  1573. int
  1574. i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
  1575. {
  1576. struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  1577. const struct drm_i915_gem_object_ops *ops = obj->ops;
  1578. int ret;
  1579. if (obj->pages)
  1580. return 0;
  1581. if (obj->madv != I915_MADV_WILLNEED) {
  1582. DRM_ERROR("Attempting to obtain a purgeable object\n");
  1583. return -EINVAL;
  1584. }
  1585. BUG_ON(obj->pages_pin_count);
  1586. ret = ops->get_pages(obj);
  1587. if (ret)
  1588. return ret;
  1589. list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list);
  1590. return 0;
  1591. }
  1592. void
  1593. i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
  1594. struct intel_ring_buffer *ring)
  1595. {
  1596. struct drm_device *dev = obj->base.dev;
  1597. struct drm_i915_private *dev_priv = dev->dev_private;
  1598. u32 seqno = intel_ring_get_seqno(ring);
  1599. BUG_ON(ring == NULL);
  1600. if (obj->ring != ring && obj->last_write_seqno) {
  1601. /* Keep the seqno relative to the current ring */
  1602. obj->last_write_seqno = seqno;
  1603. }
  1604. obj->ring = ring;
  1605. /* Add a reference if we're newly entering the active list. */
  1606. if (!obj->active) {
  1607. drm_gem_object_reference(&obj->base);
  1608. obj->active = 1;
  1609. }
  1610. list_move_tail(&obj->ring_list, &ring->active_list);
  1611. obj->last_read_seqno = seqno;
  1612. if (obj->fenced_gpu_access) {
  1613. obj->last_fenced_seqno = seqno;
  1614. /* Bump MRU to take account of the delayed flush */
  1615. if (obj->fence_reg != I915_FENCE_REG_NONE) {
  1616. struct drm_i915_fence_reg *reg;
  1617. reg = &dev_priv->fence_regs[obj->fence_reg];
  1618. list_move_tail(&reg->lru_list,
  1619. &dev_priv->mm.fence_list);
  1620. }
  1621. }
  1622. }
  1623. static void
  1624. i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
  1625. {
  1626. struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  1627. struct i915_address_space *ggtt_vm = &dev_priv->gtt.base;
  1628. struct i915_vma *vma = i915_gem_obj_to_vma(obj, ggtt_vm);
  1629. BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS);
  1630. BUG_ON(!obj->active);
  1631. list_move_tail(&vma->mm_list, &ggtt_vm->inactive_list);
  1632. list_del_init(&obj->ring_list);
  1633. obj->ring = NULL;
  1634. obj->last_read_seqno = 0;
  1635. obj->last_write_seqno = 0;
  1636. obj->base.write_domain = 0;
  1637. obj->last_fenced_seqno = 0;
  1638. obj->fenced_gpu_access = false;
  1639. obj->active = 0;
  1640. drm_gem_object_unreference(&obj->base);
  1641. WARN_ON(i915_verify_lists(dev));
  1642. }
  1643. static int
  1644. i915_gem_init_seqno(struct drm_device *dev, u32 seqno)
  1645. {
  1646. struct drm_i915_private *dev_priv = dev->dev_private;
  1647. struct intel_ring_buffer *ring;
  1648. int ret, i, j;
  1649. /* Carefully retire all requests without writing to the rings */
  1650. for_each_ring(ring, dev_priv, i) {
  1651. ret = intel_ring_idle(ring);
  1652. if (ret)
  1653. return ret;
  1654. }
  1655. i915_gem_retire_requests(dev);
  1656. /* Finally reset hw state */
  1657. for_each_ring(ring, dev_priv, i) {
  1658. intel_ring_init_seqno(ring, seqno);
  1659. for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++)
  1660. ring->sync_seqno[j] = 0;
  1661. }
  1662. return 0;
  1663. }
  1664. int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
  1665. {
  1666. struct drm_i915_private *dev_priv = dev->dev_private;
  1667. int ret;
  1668. if (seqno == 0)
  1669. return -EINVAL;
  1670. /* HWS page needs to be set less than what we
  1671. * will inject to ring
  1672. */
  1673. ret = i915_gem_init_seqno(dev, seqno - 1);
  1674. if (ret)
  1675. return ret;
  1676. /* Carefully set the last_seqno value so that wrap
  1677. * detection still works
  1678. */
  1679. dev_priv->next_seqno = seqno;
  1680. dev_priv->last_seqno = seqno - 1;
  1681. if (dev_priv->last_seqno == 0)
  1682. dev_priv->last_seqno--;
  1683. return 0;
  1684. }
  1685. int
  1686. i915_gem_get_seqno(struct drm_device *dev, u32 *seqno)
  1687. {
  1688. struct drm_i915_private *dev_priv = dev->dev_private;
  1689. /* reserve 0 for non-seqno */
  1690. if (dev_priv->next_seqno == 0) {
  1691. int ret = i915_gem_init_seqno(dev, 0);
  1692. if (ret)
  1693. return ret;
  1694. dev_priv->next_seqno = 1;
  1695. }
  1696. *seqno = dev_priv->last_seqno = dev_priv->next_seqno++;
  1697. return 0;
  1698. }
  1699. int __i915_add_request(struct intel_ring_buffer *ring,
  1700. struct drm_file *file,
  1701. struct drm_i915_gem_object *obj,
  1702. u32 *out_seqno)
  1703. {
  1704. drm_i915_private_t *dev_priv = ring->dev->dev_private;
  1705. struct drm_i915_gem_request *request;
  1706. u32 request_ring_position, request_start;
  1707. int was_empty;
  1708. int ret;
  1709. request_start = intel_ring_get_tail(ring);
  1710. /*
  1711. * Emit any outstanding flushes - execbuf can fail to emit the flush
  1712. * after having emitted the batchbuffer command. Hence we need to fix
  1713. * things up similar to emitting the lazy request. The difference here
  1714. * is that the flush _must_ happen before the next request, no matter
  1715. * what.
  1716. */
  1717. ret = intel_ring_flush_all_caches(ring);
  1718. if (ret)
  1719. return ret;
  1720. request = kmalloc(sizeof(*request), GFP_KERNEL);
  1721. if (request == NULL)
  1722. return -ENOMEM;
  1723. /* Record the position of the start of the request so that
  1724. * should we detect the updated seqno part-way through the
  1725. * GPU processing the request, we never over-estimate the
  1726. * position of the head.
  1727. */
  1728. request_ring_position = intel_ring_get_tail(ring);
  1729. ret = ring->add_request(ring);
  1730. if (ret) {
  1731. kfree(request);
  1732. return ret;
  1733. }
  1734. request->seqno = intel_ring_get_seqno(ring);
  1735. request->ring = ring;
  1736. request->head = request_start;
  1737. request->tail = request_ring_position;
  1738. request->ctx = ring->last_context;
  1739. request->batch_obj = obj;
  1740. /* Whilst this request exists, batch_obj will be on the
  1741. * active_list, and so will hold the active reference. Only when this
  1742. * request is retired will the the batch_obj be moved onto the
  1743. * inactive_list and lose its active reference. Hence we do not need
  1744. * to explicitly hold another reference here.
  1745. */
  1746. if (request->ctx)
  1747. i915_gem_context_reference(request->ctx);
  1748. request->emitted_jiffies = jiffies;
  1749. was_empty = list_empty(&ring->request_list);
  1750. list_add_tail(&request->list, &ring->request_list);
  1751. request->file_priv = NULL;
  1752. if (file) {
  1753. struct drm_i915_file_private *file_priv = file->driver_priv;
  1754. spin_lock(&file_priv->mm.lock);
  1755. request->file_priv = file_priv;
  1756. list_add_tail(&request->client_list,
  1757. &file_priv->mm.request_list);
  1758. spin_unlock(&file_priv->mm.lock);
  1759. }
  1760. trace_i915_gem_request_add(ring, request->seqno);
  1761. ring->outstanding_lazy_request = 0;
  1762. if (!dev_priv->ums.mm_suspended) {
  1763. i915_queue_hangcheck(ring->dev);
  1764. if (was_empty) {
  1765. queue_delayed_work(dev_priv->wq,
  1766. &dev_priv->mm.retire_work,
  1767. round_jiffies_up_relative(HZ));
  1768. intel_mark_busy(dev_priv->dev);
  1769. }
  1770. }
  1771. if (out_seqno)
  1772. *out_seqno = request->seqno;
  1773. return 0;
  1774. }
  1775. static inline void
  1776. i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
  1777. {
  1778. struct drm_i915_file_private *file_priv = request->file_priv;
  1779. if (!file_priv)
  1780. return;
  1781. spin_lock(&file_priv->mm.lock);
  1782. if (request->file_priv) {
  1783. list_del(&request->client_list);
  1784. request->file_priv = NULL;
  1785. }
  1786. spin_unlock(&file_priv->mm.lock);
  1787. }
  1788. static bool i915_head_inside_object(u32 acthd, struct drm_i915_gem_object *obj,
  1789. struct i915_address_space *vm)
  1790. {
  1791. if (acthd >= i915_gem_obj_offset(obj, vm) &&
  1792. acthd < i915_gem_obj_offset(obj, vm) + obj->base.size)
  1793. return true;
  1794. return false;
  1795. }
  1796. static bool i915_head_inside_request(const u32 acthd_unmasked,
  1797. const u32 request_start,
  1798. const u32 request_end)
  1799. {
  1800. const u32 acthd = acthd_unmasked & HEAD_ADDR;
  1801. if (request_start < request_end) {
  1802. if (acthd >= request_start && acthd < request_end)
  1803. return true;
  1804. } else if (request_start > request_end) {
  1805. if (acthd >= request_start || acthd < request_end)
  1806. return true;
  1807. }
  1808. return false;
  1809. }
  1810. static struct i915_address_space *
  1811. request_to_vm(struct drm_i915_gem_request *request)
  1812. {
  1813. struct drm_i915_private *dev_priv = request->ring->dev->dev_private;
  1814. struct i915_address_space *vm;
  1815. vm = &dev_priv->gtt.base;
  1816. return vm;
  1817. }
  1818. static bool i915_request_guilty(struct drm_i915_gem_request *request,
  1819. const u32 acthd, bool *inside)
  1820. {
  1821. /* There is a possibility that unmasked head address
  1822. * pointing inside the ring, matches the batch_obj address range.
  1823. * However this is extremely unlikely.
  1824. */
  1825. if (request->batch_obj) {
  1826. if (i915_head_inside_object(acthd, request->batch_obj,
  1827. request_to_vm(request))) {
  1828. *inside = true;
  1829. return true;
  1830. }
  1831. }
  1832. if (i915_head_inside_request(acthd, request->head, request->tail)) {
  1833. *inside = false;
  1834. return true;
  1835. }
  1836. return false;
  1837. }
  1838. static void i915_set_reset_status(struct intel_ring_buffer *ring,
  1839. struct drm_i915_gem_request *request,
  1840. u32 acthd)
  1841. {
  1842. struct i915_ctx_hang_stats *hs = NULL;
  1843. bool inside, guilty;
  1844. unsigned long offset = 0;
  1845. /* Innocent until proven guilty */
  1846. guilty = false;
  1847. if (request->batch_obj)
  1848. offset = i915_gem_obj_offset(request->batch_obj,
  1849. request_to_vm(request));
  1850. if (ring->hangcheck.action != HANGCHECK_WAIT &&
  1851. i915_request_guilty(request, acthd, &inside)) {
  1852. DRM_ERROR("%s hung %s bo (0x%lx ctx %d) at 0x%x\n",
  1853. ring->name,
  1854. inside ? "inside" : "flushing",
  1855. offset,
  1856. request->ctx ? request->ctx->id : 0,
  1857. acthd);
  1858. guilty = true;
  1859. }
  1860. /* If contexts are disabled or this is the default context, use
  1861. * file_priv->reset_state
  1862. */
  1863. if (request->ctx && request->ctx->id != DEFAULT_CONTEXT_ID)
  1864. hs = &request->ctx->hang_stats;
  1865. else if (request->file_priv)
  1866. hs = &request->file_priv->hang_stats;
  1867. if (hs) {
  1868. if (guilty)
  1869. hs->batch_active++;
  1870. else
  1871. hs->batch_pending++;
  1872. }
  1873. }
  1874. static void i915_gem_free_request(struct drm_i915_gem_request *request)
  1875. {
  1876. list_del(&request->list);
  1877. i915_gem_request_remove_from_client(request);
  1878. if (request->ctx)
  1879. i915_gem_context_unreference(request->ctx);
  1880. kfree(request);
  1881. }
  1882. static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
  1883. struct intel_ring_buffer *ring)
  1884. {
  1885. u32 completed_seqno;
  1886. u32 acthd;
  1887. acthd = intel_ring_get_active_head(ring);
  1888. completed_seqno = ring->get_seqno(ring, false);
  1889. while (!list_empty(&ring->request_list)) {
  1890. struct drm_i915_gem_request *request;
  1891. request = list_first_entry(&ring->request_list,
  1892. struct drm_i915_gem_request,
  1893. list);
  1894. if (request->seqno > completed_seqno)
  1895. i915_set_reset_status(ring, request, acthd);
  1896. i915_gem_free_request(request);
  1897. }
  1898. while (!list_empty(&ring->active_list)) {
  1899. struct drm_i915_gem_object *obj;
  1900. obj = list_first_entry(&ring->active_list,
  1901. struct drm_i915_gem_object,
  1902. ring_list);
  1903. i915_gem_object_move_to_inactive(obj);
  1904. }
  1905. }
  1906. void i915_gem_restore_fences(struct drm_device *dev)
  1907. {
  1908. struct drm_i915_private *dev_priv = dev->dev_private;
  1909. int i;
  1910. for (i = 0; i < dev_priv->num_fence_regs; i++) {
  1911. struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
  1912. /*
  1913. * Commit delayed tiling changes if we have an object still
  1914. * attached to the fence, otherwise just clear the fence.
  1915. */
  1916. if (reg->obj) {
  1917. i915_gem_object_update_fence(reg->obj, reg,
  1918. reg->obj->tiling_mode);
  1919. } else {
  1920. i915_gem_write_fence(dev, i, NULL);
  1921. }
  1922. }
  1923. }
  1924. void i915_gem_reset(struct drm_device *dev)
  1925. {
  1926. struct drm_i915_private *dev_priv = dev->dev_private;
  1927. struct intel_ring_buffer *ring;
  1928. int i;
  1929. for_each_ring(ring, dev_priv, i)
  1930. i915_gem_reset_ring_lists(dev_priv, ring);
  1931. i915_gem_restore_fences(dev);
  1932. }
  1933. /**
  1934. * This function clears the request list as sequence numbers are passed.
  1935. */
  1936. void
  1937. i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
  1938. {
  1939. uint32_t seqno;
  1940. if (list_empty(&ring->request_list))
  1941. return;
  1942. WARN_ON(i915_verify_lists(ring->dev));
  1943. seqno = ring->get_seqno(ring, true);
  1944. while (!list_empty(&ring->request_list)) {
  1945. struct drm_i915_gem_request *request;
  1946. request = list_first_entry(&ring->request_list,
  1947. struct drm_i915_gem_request,
  1948. list);
  1949. if (!i915_seqno_passed(seqno, request->seqno))
  1950. break;
  1951. trace_i915_gem_request_retire(ring, request->seqno);
  1952. /* We know the GPU must have read the request to have
  1953. * sent us the seqno + interrupt, so use the position
  1954. * of tail of the request to update the last known position
  1955. * of the GPU head.
  1956. */
  1957. ring->last_retired_head = request->tail;
  1958. i915_gem_free_request(request);
  1959. }
  1960. /* Move any buffers on the active list that are no longer referenced
  1961. * by the ringbuffer to the flushing/inactive lists as appropriate.
  1962. */
  1963. while (!list_empty(&ring->active_list)) {
  1964. struct drm_i915_gem_object *obj;
  1965. obj = list_first_entry(&ring->active_list,
  1966. struct drm_i915_gem_object,
  1967. ring_list);
  1968. if (!i915_seqno_passed(seqno, obj->last_read_seqno))
  1969. break;
  1970. i915_gem_object_move_to_inactive(obj);
  1971. }
  1972. if (unlikely(ring->trace_irq_seqno &&
  1973. i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
  1974. ring->irq_put(ring);
  1975. ring->trace_irq_seqno = 0;
  1976. }
  1977. WARN_ON(i915_verify_lists(ring->dev));
  1978. }
  1979. void
  1980. i915_gem_retire_requests(struct drm_device *dev)
  1981. {
  1982. drm_i915_private_t *dev_priv = dev->dev_private;
  1983. struct intel_ring_buffer *ring;
  1984. int i;
  1985. for_each_ring(ring, dev_priv, i)
  1986. i915_gem_retire_requests_ring(ring);
  1987. }
  1988. static void
  1989. i915_gem_retire_work_handler(struct work_struct *work)
  1990. {
  1991. drm_i915_private_t *dev_priv;
  1992. struct drm_device *dev;
  1993. struct intel_ring_buffer *ring;
  1994. bool idle;
  1995. int i;
  1996. dev_priv = container_of(work, drm_i915_private_t,
  1997. mm.retire_work.work);
  1998. dev = dev_priv->dev;
  1999. /* Come back later if the device is busy... */
  2000. if (!mutex_trylock(&dev->struct_mutex)) {
  2001. queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
  2002. round_jiffies_up_relative(HZ));
  2003. return;
  2004. }
  2005. i915_gem_retire_requests(dev);
  2006. /* Send a periodic flush down the ring so we don't hold onto GEM
  2007. * objects indefinitely.
  2008. */
  2009. idle = true;
  2010. for_each_ring(ring, dev_priv, i) {
  2011. if (ring->gpu_caches_dirty)
  2012. i915_add_request(ring, NULL);
  2013. idle &= list_empty(&ring->request_list);
  2014. }
  2015. if (!dev_priv->ums.mm_suspended && !idle)
  2016. queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
  2017. round_jiffies_up_relative(HZ));
  2018. if (idle)
  2019. intel_mark_idle(dev);
  2020. mutex_unlock(&dev->struct_mutex);
  2021. }
  2022. /**
  2023. * Ensures that an object will eventually get non-busy by flushing any required
  2024. * write domains, emitting any outstanding lazy request and retiring and
  2025. * completed requests.
  2026. */
  2027. static int
  2028. i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
  2029. {
  2030. int ret;
  2031. if (obj->active) {
  2032. ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno);
  2033. if (ret)
  2034. return ret;
  2035. i915_gem_retire_requests_ring(obj->ring);
  2036. }
  2037. return 0;
  2038. }
  2039. /**
  2040. * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
  2041. * @DRM_IOCTL_ARGS: standard ioctl arguments
  2042. *
  2043. * Returns 0 if successful, else an error is returned with the remaining time in
  2044. * the timeout parameter.
  2045. * -ETIME: object is still busy after timeout
  2046. * -ERESTARTSYS: signal interrupted the wait
  2047. * -ENONENT: object doesn't exist
  2048. * Also possible, but rare:
  2049. * -EAGAIN: GPU wedged
  2050. * -ENOMEM: damn
  2051. * -ENODEV: Internal IRQ fail
  2052. * -E?: The add request failed
  2053. *
  2054. * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
  2055. * non-zero timeout parameter the wait ioctl will wait for the given number of
  2056. * nanoseconds on an object becoming unbusy. Since the wait itself does so
  2057. * without holding struct_mutex the object may become re-busied before this
  2058. * function completes. A similar but shorter * race condition exists in the busy
  2059. * ioctl
  2060. */
  2061. int
  2062. i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
  2063. {
  2064. drm_i915_private_t *dev_priv = dev->dev_private;
  2065. struct drm_i915_gem_wait *args = data;
  2066. struct drm_i915_gem_object *obj;
  2067. struct intel_ring_buffer *ring = NULL;
  2068. struct timespec timeout_stack, *timeout = NULL;
  2069. unsigned reset_counter;
  2070. u32 seqno = 0;
  2071. int ret = 0;
  2072. if (args->timeout_ns >= 0) {
  2073. timeout_stack = ns_to_timespec(args->timeout_ns);
  2074. timeout = &timeout_stack;
  2075. }
  2076. ret = i915_mutex_lock_interruptible(dev);
  2077. if (ret)
  2078. return ret;
  2079. obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->bo_handle));
  2080. if (&obj->base == NULL) {
  2081. mutex_unlock(&dev->struct_mutex);
  2082. return -ENOENT;
  2083. }
  2084. /* Need to make sure the object gets inactive eventually. */
  2085. ret = i915_gem_object_flush_active(obj);
  2086. if (ret)
  2087. goto out;
  2088. if (obj->active) {
  2089. seqno = obj->last_read_seqno;
  2090. ring = obj->ring;
  2091. }
  2092. if (seqno == 0)
  2093. goto out;
  2094. /* Do this after OLR check to make sure we make forward progress polling
  2095. * on this IOCTL with a 0 timeout (like busy ioctl)
  2096. */
  2097. if (!args->timeout_ns) {
  2098. ret = -ETIME;
  2099. goto out;
  2100. }
  2101. drm_gem_object_unreference(&obj->base);
  2102. reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
  2103. mutex_unlock(&dev->struct_mutex);
  2104. ret = __wait_seqno(ring, seqno, reset_counter, true, timeout);
  2105. if (timeout)
  2106. args->timeout_ns = timespec_to_ns(timeout);
  2107. return ret;
  2108. out:
  2109. drm_gem_object_unreference(&obj->base);
  2110. mutex_unlock(&dev->struct_mutex);
  2111. return ret;
  2112. }
  2113. /**
  2114. * i915_gem_object_sync - sync an object to a ring.
  2115. *
  2116. * @obj: object which may be in use on another ring.
  2117. * @to: ring we wish to use the object on. May be NULL.
  2118. *
  2119. * This code is meant to abstract object synchronization with the GPU.
  2120. * Calling with NULL implies synchronizing the object with the CPU
  2121. * rather than a particular GPU ring.
  2122. *
  2123. * Returns 0 if successful, else propagates up the lower layer error.
  2124. */
  2125. int
  2126. i915_gem_object_sync(struct drm_i915_gem_object *obj,
  2127. struct intel_ring_buffer *to)
  2128. {
  2129. struct intel_ring_buffer *from = obj->ring;
  2130. u32 seqno;
  2131. int ret, idx;
  2132. if (from == NULL || to == from)
  2133. return 0;
  2134. if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev))
  2135. return i915_gem_object_wait_rendering(obj, false);
  2136. idx = intel_ring_sync_index(from, to);
  2137. seqno = obj->last_read_seqno;
  2138. if (seqno <= from->sync_seqno[idx])
  2139. return 0;
  2140. ret = i915_gem_check_olr(obj->ring, seqno);
  2141. if (ret)
  2142. return ret;
  2143. ret = to->sync_to(to, from, seqno);
  2144. if (!ret)
  2145. /* We use last_read_seqno because sync_to()
  2146. * might have just caused seqno wrap under
  2147. * the radar.
  2148. */
  2149. from->sync_seqno[idx] = obj->last_read_seqno;
  2150. return ret;
  2151. }
  2152. static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
  2153. {
  2154. u32 old_write_domain, old_read_domains;
  2155. /* Force a pagefault for domain tracking on next user access */
  2156. i915_gem_release_mmap(obj);
  2157. if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
  2158. return;
  2159. /* Wait for any direct GTT access to complete */
  2160. mb();
  2161. old_read_domains = obj->base.read_domains;
  2162. old_write_domain = obj->base.write_domain;
  2163. obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
  2164. obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
  2165. trace_i915_gem_object_change_domain(obj,
  2166. old_read_domains,
  2167. old_write_domain);
  2168. }
  2169. int i915_vma_unbind(struct i915_vma *vma)
  2170. {
  2171. struct drm_i915_gem_object *obj = vma->obj;
  2172. drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
  2173. int ret;
  2174. if (list_empty(&vma->vma_link))
  2175. return 0;
  2176. if (obj->pin_count)
  2177. return -EBUSY;
  2178. BUG_ON(obj->pages == NULL);
  2179. ret = i915_gem_object_finish_gpu(obj);
  2180. if (ret)
  2181. return ret;
  2182. /* Continue on if we fail due to EIO, the GPU is hung so we
  2183. * should be safe and we need to cleanup or else we might
  2184. * cause memory corruption through use-after-free.
  2185. */
  2186. i915_gem_object_finish_gtt(obj);
  2187. /* release the fence reg _after_ flushing */
  2188. ret = i915_gem_object_put_fence(obj);
  2189. if (ret)
  2190. return ret;
  2191. trace_i915_vma_unbind(vma);
  2192. if (obj->has_global_gtt_mapping)
  2193. i915_gem_gtt_unbind_object(obj);
  2194. if (obj->has_aliasing_ppgtt_mapping) {
  2195. i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
  2196. obj->has_aliasing_ppgtt_mapping = 0;
  2197. }
  2198. i915_gem_gtt_finish_object(obj);
  2199. i915_gem_object_unpin_pages(obj);
  2200. list_del(&vma->mm_list);
  2201. /* Avoid an unnecessary call to unbind on rebind. */
  2202. if (i915_is_ggtt(vma->vm))
  2203. obj->map_and_fenceable = true;
  2204. drm_mm_remove_node(&vma->node);
  2205. i915_gem_vma_destroy(vma);
  2206. /* Since the unbound list is global, only move to that list if
  2207. * no more VMAs exist.
  2208. * NB: Until we have real VMAs there will only ever be one */
  2209. WARN_ON(!list_empty(&obj->vma_list));
  2210. if (list_empty(&obj->vma_list))
  2211. list_move_tail(&obj->global_list, &dev_priv->mm.unbound_list);
  2212. return 0;
  2213. }
  2214. /**
  2215. * Unbinds an object from the global GTT aperture.
  2216. */
  2217. int
  2218. i915_gem_object_ggtt_unbind(struct drm_i915_gem_object *obj)
  2219. {
  2220. struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  2221. struct i915_address_space *ggtt = &dev_priv->gtt.base;
  2222. if (!i915_gem_obj_ggtt_bound(obj))
  2223. return 0;
  2224. if (obj->pin_count)
  2225. return -EBUSY;
  2226. BUG_ON(obj->pages == NULL);
  2227. return i915_vma_unbind(i915_gem_obj_to_vma(obj, ggtt));
  2228. }
  2229. int i915_gpu_idle(struct drm_device *dev)
  2230. {
  2231. drm_i915_private_t *dev_priv = dev->dev_private;
  2232. struct intel_ring_buffer *ring;
  2233. int ret, i;
  2234. /* Flush everything onto the inactive list. */
  2235. for_each_ring(ring, dev_priv, i) {
  2236. ret = i915_switch_context(ring, NULL, DEFAULT_CONTEXT_ID);
  2237. if (ret)
  2238. return ret;
  2239. ret = intel_ring_idle(ring);
  2240. if (ret)
  2241. return ret;
  2242. }
  2243. return 0;
  2244. }
  2245. static void i965_write_fence_reg(struct drm_device *dev, int reg,
  2246. struct drm_i915_gem_object *obj)
  2247. {
  2248. drm_i915_private_t *dev_priv = dev->dev_private;
  2249. int fence_reg;
  2250. int fence_pitch_shift;
  2251. if (INTEL_INFO(dev)->gen >= 6) {
  2252. fence_reg = FENCE_REG_SANDYBRIDGE_0;
  2253. fence_pitch_shift = SANDYBRIDGE_FENCE_PITCH_SHIFT;
  2254. } else {
  2255. fence_reg = FENCE_REG_965_0;
  2256. fence_pitch_shift = I965_FENCE_PITCH_SHIFT;
  2257. }
  2258. fence_reg += reg * 8;
  2259. /* To w/a incoherency with non-atomic 64-bit register updates,
  2260. * we split the 64-bit update into two 32-bit writes. In order
  2261. * for a partial fence not to be evaluated between writes, we
  2262. * precede the update with write to turn off the fence register,
  2263. * and only enable the fence as the last step.
  2264. *
  2265. * For extra levels of paranoia, we make sure each step lands
  2266. * before applying the next step.
  2267. */
  2268. I915_WRITE(fence_reg, 0);
  2269. POSTING_READ(fence_reg);
  2270. if (obj) {
  2271. u32 size = i915_gem_obj_ggtt_size(obj);
  2272. uint64_t val;
  2273. val = (uint64_t)((i915_gem_obj_ggtt_offset(obj) + size - 4096) &
  2274. 0xfffff000) << 32;
  2275. val |= i915_gem_obj_ggtt_offset(obj) & 0xfffff000;
  2276. val |= (uint64_t)((obj->stride / 128) - 1) << fence_pitch_shift;
  2277. if (obj->tiling_mode == I915_TILING_Y)
  2278. val |= 1 << I965_FENCE_TILING_Y_SHIFT;
  2279. val |= I965_FENCE_REG_VALID;
  2280. I915_WRITE(fence_reg + 4, val >> 32);
  2281. POSTING_READ(fence_reg + 4);
  2282. I915_WRITE(fence_reg + 0, val);
  2283. POSTING_READ(fence_reg);
  2284. } else {
  2285. I915_WRITE(fence_reg + 4, 0);
  2286. POSTING_READ(fence_reg + 4);
  2287. }
  2288. }
  2289. static void i915_write_fence_reg(struct drm_device *dev, int reg,
  2290. struct drm_i915_gem_object *obj)
  2291. {
  2292. drm_i915_private_t *dev_priv = dev->dev_private;
  2293. u32 val;
  2294. if (obj) {
  2295. u32 size = i915_gem_obj_ggtt_size(obj);
  2296. int pitch_val;
  2297. int tile_width;
  2298. WARN((i915_gem_obj_ggtt_offset(obj) & ~I915_FENCE_START_MASK) ||
  2299. (size & -size) != size ||
  2300. (i915_gem_obj_ggtt_offset(obj) & (size - 1)),
  2301. "object 0x%08lx [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
  2302. i915_gem_obj_ggtt_offset(obj), obj->map_and_fenceable, size);
  2303. if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
  2304. tile_width = 128;
  2305. else
  2306. tile_width = 512;
  2307. /* Note: pitch better be a power of two tile widths */
  2308. pitch_val = obj->stride / tile_width;
  2309. pitch_val = ffs(pitch_val) - 1;
  2310. val = i915_gem_obj_ggtt_offset(obj);
  2311. if (obj->tiling_mode == I915_TILING_Y)
  2312. val |= 1 << I830_FENCE_TILING_Y_SHIFT;
  2313. val |= I915_FENCE_SIZE_BITS(size);
  2314. val |= pitch_val << I830_FENCE_PITCH_SHIFT;
  2315. val |= I830_FENCE_REG_VALID;
  2316. } else
  2317. val = 0;
  2318. if (reg < 8)
  2319. reg = FENCE_REG_830_0 + reg * 4;
  2320. else
  2321. reg = FENCE_REG_945_8 + (reg - 8) * 4;
  2322. I915_WRITE(reg, val);
  2323. POSTING_READ(reg);
  2324. }
  2325. static void i830_write_fence_reg(struct drm_device *dev, int reg,
  2326. struct drm_i915_gem_object *obj)
  2327. {
  2328. drm_i915_private_t *dev_priv = dev->dev_private;
  2329. uint32_t val;
  2330. if (obj) {
  2331. u32 size = i915_gem_obj_ggtt_size(obj);
  2332. uint32_t pitch_val;
  2333. WARN((i915_gem_obj_ggtt_offset(obj) & ~I830_FENCE_START_MASK) ||
  2334. (size & -size) != size ||
  2335. (i915_gem_obj_ggtt_offset(obj) & (size - 1)),
  2336. "object 0x%08lx not 512K or pot-size 0x%08x aligned\n",
  2337. i915_gem_obj_ggtt_offset(obj), size);
  2338. pitch_val = obj->stride / 128;
  2339. pitch_val = ffs(pitch_val) - 1;
  2340. val = i915_gem_obj_ggtt_offset(obj);
  2341. if (obj->tiling_mode == I915_TILING_Y)
  2342. val |= 1 << I830_FENCE_TILING_Y_SHIFT;
  2343. val |= I830_FENCE_SIZE_BITS(size);
  2344. val |= pitch_val << I830_FENCE_PITCH_SHIFT;
  2345. val |= I830_FENCE_REG_VALID;
  2346. } else
  2347. val = 0;
  2348. I915_WRITE(FENCE_REG_830_0 + reg * 4, val);
  2349. POSTING_READ(FENCE_REG_830_0 + reg * 4);
  2350. }
  2351. inline static bool i915_gem_object_needs_mb(struct drm_i915_gem_object *obj)
  2352. {
  2353. return obj && obj->base.read_domains & I915_GEM_DOMAIN_GTT;
  2354. }
  2355. static void i915_gem_write_fence(struct drm_device *dev, int reg,
  2356. struct drm_i915_gem_object *obj)
  2357. {
  2358. struct drm_i915_private *dev_priv = dev->dev_private;
  2359. /* Ensure that all CPU reads are completed before installing a fence
  2360. * and all writes before removing the fence.
  2361. */
  2362. if (i915_gem_object_needs_mb(dev_priv->fence_regs[reg].obj))
  2363. mb();
  2364. WARN(obj && (!obj->stride || !obj->tiling_mode),
  2365. "bogus fence setup with stride: 0x%x, tiling mode: %i\n",
  2366. obj->stride, obj->tiling_mode);
  2367. switch (INTEL_INFO(dev)->gen) {
  2368. case 7:
  2369. case 6:
  2370. case 5:
  2371. case 4: i965_write_fence_reg(dev, reg, obj); break;
  2372. case 3: i915_write_fence_reg(dev, reg, obj); break;
  2373. case 2: i830_write_fence_reg(dev, reg, obj); break;
  2374. default: BUG();
  2375. }
  2376. /* And similarly be paranoid that no direct access to this region
  2377. * is reordered to before the fence is installed.
  2378. */
  2379. if (i915_gem_object_needs_mb(obj))
  2380. mb();
  2381. }
  2382. static inline int fence_number(struct drm_i915_private *dev_priv,
  2383. struct drm_i915_fence_reg *fence)
  2384. {
  2385. return fence - dev_priv->fence_regs;
  2386. }
  2387. static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
  2388. struct drm_i915_fence_reg *fence,
  2389. bool enable)
  2390. {
  2391. struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  2392. int reg = fence_number(dev_priv, fence);
  2393. i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL);
  2394. if (enable) {
  2395. obj->fence_reg = reg;
  2396. fence->obj = obj;
  2397. list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list);
  2398. } else {
  2399. obj->fence_reg = I915_FENCE_REG_NONE;
  2400. fence->obj = NULL;
  2401. list_del_init(&fence->lru_list);
  2402. }
  2403. obj->fence_dirty = false;
  2404. }
  2405. static int
  2406. i915_gem_object_wait_fence(struct drm_i915_gem_object *obj)
  2407. {
  2408. if (obj->last_fenced_seqno) {
  2409. int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno);
  2410. if (ret)
  2411. return ret;
  2412. obj->last_fenced_seqno = 0;
  2413. }
  2414. obj->fenced_gpu_access = false;
  2415. return 0;
  2416. }
  2417. int
  2418. i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
  2419. {
  2420. struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  2421. struct drm_i915_fence_reg *fence;
  2422. int ret;
  2423. ret = i915_gem_object_wait_fence(obj);
  2424. if (ret)
  2425. return ret;
  2426. if (obj->fence_reg == I915_FENCE_REG_NONE)
  2427. return 0;
  2428. fence = &dev_priv->fence_regs[obj->fence_reg];
  2429. i915_gem_object_fence_lost(obj);
  2430. i915_gem_object_update_fence(obj, fence, false);
  2431. return 0;
  2432. }
  2433. static struct drm_i915_fence_reg *
  2434. i915_find_fence_reg(struct drm_device *dev)
  2435. {
  2436. struct drm_i915_private *dev_priv = dev->dev_private;
  2437. struct drm_i915_fence_reg *reg, *avail;
  2438. int i;
  2439. /* First try to find a free reg */
  2440. avail = NULL;
  2441. for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
  2442. reg = &dev_priv->fence_regs[i];
  2443. if (!reg->obj)
  2444. return reg;
  2445. if (!reg->pin_count)
  2446. avail = reg;
  2447. }
  2448. if (avail == NULL)
  2449. return NULL;
  2450. /* None available, try to steal one or wait for a user to finish */
  2451. list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
  2452. if (reg->pin_count)
  2453. continue;
  2454. return reg;
  2455. }
  2456. return NULL;
  2457. }
  2458. /**
  2459. * i915_gem_object_get_fence - set up fencing for an object
  2460. * @obj: object to map through a fence reg
  2461. *
  2462. * When mapping objects through the GTT, userspace wants to be able to write
  2463. * to them without having to worry about swizzling if the object is tiled.
  2464. * This function walks the fence regs looking for a free one for @obj,
  2465. * stealing one if it can't find any.
  2466. *
  2467. * It then sets up the reg based on the object's properties: address, pitch
  2468. * and tiling format.
  2469. *
  2470. * For an untiled surface, this removes any existing fence.
  2471. */
  2472. int
  2473. i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
  2474. {
  2475. struct drm_device *dev = obj->base.dev;
  2476. struct drm_i915_private *dev_priv = dev->dev_private;
  2477. bool enable = obj->tiling_mode != I915_TILING_NONE;
  2478. struct drm_i915_fence_reg *reg;
  2479. int ret;
  2480. /* Have we updated the tiling parameters upon the object and so
  2481. * will need to serialise the write to the associated fence register?
  2482. */
  2483. if (obj->fence_dirty) {
  2484. ret = i915_gem_object_wait_fence(obj);
  2485. if (ret)
  2486. return ret;
  2487. }
  2488. /* Just update our place in the LRU if our fence is getting reused. */
  2489. if (obj->fence_reg != I915_FENCE_REG_NONE) {
  2490. reg = &dev_priv->fence_regs[obj->fence_reg];
  2491. if (!obj->fence_dirty) {
  2492. list_move_tail(&reg->lru_list,
  2493. &dev_priv->mm.fence_list);
  2494. return 0;
  2495. }
  2496. } else if (enable) {
  2497. reg = i915_find_fence_reg(dev);
  2498. if (reg == NULL)
  2499. return -EDEADLK;
  2500. if (reg->obj) {
  2501. struct drm_i915_gem_object *old = reg->obj;
  2502. ret = i915_gem_object_wait_fence(old);
  2503. if (ret)
  2504. return ret;
  2505. i915_gem_object_fence_lost(old);
  2506. }
  2507. } else
  2508. return 0;
  2509. i915_gem_object_update_fence(obj, reg, enable);
  2510. return 0;
  2511. }
  2512. static bool i915_gem_valid_gtt_space(struct drm_device *dev,
  2513. struct drm_mm_node *gtt_space,
  2514. unsigned long cache_level)
  2515. {
  2516. struct drm_mm_node *other;
  2517. /* On non-LLC machines we have to be careful when putting differing
  2518. * types of snoopable memory together to avoid the prefetcher
  2519. * crossing memory domains and dying.
  2520. */
  2521. if (HAS_LLC(dev))
  2522. return true;
  2523. if (!drm_mm_node_allocated(gtt_space))
  2524. return true;
  2525. if (list_empty(&gtt_space->node_list))
  2526. return true;
  2527. other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
  2528. if (other->allocated && !other->hole_follows && other->color != cache_level)
  2529. return false;
  2530. other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
  2531. if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
  2532. return false;
  2533. return true;
  2534. }
  2535. static void i915_gem_verify_gtt(struct drm_device *dev)
  2536. {
  2537. #if WATCH_GTT
  2538. struct drm_i915_private *dev_priv = dev->dev_private;
  2539. struct drm_i915_gem_object *obj;
  2540. int err = 0;
  2541. list_for_each_entry(obj, &dev_priv->mm.gtt_list, global_list) {
  2542. if (obj->gtt_space == NULL) {
  2543. printk(KERN_ERR "object found on GTT list with no space reserved\n");
  2544. err++;
  2545. continue;
  2546. }
  2547. if (obj->cache_level != obj->gtt_space->color) {
  2548. printk(KERN_ERR "object reserved space [%08lx, %08lx] with wrong color, cache_level=%x, color=%lx\n",
  2549. i915_gem_obj_ggtt_offset(obj),
  2550. i915_gem_obj_ggtt_offset(obj) + i915_gem_obj_ggtt_size(obj),
  2551. obj->cache_level,
  2552. obj->gtt_space->color);
  2553. err++;
  2554. continue;
  2555. }
  2556. if (!i915_gem_valid_gtt_space(dev,
  2557. obj->gtt_space,
  2558. obj->cache_level)) {
  2559. printk(KERN_ERR "invalid GTT space found at [%08lx, %08lx] - color=%x\n",
  2560. i915_gem_obj_ggtt_offset(obj),
  2561. i915_gem_obj_ggtt_offset(obj) + i915_gem_obj_ggtt_size(obj),
  2562. obj->cache_level);
  2563. err++;
  2564. continue;
  2565. }
  2566. }
  2567. WARN_ON(err);
  2568. #endif
  2569. }
  2570. /**
  2571. * Finds free space in the GTT aperture and binds the object there.
  2572. */
  2573. static int
  2574. i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
  2575. struct i915_address_space *vm,
  2576. unsigned alignment,
  2577. bool map_and_fenceable,
  2578. bool nonblocking)
  2579. {
  2580. struct drm_device *dev = obj->base.dev;
  2581. drm_i915_private_t *dev_priv = dev->dev_private;
  2582. u32 size, fence_size, fence_alignment, unfenced_alignment;
  2583. bool mappable, fenceable;
  2584. size_t gtt_max =
  2585. map_and_fenceable ? dev_priv->gtt.mappable_end : vm->total;
  2586. struct i915_vma *vma;
  2587. int ret;
  2588. if (WARN_ON(!list_empty(&obj->vma_list)))
  2589. return -EBUSY;
  2590. fence_size = i915_gem_get_gtt_size(dev,
  2591. obj->base.size,
  2592. obj->tiling_mode);
  2593. fence_alignment = i915_gem_get_gtt_alignment(dev,
  2594. obj->base.size,
  2595. obj->tiling_mode, true);
  2596. unfenced_alignment =
  2597. i915_gem_get_gtt_alignment(dev,
  2598. obj->base.size,
  2599. obj->tiling_mode, false);
  2600. if (alignment == 0)
  2601. alignment = map_and_fenceable ? fence_alignment :
  2602. unfenced_alignment;
  2603. if (map_and_fenceable && alignment & (fence_alignment - 1)) {
  2604. DRM_ERROR("Invalid object alignment requested %u\n", alignment);
  2605. return -EINVAL;
  2606. }
  2607. size = map_and_fenceable ? fence_size : obj->base.size;
  2608. /* If the object is bigger than the entire aperture, reject it early
  2609. * before evicting everything in a vain attempt to find space.
  2610. */
  2611. if (obj->base.size > gtt_max) {
  2612. DRM_ERROR("Attempting to bind an object larger than the aperture: object=%zd > %s aperture=%zu\n",
  2613. obj->base.size,
  2614. map_and_fenceable ? "mappable" : "total",
  2615. gtt_max);
  2616. return -E2BIG;
  2617. }
  2618. ret = i915_gem_object_get_pages(obj);
  2619. if (ret)
  2620. return ret;
  2621. i915_gem_object_pin_pages(obj);
  2622. /* FIXME: For now we only ever use 1 VMA per object */
  2623. BUG_ON(!i915_is_ggtt(vm));
  2624. WARN_ON(!list_empty(&obj->vma_list));
  2625. vma = i915_gem_vma_create(obj, vm);
  2626. if (IS_ERR(vma)) {
  2627. ret = PTR_ERR(vma);
  2628. goto err_unpin;
  2629. }
  2630. search_free:
  2631. ret = drm_mm_insert_node_in_range_generic(&vm->mm, &vma->node,
  2632. size, alignment,
  2633. obj->cache_level, 0, gtt_max);
  2634. if (ret) {
  2635. ret = i915_gem_evict_something(dev, vm, size, alignment,
  2636. obj->cache_level,
  2637. map_and_fenceable,
  2638. nonblocking);
  2639. if (ret == 0)
  2640. goto search_free;
  2641. goto err_free_vma;
  2642. }
  2643. if (WARN_ON(!i915_gem_valid_gtt_space(dev, &vma->node,
  2644. obj->cache_level))) {
  2645. ret = -EINVAL;
  2646. goto err_remove_node;
  2647. }
  2648. ret = i915_gem_gtt_prepare_object(obj);
  2649. if (ret)
  2650. goto err_remove_node;
  2651. list_move_tail(&obj->global_list, &dev_priv->mm.bound_list);
  2652. list_add_tail(&vma->mm_list, &vm->inactive_list);
  2653. fenceable =
  2654. i915_is_ggtt(vm) &&
  2655. i915_gem_obj_ggtt_size(obj) == fence_size &&
  2656. (i915_gem_obj_ggtt_offset(obj) & (fence_alignment - 1)) == 0;
  2657. mappable =
  2658. i915_is_ggtt(vm) &&
  2659. vma->node.start + obj->base.size <= dev_priv->gtt.mappable_end;
  2660. /* Map and fenceable only changes if the VM is the global GGTT */
  2661. if (i915_is_ggtt(vm))
  2662. obj->map_and_fenceable = mappable && fenceable;
  2663. WARN_ON(map_and_fenceable && !obj->map_and_fenceable);
  2664. trace_i915_vma_bind(vma, map_and_fenceable);
  2665. i915_gem_verify_gtt(dev);
  2666. return 0;
  2667. err_remove_node:
  2668. drm_mm_remove_node(&vma->node);
  2669. err_free_vma:
  2670. i915_gem_vma_destroy(vma);
  2671. err_unpin:
  2672. i915_gem_object_unpin_pages(obj);
  2673. return ret;
  2674. }
  2675. bool
  2676. i915_gem_clflush_object(struct drm_i915_gem_object *obj,
  2677. bool force)
  2678. {
  2679. /* If we don't have a page list set up, then we're not pinned
  2680. * to GPU, and we can ignore the cache flush because it'll happen
  2681. * again at bind time.
  2682. */
  2683. if (obj->pages == NULL)
  2684. return false;
  2685. /*
  2686. * Stolen memory is always coherent with the GPU as it is explicitly
  2687. * marked as wc by the system, or the system is cache-coherent.
  2688. */
  2689. if (obj->stolen)
  2690. return false;
  2691. /* If the GPU is snooping the contents of the CPU cache,
  2692. * we do not need to manually clear the CPU cache lines. However,
  2693. * the caches are only snooped when the render cache is
  2694. * flushed/invalidated. As we always have to emit invalidations
  2695. * and flushes when moving into and out of the RENDER domain, correct
  2696. * snooping behaviour occurs naturally as the result of our domain
  2697. * tracking.
  2698. */
  2699. if (!force && cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
  2700. return false;
  2701. trace_i915_gem_object_clflush(obj);
  2702. drm_clflush_sg(obj->pages);
  2703. return true;
  2704. }
  2705. /** Flushes the GTT write domain for the object if it's dirty. */
  2706. static void
  2707. i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
  2708. {
  2709. uint32_t old_write_domain;
  2710. if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
  2711. return;
  2712. /* No actual flushing is required for the GTT write domain. Writes
  2713. * to it immediately go to main memory as far as we know, so there's
  2714. * no chipset flush. It also doesn't land in render cache.
  2715. *
  2716. * However, we do have to enforce the order so that all writes through
  2717. * the GTT land before any writes to the device, such as updates to
  2718. * the GATT itself.
  2719. */
  2720. wmb();
  2721. old_write_domain = obj->base.write_domain;
  2722. obj->base.write_domain = 0;
  2723. trace_i915_gem_object_change_domain(obj,
  2724. obj->base.read_domains,
  2725. old_write_domain);
  2726. }
  2727. /** Flushes the CPU write domain for the object if it's dirty. */
  2728. static void
  2729. i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj,
  2730. bool force)
  2731. {
  2732. uint32_t old_write_domain;
  2733. if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
  2734. return;
  2735. if (i915_gem_clflush_object(obj, force))
  2736. i915_gem_chipset_flush(obj->base.dev);
  2737. old_write_domain = obj->base.write_domain;
  2738. obj->base.write_domain = 0;
  2739. trace_i915_gem_object_change_domain(obj,
  2740. obj->base.read_domains,
  2741. old_write_domain);
  2742. }
  2743. /**
  2744. * Moves a single object to the GTT read, and possibly write domain.
  2745. *
  2746. * This function returns when the move is complete, including waiting on
  2747. * flushes to occur.
  2748. */
  2749. int
  2750. i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
  2751. {
  2752. drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
  2753. uint32_t old_write_domain, old_read_domains;
  2754. int ret;
  2755. /* Not valid to be called on unbound objects. */
  2756. if (!i915_gem_obj_bound_any(obj))
  2757. return -EINVAL;
  2758. if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
  2759. return 0;
  2760. ret = i915_gem_object_wait_rendering(obj, !write);
  2761. if (ret)
  2762. return ret;
  2763. i915_gem_object_flush_cpu_write_domain(obj, false);
  2764. /* Serialise direct access to this object with the barriers for
  2765. * coherent writes from the GPU, by effectively invalidating the
  2766. * GTT domain upon first access.
  2767. */
  2768. if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
  2769. mb();
  2770. old_write_domain = obj->base.write_domain;
  2771. old_read_domains = obj->base.read_domains;
  2772. /* It should now be out of any other write domains, and we can update
  2773. * the domain values for our changes.
  2774. */
  2775. BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
  2776. obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
  2777. if (write) {
  2778. obj->base.read_domains = I915_GEM_DOMAIN_GTT;
  2779. obj->base.write_domain = I915_GEM_DOMAIN_GTT;
  2780. obj->dirty = 1;
  2781. }
  2782. trace_i915_gem_object_change_domain(obj,
  2783. old_read_domains,
  2784. old_write_domain);
  2785. /* And bump the LRU for this access */
  2786. if (i915_gem_object_is_inactive(obj)) {
  2787. struct i915_vma *vma = i915_gem_obj_to_vma(obj,
  2788. &dev_priv->gtt.base);
  2789. if (vma)
  2790. list_move_tail(&vma->mm_list,
  2791. &dev_priv->gtt.base.inactive_list);
  2792. }
  2793. return 0;
  2794. }
  2795. int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
  2796. enum i915_cache_level cache_level)
  2797. {
  2798. struct drm_device *dev = obj->base.dev;
  2799. drm_i915_private_t *dev_priv = dev->dev_private;
  2800. struct i915_vma *vma;
  2801. int ret;
  2802. if (obj->cache_level == cache_level)
  2803. return 0;
  2804. if (obj->pin_count) {
  2805. DRM_DEBUG("can not change the cache level of pinned objects\n");
  2806. return -EBUSY;
  2807. }
  2808. list_for_each_entry(vma, &obj->vma_list, vma_link) {
  2809. if (!i915_gem_valid_gtt_space(dev, &vma->node, cache_level)) {
  2810. ret = i915_vma_unbind(vma);
  2811. if (ret)
  2812. return ret;
  2813. break;
  2814. }
  2815. }
  2816. if (i915_gem_obj_bound_any(obj)) {
  2817. ret = i915_gem_object_finish_gpu(obj);
  2818. if (ret)
  2819. return ret;
  2820. i915_gem_object_finish_gtt(obj);
  2821. /* Before SandyBridge, you could not use tiling or fence
  2822. * registers with snooped memory, so relinquish any fences
  2823. * currently pointing to our region in the aperture.
  2824. */
  2825. if (INTEL_INFO(dev)->gen < 6) {
  2826. ret = i915_gem_object_put_fence(obj);
  2827. if (ret)
  2828. return ret;
  2829. }
  2830. if (obj->has_global_gtt_mapping)
  2831. i915_gem_gtt_bind_object(obj, cache_level);
  2832. if (obj->has_aliasing_ppgtt_mapping)
  2833. i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
  2834. obj, cache_level);
  2835. }
  2836. list_for_each_entry(vma, &obj->vma_list, vma_link)
  2837. vma->node.color = cache_level;
  2838. obj->cache_level = cache_level;
  2839. if (cpu_write_needs_clflush(obj)) {
  2840. u32 old_read_domains, old_write_domain;
  2841. /* If we're coming from LLC cached, then we haven't
  2842. * actually been tracking whether the data is in the
  2843. * CPU cache or not, since we only allow one bit set
  2844. * in obj->write_domain and have been skipping the clflushes.
  2845. * Just set it to the CPU cache for now.
  2846. */
  2847. WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
  2848. WARN_ON(obj->base.read_domains & ~I915_GEM_DOMAIN_CPU);
  2849. old_read_domains = obj->base.read_domains;
  2850. old_write_domain = obj->base.write_domain;
  2851. obj->base.read_domains = I915_GEM_DOMAIN_CPU;
  2852. obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  2853. trace_i915_gem_object_change_domain(obj,
  2854. old_read_domains,
  2855. old_write_domain);
  2856. }
  2857. i915_gem_verify_gtt(dev);
  2858. return 0;
  2859. }
  2860. int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
  2861. struct drm_file *file)
  2862. {
  2863. struct drm_i915_gem_caching *args = data;
  2864. struct drm_i915_gem_object *obj;
  2865. int ret;
  2866. ret = i915_mutex_lock_interruptible(dev);
  2867. if (ret)
  2868. return ret;
  2869. obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  2870. if (&obj->base == NULL) {
  2871. ret = -ENOENT;
  2872. goto unlock;
  2873. }
  2874. switch (obj->cache_level) {
  2875. case I915_CACHE_LLC:
  2876. case I915_CACHE_L3_LLC:
  2877. args->caching = I915_CACHING_CACHED;
  2878. break;
  2879. case I915_CACHE_WT:
  2880. args->caching = I915_CACHING_DISPLAY;
  2881. break;
  2882. default:
  2883. args->caching = I915_CACHING_NONE;
  2884. break;
  2885. }
  2886. drm_gem_object_unreference(&obj->base);
  2887. unlock:
  2888. mutex_unlock(&dev->struct_mutex);
  2889. return ret;
  2890. }
  2891. int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
  2892. struct drm_file *file)
  2893. {
  2894. struct drm_i915_gem_caching *args = data;
  2895. struct drm_i915_gem_object *obj;
  2896. enum i915_cache_level level;
  2897. int ret;
  2898. switch (args->caching) {
  2899. case I915_CACHING_NONE:
  2900. level = I915_CACHE_NONE;
  2901. break;
  2902. case I915_CACHING_CACHED:
  2903. level = I915_CACHE_LLC;
  2904. break;
  2905. case I915_CACHING_DISPLAY:
  2906. level = HAS_WT(dev) ? I915_CACHE_WT : I915_CACHE_NONE;
  2907. break;
  2908. default:
  2909. return -EINVAL;
  2910. }
  2911. ret = i915_mutex_lock_interruptible(dev);
  2912. if (ret)
  2913. return ret;
  2914. obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  2915. if (&obj->base == NULL) {
  2916. ret = -ENOENT;
  2917. goto unlock;
  2918. }
  2919. ret = i915_gem_object_set_cache_level(obj, level);
  2920. drm_gem_object_unreference(&obj->base);
  2921. unlock:
  2922. mutex_unlock(&dev->struct_mutex);
  2923. return ret;
  2924. }
  2925. static bool is_pin_display(struct drm_i915_gem_object *obj)
  2926. {
  2927. /* There are 3 sources that pin objects:
  2928. * 1. The display engine (scanouts, sprites, cursors);
  2929. * 2. Reservations for execbuffer;
  2930. * 3. The user.
  2931. *
  2932. * We can ignore reservations as we hold the struct_mutex and
  2933. * are only called outside of the reservation path. The user
  2934. * can only increment pin_count once, and so if after
  2935. * subtracting the potential reference by the user, any pin_count
  2936. * remains, it must be due to another use by the display engine.
  2937. */
  2938. return obj->pin_count - !!obj->user_pin_count;
  2939. }
  2940. /*
  2941. * Prepare buffer for display plane (scanout, cursors, etc).
  2942. * Can be called from an uninterruptible phase (modesetting) and allows
  2943. * any flushes to be pipelined (for pageflips).
  2944. */
  2945. int
  2946. i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
  2947. u32 alignment,
  2948. struct intel_ring_buffer *pipelined)
  2949. {
  2950. u32 old_read_domains, old_write_domain;
  2951. int ret;
  2952. if (pipelined != obj->ring) {
  2953. ret = i915_gem_object_sync(obj, pipelined);
  2954. if (ret)
  2955. return ret;
  2956. }
  2957. /* Mark the pin_display early so that we account for the
  2958. * display coherency whilst setting up the cache domains.
  2959. */
  2960. obj->pin_display = true;
  2961. /* The display engine is not coherent with the LLC cache on gen6. As
  2962. * a result, we make sure that the pinning that is about to occur is
  2963. * done with uncached PTEs. This is lowest common denominator for all
  2964. * chipsets.
  2965. *
  2966. * However for gen6+, we could do better by using the GFDT bit instead
  2967. * of uncaching, which would allow us to flush all the LLC-cached data
  2968. * with that bit in the PTE to main memory with just one PIPE_CONTROL.
  2969. */
  2970. ret = i915_gem_object_set_cache_level(obj,
  2971. HAS_WT(obj->base.dev) ? I915_CACHE_WT : I915_CACHE_NONE);
  2972. if (ret)
  2973. goto err_unpin_display;
  2974. /* As the user may map the buffer once pinned in the display plane
  2975. * (e.g. libkms for the bootup splash), we have to ensure that we
  2976. * always use map_and_fenceable for all scanout buffers.
  2977. */
  2978. ret = i915_gem_obj_ggtt_pin(obj, alignment, true, false);
  2979. if (ret)
  2980. goto err_unpin_display;
  2981. i915_gem_object_flush_cpu_write_domain(obj, true);
  2982. old_write_domain = obj->base.write_domain;
  2983. old_read_domains = obj->base.read_domains;
  2984. /* It should now be out of any other write domains, and we can update
  2985. * the domain values for our changes.
  2986. */
  2987. obj->base.write_domain = 0;
  2988. obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
  2989. trace_i915_gem_object_change_domain(obj,
  2990. old_read_domains,
  2991. old_write_domain);
  2992. return 0;
  2993. err_unpin_display:
  2994. obj->pin_display = is_pin_display(obj);
  2995. return ret;
  2996. }
  2997. void
  2998. i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj)
  2999. {
  3000. i915_gem_object_unpin(obj);
  3001. obj->pin_display = is_pin_display(obj);
  3002. }
  3003. int
  3004. i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
  3005. {
  3006. int ret;
  3007. if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
  3008. return 0;
  3009. ret = i915_gem_object_wait_rendering(obj, false);
  3010. if (ret)
  3011. return ret;
  3012. /* Ensure that we invalidate the GPU's caches and TLBs. */
  3013. obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
  3014. return 0;
  3015. }
  3016. /**
  3017. * Moves a single object to the CPU read, and possibly write domain.
  3018. *
  3019. * This function returns when the move is complete, including waiting on
  3020. * flushes to occur.
  3021. */
  3022. int
  3023. i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
  3024. {
  3025. uint32_t old_write_domain, old_read_domains;
  3026. int ret;
  3027. if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
  3028. return 0;
  3029. ret = i915_gem_object_wait_rendering(obj, !write);
  3030. if (ret)
  3031. return ret;
  3032. i915_gem_object_flush_gtt_write_domain(obj);
  3033. old_write_domain = obj->base.write_domain;
  3034. old_read_domains = obj->base.read_domains;
  3035. /* Flush the CPU cache if it's still invalid. */
  3036. if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
  3037. i915_gem_clflush_object(obj, false);
  3038. obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
  3039. }
  3040. /* It should now be out of any other write domains, and we can update
  3041. * the domain values for our changes.
  3042. */
  3043. BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
  3044. /* If we're writing through the CPU, then the GPU read domains will
  3045. * need to be invalidated at next use.
  3046. */
  3047. if (write) {
  3048. obj->base.read_domains = I915_GEM_DOMAIN_CPU;
  3049. obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  3050. }
  3051. trace_i915_gem_object_change_domain(obj,
  3052. old_read_domains,
  3053. old_write_domain);
  3054. return 0;
  3055. }
  3056. /* Throttle our rendering by waiting until the ring has completed our requests
  3057. * emitted over 20 msec ago.
  3058. *
  3059. * Note that if we were to use the current jiffies each time around the loop,
  3060. * we wouldn't escape the function with any frames outstanding if the time to
  3061. * render a frame was over 20ms.
  3062. *
  3063. * This should get us reasonable parallelism between CPU and GPU but also
  3064. * relatively low latency when blocking on a particular request to finish.
  3065. */
  3066. static int
  3067. i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
  3068. {
  3069. struct drm_i915_private *dev_priv = dev->dev_private;
  3070. struct drm_i915_file_private *file_priv = file->driver_priv;
  3071. unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
  3072. struct drm_i915_gem_request *request;
  3073. struct intel_ring_buffer *ring = NULL;
  3074. unsigned reset_counter;
  3075. u32 seqno = 0;
  3076. int ret;
  3077. ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
  3078. if (ret)
  3079. return ret;
  3080. ret = i915_gem_check_wedge(&dev_priv->gpu_error, false);
  3081. if (ret)
  3082. return ret;
  3083. spin_lock(&file_priv->mm.lock);
  3084. list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
  3085. if (time_after_eq(request->emitted_jiffies, recent_enough))
  3086. break;
  3087. ring = request->ring;
  3088. seqno = request->seqno;
  3089. }
  3090. reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
  3091. spin_unlock(&file_priv->mm.lock);
  3092. if (seqno == 0)
  3093. return 0;
  3094. ret = __wait_seqno(ring, seqno, reset_counter, true, NULL);
  3095. if (ret == 0)
  3096. queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
  3097. return ret;
  3098. }
  3099. int
  3100. i915_gem_object_pin(struct drm_i915_gem_object *obj,
  3101. struct i915_address_space *vm,
  3102. uint32_t alignment,
  3103. bool map_and_fenceable,
  3104. bool nonblocking)
  3105. {
  3106. struct i915_vma *vma;
  3107. int ret;
  3108. if (WARN_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
  3109. return -EBUSY;
  3110. WARN_ON(map_and_fenceable && !i915_is_ggtt(vm));
  3111. vma = i915_gem_obj_to_vma(obj, vm);
  3112. if (vma) {
  3113. if ((alignment &&
  3114. vma->node.start & (alignment - 1)) ||
  3115. (map_and_fenceable && !obj->map_and_fenceable)) {
  3116. WARN(obj->pin_count,
  3117. "bo is already pinned with incorrect alignment:"
  3118. " offset=%lx, req.alignment=%x, req.map_and_fenceable=%d,"
  3119. " obj->map_and_fenceable=%d\n",
  3120. i915_gem_obj_offset(obj, vm), alignment,
  3121. map_and_fenceable,
  3122. obj->map_and_fenceable);
  3123. ret = i915_vma_unbind(vma);
  3124. if (ret)
  3125. return ret;
  3126. }
  3127. }
  3128. if (!i915_gem_obj_bound(obj, vm)) {
  3129. struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  3130. ret = i915_gem_object_bind_to_vm(obj, vm, alignment,
  3131. map_and_fenceable,
  3132. nonblocking);
  3133. if (ret)
  3134. return ret;
  3135. if (!dev_priv->mm.aliasing_ppgtt)
  3136. i915_gem_gtt_bind_object(obj, obj->cache_level);
  3137. }
  3138. if (!obj->has_global_gtt_mapping && map_and_fenceable)
  3139. i915_gem_gtt_bind_object(obj, obj->cache_level);
  3140. obj->pin_count++;
  3141. obj->pin_mappable |= map_and_fenceable;
  3142. return 0;
  3143. }
  3144. void
  3145. i915_gem_object_unpin(struct drm_i915_gem_object *obj)
  3146. {
  3147. BUG_ON(obj->pin_count == 0);
  3148. BUG_ON(!i915_gem_obj_bound_any(obj));
  3149. if (--obj->pin_count == 0)
  3150. obj->pin_mappable = false;
  3151. }
  3152. int
  3153. i915_gem_pin_ioctl(struct drm_device *dev, void *data,
  3154. struct drm_file *file)
  3155. {
  3156. struct drm_i915_gem_pin *args = data;
  3157. struct drm_i915_gem_object *obj;
  3158. int ret;
  3159. ret = i915_mutex_lock_interruptible(dev);
  3160. if (ret)
  3161. return ret;
  3162. obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  3163. if (&obj->base == NULL) {
  3164. ret = -ENOENT;
  3165. goto unlock;
  3166. }
  3167. if (obj->madv != I915_MADV_WILLNEED) {
  3168. DRM_ERROR("Attempting to pin a purgeable buffer\n");
  3169. ret = -EINVAL;
  3170. goto out;
  3171. }
  3172. if (obj->pin_filp != NULL && obj->pin_filp != file) {
  3173. DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
  3174. args->handle);
  3175. ret = -EINVAL;
  3176. goto out;
  3177. }
  3178. if (obj->user_pin_count == 0) {
  3179. ret = i915_gem_obj_ggtt_pin(obj, args->alignment, true, false);
  3180. if (ret)
  3181. goto out;
  3182. }
  3183. obj->user_pin_count++;
  3184. obj->pin_filp = file;
  3185. args->offset = i915_gem_obj_ggtt_offset(obj);
  3186. out:
  3187. drm_gem_object_unreference(&obj->base);
  3188. unlock:
  3189. mutex_unlock(&dev->struct_mutex);
  3190. return ret;
  3191. }
  3192. int
  3193. i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
  3194. struct drm_file *file)
  3195. {
  3196. struct drm_i915_gem_pin *args = data;
  3197. struct drm_i915_gem_object *obj;
  3198. int ret;
  3199. ret = i915_mutex_lock_interruptible(dev);
  3200. if (ret)
  3201. return ret;
  3202. obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  3203. if (&obj->base == NULL) {
  3204. ret = -ENOENT;
  3205. goto unlock;
  3206. }
  3207. if (obj->pin_filp != file) {
  3208. DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
  3209. args->handle);
  3210. ret = -EINVAL;
  3211. goto out;
  3212. }
  3213. obj->user_pin_count--;
  3214. if (obj->user_pin_count == 0) {
  3215. obj->pin_filp = NULL;
  3216. i915_gem_object_unpin(obj);
  3217. }
  3218. out:
  3219. drm_gem_object_unreference(&obj->base);
  3220. unlock:
  3221. mutex_unlock(&dev->struct_mutex);
  3222. return ret;
  3223. }
  3224. int
  3225. i915_gem_busy_ioctl(struct drm_device *dev, void *data,
  3226. struct drm_file *file)
  3227. {
  3228. struct drm_i915_gem_busy *args = data;
  3229. struct drm_i915_gem_object *obj;
  3230. int ret;
  3231. ret = i915_mutex_lock_interruptible(dev);
  3232. if (ret)
  3233. return ret;
  3234. obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  3235. if (&obj->base == NULL) {
  3236. ret = -ENOENT;
  3237. goto unlock;
  3238. }
  3239. /* Count all active objects as busy, even if they are currently not used
  3240. * by the gpu. Users of this interface expect objects to eventually
  3241. * become non-busy without any further actions, therefore emit any
  3242. * necessary flushes here.
  3243. */
  3244. ret = i915_gem_object_flush_active(obj);
  3245. args->busy = obj->active;
  3246. if (obj->ring) {
  3247. BUILD_BUG_ON(I915_NUM_RINGS > 16);
  3248. args->busy |= intel_ring_flag(obj->ring) << 16;
  3249. }
  3250. drm_gem_object_unreference(&obj->base);
  3251. unlock:
  3252. mutex_unlock(&dev->struct_mutex);
  3253. return ret;
  3254. }
  3255. int
  3256. i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
  3257. struct drm_file *file_priv)
  3258. {
  3259. return i915_gem_ring_throttle(dev, file_priv);
  3260. }
  3261. int
  3262. i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
  3263. struct drm_file *file_priv)
  3264. {
  3265. struct drm_i915_gem_madvise *args = data;
  3266. struct drm_i915_gem_object *obj;
  3267. int ret;
  3268. switch (args->madv) {
  3269. case I915_MADV_DONTNEED:
  3270. case I915_MADV_WILLNEED:
  3271. break;
  3272. default:
  3273. return -EINVAL;
  3274. }
  3275. ret = i915_mutex_lock_interruptible(dev);
  3276. if (ret)
  3277. return ret;
  3278. obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
  3279. if (&obj->base == NULL) {
  3280. ret = -ENOENT;
  3281. goto unlock;
  3282. }
  3283. if (obj->pin_count) {
  3284. ret = -EINVAL;
  3285. goto out;
  3286. }
  3287. if (obj->madv != __I915_MADV_PURGED)
  3288. obj->madv = args->madv;
  3289. /* if the object is no longer attached, discard its backing storage */
  3290. if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL)
  3291. i915_gem_object_truncate(obj);
  3292. args->retained = obj->madv != __I915_MADV_PURGED;
  3293. out:
  3294. drm_gem_object_unreference(&obj->base);
  3295. unlock:
  3296. mutex_unlock(&dev->struct_mutex);
  3297. return ret;
  3298. }
  3299. void i915_gem_object_init(struct drm_i915_gem_object *obj,
  3300. const struct drm_i915_gem_object_ops *ops)
  3301. {
  3302. INIT_LIST_HEAD(&obj->global_list);
  3303. INIT_LIST_HEAD(&obj->ring_list);
  3304. INIT_LIST_HEAD(&obj->exec_list);
  3305. INIT_LIST_HEAD(&obj->vma_list);
  3306. obj->ops = ops;
  3307. obj->fence_reg = I915_FENCE_REG_NONE;
  3308. obj->madv = I915_MADV_WILLNEED;
  3309. /* Avoid an unnecessary call to unbind on the first bind. */
  3310. obj->map_and_fenceable = true;
  3311. i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
  3312. }
  3313. static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
  3314. .get_pages = i915_gem_object_get_pages_gtt,
  3315. .put_pages = i915_gem_object_put_pages_gtt,
  3316. };
  3317. struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
  3318. size_t size)
  3319. {
  3320. struct drm_i915_gem_object *obj;
  3321. struct address_space *mapping;
  3322. gfp_t mask;
  3323. obj = i915_gem_object_alloc(dev);
  3324. if (obj == NULL)
  3325. return NULL;
  3326. if (drm_gem_object_init(dev, &obj->base, size) != 0) {
  3327. i915_gem_object_free(obj);
  3328. return NULL;
  3329. }
  3330. mask = GFP_HIGHUSER | __GFP_RECLAIMABLE;
  3331. if (IS_CRESTLINE(dev) || IS_BROADWATER(dev)) {
  3332. /* 965gm cannot relocate objects above 4GiB. */
  3333. mask &= ~__GFP_HIGHMEM;
  3334. mask |= __GFP_DMA32;
  3335. }
  3336. mapping = file_inode(obj->base.filp)->i_mapping;
  3337. mapping_set_gfp_mask(mapping, mask);
  3338. i915_gem_object_init(obj, &i915_gem_object_ops);
  3339. obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  3340. obj->base.read_domains = I915_GEM_DOMAIN_CPU;
  3341. if (HAS_LLC(dev)) {
  3342. /* On some devices, we can have the GPU use the LLC (the CPU
  3343. * cache) for about a 10% performance improvement
  3344. * compared to uncached. Graphics requests other than
  3345. * display scanout are coherent with the CPU in
  3346. * accessing this cache. This means in this mode we
  3347. * don't need to clflush on the CPU side, and on the
  3348. * GPU side we only need to flush internal caches to
  3349. * get data visible to the CPU.
  3350. *
  3351. * However, we maintain the display planes as UC, and so
  3352. * need to rebind when first used as such.
  3353. */
  3354. obj->cache_level = I915_CACHE_LLC;
  3355. } else
  3356. obj->cache_level = I915_CACHE_NONE;
  3357. trace_i915_gem_object_create(obj);
  3358. return obj;
  3359. }
  3360. int i915_gem_init_object(struct drm_gem_object *obj)
  3361. {
  3362. BUG();
  3363. return 0;
  3364. }
  3365. void i915_gem_free_object(struct drm_gem_object *gem_obj)
  3366. {
  3367. struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
  3368. struct drm_device *dev = obj->base.dev;
  3369. drm_i915_private_t *dev_priv = dev->dev_private;
  3370. struct i915_vma *vma, *next;
  3371. trace_i915_gem_object_destroy(obj);
  3372. if (obj->phys_obj)
  3373. i915_gem_detach_phys_object(dev, obj);
  3374. obj->pin_count = 0;
  3375. /* NB: 0 or 1 elements */
  3376. WARN_ON(!list_empty(&obj->vma_list) &&
  3377. !list_is_singular(&obj->vma_list));
  3378. list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) {
  3379. int ret = i915_vma_unbind(vma);
  3380. if (WARN_ON(ret == -ERESTARTSYS)) {
  3381. bool was_interruptible;
  3382. was_interruptible = dev_priv->mm.interruptible;
  3383. dev_priv->mm.interruptible = false;
  3384. WARN_ON(i915_vma_unbind(vma));
  3385. dev_priv->mm.interruptible = was_interruptible;
  3386. }
  3387. }
  3388. /* Stolen objects don't hold a ref, but do hold pin count. Fix that up
  3389. * before progressing. */
  3390. if (obj->stolen)
  3391. i915_gem_object_unpin_pages(obj);
  3392. if (WARN_ON(obj->pages_pin_count))
  3393. obj->pages_pin_count = 0;
  3394. i915_gem_object_put_pages(obj);
  3395. i915_gem_object_free_mmap_offset(obj);
  3396. i915_gem_object_release_stolen(obj);
  3397. BUG_ON(obj->pages);
  3398. if (obj->base.import_attach)
  3399. drm_prime_gem_destroy(&obj->base, NULL);
  3400. drm_gem_object_release(&obj->base);
  3401. i915_gem_info_remove_obj(dev_priv, obj->base.size);
  3402. kfree(obj->bit_17);
  3403. i915_gem_object_free(obj);
  3404. }
  3405. struct i915_vma *i915_gem_vma_create(struct drm_i915_gem_object *obj,
  3406. struct i915_address_space *vm)
  3407. {
  3408. struct i915_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL);
  3409. if (vma == NULL)
  3410. return ERR_PTR(-ENOMEM);
  3411. INIT_LIST_HEAD(&vma->vma_link);
  3412. INIT_LIST_HEAD(&vma->mm_list);
  3413. vma->vm = vm;
  3414. vma->obj = obj;
  3415. /* Keep GGTT vmas first to make debug easier */
  3416. if (i915_is_ggtt(vm))
  3417. list_add(&vma->vma_link, &obj->vma_list);
  3418. else
  3419. list_add_tail(&vma->vma_link, &obj->vma_list);
  3420. return vma;
  3421. }
  3422. void i915_gem_vma_destroy(struct i915_vma *vma)
  3423. {
  3424. WARN_ON(vma->node.allocated);
  3425. list_del(&vma->vma_link);
  3426. kfree(vma);
  3427. }
  3428. int
  3429. i915_gem_idle(struct drm_device *dev)
  3430. {
  3431. drm_i915_private_t *dev_priv = dev->dev_private;
  3432. int ret;
  3433. if (dev_priv->ums.mm_suspended) {
  3434. mutex_unlock(&dev->struct_mutex);
  3435. return 0;
  3436. }
  3437. ret = i915_gpu_idle(dev);
  3438. if (ret) {
  3439. mutex_unlock(&dev->struct_mutex);
  3440. return ret;
  3441. }
  3442. i915_gem_retire_requests(dev);
  3443. /* Under UMS, be paranoid and evict. */
  3444. if (!drm_core_check_feature(dev, DRIVER_MODESET))
  3445. i915_gem_evict_everything(dev);
  3446. del_timer_sync(&dev_priv->gpu_error.hangcheck_timer);
  3447. i915_kernel_lost_context(dev);
  3448. i915_gem_cleanup_ringbuffer(dev);
  3449. /* Cancel the retire work handler, which should be idle now. */
  3450. cancel_delayed_work_sync(&dev_priv->mm.retire_work);
  3451. return 0;
  3452. }
  3453. void i915_gem_l3_remap(struct drm_device *dev)
  3454. {
  3455. drm_i915_private_t *dev_priv = dev->dev_private;
  3456. u32 misccpctl;
  3457. int i;
  3458. if (!HAS_L3_GPU_CACHE(dev))
  3459. return;
  3460. if (!dev_priv->l3_parity.remap_info)
  3461. return;
  3462. misccpctl = I915_READ(GEN7_MISCCPCTL);
  3463. I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
  3464. POSTING_READ(GEN7_MISCCPCTL);
  3465. for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
  3466. u32 remap = I915_READ(GEN7_L3LOG_BASE + i);
  3467. if (remap && remap != dev_priv->l3_parity.remap_info[i/4])
  3468. DRM_DEBUG("0x%x was already programmed to %x\n",
  3469. GEN7_L3LOG_BASE + i, remap);
  3470. if (remap && !dev_priv->l3_parity.remap_info[i/4])
  3471. DRM_DEBUG_DRIVER("Clearing remapped register\n");
  3472. I915_WRITE(GEN7_L3LOG_BASE + i, dev_priv->l3_parity.remap_info[i/4]);
  3473. }
  3474. /* Make sure all the writes land before disabling dop clock gating */
  3475. POSTING_READ(GEN7_L3LOG_BASE);
  3476. I915_WRITE(GEN7_MISCCPCTL, misccpctl);
  3477. }
  3478. void i915_gem_init_swizzling(struct drm_device *dev)
  3479. {
  3480. drm_i915_private_t *dev_priv = dev->dev_private;
  3481. if (INTEL_INFO(dev)->gen < 5 ||
  3482. dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
  3483. return;
  3484. I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
  3485. DISP_TILE_SURFACE_SWIZZLING);
  3486. if (IS_GEN5(dev))
  3487. return;
  3488. I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
  3489. if (IS_GEN6(dev))
  3490. I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
  3491. else if (IS_GEN7(dev))
  3492. I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
  3493. else
  3494. BUG();
  3495. }
  3496. static bool
  3497. intel_enable_blt(struct drm_device *dev)
  3498. {
  3499. if (!HAS_BLT(dev))
  3500. return false;
  3501. /* The blitter was dysfunctional on early prototypes */
  3502. if (IS_GEN6(dev) && dev->pdev->revision < 8) {
  3503. DRM_INFO("BLT not supported on this pre-production hardware;"
  3504. " graphics performance will be degraded.\n");
  3505. return false;
  3506. }
  3507. return true;
  3508. }
  3509. static int i915_gem_init_rings(struct drm_device *dev)
  3510. {
  3511. struct drm_i915_private *dev_priv = dev->dev_private;
  3512. int ret;
  3513. ret = intel_init_render_ring_buffer(dev);
  3514. if (ret)
  3515. return ret;
  3516. if (HAS_BSD(dev)) {
  3517. ret = intel_init_bsd_ring_buffer(dev);
  3518. if (ret)
  3519. goto cleanup_render_ring;
  3520. }
  3521. if (intel_enable_blt(dev)) {
  3522. ret = intel_init_blt_ring_buffer(dev);
  3523. if (ret)
  3524. goto cleanup_bsd_ring;
  3525. }
  3526. if (HAS_VEBOX(dev)) {
  3527. ret = intel_init_vebox_ring_buffer(dev);
  3528. if (ret)
  3529. goto cleanup_blt_ring;
  3530. }
  3531. ret = i915_gem_set_seqno(dev, ((u32)~0 - 0x1000));
  3532. if (ret)
  3533. goto cleanup_vebox_ring;
  3534. return 0;
  3535. cleanup_vebox_ring:
  3536. intel_cleanup_ring_buffer(&dev_priv->ring[VECS]);
  3537. cleanup_blt_ring:
  3538. intel_cleanup_ring_buffer(&dev_priv->ring[BCS]);
  3539. cleanup_bsd_ring:
  3540. intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
  3541. cleanup_render_ring:
  3542. intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
  3543. return ret;
  3544. }
  3545. int
  3546. i915_gem_init_hw(struct drm_device *dev)
  3547. {
  3548. drm_i915_private_t *dev_priv = dev->dev_private;
  3549. int ret;
  3550. if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt())
  3551. return -EIO;
  3552. if (dev_priv->ellc_size)
  3553. I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) | IDIHASHMSK(0xf));
  3554. if (HAS_PCH_NOP(dev)) {
  3555. u32 temp = I915_READ(GEN7_MSG_CTL);
  3556. temp &= ~(WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK);
  3557. I915_WRITE(GEN7_MSG_CTL, temp);
  3558. }
  3559. i915_gem_l3_remap(dev);
  3560. i915_gem_init_swizzling(dev);
  3561. ret = i915_gem_init_rings(dev);
  3562. if (ret)
  3563. return ret;
  3564. /*
  3565. * XXX: There was some w/a described somewhere suggesting loading
  3566. * contexts before PPGTT.
  3567. */
  3568. i915_gem_context_init(dev);
  3569. if (dev_priv->mm.aliasing_ppgtt) {
  3570. ret = dev_priv->mm.aliasing_ppgtt->enable(dev);
  3571. if (ret) {
  3572. i915_gem_cleanup_aliasing_ppgtt(dev);
  3573. DRM_INFO("PPGTT enable failed. This is not fatal, but unexpected\n");
  3574. }
  3575. }
  3576. return 0;
  3577. }
  3578. int i915_gem_init(struct drm_device *dev)
  3579. {
  3580. struct drm_i915_private *dev_priv = dev->dev_private;
  3581. int ret;
  3582. mutex_lock(&dev->struct_mutex);
  3583. if (IS_VALLEYVIEW(dev)) {
  3584. /* VLVA0 (potential hack), BIOS isn't actually waking us */
  3585. I915_WRITE(VLV_GTLC_WAKE_CTRL, 1);
  3586. if (wait_for((I915_READ(VLV_GTLC_PW_STATUS) & 1) == 1, 10))
  3587. DRM_DEBUG_DRIVER("allow wake ack timed out\n");
  3588. }
  3589. i915_gem_init_global_gtt(dev);
  3590. ret = i915_gem_init_hw(dev);
  3591. mutex_unlock(&dev->struct_mutex);
  3592. if (ret) {
  3593. i915_gem_cleanup_aliasing_ppgtt(dev);
  3594. return ret;
  3595. }
  3596. /* Allow hardware batchbuffers unless told otherwise, but not for KMS. */
  3597. if (!drm_core_check_feature(dev, DRIVER_MODESET))
  3598. dev_priv->dri1.allow_batchbuffer = 1;
  3599. return 0;
  3600. }
  3601. void
  3602. i915_gem_cleanup_ringbuffer(struct drm_device *dev)
  3603. {
  3604. drm_i915_private_t *dev_priv = dev->dev_private;
  3605. struct intel_ring_buffer *ring;
  3606. int i;
  3607. for_each_ring(ring, dev_priv, i)
  3608. intel_cleanup_ring_buffer(ring);
  3609. }
  3610. int
  3611. i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
  3612. struct drm_file *file_priv)
  3613. {
  3614. struct drm_i915_private *dev_priv = dev->dev_private;
  3615. int ret;
  3616. if (drm_core_check_feature(dev, DRIVER_MODESET))
  3617. return 0;
  3618. if (i915_reset_in_progress(&dev_priv->gpu_error)) {
  3619. DRM_ERROR("Reenabling wedged hardware, good luck\n");
  3620. atomic_set(&dev_priv->gpu_error.reset_counter, 0);
  3621. }
  3622. mutex_lock(&dev->struct_mutex);
  3623. dev_priv->ums.mm_suspended = 0;
  3624. ret = i915_gem_init_hw(dev);
  3625. if (ret != 0) {
  3626. mutex_unlock(&dev->struct_mutex);
  3627. return ret;
  3628. }
  3629. BUG_ON(!list_empty(&dev_priv->gtt.base.active_list));
  3630. mutex_unlock(&dev->struct_mutex);
  3631. ret = drm_irq_install(dev);
  3632. if (ret)
  3633. goto cleanup_ringbuffer;
  3634. return 0;
  3635. cleanup_ringbuffer:
  3636. mutex_lock(&dev->struct_mutex);
  3637. i915_gem_cleanup_ringbuffer(dev);
  3638. dev_priv->ums.mm_suspended = 1;
  3639. mutex_unlock(&dev->struct_mutex);
  3640. return ret;
  3641. }
  3642. int
  3643. i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
  3644. struct drm_file *file_priv)
  3645. {
  3646. struct drm_i915_private *dev_priv = dev->dev_private;
  3647. int ret;
  3648. if (drm_core_check_feature(dev, DRIVER_MODESET))
  3649. return 0;
  3650. drm_irq_uninstall(dev);
  3651. mutex_lock(&dev->struct_mutex);
  3652. ret = i915_gem_idle(dev);
  3653. /* Hack! Don't let anybody do execbuf while we don't control the chip.
  3654. * We need to replace this with a semaphore, or something.
  3655. * And not confound ums.mm_suspended!
  3656. */
  3657. if (ret != 0)
  3658. dev_priv->ums.mm_suspended = 1;
  3659. mutex_unlock(&dev->struct_mutex);
  3660. return ret;
  3661. }
  3662. void
  3663. i915_gem_lastclose(struct drm_device *dev)
  3664. {
  3665. int ret;
  3666. if (drm_core_check_feature(dev, DRIVER_MODESET))
  3667. return;
  3668. mutex_lock(&dev->struct_mutex);
  3669. ret = i915_gem_idle(dev);
  3670. if (ret)
  3671. DRM_ERROR("failed to idle hardware: %d\n", ret);
  3672. mutex_unlock(&dev->struct_mutex);
  3673. }
  3674. static void
  3675. init_ring_lists(struct intel_ring_buffer *ring)
  3676. {
  3677. INIT_LIST_HEAD(&ring->active_list);
  3678. INIT_LIST_HEAD(&ring->request_list);
  3679. }
  3680. static void i915_init_vm(struct drm_i915_private *dev_priv,
  3681. struct i915_address_space *vm)
  3682. {
  3683. vm->dev = dev_priv->dev;
  3684. INIT_LIST_HEAD(&vm->active_list);
  3685. INIT_LIST_HEAD(&vm->inactive_list);
  3686. INIT_LIST_HEAD(&vm->global_link);
  3687. list_add(&vm->global_link, &dev_priv->vm_list);
  3688. }
  3689. void
  3690. i915_gem_load(struct drm_device *dev)
  3691. {
  3692. drm_i915_private_t *dev_priv = dev->dev_private;
  3693. int i;
  3694. dev_priv->slab =
  3695. kmem_cache_create("i915_gem_object",
  3696. sizeof(struct drm_i915_gem_object), 0,
  3697. SLAB_HWCACHE_ALIGN,
  3698. NULL);
  3699. INIT_LIST_HEAD(&dev_priv->vm_list);
  3700. i915_init_vm(dev_priv, &dev_priv->gtt.base);
  3701. INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
  3702. INIT_LIST_HEAD(&dev_priv->mm.bound_list);
  3703. INIT_LIST_HEAD(&dev_priv->mm.fence_list);
  3704. for (i = 0; i < I915_NUM_RINGS; i++)
  3705. init_ring_lists(&dev_priv->ring[i]);
  3706. for (i = 0; i < I915_MAX_NUM_FENCES; i++)
  3707. INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
  3708. INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
  3709. i915_gem_retire_work_handler);
  3710. init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
  3711. /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
  3712. if (IS_GEN3(dev)) {
  3713. I915_WRITE(MI_ARB_STATE,
  3714. _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
  3715. }
  3716. dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
  3717. /* Old X drivers will take 0-2 for front, back, depth buffers */
  3718. if (!drm_core_check_feature(dev, DRIVER_MODESET))
  3719. dev_priv->fence_reg_start = 3;
  3720. if (INTEL_INFO(dev)->gen >= 7 && !IS_VALLEYVIEW(dev))
  3721. dev_priv->num_fence_regs = 32;
  3722. else if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
  3723. dev_priv->num_fence_regs = 16;
  3724. else
  3725. dev_priv->num_fence_regs = 8;
  3726. /* Initialize fence registers to zero */
  3727. INIT_LIST_HEAD(&dev_priv->mm.fence_list);
  3728. i915_gem_restore_fences(dev);
  3729. i915_gem_detect_bit_6_swizzle(dev);
  3730. init_waitqueue_head(&dev_priv->pending_flip_queue);
  3731. dev_priv->mm.interruptible = true;
  3732. dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
  3733. dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
  3734. register_shrinker(&dev_priv->mm.inactive_shrinker);
  3735. }
  3736. /*
  3737. * Create a physically contiguous memory object for this object
  3738. * e.g. for cursor + overlay regs
  3739. */
  3740. static int i915_gem_init_phys_object(struct drm_device *dev,
  3741. int id, int size, int align)
  3742. {
  3743. drm_i915_private_t *dev_priv = dev->dev_private;
  3744. struct drm_i915_gem_phys_object *phys_obj;
  3745. int ret;
  3746. if (dev_priv->mm.phys_objs[id - 1] || !size)
  3747. return 0;
  3748. phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
  3749. if (!phys_obj)
  3750. return -ENOMEM;
  3751. phys_obj->id = id;
  3752. phys_obj->handle = drm_pci_alloc(dev, size, align);
  3753. if (!phys_obj->handle) {
  3754. ret = -ENOMEM;
  3755. goto kfree_obj;
  3756. }
  3757. #ifdef CONFIG_X86
  3758. set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
  3759. #endif
  3760. dev_priv->mm.phys_objs[id - 1] = phys_obj;
  3761. return 0;
  3762. kfree_obj:
  3763. kfree(phys_obj);
  3764. return ret;
  3765. }
  3766. static void i915_gem_free_phys_object(struct drm_device *dev, int id)
  3767. {
  3768. drm_i915_private_t *dev_priv = dev->dev_private;
  3769. struct drm_i915_gem_phys_object *phys_obj;
  3770. if (!dev_priv->mm.phys_objs[id - 1])
  3771. return;
  3772. phys_obj = dev_priv->mm.phys_objs[id - 1];
  3773. if (phys_obj->cur_obj) {
  3774. i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
  3775. }
  3776. #ifdef CONFIG_X86
  3777. set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
  3778. #endif
  3779. drm_pci_free(dev, phys_obj->handle);
  3780. kfree(phys_obj);
  3781. dev_priv->mm.phys_objs[id - 1] = NULL;
  3782. }
  3783. void i915_gem_free_all_phys_object(struct drm_device *dev)
  3784. {
  3785. int i;
  3786. for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
  3787. i915_gem_free_phys_object(dev, i);
  3788. }
  3789. void i915_gem_detach_phys_object(struct drm_device *dev,
  3790. struct drm_i915_gem_object *obj)
  3791. {
  3792. struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
  3793. char *vaddr;
  3794. int i;
  3795. int page_count;
  3796. if (!obj->phys_obj)
  3797. return;
  3798. vaddr = obj->phys_obj->handle->vaddr;
  3799. page_count = obj->base.size / PAGE_SIZE;
  3800. for (i = 0; i < page_count; i++) {
  3801. struct page *page = shmem_read_mapping_page(mapping, i);
  3802. if (!IS_ERR(page)) {
  3803. char *dst = kmap_atomic(page);
  3804. memcpy(dst, vaddr + i*PAGE_SIZE, PAGE_SIZE);
  3805. kunmap_atomic(dst);
  3806. drm_clflush_pages(&page, 1);
  3807. set_page_dirty(page);
  3808. mark_page_accessed(page);
  3809. page_cache_release(page);
  3810. }
  3811. }
  3812. i915_gem_chipset_flush(dev);
  3813. obj->phys_obj->cur_obj = NULL;
  3814. obj->phys_obj = NULL;
  3815. }
  3816. int
  3817. i915_gem_attach_phys_object(struct drm_device *dev,
  3818. struct drm_i915_gem_object *obj,
  3819. int id,
  3820. int align)
  3821. {
  3822. struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
  3823. drm_i915_private_t *dev_priv = dev->dev_private;
  3824. int ret = 0;
  3825. int page_count;
  3826. int i;
  3827. if (id > I915_MAX_PHYS_OBJECT)
  3828. return -EINVAL;
  3829. if (obj->phys_obj) {
  3830. if (obj->phys_obj->id == id)
  3831. return 0;
  3832. i915_gem_detach_phys_object(dev, obj);
  3833. }
  3834. /* create a new object */
  3835. if (!dev_priv->mm.phys_objs[id - 1]) {
  3836. ret = i915_gem_init_phys_object(dev, id,
  3837. obj->base.size, align);
  3838. if (ret) {
  3839. DRM_ERROR("failed to init phys object %d size: %zu\n",
  3840. id, obj->base.size);
  3841. return ret;
  3842. }
  3843. }
  3844. /* bind to the object */
  3845. obj->phys_obj = dev_priv->mm.phys_objs[id - 1];
  3846. obj->phys_obj->cur_obj = obj;
  3847. page_count = obj->base.size / PAGE_SIZE;
  3848. for (i = 0; i < page_count; i++) {
  3849. struct page *page;
  3850. char *dst, *src;
  3851. page = shmem_read_mapping_page(mapping, i);
  3852. if (IS_ERR(page))
  3853. return PTR_ERR(page);
  3854. src = kmap_atomic(page);
  3855. dst = obj->phys_obj->handle->vaddr + (i * PAGE_SIZE);
  3856. memcpy(dst, src, PAGE_SIZE);
  3857. kunmap_atomic(src);
  3858. mark_page_accessed(page);
  3859. page_cache_release(page);
  3860. }
  3861. return 0;
  3862. }
  3863. static int
  3864. i915_gem_phys_pwrite(struct drm_device *dev,
  3865. struct drm_i915_gem_object *obj,
  3866. struct drm_i915_gem_pwrite *args,
  3867. struct drm_file *file_priv)
  3868. {
  3869. void *vaddr = obj->phys_obj->handle->vaddr + args->offset;
  3870. char __user *user_data = to_user_ptr(args->data_ptr);
  3871. if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
  3872. unsigned long unwritten;
  3873. /* The physical object once assigned is fixed for the lifetime
  3874. * of the obj, so we can safely drop the lock and continue
  3875. * to access vaddr.
  3876. */
  3877. mutex_unlock(&dev->struct_mutex);
  3878. unwritten = copy_from_user(vaddr, user_data, args->size);
  3879. mutex_lock(&dev->struct_mutex);
  3880. if (unwritten)
  3881. return -EFAULT;
  3882. }
  3883. i915_gem_chipset_flush(dev);
  3884. return 0;
  3885. }
  3886. void i915_gem_release(struct drm_device *dev, struct drm_file *file)
  3887. {
  3888. struct drm_i915_file_private *file_priv = file->driver_priv;
  3889. /* Clean up our request list when the client is going away, so that
  3890. * later retire_requests won't dereference our soon-to-be-gone
  3891. * file_priv.
  3892. */
  3893. spin_lock(&file_priv->mm.lock);
  3894. while (!list_empty(&file_priv->mm.request_list)) {
  3895. struct drm_i915_gem_request *request;
  3896. request = list_first_entry(&file_priv->mm.request_list,
  3897. struct drm_i915_gem_request,
  3898. client_list);
  3899. list_del(&request->client_list);
  3900. request->file_priv = NULL;
  3901. }
  3902. spin_unlock(&file_priv->mm.lock);
  3903. }
  3904. static bool mutex_is_locked_by(struct mutex *mutex, struct task_struct *task)
  3905. {
  3906. if (!mutex_is_locked(mutex))
  3907. return false;
  3908. #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_MUTEXES)
  3909. return mutex->owner == task;
  3910. #else
  3911. /* Since UP may be pre-empted, we cannot assume that we own the lock */
  3912. return false;
  3913. #endif
  3914. }
  3915. static int
  3916. i915_gem_inactive_shrink(struct shrinker *shrinker, struct shrink_control *sc)
  3917. {
  3918. struct drm_i915_private *dev_priv =
  3919. container_of(shrinker,
  3920. struct drm_i915_private,
  3921. mm.inactive_shrinker);
  3922. struct drm_device *dev = dev_priv->dev;
  3923. struct drm_i915_gem_object *obj;
  3924. int nr_to_scan = sc->nr_to_scan;
  3925. bool unlock = true;
  3926. int cnt;
  3927. if (!mutex_trylock(&dev->struct_mutex)) {
  3928. if (!mutex_is_locked_by(&dev->struct_mutex, current))
  3929. return 0;
  3930. if (dev_priv->mm.shrinker_no_lock_stealing)
  3931. return 0;
  3932. unlock = false;
  3933. }
  3934. if (nr_to_scan) {
  3935. nr_to_scan -= i915_gem_purge(dev_priv, nr_to_scan);
  3936. if (nr_to_scan > 0)
  3937. nr_to_scan -= __i915_gem_shrink(dev_priv, nr_to_scan,
  3938. false);
  3939. if (nr_to_scan > 0)
  3940. i915_gem_shrink_all(dev_priv);
  3941. }
  3942. cnt = 0;
  3943. list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list)
  3944. if (obj->pages_pin_count == 0)
  3945. cnt += obj->base.size >> PAGE_SHIFT;
  3946. list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
  3947. if (obj->active)
  3948. continue;
  3949. if (obj->pin_count == 0 && obj->pages_pin_count == 0)
  3950. cnt += obj->base.size >> PAGE_SHIFT;
  3951. }
  3952. if (unlock)
  3953. mutex_unlock(&dev->struct_mutex);
  3954. return cnt;
  3955. }
  3956. /* All the new VM stuff */
  3957. unsigned long i915_gem_obj_offset(struct drm_i915_gem_object *o,
  3958. struct i915_address_space *vm)
  3959. {
  3960. struct drm_i915_private *dev_priv = o->base.dev->dev_private;
  3961. struct i915_vma *vma;
  3962. if (vm == &dev_priv->mm.aliasing_ppgtt->base)
  3963. vm = &dev_priv->gtt.base;
  3964. BUG_ON(list_empty(&o->vma_list));
  3965. list_for_each_entry(vma, &o->vma_list, vma_link) {
  3966. if (vma->vm == vm)
  3967. return vma->node.start;
  3968. }
  3969. return -1;
  3970. }
  3971. bool i915_gem_obj_bound(struct drm_i915_gem_object *o,
  3972. struct i915_address_space *vm)
  3973. {
  3974. struct i915_vma *vma;
  3975. list_for_each_entry(vma, &o->vma_list, vma_link)
  3976. if (vma->vm == vm && drm_mm_node_allocated(&vma->node))
  3977. return true;
  3978. return false;
  3979. }
  3980. bool i915_gem_obj_bound_any(struct drm_i915_gem_object *o)
  3981. {
  3982. struct drm_i915_private *dev_priv = o->base.dev->dev_private;
  3983. struct i915_address_space *vm;
  3984. list_for_each_entry(vm, &dev_priv->vm_list, global_link)
  3985. if (i915_gem_obj_bound(o, vm))
  3986. return true;
  3987. return false;
  3988. }
  3989. unsigned long i915_gem_obj_size(struct drm_i915_gem_object *o,
  3990. struct i915_address_space *vm)
  3991. {
  3992. struct drm_i915_private *dev_priv = o->base.dev->dev_private;
  3993. struct i915_vma *vma;
  3994. if (vm == &dev_priv->mm.aliasing_ppgtt->base)
  3995. vm = &dev_priv->gtt.base;
  3996. BUG_ON(list_empty(&o->vma_list));
  3997. list_for_each_entry(vma, &o->vma_list, vma_link)
  3998. if (vma->vm == vm)
  3999. return vma->node.size;
  4000. return 0;
  4001. }
  4002. struct i915_vma *i915_gem_obj_to_vma(struct drm_i915_gem_object *obj,
  4003. struct i915_address_space *vm)
  4004. {
  4005. struct i915_vma *vma;
  4006. list_for_each_entry(vma, &obj->vma_list, vma_link)
  4007. if (vma->vm == vm)
  4008. return vma;
  4009. return NULL;
  4010. }