i915_gem.c 110 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419
  1. /*
  2. * Copyright © 2008 Intel Corporation
  3. *
  4. * Permission is hereby granted, free of charge, to any person obtaining a
  5. * copy of this software and associated documentation files (the "Software"),
  6. * to deal in the Software without restriction, including without limitation
  7. * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8. * and/or sell copies of the Software, and to permit persons to whom the
  9. * Software is furnished to do so, subject to the following conditions:
  10. *
  11. * The above copyright notice and this permission notice (including the next
  12. * paragraph) shall be included in all copies or substantial portions of the
  13. * Software.
  14. *
  15. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17. * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  18. * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19. * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20. * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21. * IN THE SOFTWARE.
  22. *
  23. * Authors:
  24. * Eric Anholt <eric@anholt.net>
  25. *
  26. */
  27. #include <drm/drmP.h>
  28. #include <drm/i915_drm.h>
  29. #include "i915_drv.h"
  30. #include "i915_trace.h"
  31. #include "intel_drv.h"
  32. #include <linux/shmem_fs.h>
  33. #include <linux/slab.h>
  34. #include <linux/swap.h>
  35. #include <linux/pci.h>
  36. #include <linux/dma-buf.h>
  37. static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
  38. static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
  39. static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
  40. unsigned alignment,
  41. bool map_and_fenceable,
  42. bool nonblocking);
  43. static int i915_gem_phys_pwrite(struct drm_device *dev,
  44. struct drm_i915_gem_object *obj,
  45. struct drm_i915_gem_pwrite *args,
  46. struct drm_file *file);
  47. static void i915_gem_write_fence(struct drm_device *dev, int reg,
  48. struct drm_i915_gem_object *obj);
  49. static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
  50. struct drm_i915_fence_reg *fence,
  51. bool enable);
  52. static int i915_gem_inactive_shrink(struct shrinker *shrinker,
  53. struct shrink_control *sc);
  54. static long i915_gem_purge(struct drm_i915_private *dev_priv, long target);
  55. static void i915_gem_shrink_all(struct drm_i915_private *dev_priv);
  56. static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
  57. static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
  58. {
  59. if (obj->tiling_mode)
  60. i915_gem_release_mmap(obj);
  61. /* As we do not have an associated fence register, we will force
  62. * a tiling change if we ever need to acquire one.
  63. */
  64. obj->fence_dirty = false;
  65. obj->fence_reg = I915_FENCE_REG_NONE;
  66. }
  67. /* some bookkeeping */
  68. static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
  69. size_t size)
  70. {
  71. dev_priv->mm.object_count++;
  72. dev_priv->mm.object_memory += size;
  73. }
  74. static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
  75. size_t size)
  76. {
  77. dev_priv->mm.object_count--;
  78. dev_priv->mm.object_memory -= size;
  79. }
  80. static int
  81. i915_gem_wait_for_error(struct drm_device *dev)
  82. {
  83. struct drm_i915_private *dev_priv = dev->dev_private;
  84. struct completion *x = &dev_priv->error_completion;
  85. unsigned long flags;
  86. int ret;
  87. if (!atomic_read(&dev_priv->mm.wedged))
  88. return 0;
  89. /*
  90. * Only wait 10 seconds for the gpu reset to complete to avoid hanging
  91. * userspace. If it takes that long something really bad is going on and
  92. * we should simply try to bail out and fail as gracefully as possible.
  93. */
  94. ret = wait_for_completion_interruptible_timeout(x, 10*HZ);
  95. if (ret == 0) {
  96. DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
  97. return -EIO;
  98. } else if (ret < 0) {
  99. return ret;
  100. }
  101. if (atomic_read(&dev_priv->mm.wedged)) {
  102. /* GPU is hung, bump the completion count to account for
  103. * the token we just consumed so that we never hit zero and
  104. * end up waiting upon a subsequent completion event that
  105. * will never happen.
  106. */
  107. spin_lock_irqsave(&x->wait.lock, flags);
  108. x->done++;
  109. spin_unlock_irqrestore(&x->wait.lock, flags);
  110. }
  111. return 0;
  112. }
  113. int i915_mutex_lock_interruptible(struct drm_device *dev)
  114. {
  115. int ret;
  116. ret = i915_gem_wait_for_error(dev);
  117. if (ret)
  118. return ret;
  119. ret = mutex_lock_interruptible(&dev->struct_mutex);
  120. if (ret)
  121. return ret;
  122. WARN_ON(i915_verify_lists(dev));
  123. return 0;
  124. }
  125. static inline bool
  126. i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
  127. {
  128. return obj->gtt_space && !obj->active;
  129. }
  130. int
  131. i915_gem_init_ioctl(struct drm_device *dev, void *data,
  132. struct drm_file *file)
  133. {
  134. struct drm_i915_gem_init *args = data;
  135. if (drm_core_check_feature(dev, DRIVER_MODESET))
  136. return -ENODEV;
  137. if (args->gtt_start >= args->gtt_end ||
  138. (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
  139. return -EINVAL;
  140. /* GEM with user mode setting was never supported on ilk and later. */
  141. if (INTEL_INFO(dev)->gen >= 5)
  142. return -ENODEV;
  143. mutex_lock(&dev->struct_mutex);
  144. i915_gem_init_global_gtt(dev, args->gtt_start,
  145. args->gtt_end, args->gtt_end);
  146. mutex_unlock(&dev->struct_mutex);
  147. return 0;
  148. }
  149. int
  150. i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
  151. struct drm_file *file)
  152. {
  153. struct drm_i915_private *dev_priv = dev->dev_private;
  154. struct drm_i915_gem_get_aperture *args = data;
  155. struct drm_i915_gem_object *obj;
  156. size_t pinned;
  157. pinned = 0;
  158. mutex_lock(&dev->struct_mutex);
  159. list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list)
  160. if (obj->pin_count)
  161. pinned += obj->gtt_space->size;
  162. mutex_unlock(&dev->struct_mutex);
  163. args->aper_size = dev_priv->mm.gtt_total;
  164. args->aper_available_size = args->aper_size - pinned;
  165. return 0;
  166. }
  167. void *i915_gem_object_alloc(struct drm_device *dev)
  168. {
  169. struct drm_i915_private *dev_priv = dev->dev_private;
  170. return kmem_cache_alloc(dev_priv->slab, GFP_KERNEL | __GFP_ZERO);
  171. }
  172. void i915_gem_object_free(struct drm_i915_gem_object *obj)
  173. {
  174. struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  175. kmem_cache_free(dev_priv->slab, obj);
  176. }
  177. static int
  178. i915_gem_create(struct drm_file *file,
  179. struct drm_device *dev,
  180. uint64_t size,
  181. uint32_t *handle_p)
  182. {
  183. struct drm_i915_gem_object *obj;
  184. int ret;
  185. u32 handle;
  186. size = roundup(size, PAGE_SIZE);
  187. if (size == 0)
  188. return -EINVAL;
  189. /* Allocate the new object */
  190. obj = i915_gem_alloc_object(dev, size);
  191. if (obj == NULL)
  192. return -ENOMEM;
  193. ret = drm_gem_handle_create(file, &obj->base, &handle);
  194. if (ret) {
  195. drm_gem_object_release(&obj->base);
  196. i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
  197. i915_gem_object_free(obj);
  198. return ret;
  199. }
  200. /* drop reference from allocate - handle holds it now */
  201. drm_gem_object_unreference(&obj->base);
  202. trace_i915_gem_object_create(obj);
  203. *handle_p = handle;
  204. return 0;
  205. }
  206. int
  207. i915_gem_dumb_create(struct drm_file *file,
  208. struct drm_device *dev,
  209. struct drm_mode_create_dumb *args)
  210. {
  211. /* have to work out size/pitch and return them */
  212. args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
  213. args->size = args->pitch * args->height;
  214. return i915_gem_create(file, dev,
  215. args->size, &args->handle);
  216. }
  217. int i915_gem_dumb_destroy(struct drm_file *file,
  218. struct drm_device *dev,
  219. uint32_t handle)
  220. {
  221. return drm_gem_handle_delete(file, handle);
  222. }
  223. /**
  224. * Creates a new mm object and returns a handle to it.
  225. */
  226. int
  227. i915_gem_create_ioctl(struct drm_device *dev, void *data,
  228. struct drm_file *file)
  229. {
  230. struct drm_i915_gem_create *args = data;
  231. return i915_gem_create(file, dev,
  232. args->size, &args->handle);
  233. }
  234. static inline int
  235. __copy_to_user_swizzled(char __user *cpu_vaddr,
  236. const char *gpu_vaddr, int gpu_offset,
  237. int length)
  238. {
  239. int ret, cpu_offset = 0;
  240. while (length > 0) {
  241. int cacheline_end = ALIGN(gpu_offset + 1, 64);
  242. int this_length = min(cacheline_end - gpu_offset, length);
  243. int swizzled_gpu_offset = gpu_offset ^ 64;
  244. ret = __copy_to_user(cpu_vaddr + cpu_offset,
  245. gpu_vaddr + swizzled_gpu_offset,
  246. this_length);
  247. if (ret)
  248. return ret + length;
  249. cpu_offset += this_length;
  250. gpu_offset += this_length;
  251. length -= this_length;
  252. }
  253. return 0;
  254. }
  255. static inline int
  256. __copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
  257. const char __user *cpu_vaddr,
  258. int length)
  259. {
  260. int ret, cpu_offset = 0;
  261. while (length > 0) {
  262. int cacheline_end = ALIGN(gpu_offset + 1, 64);
  263. int this_length = min(cacheline_end - gpu_offset, length);
  264. int swizzled_gpu_offset = gpu_offset ^ 64;
  265. ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
  266. cpu_vaddr + cpu_offset,
  267. this_length);
  268. if (ret)
  269. return ret + length;
  270. cpu_offset += this_length;
  271. gpu_offset += this_length;
  272. length -= this_length;
  273. }
  274. return 0;
  275. }
  276. /* Per-page copy function for the shmem pread fastpath.
  277. * Flushes invalid cachelines before reading the target if
  278. * needs_clflush is set. */
  279. static int
  280. shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
  281. char __user *user_data,
  282. bool page_do_bit17_swizzling, bool needs_clflush)
  283. {
  284. char *vaddr;
  285. int ret;
  286. if (unlikely(page_do_bit17_swizzling))
  287. return -EINVAL;
  288. vaddr = kmap_atomic(page);
  289. if (needs_clflush)
  290. drm_clflush_virt_range(vaddr + shmem_page_offset,
  291. page_length);
  292. ret = __copy_to_user_inatomic(user_data,
  293. vaddr + shmem_page_offset,
  294. page_length);
  295. kunmap_atomic(vaddr);
  296. return ret ? -EFAULT : 0;
  297. }
  298. static void
  299. shmem_clflush_swizzled_range(char *addr, unsigned long length,
  300. bool swizzled)
  301. {
  302. if (unlikely(swizzled)) {
  303. unsigned long start = (unsigned long) addr;
  304. unsigned long end = (unsigned long) addr + length;
  305. /* For swizzling simply ensure that we always flush both
  306. * channels. Lame, but simple and it works. Swizzled
  307. * pwrite/pread is far from a hotpath - current userspace
  308. * doesn't use it at all. */
  309. start = round_down(start, 128);
  310. end = round_up(end, 128);
  311. drm_clflush_virt_range((void *)start, end - start);
  312. } else {
  313. drm_clflush_virt_range(addr, length);
  314. }
  315. }
  316. /* Only difference to the fast-path function is that this can handle bit17
  317. * and uses non-atomic copy and kmap functions. */
  318. static int
  319. shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
  320. char __user *user_data,
  321. bool page_do_bit17_swizzling, bool needs_clflush)
  322. {
  323. char *vaddr;
  324. int ret;
  325. vaddr = kmap(page);
  326. if (needs_clflush)
  327. shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
  328. page_length,
  329. page_do_bit17_swizzling);
  330. if (page_do_bit17_swizzling)
  331. ret = __copy_to_user_swizzled(user_data,
  332. vaddr, shmem_page_offset,
  333. page_length);
  334. else
  335. ret = __copy_to_user(user_data,
  336. vaddr + shmem_page_offset,
  337. page_length);
  338. kunmap(page);
  339. return ret ? - EFAULT : 0;
  340. }
  341. static int
  342. i915_gem_shmem_pread(struct drm_device *dev,
  343. struct drm_i915_gem_object *obj,
  344. struct drm_i915_gem_pread *args,
  345. struct drm_file *file)
  346. {
  347. char __user *user_data;
  348. ssize_t remain;
  349. loff_t offset;
  350. int shmem_page_offset, page_length, ret = 0;
  351. int obj_do_bit17_swizzling, page_do_bit17_swizzling;
  352. int prefaulted = 0;
  353. int needs_clflush = 0;
  354. struct scatterlist *sg;
  355. int i;
  356. user_data = (char __user *) (uintptr_t) args->data_ptr;
  357. remain = args->size;
  358. obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
  359. if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
  360. /* If we're not in the cpu read domain, set ourself into the gtt
  361. * read domain and manually flush cachelines (if required). This
  362. * optimizes for the case when the gpu will dirty the data
  363. * anyway again before the next pread happens. */
  364. if (obj->cache_level == I915_CACHE_NONE)
  365. needs_clflush = 1;
  366. if (obj->gtt_space) {
  367. ret = i915_gem_object_set_to_gtt_domain(obj, false);
  368. if (ret)
  369. return ret;
  370. }
  371. }
  372. ret = i915_gem_object_get_pages(obj);
  373. if (ret)
  374. return ret;
  375. i915_gem_object_pin_pages(obj);
  376. offset = args->offset;
  377. for_each_sg(obj->pages->sgl, sg, obj->pages->nents, i) {
  378. struct page *page;
  379. if (i < offset >> PAGE_SHIFT)
  380. continue;
  381. if (remain <= 0)
  382. break;
  383. /* Operation in this page
  384. *
  385. * shmem_page_offset = offset within page in shmem file
  386. * page_length = bytes to copy for this page
  387. */
  388. shmem_page_offset = offset_in_page(offset);
  389. page_length = remain;
  390. if ((shmem_page_offset + page_length) > PAGE_SIZE)
  391. page_length = PAGE_SIZE - shmem_page_offset;
  392. page = sg_page(sg);
  393. page_do_bit17_swizzling = obj_do_bit17_swizzling &&
  394. (page_to_phys(page) & (1 << 17)) != 0;
  395. ret = shmem_pread_fast(page, shmem_page_offset, page_length,
  396. user_data, page_do_bit17_swizzling,
  397. needs_clflush);
  398. if (ret == 0)
  399. goto next_page;
  400. mutex_unlock(&dev->struct_mutex);
  401. if (!prefaulted) {
  402. ret = fault_in_multipages_writeable(user_data, remain);
  403. /* Userspace is tricking us, but we've already clobbered
  404. * its pages with the prefault and promised to write the
  405. * data up to the first fault. Hence ignore any errors
  406. * and just continue. */
  407. (void)ret;
  408. prefaulted = 1;
  409. }
  410. ret = shmem_pread_slow(page, shmem_page_offset, page_length,
  411. user_data, page_do_bit17_swizzling,
  412. needs_clflush);
  413. mutex_lock(&dev->struct_mutex);
  414. next_page:
  415. mark_page_accessed(page);
  416. if (ret)
  417. goto out;
  418. remain -= page_length;
  419. user_data += page_length;
  420. offset += page_length;
  421. }
  422. out:
  423. i915_gem_object_unpin_pages(obj);
  424. return ret;
  425. }
  426. /**
  427. * Reads data from the object referenced by handle.
  428. *
  429. * On error, the contents of *data are undefined.
  430. */
  431. int
  432. i915_gem_pread_ioctl(struct drm_device *dev, void *data,
  433. struct drm_file *file)
  434. {
  435. struct drm_i915_gem_pread *args = data;
  436. struct drm_i915_gem_object *obj;
  437. int ret = 0;
  438. if (args->size == 0)
  439. return 0;
  440. if (!access_ok(VERIFY_WRITE,
  441. (char __user *)(uintptr_t)args->data_ptr,
  442. args->size))
  443. return -EFAULT;
  444. ret = i915_mutex_lock_interruptible(dev);
  445. if (ret)
  446. return ret;
  447. obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  448. if (&obj->base == NULL) {
  449. ret = -ENOENT;
  450. goto unlock;
  451. }
  452. /* Bounds check source. */
  453. if (args->offset > obj->base.size ||
  454. args->size > obj->base.size - args->offset) {
  455. ret = -EINVAL;
  456. goto out;
  457. }
  458. /* prime objects have no backing filp to GEM pread/pwrite
  459. * pages from.
  460. */
  461. if (!obj->base.filp) {
  462. ret = -EINVAL;
  463. goto out;
  464. }
  465. trace_i915_gem_object_pread(obj, args->offset, args->size);
  466. ret = i915_gem_shmem_pread(dev, obj, args, file);
  467. out:
  468. drm_gem_object_unreference(&obj->base);
  469. unlock:
  470. mutex_unlock(&dev->struct_mutex);
  471. return ret;
  472. }
  473. /* This is the fast write path which cannot handle
  474. * page faults in the source data
  475. */
  476. static inline int
  477. fast_user_write(struct io_mapping *mapping,
  478. loff_t page_base, int page_offset,
  479. char __user *user_data,
  480. int length)
  481. {
  482. void __iomem *vaddr_atomic;
  483. void *vaddr;
  484. unsigned long unwritten;
  485. vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
  486. /* We can use the cpu mem copy function because this is X86. */
  487. vaddr = (void __force*)vaddr_atomic + page_offset;
  488. unwritten = __copy_from_user_inatomic_nocache(vaddr,
  489. user_data, length);
  490. io_mapping_unmap_atomic(vaddr_atomic);
  491. return unwritten;
  492. }
  493. /**
  494. * This is the fast pwrite path, where we copy the data directly from the
  495. * user into the GTT, uncached.
  496. */
  497. static int
  498. i915_gem_gtt_pwrite_fast(struct drm_device *dev,
  499. struct drm_i915_gem_object *obj,
  500. struct drm_i915_gem_pwrite *args,
  501. struct drm_file *file)
  502. {
  503. drm_i915_private_t *dev_priv = dev->dev_private;
  504. ssize_t remain;
  505. loff_t offset, page_base;
  506. char __user *user_data;
  507. int page_offset, page_length, ret;
  508. ret = i915_gem_object_pin(obj, 0, true, true);
  509. if (ret)
  510. goto out;
  511. ret = i915_gem_object_set_to_gtt_domain(obj, true);
  512. if (ret)
  513. goto out_unpin;
  514. ret = i915_gem_object_put_fence(obj);
  515. if (ret)
  516. goto out_unpin;
  517. user_data = (char __user *) (uintptr_t) args->data_ptr;
  518. remain = args->size;
  519. offset = obj->gtt_offset + args->offset;
  520. while (remain > 0) {
  521. /* Operation in this page
  522. *
  523. * page_base = page offset within aperture
  524. * page_offset = offset within page
  525. * page_length = bytes to copy for this page
  526. */
  527. page_base = offset & PAGE_MASK;
  528. page_offset = offset_in_page(offset);
  529. page_length = remain;
  530. if ((page_offset + remain) > PAGE_SIZE)
  531. page_length = PAGE_SIZE - page_offset;
  532. /* If we get a fault while copying data, then (presumably) our
  533. * source page isn't available. Return the error and we'll
  534. * retry in the slow path.
  535. */
  536. if (fast_user_write(dev_priv->mm.gtt_mapping, page_base,
  537. page_offset, user_data, page_length)) {
  538. ret = -EFAULT;
  539. goto out_unpin;
  540. }
  541. remain -= page_length;
  542. user_data += page_length;
  543. offset += page_length;
  544. }
  545. out_unpin:
  546. i915_gem_object_unpin(obj);
  547. out:
  548. return ret;
  549. }
  550. /* Per-page copy function for the shmem pwrite fastpath.
  551. * Flushes invalid cachelines before writing to the target if
  552. * needs_clflush_before is set and flushes out any written cachelines after
  553. * writing if needs_clflush is set. */
  554. static int
  555. shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
  556. char __user *user_data,
  557. bool page_do_bit17_swizzling,
  558. bool needs_clflush_before,
  559. bool needs_clflush_after)
  560. {
  561. char *vaddr;
  562. int ret;
  563. if (unlikely(page_do_bit17_swizzling))
  564. return -EINVAL;
  565. vaddr = kmap_atomic(page);
  566. if (needs_clflush_before)
  567. drm_clflush_virt_range(vaddr + shmem_page_offset,
  568. page_length);
  569. ret = __copy_from_user_inatomic_nocache(vaddr + shmem_page_offset,
  570. user_data,
  571. page_length);
  572. if (needs_clflush_after)
  573. drm_clflush_virt_range(vaddr + shmem_page_offset,
  574. page_length);
  575. kunmap_atomic(vaddr);
  576. return ret ? -EFAULT : 0;
  577. }
  578. /* Only difference to the fast-path function is that this can handle bit17
  579. * and uses non-atomic copy and kmap functions. */
  580. static int
  581. shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
  582. char __user *user_data,
  583. bool page_do_bit17_swizzling,
  584. bool needs_clflush_before,
  585. bool needs_clflush_after)
  586. {
  587. char *vaddr;
  588. int ret;
  589. vaddr = kmap(page);
  590. if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
  591. shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
  592. page_length,
  593. page_do_bit17_swizzling);
  594. if (page_do_bit17_swizzling)
  595. ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
  596. user_data,
  597. page_length);
  598. else
  599. ret = __copy_from_user(vaddr + shmem_page_offset,
  600. user_data,
  601. page_length);
  602. if (needs_clflush_after)
  603. shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
  604. page_length,
  605. page_do_bit17_swizzling);
  606. kunmap(page);
  607. return ret ? -EFAULT : 0;
  608. }
  609. static int
  610. i915_gem_shmem_pwrite(struct drm_device *dev,
  611. struct drm_i915_gem_object *obj,
  612. struct drm_i915_gem_pwrite *args,
  613. struct drm_file *file)
  614. {
  615. ssize_t remain;
  616. loff_t offset;
  617. char __user *user_data;
  618. int shmem_page_offset, page_length, ret = 0;
  619. int obj_do_bit17_swizzling, page_do_bit17_swizzling;
  620. int hit_slowpath = 0;
  621. int needs_clflush_after = 0;
  622. int needs_clflush_before = 0;
  623. int i;
  624. struct scatterlist *sg;
  625. user_data = (char __user *) (uintptr_t) args->data_ptr;
  626. remain = args->size;
  627. obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
  628. if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
  629. /* If we're not in the cpu write domain, set ourself into the gtt
  630. * write domain and manually flush cachelines (if required). This
  631. * optimizes for the case when the gpu will use the data
  632. * right away and we therefore have to clflush anyway. */
  633. if (obj->cache_level == I915_CACHE_NONE)
  634. needs_clflush_after = 1;
  635. if (obj->gtt_space) {
  636. ret = i915_gem_object_set_to_gtt_domain(obj, true);
  637. if (ret)
  638. return ret;
  639. }
  640. }
  641. /* Same trick applies for invalidate partially written cachelines before
  642. * writing. */
  643. if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)
  644. && obj->cache_level == I915_CACHE_NONE)
  645. needs_clflush_before = 1;
  646. ret = i915_gem_object_get_pages(obj);
  647. if (ret)
  648. return ret;
  649. i915_gem_object_pin_pages(obj);
  650. offset = args->offset;
  651. obj->dirty = 1;
  652. for_each_sg(obj->pages->sgl, sg, obj->pages->nents, i) {
  653. struct page *page;
  654. int partial_cacheline_write;
  655. if (i < offset >> PAGE_SHIFT)
  656. continue;
  657. if (remain <= 0)
  658. break;
  659. /* Operation in this page
  660. *
  661. * shmem_page_offset = offset within page in shmem file
  662. * page_length = bytes to copy for this page
  663. */
  664. shmem_page_offset = offset_in_page(offset);
  665. page_length = remain;
  666. if ((shmem_page_offset + page_length) > PAGE_SIZE)
  667. page_length = PAGE_SIZE - shmem_page_offset;
  668. /* If we don't overwrite a cacheline completely we need to be
  669. * careful to have up-to-date data by first clflushing. Don't
  670. * overcomplicate things and flush the entire patch. */
  671. partial_cacheline_write = needs_clflush_before &&
  672. ((shmem_page_offset | page_length)
  673. & (boot_cpu_data.x86_clflush_size - 1));
  674. page = sg_page(sg);
  675. page_do_bit17_swizzling = obj_do_bit17_swizzling &&
  676. (page_to_phys(page) & (1 << 17)) != 0;
  677. ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
  678. user_data, page_do_bit17_swizzling,
  679. partial_cacheline_write,
  680. needs_clflush_after);
  681. if (ret == 0)
  682. goto next_page;
  683. hit_slowpath = 1;
  684. mutex_unlock(&dev->struct_mutex);
  685. ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
  686. user_data, page_do_bit17_swizzling,
  687. partial_cacheline_write,
  688. needs_clflush_after);
  689. mutex_lock(&dev->struct_mutex);
  690. next_page:
  691. set_page_dirty(page);
  692. mark_page_accessed(page);
  693. if (ret)
  694. goto out;
  695. remain -= page_length;
  696. user_data += page_length;
  697. offset += page_length;
  698. }
  699. out:
  700. i915_gem_object_unpin_pages(obj);
  701. if (hit_slowpath) {
  702. /*
  703. * Fixup: Flush cpu caches in case we didn't flush the dirty
  704. * cachelines in-line while writing and the object moved
  705. * out of the cpu write domain while we've dropped the lock.
  706. */
  707. if (!needs_clflush_after &&
  708. obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
  709. i915_gem_clflush_object(obj);
  710. i915_gem_chipset_flush(dev);
  711. }
  712. }
  713. if (needs_clflush_after)
  714. i915_gem_chipset_flush(dev);
  715. return ret;
  716. }
  717. /**
  718. * Writes data to the object referenced by handle.
  719. *
  720. * On error, the contents of the buffer that were to be modified are undefined.
  721. */
  722. int
  723. i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
  724. struct drm_file *file)
  725. {
  726. struct drm_i915_gem_pwrite *args = data;
  727. struct drm_i915_gem_object *obj;
  728. int ret;
  729. if (args->size == 0)
  730. return 0;
  731. if (!access_ok(VERIFY_READ,
  732. (char __user *)(uintptr_t)args->data_ptr,
  733. args->size))
  734. return -EFAULT;
  735. ret = fault_in_multipages_readable((char __user *)(uintptr_t)args->data_ptr,
  736. args->size);
  737. if (ret)
  738. return -EFAULT;
  739. ret = i915_mutex_lock_interruptible(dev);
  740. if (ret)
  741. return ret;
  742. obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  743. if (&obj->base == NULL) {
  744. ret = -ENOENT;
  745. goto unlock;
  746. }
  747. /* Bounds check destination. */
  748. if (args->offset > obj->base.size ||
  749. args->size > obj->base.size - args->offset) {
  750. ret = -EINVAL;
  751. goto out;
  752. }
  753. /* prime objects have no backing filp to GEM pread/pwrite
  754. * pages from.
  755. */
  756. if (!obj->base.filp) {
  757. ret = -EINVAL;
  758. goto out;
  759. }
  760. trace_i915_gem_object_pwrite(obj, args->offset, args->size);
  761. ret = -EFAULT;
  762. /* We can only do the GTT pwrite on untiled buffers, as otherwise
  763. * it would end up going through the fenced access, and we'll get
  764. * different detiling behavior between reading and writing.
  765. * pread/pwrite currently are reading and writing from the CPU
  766. * perspective, requiring manual detiling by the client.
  767. */
  768. if (obj->phys_obj) {
  769. ret = i915_gem_phys_pwrite(dev, obj, args, file);
  770. goto out;
  771. }
  772. if (obj->cache_level == I915_CACHE_NONE &&
  773. obj->tiling_mode == I915_TILING_NONE &&
  774. obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
  775. ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
  776. /* Note that the gtt paths might fail with non-page-backed user
  777. * pointers (e.g. gtt mappings when moving data between
  778. * textures). Fallback to the shmem path in that case. */
  779. }
  780. if (ret == -EFAULT || ret == -ENOSPC)
  781. ret = i915_gem_shmem_pwrite(dev, obj, args, file);
  782. out:
  783. drm_gem_object_unreference(&obj->base);
  784. unlock:
  785. mutex_unlock(&dev->struct_mutex);
  786. return ret;
  787. }
  788. int
  789. i915_gem_check_wedge(struct drm_i915_private *dev_priv,
  790. bool interruptible)
  791. {
  792. if (atomic_read(&dev_priv->mm.wedged)) {
  793. struct completion *x = &dev_priv->error_completion;
  794. bool recovery_complete;
  795. unsigned long flags;
  796. /* Give the error handler a chance to run. */
  797. spin_lock_irqsave(&x->wait.lock, flags);
  798. recovery_complete = x->done > 0;
  799. spin_unlock_irqrestore(&x->wait.lock, flags);
  800. /* Non-interruptible callers can't handle -EAGAIN, hence return
  801. * -EIO unconditionally for these. */
  802. if (!interruptible)
  803. return -EIO;
  804. /* Recovery complete, but still wedged means reset failure. */
  805. if (recovery_complete)
  806. return -EIO;
  807. return -EAGAIN;
  808. }
  809. return 0;
  810. }
  811. /*
  812. * Compare seqno against outstanding lazy request. Emit a request if they are
  813. * equal.
  814. */
  815. static int
  816. i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno)
  817. {
  818. int ret;
  819. BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex));
  820. ret = 0;
  821. if (seqno == ring->outstanding_lazy_request)
  822. ret = i915_add_request(ring, NULL, NULL);
  823. return ret;
  824. }
  825. /**
  826. * __wait_seqno - wait until execution of seqno has finished
  827. * @ring: the ring expected to report seqno
  828. * @seqno: duh!
  829. * @interruptible: do an interruptible wait (normally yes)
  830. * @timeout: in - how long to wait (NULL forever); out - how much time remaining
  831. *
  832. * Returns 0 if the seqno was found within the alloted time. Else returns the
  833. * errno with remaining time filled in timeout argument.
  834. */
  835. static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
  836. bool interruptible, struct timespec *timeout)
  837. {
  838. drm_i915_private_t *dev_priv = ring->dev->dev_private;
  839. struct timespec before, now, wait_time={1,0};
  840. unsigned long timeout_jiffies;
  841. long end;
  842. bool wait_forever = true;
  843. int ret;
  844. if (i915_seqno_passed(ring->get_seqno(ring, true), seqno))
  845. return 0;
  846. trace_i915_gem_request_wait_begin(ring, seqno);
  847. if (timeout != NULL) {
  848. wait_time = *timeout;
  849. wait_forever = false;
  850. }
  851. timeout_jiffies = timespec_to_jiffies(&wait_time);
  852. if (WARN_ON(!ring->irq_get(ring)))
  853. return -ENODEV;
  854. /* Record current time in case interrupted by signal, or wedged * */
  855. getrawmonotonic(&before);
  856. #define EXIT_COND \
  857. (i915_seqno_passed(ring->get_seqno(ring, false), seqno) || \
  858. atomic_read(&dev_priv->mm.wedged))
  859. do {
  860. if (interruptible)
  861. end = wait_event_interruptible_timeout(ring->irq_queue,
  862. EXIT_COND,
  863. timeout_jiffies);
  864. else
  865. end = wait_event_timeout(ring->irq_queue, EXIT_COND,
  866. timeout_jiffies);
  867. ret = i915_gem_check_wedge(dev_priv, interruptible);
  868. if (ret)
  869. end = ret;
  870. } while (end == 0 && wait_forever);
  871. getrawmonotonic(&now);
  872. ring->irq_put(ring);
  873. trace_i915_gem_request_wait_end(ring, seqno);
  874. #undef EXIT_COND
  875. if (timeout) {
  876. struct timespec sleep_time = timespec_sub(now, before);
  877. *timeout = timespec_sub(*timeout, sleep_time);
  878. }
  879. switch (end) {
  880. case -EIO:
  881. case -EAGAIN: /* Wedged */
  882. case -ERESTARTSYS: /* Signal */
  883. return (int)end;
  884. case 0: /* Timeout */
  885. if (timeout)
  886. set_normalized_timespec(timeout, 0, 0);
  887. return -ETIME;
  888. default: /* Completed */
  889. WARN_ON(end < 0); /* We're not aware of other errors */
  890. return 0;
  891. }
  892. }
  893. /**
  894. * Waits for a sequence number to be signaled, and cleans up the
  895. * request and object lists appropriately for that event.
  896. */
  897. int
  898. i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno)
  899. {
  900. struct drm_device *dev = ring->dev;
  901. struct drm_i915_private *dev_priv = dev->dev_private;
  902. bool interruptible = dev_priv->mm.interruptible;
  903. int ret;
  904. BUG_ON(!mutex_is_locked(&dev->struct_mutex));
  905. BUG_ON(seqno == 0);
  906. ret = i915_gem_check_wedge(dev_priv, interruptible);
  907. if (ret)
  908. return ret;
  909. ret = i915_gem_check_olr(ring, seqno);
  910. if (ret)
  911. return ret;
  912. return __wait_seqno(ring, seqno, interruptible, NULL);
  913. }
  914. /**
  915. * Ensures that all rendering to the object has completed and the object is
  916. * safe to unbind from the GTT or access from the CPU.
  917. */
  918. static __must_check int
  919. i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
  920. bool readonly)
  921. {
  922. struct intel_ring_buffer *ring = obj->ring;
  923. u32 seqno;
  924. int ret;
  925. seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
  926. if (seqno == 0)
  927. return 0;
  928. ret = i915_wait_seqno(ring, seqno);
  929. if (ret)
  930. return ret;
  931. i915_gem_retire_requests_ring(ring);
  932. /* Manually manage the write flush as we may have not yet
  933. * retired the buffer.
  934. */
  935. if (obj->last_write_seqno &&
  936. i915_seqno_passed(seqno, obj->last_write_seqno)) {
  937. obj->last_write_seqno = 0;
  938. obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
  939. }
  940. return 0;
  941. }
  942. /* A nonblocking variant of the above wait. This is a highly dangerous routine
  943. * as the object state may change during this call.
  944. */
  945. static __must_check int
  946. i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
  947. bool readonly)
  948. {
  949. struct drm_device *dev = obj->base.dev;
  950. struct drm_i915_private *dev_priv = dev->dev_private;
  951. struct intel_ring_buffer *ring = obj->ring;
  952. u32 seqno;
  953. int ret;
  954. BUG_ON(!mutex_is_locked(&dev->struct_mutex));
  955. BUG_ON(!dev_priv->mm.interruptible);
  956. seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
  957. if (seqno == 0)
  958. return 0;
  959. ret = i915_gem_check_wedge(dev_priv, true);
  960. if (ret)
  961. return ret;
  962. ret = i915_gem_check_olr(ring, seqno);
  963. if (ret)
  964. return ret;
  965. mutex_unlock(&dev->struct_mutex);
  966. ret = __wait_seqno(ring, seqno, true, NULL);
  967. mutex_lock(&dev->struct_mutex);
  968. i915_gem_retire_requests_ring(ring);
  969. /* Manually manage the write flush as we may have not yet
  970. * retired the buffer.
  971. */
  972. if (obj->last_write_seqno &&
  973. i915_seqno_passed(seqno, obj->last_write_seqno)) {
  974. obj->last_write_seqno = 0;
  975. obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
  976. }
  977. return ret;
  978. }
  979. /**
  980. * Called when user space prepares to use an object with the CPU, either
  981. * through the mmap ioctl's mapping or a GTT mapping.
  982. */
  983. int
  984. i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
  985. struct drm_file *file)
  986. {
  987. struct drm_i915_gem_set_domain *args = data;
  988. struct drm_i915_gem_object *obj;
  989. uint32_t read_domains = args->read_domains;
  990. uint32_t write_domain = args->write_domain;
  991. int ret;
  992. /* Only handle setting domains to types used by the CPU. */
  993. if (write_domain & I915_GEM_GPU_DOMAINS)
  994. return -EINVAL;
  995. if (read_domains & I915_GEM_GPU_DOMAINS)
  996. return -EINVAL;
  997. /* Having something in the write domain implies it's in the read
  998. * domain, and only that read domain. Enforce that in the request.
  999. */
  1000. if (write_domain != 0 && read_domains != write_domain)
  1001. return -EINVAL;
  1002. ret = i915_mutex_lock_interruptible(dev);
  1003. if (ret)
  1004. return ret;
  1005. obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  1006. if (&obj->base == NULL) {
  1007. ret = -ENOENT;
  1008. goto unlock;
  1009. }
  1010. /* Try to flush the object off the GPU without holding the lock.
  1011. * We will repeat the flush holding the lock in the normal manner
  1012. * to catch cases where we are gazumped.
  1013. */
  1014. ret = i915_gem_object_wait_rendering__nonblocking(obj, !write_domain);
  1015. if (ret)
  1016. goto unref;
  1017. if (read_domains & I915_GEM_DOMAIN_GTT) {
  1018. ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
  1019. /* Silently promote "you're not bound, there was nothing to do"
  1020. * to success, since the client was just asking us to
  1021. * make sure everything was done.
  1022. */
  1023. if (ret == -EINVAL)
  1024. ret = 0;
  1025. } else {
  1026. ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
  1027. }
  1028. unref:
  1029. drm_gem_object_unreference(&obj->base);
  1030. unlock:
  1031. mutex_unlock(&dev->struct_mutex);
  1032. return ret;
  1033. }
  1034. /**
  1035. * Called when user space has done writes to this buffer
  1036. */
  1037. int
  1038. i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
  1039. struct drm_file *file)
  1040. {
  1041. struct drm_i915_gem_sw_finish *args = data;
  1042. struct drm_i915_gem_object *obj;
  1043. int ret = 0;
  1044. ret = i915_mutex_lock_interruptible(dev);
  1045. if (ret)
  1046. return ret;
  1047. obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  1048. if (&obj->base == NULL) {
  1049. ret = -ENOENT;
  1050. goto unlock;
  1051. }
  1052. /* Pinned buffers may be scanout, so flush the cache */
  1053. if (obj->pin_count)
  1054. i915_gem_object_flush_cpu_write_domain(obj);
  1055. drm_gem_object_unreference(&obj->base);
  1056. unlock:
  1057. mutex_unlock(&dev->struct_mutex);
  1058. return ret;
  1059. }
  1060. /**
  1061. * Maps the contents of an object, returning the address it is mapped
  1062. * into.
  1063. *
  1064. * While the mapping holds a reference on the contents of the object, it doesn't
  1065. * imply a ref on the object itself.
  1066. */
  1067. int
  1068. i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
  1069. struct drm_file *file)
  1070. {
  1071. struct drm_i915_gem_mmap *args = data;
  1072. struct drm_gem_object *obj;
  1073. unsigned long addr;
  1074. obj = drm_gem_object_lookup(dev, file, args->handle);
  1075. if (obj == NULL)
  1076. return -ENOENT;
  1077. /* prime objects have no backing filp to GEM mmap
  1078. * pages from.
  1079. */
  1080. if (!obj->filp) {
  1081. drm_gem_object_unreference_unlocked(obj);
  1082. return -EINVAL;
  1083. }
  1084. addr = vm_mmap(obj->filp, 0, args->size,
  1085. PROT_READ | PROT_WRITE, MAP_SHARED,
  1086. args->offset);
  1087. drm_gem_object_unreference_unlocked(obj);
  1088. if (IS_ERR((void *)addr))
  1089. return addr;
  1090. args->addr_ptr = (uint64_t) addr;
  1091. return 0;
  1092. }
  1093. /**
  1094. * i915_gem_fault - fault a page into the GTT
  1095. * vma: VMA in question
  1096. * vmf: fault info
  1097. *
  1098. * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
  1099. * from userspace. The fault handler takes care of binding the object to
  1100. * the GTT (if needed), allocating and programming a fence register (again,
  1101. * only if needed based on whether the old reg is still valid or the object
  1102. * is tiled) and inserting a new PTE into the faulting process.
  1103. *
  1104. * Note that the faulting process may involve evicting existing objects
  1105. * from the GTT and/or fence registers to make room. So performance may
  1106. * suffer if the GTT working set is large or there are few fence registers
  1107. * left.
  1108. */
  1109. int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
  1110. {
  1111. struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
  1112. struct drm_device *dev = obj->base.dev;
  1113. drm_i915_private_t *dev_priv = dev->dev_private;
  1114. pgoff_t page_offset;
  1115. unsigned long pfn;
  1116. int ret = 0;
  1117. bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
  1118. /* We don't use vmf->pgoff since that has the fake offset */
  1119. page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
  1120. PAGE_SHIFT;
  1121. ret = i915_mutex_lock_interruptible(dev);
  1122. if (ret)
  1123. goto out;
  1124. trace_i915_gem_object_fault(obj, page_offset, true, write);
  1125. /* Now bind it into the GTT if needed */
  1126. ret = i915_gem_object_pin(obj, 0, true, false);
  1127. if (ret)
  1128. goto unlock;
  1129. ret = i915_gem_object_set_to_gtt_domain(obj, write);
  1130. if (ret)
  1131. goto unpin;
  1132. ret = i915_gem_object_get_fence(obj);
  1133. if (ret)
  1134. goto unpin;
  1135. obj->fault_mappable = true;
  1136. pfn = ((dev_priv->mm.gtt_base_addr + obj->gtt_offset) >> PAGE_SHIFT) +
  1137. page_offset;
  1138. /* Finally, remap it using the new GTT offset */
  1139. ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
  1140. unpin:
  1141. i915_gem_object_unpin(obj);
  1142. unlock:
  1143. mutex_unlock(&dev->struct_mutex);
  1144. out:
  1145. switch (ret) {
  1146. case -EIO:
  1147. /* If this -EIO is due to a gpu hang, give the reset code a
  1148. * chance to clean up the mess. Otherwise return the proper
  1149. * SIGBUS. */
  1150. if (!atomic_read(&dev_priv->mm.wedged))
  1151. return VM_FAULT_SIGBUS;
  1152. case -EAGAIN:
  1153. /* Give the error handler a chance to run and move the
  1154. * objects off the GPU active list. Next time we service the
  1155. * fault, we should be able to transition the page into the
  1156. * GTT without touching the GPU (and so avoid further
  1157. * EIO/EGAIN). If the GPU is wedged, then there is no issue
  1158. * with coherency, just lost writes.
  1159. */
  1160. set_need_resched();
  1161. case 0:
  1162. case -ERESTARTSYS:
  1163. case -EINTR:
  1164. case -EBUSY:
  1165. /*
  1166. * EBUSY is ok: this just means that another thread
  1167. * already did the job.
  1168. */
  1169. return VM_FAULT_NOPAGE;
  1170. case -ENOMEM:
  1171. return VM_FAULT_OOM;
  1172. case -ENOSPC:
  1173. return VM_FAULT_SIGBUS;
  1174. default:
  1175. WARN_ONCE(ret, "unhandled error in i915_gem_fault: %i\n", ret);
  1176. return VM_FAULT_SIGBUS;
  1177. }
  1178. }
  1179. /**
  1180. * i915_gem_release_mmap - remove physical page mappings
  1181. * @obj: obj in question
  1182. *
  1183. * Preserve the reservation of the mmapping with the DRM core code, but
  1184. * relinquish ownership of the pages back to the system.
  1185. *
  1186. * It is vital that we remove the page mapping if we have mapped a tiled
  1187. * object through the GTT and then lose the fence register due to
  1188. * resource pressure. Similarly if the object has been moved out of the
  1189. * aperture, than pages mapped into userspace must be revoked. Removing the
  1190. * mapping will then trigger a page fault on the next user access, allowing
  1191. * fixup by i915_gem_fault().
  1192. */
  1193. void
  1194. i915_gem_release_mmap(struct drm_i915_gem_object *obj)
  1195. {
  1196. if (!obj->fault_mappable)
  1197. return;
  1198. if (obj->base.dev->dev_mapping)
  1199. unmap_mapping_range(obj->base.dev->dev_mapping,
  1200. (loff_t)obj->base.map_list.hash.key<<PAGE_SHIFT,
  1201. obj->base.size, 1);
  1202. obj->fault_mappable = false;
  1203. }
  1204. static uint32_t
  1205. i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
  1206. {
  1207. uint32_t gtt_size;
  1208. if (INTEL_INFO(dev)->gen >= 4 ||
  1209. tiling_mode == I915_TILING_NONE)
  1210. return size;
  1211. /* Previous chips need a power-of-two fence region when tiling */
  1212. if (INTEL_INFO(dev)->gen == 3)
  1213. gtt_size = 1024*1024;
  1214. else
  1215. gtt_size = 512*1024;
  1216. while (gtt_size < size)
  1217. gtt_size <<= 1;
  1218. return gtt_size;
  1219. }
  1220. /**
  1221. * i915_gem_get_gtt_alignment - return required GTT alignment for an object
  1222. * @obj: object to check
  1223. *
  1224. * Return the required GTT alignment for an object, taking into account
  1225. * potential fence register mapping.
  1226. */
  1227. static uint32_t
  1228. i915_gem_get_gtt_alignment(struct drm_device *dev,
  1229. uint32_t size,
  1230. int tiling_mode)
  1231. {
  1232. /*
  1233. * Minimum alignment is 4k (GTT page size), but might be greater
  1234. * if a fence register is needed for the object.
  1235. */
  1236. if (INTEL_INFO(dev)->gen >= 4 ||
  1237. tiling_mode == I915_TILING_NONE)
  1238. return 4096;
  1239. /*
  1240. * Previous chips need to be aligned to the size of the smallest
  1241. * fence register that can contain the object.
  1242. */
  1243. return i915_gem_get_gtt_size(dev, size, tiling_mode);
  1244. }
  1245. /**
  1246. * i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
  1247. * unfenced object
  1248. * @dev: the device
  1249. * @size: size of the object
  1250. * @tiling_mode: tiling mode of the object
  1251. *
  1252. * Return the required GTT alignment for an object, only taking into account
  1253. * unfenced tiled surface requirements.
  1254. */
  1255. uint32_t
  1256. i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,
  1257. uint32_t size,
  1258. int tiling_mode)
  1259. {
  1260. /*
  1261. * Minimum alignment is 4k (GTT page size) for sane hw.
  1262. */
  1263. if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev) ||
  1264. tiling_mode == I915_TILING_NONE)
  1265. return 4096;
  1266. /* Previous hardware however needs to be aligned to a power-of-two
  1267. * tile height. The simplest method for determining this is to reuse
  1268. * the power-of-tile object size.
  1269. */
  1270. return i915_gem_get_gtt_size(dev, size, tiling_mode);
  1271. }
  1272. static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj)
  1273. {
  1274. struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  1275. int ret;
  1276. if (obj->base.map_list.map)
  1277. return 0;
  1278. ret = drm_gem_create_mmap_offset(&obj->base);
  1279. if (ret != -ENOSPC)
  1280. return ret;
  1281. /* Badly fragmented mmap space? The only way we can recover
  1282. * space is by destroying unwanted objects. We can't randomly release
  1283. * mmap_offsets as userspace expects them to be persistent for the
  1284. * lifetime of the objects. The closest we can is to release the
  1285. * offsets on purgeable objects by truncating it and marking it purged,
  1286. * which prevents userspace from ever using that object again.
  1287. */
  1288. i915_gem_purge(dev_priv, obj->base.size >> PAGE_SHIFT);
  1289. ret = drm_gem_create_mmap_offset(&obj->base);
  1290. if (ret != -ENOSPC)
  1291. return ret;
  1292. i915_gem_shrink_all(dev_priv);
  1293. return drm_gem_create_mmap_offset(&obj->base);
  1294. }
  1295. static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj)
  1296. {
  1297. if (!obj->base.map_list.map)
  1298. return;
  1299. drm_gem_free_mmap_offset(&obj->base);
  1300. }
  1301. int
  1302. i915_gem_mmap_gtt(struct drm_file *file,
  1303. struct drm_device *dev,
  1304. uint32_t handle,
  1305. uint64_t *offset)
  1306. {
  1307. struct drm_i915_private *dev_priv = dev->dev_private;
  1308. struct drm_i915_gem_object *obj;
  1309. int ret;
  1310. ret = i915_mutex_lock_interruptible(dev);
  1311. if (ret)
  1312. return ret;
  1313. obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
  1314. if (&obj->base == NULL) {
  1315. ret = -ENOENT;
  1316. goto unlock;
  1317. }
  1318. if (obj->base.size > dev_priv->mm.gtt_mappable_end) {
  1319. ret = -E2BIG;
  1320. goto out;
  1321. }
  1322. if (obj->madv != I915_MADV_WILLNEED) {
  1323. DRM_ERROR("Attempting to mmap a purgeable buffer\n");
  1324. ret = -EINVAL;
  1325. goto out;
  1326. }
  1327. ret = i915_gem_object_create_mmap_offset(obj);
  1328. if (ret)
  1329. goto out;
  1330. *offset = (u64)obj->base.map_list.hash.key << PAGE_SHIFT;
  1331. out:
  1332. drm_gem_object_unreference(&obj->base);
  1333. unlock:
  1334. mutex_unlock(&dev->struct_mutex);
  1335. return ret;
  1336. }
  1337. /**
  1338. * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
  1339. * @dev: DRM device
  1340. * @data: GTT mapping ioctl data
  1341. * @file: GEM object info
  1342. *
  1343. * Simply returns the fake offset to userspace so it can mmap it.
  1344. * The mmap call will end up in drm_gem_mmap(), which will set things
  1345. * up so we can get faults in the handler above.
  1346. *
  1347. * The fault handler will take care of binding the object into the GTT
  1348. * (since it may have been evicted to make room for something), allocating
  1349. * a fence register, and mapping the appropriate aperture address into
  1350. * userspace.
  1351. */
  1352. int
  1353. i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
  1354. struct drm_file *file)
  1355. {
  1356. struct drm_i915_gem_mmap_gtt *args = data;
  1357. return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
  1358. }
  1359. /* Immediately discard the backing storage */
  1360. static void
  1361. i915_gem_object_truncate(struct drm_i915_gem_object *obj)
  1362. {
  1363. struct inode *inode;
  1364. i915_gem_object_free_mmap_offset(obj);
  1365. if (obj->base.filp == NULL)
  1366. return;
  1367. /* Our goal here is to return as much of the memory as
  1368. * is possible back to the system as we are called from OOM.
  1369. * To do this we must instruct the shmfs to drop all of its
  1370. * backing pages, *now*.
  1371. */
  1372. inode = obj->base.filp->f_path.dentry->d_inode;
  1373. shmem_truncate_range(inode, 0, (loff_t)-1);
  1374. obj->madv = __I915_MADV_PURGED;
  1375. }
  1376. static inline int
  1377. i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
  1378. {
  1379. return obj->madv == I915_MADV_DONTNEED;
  1380. }
  1381. static void
  1382. i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
  1383. {
  1384. int page_count = obj->base.size / PAGE_SIZE;
  1385. struct scatterlist *sg;
  1386. int ret, i;
  1387. BUG_ON(obj->madv == __I915_MADV_PURGED);
  1388. ret = i915_gem_object_set_to_cpu_domain(obj, true);
  1389. if (ret) {
  1390. /* In the event of a disaster, abandon all caches and
  1391. * hope for the best.
  1392. */
  1393. WARN_ON(ret != -EIO);
  1394. i915_gem_clflush_object(obj);
  1395. obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  1396. }
  1397. if (i915_gem_object_needs_bit17_swizzle(obj))
  1398. i915_gem_object_save_bit_17_swizzle(obj);
  1399. if (obj->madv == I915_MADV_DONTNEED)
  1400. obj->dirty = 0;
  1401. for_each_sg(obj->pages->sgl, sg, page_count, i) {
  1402. struct page *page = sg_page(sg);
  1403. if (obj->dirty)
  1404. set_page_dirty(page);
  1405. if (obj->madv == I915_MADV_WILLNEED)
  1406. mark_page_accessed(page);
  1407. page_cache_release(page);
  1408. }
  1409. obj->dirty = 0;
  1410. sg_free_table(obj->pages);
  1411. kfree(obj->pages);
  1412. }
  1413. static int
  1414. i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
  1415. {
  1416. const struct drm_i915_gem_object_ops *ops = obj->ops;
  1417. if (obj->pages == NULL)
  1418. return 0;
  1419. BUG_ON(obj->gtt_space);
  1420. if (obj->pages_pin_count)
  1421. return -EBUSY;
  1422. ops->put_pages(obj);
  1423. obj->pages = NULL;
  1424. list_del(&obj->gtt_list);
  1425. if (i915_gem_object_is_purgeable(obj))
  1426. i915_gem_object_truncate(obj);
  1427. return 0;
  1428. }
  1429. static long
  1430. i915_gem_purge(struct drm_i915_private *dev_priv, long target)
  1431. {
  1432. struct drm_i915_gem_object *obj, *next;
  1433. long count = 0;
  1434. list_for_each_entry_safe(obj, next,
  1435. &dev_priv->mm.unbound_list,
  1436. gtt_list) {
  1437. if (i915_gem_object_is_purgeable(obj) &&
  1438. i915_gem_object_put_pages(obj) == 0) {
  1439. count += obj->base.size >> PAGE_SHIFT;
  1440. if (count >= target)
  1441. return count;
  1442. }
  1443. }
  1444. list_for_each_entry_safe(obj, next,
  1445. &dev_priv->mm.inactive_list,
  1446. mm_list) {
  1447. if (i915_gem_object_is_purgeable(obj) &&
  1448. i915_gem_object_unbind(obj) == 0 &&
  1449. i915_gem_object_put_pages(obj) == 0) {
  1450. count += obj->base.size >> PAGE_SHIFT;
  1451. if (count >= target)
  1452. return count;
  1453. }
  1454. }
  1455. return count;
  1456. }
  1457. static void
  1458. i915_gem_shrink_all(struct drm_i915_private *dev_priv)
  1459. {
  1460. struct drm_i915_gem_object *obj, *next;
  1461. i915_gem_evict_everything(dev_priv->dev);
  1462. list_for_each_entry_safe(obj, next, &dev_priv->mm.unbound_list, gtt_list)
  1463. i915_gem_object_put_pages(obj);
  1464. }
  1465. static int
  1466. i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
  1467. {
  1468. struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  1469. int page_count, i;
  1470. struct address_space *mapping;
  1471. struct sg_table *st;
  1472. struct scatterlist *sg;
  1473. struct page *page;
  1474. gfp_t gfp;
  1475. /* Assert that the object is not currently in any GPU domain. As it
  1476. * wasn't in the GTT, there shouldn't be any way it could have been in
  1477. * a GPU cache
  1478. */
  1479. BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
  1480. BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
  1481. st = kmalloc(sizeof(*st), GFP_KERNEL);
  1482. if (st == NULL)
  1483. return -ENOMEM;
  1484. page_count = obj->base.size / PAGE_SIZE;
  1485. if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
  1486. sg_free_table(st);
  1487. kfree(st);
  1488. return -ENOMEM;
  1489. }
  1490. /* Get the list of pages out of our struct file. They'll be pinned
  1491. * at this point until we release them.
  1492. *
  1493. * Fail silently without starting the shrinker
  1494. */
  1495. mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
  1496. gfp = mapping_gfp_mask(mapping);
  1497. gfp |= __GFP_NORETRY | __GFP_NOWARN;
  1498. gfp &= ~(__GFP_IO | __GFP_WAIT);
  1499. for_each_sg(st->sgl, sg, page_count, i) {
  1500. page = shmem_read_mapping_page_gfp(mapping, i, gfp);
  1501. if (IS_ERR(page)) {
  1502. i915_gem_purge(dev_priv, page_count);
  1503. page = shmem_read_mapping_page_gfp(mapping, i, gfp);
  1504. }
  1505. if (IS_ERR(page)) {
  1506. /* We've tried hard to allocate the memory by reaping
  1507. * our own buffer, now let the real VM do its job and
  1508. * go down in flames if truly OOM.
  1509. */
  1510. gfp &= ~(__GFP_NORETRY | __GFP_NOWARN);
  1511. gfp |= __GFP_IO | __GFP_WAIT;
  1512. i915_gem_shrink_all(dev_priv);
  1513. page = shmem_read_mapping_page_gfp(mapping, i, gfp);
  1514. if (IS_ERR(page))
  1515. goto err_pages;
  1516. gfp |= __GFP_NORETRY | __GFP_NOWARN;
  1517. gfp &= ~(__GFP_IO | __GFP_WAIT);
  1518. }
  1519. sg_set_page(sg, page, PAGE_SIZE, 0);
  1520. }
  1521. obj->pages = st;
  1522. if (i915_gem_object_needs_bit17_swizzle(obj))
  1523. i915_gem_object_do_bit_17_swizzle(obj);
  1524. return 0;
  1525. err_pages:
  1526. for_each_sg(st->sgl, sg, i, page_count)
  1527. page_cache_release(sg_page(sg));
  1528. sg_free_table(st);
  1529. kfree(st);
  1530. return PTR_ERR(page);
  1531. }
  1532. /* Ensure that the associated pages are gathered from the backing storage
  1533. * and pinned into our object. i915_gem_object_get_pages() may be called
  1534. * multiple times before they are released by a single call to
  1535. * i915_gem_object_put_pages() - once the pages are no longer referenced
  1536. * either as a result of memory pressure (reaping pages under the shrinker)
  1537. * or as the object is itself released.
  1538. */
  1539. int
  1540. i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
  1541. {
  1542. struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  1543. const struct drm_i915_gem_object_ops *ops = obj->ops;
  1544. int ret;
  1545. if (obj->pages)
  1546. return 0;
  1547. BUG_ON(obj->pages_pin_count);
  1548. ret = ops->get_pages(obj);
  1549. if (ret)
  1550. return ret;
  1551. list_add_tail(&obj->gtt_list, &dev_priv->mm.unbound_list);
  1552. return 0;
  1553. }
  1554. void
  1555. i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
  1556. struct intel_ring_buffer *ring)
  1557. {
  1558. struct drm_device *dev = obj->base.dev;
  1559. struct drm_i915_private *dev_priv = dev->dev_private;
  1560. u32 seqno = intel_ring_get_seqno(ring);
  1561. BUG_ON(ring == NULL);
  1562. obj->ring = ring;
  1563. /* Add a reference if we're newly entering the active list. */
  1564. if (!obj->active) {
  1565. drm_gem_object_reference(&obj->base);
  1566. obj->active = 1;
  1567. }
  1568. /* Move from whatever list we were on to the tail of execution. */
  1569. list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
  1570. list_move_tail(&obj->ring_list, &ring->active_list);
  1571. obj->last_read_seqno = seqno;
  1572. if (obj->fenced_gpu_access) {
  1573. obj->last_fenced_seqno = seqno;
  1574. /* Bump MRU to take account of the delayed flush */
  1575. if (obj->fence_reg != I915_FENCE_REG_NONE) {
  1576. struct drm_i915_fence_reg *reg;
  1577. reg = &dev_priv->fence_regs[obj->fence_reg];
  1578. list_move_tail(&reg->lru_list,
  1579. &dev_priv->mm.fence_list);
  1580. }
  1581. }
  1582. }
  1583. static void
  1584. i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
  1585. {
  1586. struct drm_device *dev = obj->base.dev;
  1587. struct drm_i915_private *dev_priv = dev->dev_private;
  1588. BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS);
  1589. BUG_ON(!obj->active);
  1590. if (obj->pin_count) /* are we a framebuffer? */
  1591. intel_mark_fb_idle(obj);
  1592. list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
  1593. list_del_init(&obj->ring_list);
  1594. obj->ring = NULL;
  1595. obj->last_read_seqno = 0;
  1596. obj->last_write_seqno = 0;
  1597. obj->base.write_domain = 0;
  1598. obj->last_fenced_seqno = 0;
  1599. obj->fenced_gpu_access = false;
  1600. obj->active = 0;
  1601. drm_gem_object_unreference(&obj->base);
  1602. WARN_ON(i915_verify_lists(dev));
  1603. }
  1604. static int
  1605. i915_gem_handle_seqno_wrap(struct drm_device *dev)
  1606. {
  1607. struct drm_i915_private *dev_priv = dev->dev_private;
  1608. struct intel_ring_buffer *ring;
  1609. int ret, i, j;
  1610. /* The hardware uses various monotonic 32-bit counters, if we
  1611. * detect that they will wraparound we need to idle the GPU
  1612. * and reset those counters.
  1613. */
  1614. ret = 0;
  1615. for_each_ring(ring, dev_priv, i) {
  1616. for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++)
  1617. ret |= ring->sync_seqno[j] != 0;
  1618. }
  1619. if (ret == 0)
  1620. return ret;
  1621. ret = i915_gpu_idle(dev);
  1622. if (ret)
  1623. return ret;
  1624. i915_gem_retire_requests(dev);
  1625. for_each_ring(ring, dev_priv, i) {
  1626. ret = intel_ring_handle_seqno_wrap(ring);
  1627. if (ret)
  1628. return ret;
  1629. for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++)
  1630. ring->sync_seqno[j] = 0;
  1631. }
  1632. return 0;
  1633. }
  1634. int
  1635. i915_gem_get_seqno(struct drm_device *dev, u32 *seqno)
  1636. {
  1637. struct drm_i915_private *dev_priv = dev->dev_private;
  1638. /* reserve 0 for non-seqno */
  1639. if (dev_priv->next_seqno == 0) {
  1640. int ret = i915_gem_handle_seqno_wrap(dev);
  1641. if (ret)
  1642. return ret;
  1643. dev_priv->next_seqno = 1;
  1644. }
  1645. *seqno = dev_priv->last_seqno = dev_priv->next_seqno++;
  1646. return 0;
  1647. }
  1648. int
  1649. i915_add_request(struct intel_ring_buffer *ring,
  1650. struct drm_file *file,
  1651. u32 *out_seqno)
  1652. {
  1653. drm_i915_private_t *dev_priv = ring->dev->dev_private;
  1654. struct drm_i915_gem_request *request;
  1655. u32 request_ring_position;
  1656. int was_empty;
  1657. int ret;
  1658. /*
  1659. * Emit any outstanding flushes - execbuf can fail to emit the flush
  1660. * after having emitted the batchbuffer command. Hence we need to fix
  1661. * things up similar to emitting the lazy request. The difference here
  1662. * is that the flush _must_ happen before the next request, no matter
  1663. * what.
  1664. */
  1665. ret = intel_ring_flush_all_caches(ring);
  1666. if (ret)
  1667. return ret;
  1668. request = kmalloc(sizeof(*request), GFP_KERNEL);
  1669. if (request == NULL)
  1670. return -ENOMEM;
  1671. /* Record the position of the start of the request so that
  1672. * should we detect the updated seqno part-way through the
  1673. * GPU processing the request, we never over-estimate the
  1674. * position of the head.
  1675. */
  1676. request_ring_position = intel_ring_get_tail(ring);
  1677. ret = ring->add_request(ring);
  1678. if (ret) {
  1679. kfree(request);
  1680. return ret;
  1681. }
  1682. request->seqno = intel_ring_get_seqno(ring);
  1683. request->ring = ring;
  1684. request->tail = request_ring_position;
  1685. request->emitted_jiffies = jiffies;
  1686. was_empty = list_empty(&ring->request_list);
  1687. list_add_tail(&request->list, &ring->request_list);
  1688. request->file_priv = NULL;
  1689. if (file) {
  1690. struct drm_i915_file_private *file_priv = file->driver_priv;
  1691. spin_lock(&file_priv->mm.lock);
  1692. request->file_priv = file_priv;
  1693. list_add_tail(&request->client_list,
  1694. &file_priv->mm.request_list);
  1695. spin_unlock(&file_priv->mm.lock);
  1696. }
  1697. trace_i915_gem_request_add(ring, request->seqno);
  1698. ring->outstanding_lazy_request = 0;
  1699. if (!dev_priv->mm.suspended) {
  1700. if (i915_enable_hangcheck) {
  1701. mod_timer(&dev_priv->hangcheck_timer,
  1702. round_jiffies_up(jiffies + DRM_I915_HANGCHECK_JIFFIES));
  1703. }
  1704. if (was_empty) {
  1705. queue_delayed_work(dev_priv->wq,
  1706. &dev_priv->mm.retire_work,
  1707. round_jiffies_up_relative(HZ));
  1708. intel_mark_busy(dev_priv->dev);
  1709. }
  1710. }
  1711. if (out_seqno)
  1712. *out_seqno = request->seqno;
  1713. return 0;
  1714. }
  1715. static inline void
  1716. i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
  1717. {
  1718. struct drm_i915_file_private *file_priv = request->file_priv;
  1719. if (!file_priv)
  1720. return;
  1721. spin_lock(&file_priv->mm.lock);
  1722. if (request->file_priv) {
  1723. list_del(&request->client_list);
  1724. request->file_priv = NULL;
  1725. }
  1726. spin_unlock(&file_priv->mm.lock);
  1727. }
  1728. static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
  1729. struct intel_ring_buffer *ring)
  1730. {
  1731. while (!list_empty(&ring->request_list)) {
  1732. struct drm_i915_gem_request *request;
  1733. request = list_first_entry(&ring->request_list,
  1734. struct drm_i915_gem_request,
  1735. list);
  1736. list_del(&request->list);
  1737. i915_gem_request_remove_from_client(request);
  1738. kfree(request);
  1739. }
  1740. while (!list_empty(&ring->active_list)) {
  1741. struct drm_i915_gem_object *obj;
  1742. obj = list_first_entry(&ring->active_list,
  1743. struct drm_i915_gem_object,
  1744. ring_list);
  1745. i915_gem_object_move_to_inactive(obj);
  1746. }
  1747. }
  1748. static void i915_gem_reset_fences(struct drm_device *dev)
  1749. {
  1750. struct drm_i915_private *dev_priv = dev->dev_private;
  1751. int i;
  1752. for (i = 0; i < dev_priv->num_fence_regs; i++) {
  1753. struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
  1754. i915_gem_write_fence(dev, i, NULL);
  1755. if (reg->obj)
  1756. i915_gem_object_fence_lost(reg->obj);
  1757. reg->pin_count = 0;
  1758. reg->obj = NULL;
  1759. INIT_LIST_HEAD(&reg->lru_list);
  1760. }
  1761. INIT_LIST_HEAD(&dev_priv->mm.fence_list);
  1762. }
  1763. void i915_gem_reset(struct drm_device *dev)
  1764. {
  1765. struct drm_i915_private *dev_priv = dev->dev_private;
  1766. struct drm_i915_gem_object *obj;
  1767. struct intel_ring_buffer *ring;
  1768. int i;
  1769. for_each_ring(ring, dev_priv, i)
  1770. i915_gem_reset_ring_lists(dev_priv, ring);
  1771. /* Move everything out of the GPU domains to ensure we do any
  1772. * necessary invalidation upon reuse.
  1773. */
  1774. list_for_each_entry(obj,
  1775. &dev_priv->mm.inactive_list,
  1776. mm_list)
  1777. {
  1778. obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
  1779. }
  1780. /* The fence registers are invalidated so clear them out */
  1781. i915_gem_reset_fences(dev);
  1782. }
  1783. /**
  1784. * This function clears the request list as sequence numbers are passed.
  1785. */
  1786. void
  1787. i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
  1788. {
  1789. uint32_t seqno;
  1790. if (list_empty(&ring->request_list))
  1791. return;
  1792. WARN_ON(i915_verify_lists(ring->dev));
  1793. seqno = ring->get_seqno(ring, true);
  1794. while (!list_empty(&ring->request_list)) {
  1795. struct drm_i915_gem_request *request;
  1796. request = list_first_entry(&ring->request_list,
  1797. struct drm_i915_gem_request,
  1798. list);
  1799. if (!i915_seqno_passed(seqno, request->seqno))
  1800. break;
  1801. trace_i915_gem_request_retire(ring, request->seqno);
  1802. /* We know the GPU must have read the request to have
  1803. * sent us the seqno + interrupt, so use the position
  1804. * of tail of the request to update the last known position
  1805. * of the GPU head.
  1806. */
  1807. ring->last_retired_head = request->tail;
  1808. list_del(&request->list);
  1809. i915_gem_request_remove_from_client(request);
  1810. kfree(request);
  1811. }
  1812. /* Move any buffers on the active list that are no longer referenced
  1813. * by the ringbuffer to the flushing/inactive lists as appropriate.
  1814. */
  1815. while (!list_empty(&ring->active_list)) {
  1816. struct drm_i915_gem_object *obj;
  1817. obj = list_first_entry(&ring->active_list,
  1818. struct drm_i915_gem_object,
  1819. ring_list);
  1820. if (!i915_seqno_passed(seqno, obj->last_read_seqno))
  1821. break;
  1822. i915_gem_object_move_to_inactive(obj);
  1823. }
  1824. if (unlikely(ring->trace_irq_seqno &&
  1825. i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
  1826. ring->irq_put(ring);
  1827. ring->trace_irq_seqno = 0;
  1828. }
  1829. WARN_ON(i915_verify_lists(ring->dev));
  1830. }
  1831. void
  1832. i915_gem_retire_requests(struct drm_device *dev)
  1833. {
  1834. drm_i915_private_t *dev_priv = dev->dev_private;
  1835. struct intel_ring_buffer *ring;
  1836. int i;
  1837. for_each_ring(ring, dev_priv, i)
  1838. i915_gem_retire_requests_ring(ring);
  1839. }
  1840. static void
  1841. i915_gem_retire_work_handler(struct work_struct *work)
  1842. {
  1843. drm_i915_private_t *dev_priv;
  1844. struct drm_device *dev;
  1845. struct intel_ring_buffer *ring;
  1846. bool idle;
  1847. int i;
  1848. dev_priv = container_of(work, drm_i915_private_t,
  1849. mm.retire_work.work);
  1850. dev = dev_priv->dev;
  1851. /* Come back later if the device is busy... */
  1852. if (!mutex_trylock(&dev->struct_mutex)) {
  1853. queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
  1854. round_jiffies_up_relative(HZ));
  1855. return;
  1856. }
  1857. i915_gem_retire_requests(dev);
  1858. /* Send a periodic flush down the ring so we don't hold onto GEM
  1859. * objects indefinitely.
  1860. */
  1861. idle = true;
  1862. for_each_ring(ring, dev_priv, i) {
  1863. if (ring->gpu_caches_dirty)
  1864. i915_add_request(ring, NULL, NULL);
  1865. idle &= list_empty(&ring->request_list);
  1866. }
  1867. if (!dev_priv->mm.suspended && !idle)
  1868. queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
  1869. round_jiffies_up_relative(HZ));
  1870. if (idle)
  1871. intel_mark_idle(dev);
  1872. mutex_unlock(&dev->struct_mutex);
  1873. }
  1874. /**
  1875. * Ensures that an object will eventually get non-busy by flushing any required
  1876. * write domains, emitting any outstanding lazy request and retiring and
  1877. * completed requests.
  1878. */
  1879. static int
  1880. i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
  1881. {
  1882. int ret;
  1883. if (obj->active) {
  1884. ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno);
  1885. if (ret)
  1886. return ret;
  1887. i915_gem_retire_requests_ring(obj->ring);
  1888. }
  1889. return 0;
  1890. }
  1891. /**
  1892. * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
  1893. * @DRM_IOCTL_ARGS: standard ioctl arguments
  1894. *
  1895. * Returns 0 if successful, else an error is returned with the remaining time in
  1896. * the timeout parameter.
  1897. * -ETIME: object is still busy after timeout
  1898. * -ERESTARTSYS: signal interrupted the wait
  1899. * -ENONENT: object doesn't exist
  1900. * Also possible, but rare:
  1901. * -EAGAIN: GPU wedged
  1902. * -ENOMEM: damn
  1903. * -ENODEV: Internal IRQ fail
  1904. * -E?: The add request failed
  1905. *
  1906. * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
  1907. * non-zero timeout parameter the wait ioctl will wait for the given number of
  1908. * nanoseconds on an object becoming unbusy. Since the wait itself does so
  1909. * without holding struct_mutex the object may become re-busied before this
  1910. * function completes. A similar but shorter * race condition exists in the busy
  1911. * ioctl
  1912. */
  1913. int
  1914. i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
  1915. {
  1916. struct drm_i915_gem_wait *args = data;
  1917. struct drm_i915_gem_object *obj;
  1918. struct intel_ring_buffer *ring = NULL;
  1919. struct timespec timeout_stack, *timeout = NULL;
  1920. u32 seqno = 0;
  1921. int ret = 0;
  1922. if (args->timeout_ns >= 0) {
  1923. timeout_stack = ns_to_timespec(args->timeout_ns);
  1924. timeout = &timeout_stack;
  1925. }
  1926. ret = i915_mutex_lock_interruptible(dev);
  1927. if (ret)
  1928. return ret;
  1929. obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->bo_handle));
  1930. if (&obj->base == NULL) {
  1931. mutex_unlock(&dev->struct_mutex);
  1932. return -ENOENT;
  1933. }
  1934. /* Need to make sure the object gets inactive eventually. */
  1935. ret = i915_gem_object_flush_active(obj);
  1936. if (ret)
  1937. goto out;
  1938. if (obj->active) {
  1939. seqno = obj->last_read_seqno;
  1940. ring = obj->ring;
  1941. }
  1942. if (seqno == 0)
  1943. goto out;
  1944. /* Do this after OLR check to make sure we make forward progress polling
  1945. * on this IOCTL with a 0 timeout (like busy ioctl)
  1946. */
  1947. if (!args->timeout_ns) {
  1948. ret = -ETIME;
  1949. goto out;
  1950. }
  1951. drm_gem_object_unreference(&obj->base);
  1952. mutex_unlock(&dev->struct_mutex);
  1953. ret = __wait_seqno(ring, seqno, true, timeout);
  1954. if (timeout) {
  1955. WARN_ON(!timespec_valid(timeout));
  1956. args->timeout_ns = timespec_to_ns(timeout);
  1957. }
  1958. return ret;
  1959. out:
  1960. drm_gem_object_unreference(&obj->base);
  1961. mutex_unlock(&dev->struct_mutex);
  1962. return ret;
  1963. }
  1964. /**
  1965. * i915_gem_object_sync - sync an object to a ring.
  1966. *
  1967. * @obj: object which may be in use on another ring.
  1968. * @to: ring we wish to use the object on. May be NULL.
  1969. *
  1970. * This code is meant to abstract object synchronization with the GPU.
  1971. * Calling with NULL implies synchronizing the object with the CPU
  1972. * rather than a particular GPU ring.
  1973. *
  1974. * Returns 0 if successful, else propagates up the lower layer error.
  1975. */
  1976. int
  1977. i915_gem_object_sync(struct drm_i915_gem_object *obj,
  1978. struct intel_ring_buffer *to)
  1979. {
  1980. struct intel_ring_buffer *from = obj->ring;
  1981. u32 seqno;
  1982. int ret, idx;
  1983. if (from == NULL || to == from)
  1984. return 0;
  1985. if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev))
  1986. return i915_gem_object_wait_rendering(obj, false);
  1987. idx = intel_ring_sync_index(from, to);
  1988. seqno = obj->last_read_seqno;
  1989. if (seqno <= from->sync_seqno[idx])
  1990. return 0;
  1991. ret = i915_gem_check_olr(obj->ring, seqno);
  1992. if (ret)
  1993. return ret;
  1994. ret = to->sync_to(to, from, seqno);
  1995. if (!ret)
  1996. /* We use last_read_seqno because sync_to()
  1997. * might have just caused seqno wrap under
  1998. * the radar.
  1999. */
  2000. from->sync_seqno[idx] = obj->last_read_seqno;
  2001. return ret;
  2002. }
  2003. static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
  2004. {
  2005. u32 old_write_domain, old_read_domains;
  2006. /* Act a barrier for all accesses through the GTT */
  2007. mb();
  2008. /* Force a pagefault for domain tracking on next user access */
  2009. i915_gem_release_mmap(obj);
  2010. if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
  2011. return;
  2012. old_read_domains = obj->base.read_domains;
  2013. old_write_domain = obj->base.write_domain;
  2014. obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
  2015. obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
  2016. trace_i915_gem_object_change_domain(obj,
  2017. old_read_domains,
  2018. old_write_domain);
  2019. }
  2020. /**
  2021. * Unbinds an object from the GTT aperture.
  2022. */
  2023. int
  2024. i915_gem_object_unbind(struct drm_i915_gem_object *obj)
  2025. {
  2026. drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
  2027. int ret = 0;
  2028. if (obj->gtt_space == NULL)
  2029. return 0;
  2030. if (obj->pin_count)
  2031. return -EBUSY;
  2032. BUG_ON(obj->pages == NULL);
  2033. ret = i915_gem_object_finish_gpu(obj);
  2034. if (ret)
  2035. return ret;
  2036. /* Continue on if we fail due to EIO, the GPU is hung so we
  2037. * should be safe and we need to cleanup or else we might
  2038. * cause memory corruption through use-after-free.
  2039. */
  2040. i915_gem_object_finish_gtt(obj);
  2041. /* release the fence reg _after_ flushing */
  2042. ret = i915_gem_object_put_fence(obj);
  2043. if (ret)
  2044. return ret;
  2045. trace_i915_gem_object_unbind(obj);
  2046. if (obj->has_global_gtt_mapping)
  2047. i915_gem_gtt_unbind_object(obj);
  2048. if (obj->has_aliasing_ppgtt_mapping) {
  2049. i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
  2050. obj->has_aliasing_ppgtt_mapping = 0;
  2051. }
  2052. i915_gem_gtt_finish_object(obj);
  2053. list_del(&obj->mm_list);
  2054. list_move_tail(&obj->gtt_list, &dev_priv->mm.unbound_list);
  2055. /* Avoid an unnecessary call to unbind on rebind. */
  2056. obj->map_and_fenceable = true;
  2057. drm_mm_put_block(obj->gtt_space);
  2058. obj->gtt_space = NULL;
  2059. obj->gtt_offset = 0;
  2060. return 0;
  2061. }
  2062. int i915_gpu_idle(struct drm_device *dev)
  2063. {
  2064. drm_i915_private_t *dev_priv = dev->dev_private;
  2065. struct intel_ring_buffer *ring;
  2066. int ret, i;
  2067. /* Flush everything onto the inactive list. */
  2068. for_each_ring(ring, dev_priv, i) {
  2069. ret = i915_switch_context(ring, NULL, DEFAULT_CONTEXT_ID);
  2070. if (ret)
  2071. return ret;
  2072. ret = intel_ring_idle(ring);
  2073. if (ret)
  2074. return ret;
  2075. }
  2076. return 0;
  2077. }
  2078. static void sandybridge_write_fence_reg(struct drm_device *dev, int reg,
  2079. struct drm_i915_gem_object *obj)
  2080. {
  2081. drm_i915_private_t *dev_priv = dev->dev_private;
  2082. uint64_t val;
  2083. if (obj) {
  2084. u32 size = obj->gtt_space->size;
  2085. val = (uint64_t)((obj->gtt_offset + size - 4096) &
  2086. 0xfffff000) << 32;
  2087. val |= obj->gtt_offset & 0xfffff000;
  2088. val |= (uint64_t)((obj->stride / 128) - 1) <<
  2089. SANDYBRIDGE_FENCE_PITCH_SHIFT;
  2090. if (obj->tiling_mode == I915_TILING_Y)
  2091. val |= 1 << I965_FENCE_TILING_Y_SHIFT;
  2092. val |= I965_FENCE_REG_VALID;
  2093. } else
  2094. val = 0;
  2095. I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + reg * 8, val);
  2096. POSTING_READ(FENCE_REG_SANDYBRIDGE_0 + reg * 8);
  2097. }
  2098. static void i965_write_fence_reg(struct drm_device *dev, int reg,
  2099. struct drm_i915_gem_object *obj)
  2100. {
  2101. drm_i915_private_t *dev_priv = dev->dev_private;
  2102. uint64_t val;
  2103. if (obj) {
  2104. u32 size = obj->gtt_space->size;
  2105. val = (uint64_t)((obj->gtt_offset + size - 4096) &
  2106. 0xfffff000) << 32;
  2107. val |= obj->gtt_offset & 0xfffff000;
  2108. val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
  2109. if (obj->tiling_mode == I915_TILING_Y)
  2110. val |= 1 << I965_FENCE_TILING_Y_SHIFT;
  2111. val |= I965_FENCE_REG_VALID;
  2112. } else
  2113. val = 0;
  2114. I915_WRITE64(FENCE_REG_965_0 + reg * 8, val);
  2115. POSTING_READ(FENCE_REG_965_0 + reg * 8);
  2116. }
  2117. static void i915_write_fence_reg(struct drm_device *dev, int reg,
  2118. struct drm_i915_gem_object *obj)
  2119. {
  2120. drm_i915_private_t *dev_priv = dev->dev_private;
  2121. u32 val;
  2122. if (obj) {
  2123. u32 size = obj->gtt_space->size;
  2124. int pitch_val;
  2125. int tile_width;
  2126. WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) ||
  2127. (size & -size) != size ||
  2128. (obj->gtt_offset & (size - 1)),
  2129. "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
  2130. obj->gtt_offset, obj->map_and_fenceable, size);
  2131. if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
  2132. tile_width = 128;
  2133. else
  2134. tile_width = 512;
  2135. /* Note: pitch better be a power of two tile widths */
  2136. pitch_val = obj->stride / tile_width;
  2137. pitch_val = ffs(pitch_val) - 1;
  2138. val = obj->gtt_offset;
  2139. if (obj->tiling_mode == I915_TILING_Y)
  2140. val |= 1 << I830_FENCE_TILING_Y_SHIFT;
  2141. val |= I915_FENCE_SIZE_BITS(size);
  2142. val |= pitch_val << I830_FENCE_PITCH_SHIFT;
  2143. val |= I830_FENCE_REG_VALID;
  2144. } else
  2145. val = 0;
  2146. if (reg < 8)
  2147. reg = FENCE_REG_830_0 + reg * 4;
  2148. else
  2149. reg = FENCE_REG_945_8 + (reg - 8) * 4;
  2150. I915_WRITE(reg, val);
  2151. POSTING_READ(reg);
  2152. }
  2153. static void i830_write_fence_reg(struct drm_device *dev, int reg,
  2154. struct drm_i915_gem_object *obj)
  2155. {
  2156. drm_i915_private_t *dev_priv = dev->dev_private;
  2157. uint32_t val;
  2158. if (obj) {
  2159. u32 size = obj->gtt_space->size;
  2160. uint32_t pitch_val;
  2161. WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) ||
  2162. (size & -size) != size ||
  2163. (obj->gtt_offset & (size - 1)),
  2164. "object 0x%08x not 512K or pot-size 0x%08x aligned\n",
  2165. obj->gtt_offset, size);
  2166. pitch_val = obj->stride / 128;
  2167. pitch_val = ffs(pitch_val) - 1;
  2168. val = obj->gtt_offset;
  2169. if (obj->tiling_mode == I915_TILING_Y)
  2170. val |= 1 << I830_FENCE_TILING_Y_SHIFT;
  2171. val |= I830_FENCE_SIZE_BITS(size);
  2172. val |= pitch_val << I830_FENCE_PITCH_SHIFT;
  2173. val |= I830_FENCE_REG_VALID;
  2174. } else
  2175. val = 0;
  2176. I915_WRITE(FENCE_REG_830_0 + reg * 4, val);
  2177. POSTING_READ(FENCE_REG_830_0 + reg * 4);
  2178. }
  2179. static void i915_gem_write_fence(struct drm_device *dev, int reg,
  2180. struct drm_i915_gem_object *obj)
  2181. {
  2182. switch (INTEL_INFO(dev)->gen) {
  2183. case 7:
  2184. case 6: sandybridge_write_fence_reg(dev, reg, obj); break;
  2185. case 5:
  2186. case 4: i965_write_fence_reg(dev, reg, obj); break;
  2187. case 3: i915_write_fence_reg(dev, reg, obj); break;
  2188. case 2: i830_write_fence_reg(dev, reg, obj); break;
  2189. default: break;
  2190. }
  2191. }
  2192. static inline int fence_number(struct drm_i915_private *dev_priv,
  2193. struct drm_i915_fence_reg *fence)
  2194. {
  2195. return fence - dev_priv->fence_regs;
  2196. }
  2197. static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
  2198. struct drm_i915_fence_reg *fence,
  2199. bool enable)
  2200. {
  2201. struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  2202. int reg = fence_number(dev_priv, fence);
  2203. i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL);
  2204. if (enable) {
  2205. obj->fence_reg = reg;
  2206. fence->obj = obj;
  2207. list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list);
  2208. } else {
  2209. obj->fence_reg = I915_FENCE_REG_NONE;
  2210. fence->obj = NULL;
  2211. list_del_init(&fence->lru_list);
  2212. }
  2213. }
  2214. static int
  2215. i915_gem_object_flush_fence(struct drm_i915_gem_object *obj)
  2216. {
  2217. if (obj->last_fenced_seqno) {
  2218. int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno);
  2219. if (ret)
  2220. return ret;
  2221. obj->last_fenced_seqno = 0;
  2222. }
  2223. /* Ensure that all CPU reads are completed before installing a fence
  2224. * and all writes before removing the fence.
  2225. */
  2226. if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
  2227. mb();
  2228. obj->fenced_gpu_access = false;
  2229. return 0;
  2230. }
  2231. int
  2232. i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
  2233. {
  2234. struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  2235. int ret;
  2236. ret = i915_gem_object_flush_fence(obj);
  2237. if (ret)
  2238. return ret;
  2239. if (obj->fence_reg == I915_FENCE_REG_NONE)
  2240. return 0;
  2241. i915_gem_object_update_fence(obj,
  2242. &dev_priv->fence_regs[obj->fence_reg],
  2243. false);
  2244. i915_gem_object_fence_lost(obj);
  2245. return 0;
  2246. }
  2247. static struct drm_i915_fence_reg *
  2248. i915_find_fence_reg(struct drm_device *dev)
  2249. {
  2250. struct drm_i915_private *dev_priv = dev->dev_private;
  2251. struct drm_i915_fence_reg *reg, *avail;
  2252. int i;
  2253. /* First try to find a free reg */
  2254. avail = NULL;
  2255. for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
  2256. reg = &dev_priv->fence_regs[i];
  2257. if (!reg->obj)
  2258. return reg;
  2259. if (!reg->pin_count)
  2260. avail = reg;
  2261. }
  2262. if (avail == NULL)
  2263. return NULL;
  2264. /* None available, try to steal one or wait for a user to finish */
  2265. list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
  2266. if (reg->pin_count)
  2267. continue;
  2268. return reg;
  2269. }
  2270. return NULL;
  2271. }
  2272. /**
  2273. * i915_gem_object_get_fence - set up fencing for an object
  2274. * @obj: object to map through a fence reg
  2275. *
  2276. * When mapping objects through the GTT, userspace wants to be able to write
  2277. * to them without having to worry about swizzling if the object is tiled.
  2278. * This function walks the fence regs looking for a free one for @obj,
  2279. * stealing one if it can't find any.
  2280. *
  2281. * It then sets up the reg based on the object's properties: address, pitch
  2282. * and tiling format.
  2283. *
  2284. * For an untiled surface, this removes any existing fence.
  2285. */
  2286. int
  2287. i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
  2288. {
  2289. struct drm_device *dev = obj->base.dev;
  2290. struct drm_i915_private *dev_priv = dev->dev_private;
  2291. bool enable = obj->tiling_mode != I915_TILING_NONE;
  2292. struct drm_i915_fence_reg *reg;
  2293. int ret;
  2294. /* Have we updated the tiling parameters upon the object and so
  2295. * will need to serialise the write to the associated fence register?
  2296. */
  2297. if (obj->fence_dirty) {
  2298. ret = i915_gem_object_flush_fence(obj);
  2299. if (ret)
  2300. return ret;
  2301. }
  2302. /* Just update our place in the LRU if our fence is getting reused. */
  2303. if (obj->fence_reg != I915_FENCE_REG_NONE) {
  2304. reg = &dev_priv->fence_regs[obj->fence_reg];
  2305. if (!obj->fence_dirty) {
  2306. list_move_tail(&reg->lru_list,
  2307. &dev_priv->mm.fence_list);
  2308. return 0;
  2309. }
  2310. } else if (enable) {
  2311. reg = i915_find_fence_reg(dev);
  2312. if (reg == NULL)
  2313. return -EDEADLK;
  2314. if (reg->obj) {
  2315. struct drm_i915_gem_object *old = reg->obj;
  2316. ret = i915_gem_object_flush_fence(old);
  2317. if (ret)
  2318. return ret;
  2319. i915_gem_object_fence_lost(old);
  2320. }
  2321. } else
  2322. return 0;
  2323. i915_gem_object_update_fence(obj, reg, enable);
  2324. obj->fence_dirty = false;
  2325. return 0;
  2326. }
  2327. static bool i915_gem_valid_gtt_space(struct drm_device *dev,
  2328. struct drm_mm_node *gtt_space,
  2329. unsigned long cache_level)
  2330. {
  2331. struct drm_mm_node *other;
  2332. /* On non-LLC machines we have to be careful when putting differing
  2333. * types of snoopable memory together to avoid the prefetcher
  2334. * crossing memory domains and dying.
  2335. */
  2336. if (HAS_LLC(dev))
  2337. return true;
  2338. if (gtt_space == NULL)
  2339. return true;
  2340. if (list_empty(&gtt_space->node_list))
  2341. return true;
  2342. other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
  2343. if (other->allocated && !other->hole_follows && other->color != cache_level)
  2344. return false;
  2345. other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
  2346. if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
  2347. return false;
  2348. return true;
  2349. }
  2350. static void i915_gem_verify_gtt(struct drm_device *dev)
  2351. {
  2352. #if WATCH_GTT
  2353. struct drm_i915_private *dev_priv = dev->dev_private;
  2354. struct drm_i915_gem_object *obj;
  2355. int err = 0;
  2356. list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) {
  2357. if (obj->gtt_space == NULL) {
  2358. printk(KERN_ERR "object found on GTT list with no space reserved\n");
  2359. err++;
  2360. continue;
  2361. }
  2362. if (obj->cache_level != obj->gtt_space->color) {
  2363. printk(KERN_ERR "object reserved space [%08lx, %08lx] with wrong color, cache_level=%x, color=%lx\n",
  2364. obj->gtt_space->start,
  2365. obj->gtt_space->start + obj->gtt_space->size,
  2366. obj->cache_level,
  2367. obj->gtt_space->color);
  2368. err++;
  2369. continue;
  2370. }
  2371. if (!i915_gem_valid_gtt_space(dev,
  2372. obj->gtt_space,
  2373. obj->cache_level)) {
  2374. printk(KERN_ERR "invalid GTT space found at [%08lx, %08lx] - color=%x\n",
  2375. obj->gtt_space->start,
  2376. obj->gtt_space->start + obj->gtt_space->size,
  2377. obj->cache_level);
  2378. err++;
  2379. continue;
  2380. }
  2381. }
  2382. WARN_ON(err);
  2383. #endif
  2384. }
  2385. /**
  2386. * Finds free space in the GTT aperture and binds the object there.
  2387. */
  2388. static int
  2389. i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
  2390. unsigned alignment,
  2391. bool map_and_fenceable,
  2392. bool nonblocking)
  2393. {
  2394. struct drm_device *dev = obj->base.dev;
  2395. drm_i915_private_t *dev_priv = dev->dev_private;
  2396. struct drm_mm_node *free_space;
  2397. u32 size, fence_size, fence_alignment, unfenced_alignment;
  2398. bool mappable, fenceable;
  2399. int ret;
  2400. if (obj->madv != I915_MADV_WILLNEED) {
  2401. DRM_ERROR("Attempting to bind a purgeable object\n");
  2402. return -EINVAL;
  2403. }
  2404. fence_size = i915_gem_get_gtt_size(dev,
  2405. obj->base.size,
  2406. obj->tiling_mode);
  2407. fence_alignment = i915_gem_get_gtt_alignment(dev,
  2408. obj->base.size,
  2409. obj->tiling_mode);
  2410. unfenced_alignment =
  2411. i915_gem_get_unfenced_gtt_alignment(dev,
  2412. obj->base.size,
  2413. obj->tiling_mode);
  2414. if (alignment == 0)
  2415. alignment = map_and_fenceable ? fence_alignment :
  2416. unfenced_alignment;
  2417. if (map_and_fenceable && alignment & (fence_alignment - 1)) {
  2418. DRM_ERROR("Invalid object alignment requested %u\n", alignment);
  2419. return -EINVAL;
  2420. }
  2421. size = map_and_fenceable ? fence_size : obj->base.size;
  2422. /* If the object is bigger than the entire aperture, reject it early
  2423. * before evicting everything in a vain attempt to find space.
  2424. */
  2425. if (obj->base.size >
  2426. (map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
  2427. DRM_ERROR("Attempting to bind an object larger than the aperture\n");
  2428. return -E2BIG;
  2429. }
  2430. ret = i915_gem_object_get_pages(obj);
  2431. if (ret)
  2432. return ret;
  2433. i915_gem_object_pin_pages(obj);
  2434. search_free:
  2435. if (map_and_fenceable)
  2436. free_space = drm_mm_search_free_in_range_color(&dev_priv->mm.gtt_space,
  2437. size, alignment, obj->cache_level,
  2438. 0, dev_priv->mm.gtt_mappable_end,
  2439. false);
  2440. else
  2441. free_space = drm_mm_search_free_color(&dev_priv->mm.gtt_space,
  2442. size, alignment, obj->cache_level,
  2443. false);
  2444. if (free_space != NULL) {
  2445. if (map_and_fenceable)
  2446. free_space =
  2447. drm_mm_get_block_range_generic(free_space,
  2448. size, alignment, obj->cache_level,
  2449. 0, dev_priv->mm.gtt_mappable_end,
  2450. false);
  2451. else
  2452. free_space =
  2453. drm_mm_get_block_generic(free_space,
  2454. size, alignment, obj->cache_level,
  2455. false);
  2456. }
  2457. if (free_space == NULL) {
  2458. ret = i915_gem_evict_something(dev, size, alignment,
  2459. obj->cache_level,
  2460. map_and_fenceable,
  2461. nonblocking);
  2462. if (ret) {
  2463. i915_gem_object_unpin_pages(obj);
  2464. return ret;
  2465. }
  2466. goto search_free;
  2467. }
  2468. if (WARN_ON(!i915_gem_valid_gtt_space(dev,
  2469. free_space,
  2470. obj->cache_level))) {
  2471. i915_gem_object_unpin_pages(obj);
  2472. drm_mm_put_block(free_space);
  2473. return -EINVAL;
  2474. }
  2475. ret = i915_gem_gtt_prepare_object(obj);
  2476. if (ret) {
  2477. i915_gem_object_unpin_pages(obj);
  2478. drm_mm_put_block(free_space);
  2479. return ret;
  2480. }
  2481. list_move_tail(&obj->gtt_list, &dev_priv->mm.bound_list);
  2482. list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
  2483. obj->gtt_space = free_space;
  2484. obj->gtt_offset = free_space->start;
  2485. fenceable =
  2486. free_space->size == fence_size &&
  2487. (free_space->start & (fence_alignment - 1)) == 0;
  2488. mappable =
  2489. obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
  2490. obj->map_and_fenceable = mappable && fenceable;
  2491. i915_gem_object_unpin_pages(obj);
  2492. trace_i915_gem_object_bind(obj, map_and_fenceable);
  2493. i915_gem_verify_gtt(dev);
  2494. return 0;
  2495. }
  2496. void
  2497. i915_gem_clflush_object(struct drm_i915_gem_object *obj)
  2498. {
  2499. /* If we don't have a page list set up, then we're not pinned
  2500. * to GPU, and we can ignore the cache flush because it'll happen
  2501. * again at bind time.
  2502. */
  2503. if (obj->pages == NULL)
  2504. return;
  2505. /* If the GPU is snooping the contents of the CPU cache,
  2506. * we do not need to manually clear the CPU cache lines. However,
  2507. * the caches are only snooped when the render cache is
  2508. * flushed/invalidated. As we always have to emit invalidations
  2509. * and flushes when moving into and out of the RENDER domain, correct
  2510. * snooping behaviour occurs naturally as the result of our domain
  2511. * tracking.
  2512. */
  2513. if (obj->cache_level != I915_CACHE_NONE)
  2514. return;
  2515. trace_i915_gem_object_clflush(obj);
  2516. drm_clflush_sg(obj->pages);
  2517. }
  2518. /** Flushes the GTT write domain for the object if it's dirty. */
  2519. static void
  2520. i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
  2521. {
  2522. uint32_t old_write_domain;
  2523. if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
  2524. return;
  2525. /* No actual flushing is required for the GTT write domain. Writes
  2526. * to it immediately go to main memory as far as we know, so there's
  2527. * no chipset flush. It also doesn't land in render cache.
  2528. *
  2529. * However, we do have to enforce the order so that all writes through
  2530. * the GTT land before any writes to the device, such as updates to
  2531. * the GATT itself.
  2532. */
  2533. wmb();
  2534. old_write_domain = obj->base.write_domain;
  2535. obj->base.write_domain = 0;
  2536. trace_i915_gem_object_change_domain(obj,
  2537. obj->base.read_domains,
  2538. old_write_domain);
  2539. }
  2540. /** Flushes the CPU write domain for the object if it's dirty. */
  2541. static void
  2542. i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
  2543. {
  2544. uint32_t old_write_domain;
  2545. if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
  2546. return;
  2547. i915_gem_clflush_object(obj);
  2548. i915_gem_chipset_flush(obj->base.dev);
  2549. old_write_domain = obj->base.write_domain;
  2550. obj->base.write_domain = 0;
  2551. trace_i915_gem_object_change_domain(obj,
  2552. obj->base.read_domains,
  2553. old_write_domain);
  2554. }
  2555. /**
  2556. * Moves a single object to the GTT read, and possibly write domain.
  2557. *
  2558. * This function returns when the move is complete, including waiting on
  2559. * flushes to occur.
  2560. */
  2561. int
  2562. i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
  2563. {
  2564. drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
  2565. uint32_t old_write_domain, old_read_domains;
  2566. int ret;
  2567. /* Not valid to be called on unbound objects. */
  2568. if (obj->gtt_space == NULL)
  2569. return -EINVAL;
  2570. if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
  2571. return 0;
  2572. ret = i915_gem_object_wait_rendering(obj, !write);
  2573. if (ret)
  2574. return ret;
  2575. i915_gem_object_flush_cpu_write_domain(obj);
  2576. old_write_domain = obj->base.write_domain;
  2577. old_read_domains = obj->base.read_domains;
  2578. /* It should now be out of any other write domains, and we can update
  2579. * the domain values for our changes.
  2580. */
  2581. BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
  2582. obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
  2583. if (write) {
  2584. obj->base.read_domains = I915_GEM_DOMAIN_GTT;
  2585. obj->base.write_domain = I915_GEM_DOMAIN_GTT;
  2586. obj->dirty = 1;
  2587. }
  2588. trace_i915_gem_object_change_domain(obj,
  2589. old_read_domains,
  2590. old_write_domain);
  2591. /* And bump the LRU for this access */
  2592. if (i915_gem_object_is_inactive(obj))
  2593. list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
  2594. return 0;
  2595. }
  2596. int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
  2597. enum i915_cache_level cache_level)
  2598. {
  2599. struct drm_device *dev = obj->base.dev;
  2600. drm_i915_private_t *dev_priv = dev->dev_private;
  2601. int ret;
  2602. if (obj->cache_level == cache_level)
  2603. return 0;
  2604. if (obj->pin_count) {
  2605. DRM_DEBUG("can not change the cache level of pinned objects\n");
  2606. return -EBUSY;
  2607. }
  2608. if (!i915_gem_valid_gtt_space(dev, obj->gtt_space, cache_level)) {
  2609. ret = i915_gem_object_unbind(obj);
  2610. if (ret)
  2611. return ret;
  2612. }
  2613. if (obj->gtt_space) {
  2614. ret = i915_gem_object_finish_gpu(obj);
  2615. if (ret)
  2616. return ret;
  2617. i915_gem_object_finish_gtt(obj);
  2618. /* Before SandyBridge, you could not use tiling or fence
  2619. * registers with snooped memory, so relinquish any fences
  2620. * currently pointing to our region in the aperture.
  2621. */
  2622. if (INTEL_INFO(dev)->gen < 6) {
  2623. ret = i915_gem_object_put_fence(obj);
  2624. if (ret)
  2625. return ret;
  2626. }
  2627. if (obj->has_global_gtt_mapping)
  2628. i915_gem_gtt_bind_object(obj, cache_level);
  2629. if (obj->has_aliasing_ppgtt_mapping)
  2630. i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
  2631. obj, cache_level);
  2632. obj->gtt_space->color = cache_level;
  2633. }
  2634. if (cache_level == I915_CACHE_NONE) {
  2635. u32 old_read_domains, old_write_domain;
  2636. /* If we're coming from LLC cached, then we haven't
  2637. * actually been tracking whether the data is in the
  2638. * CPU cache or not, since we only allow one bit set
  2639. * in obj->write_domain and have been skipping the clflushes.
  2640. * Just set it to the CPU cache for now.
  2641. */
  2642. WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
  2643. WARN_ON(obj->base.read_domains & ~I915_GEM_DOMAIN_CPU);
  2644. old_read_domains = obj->base.read_domains;
  2645. old_write_domain = obj->base.write_domain;
  2646. obj->base.read_domains = I915_GEM_DOMAIN_CPU;
  2647. obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  2648. trace_i915_gem_object_change_domain(obj,
  2649. old_read_domains,
  2650. old_write_domain);
  2651. }
  2652. obj->cache_level = cache_level;
  2653. i915_gem_verify_gtt(dev);
  2654. return 0;
  2655. }
  2656. int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
  2657. struct drm_file *file)
  2658. {
  2659. struct drm_i915_gem_caching *args = data;
  2660. struct drm_i915_gem_object *obj;
  2661. int ret;
  2662. ret = i915_mutex_lock_interruptible(dev);
  2663. if (ret)
  2664. return ret;
  2665. obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  2666. if (&obj->base == NULL) {
  2667. ret = -ENOENT;
  2668. goto unlock;
  2669. }
  2670. args->caching = obj->cache_level != I915_CACHE_NONE;
  2671. drm_gem_object_unreference(&obj->base);
  2672. unlock:
  2673. mutex_unlock(&dev->struct_mutex);
  2674. return ret;
  2675. }
  2676. int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
  2677. struct drm_file *file)
  2678. {
  2679. struct drm_i915_gem_caching *args = data;
  2680. struct drm_i915_gem_object *obj;
  2681. enum i915_cache_level level;
  2682. int ret;
  2683. switch (args->caching) {
  2684. case I915_CACHING_NONE:
  2685. level = I915_CACHE_NONE;
  2686. break;
  2687. case I915_CACHING_CACHED:
  2688. level = I915_CACHE_LLC;
  2689. break;
  2690. default:
  2691. return -EINVAL;
  2692. }
  2693. ret = i915_mutex_lock_interruptible(dev);
  2694. if (ret)
  2695. return ret;
  2696. obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  2697. if (&obj->base == NULL) {
  2698. ret = -ENOENT;
  2699. goto unlock;
  2700. }
  2701. ret = i915_gem_object_set_cache_level(obj, level);
  2702. drm_gem_object_unreference(&obj->base);
  2703. unlock:
  2704. mutex_unlock(&dev->struct_mutex);
  2705. return ret;
  2706. }
  2707. /*
  2708. * Prepare buffer for display plane (scanout, cursors, etc).
  2709. * Can be called from an uninterruptible phase (modesetting) and allows
  2710. * any flushes to be pipelined (for pageflips).
  2711. */
  2712. int
  2713. i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
  2714. u32 alignment,
  2715. struct intel_ring_buffer *pipelined)
  2716. {
  2717. u32 old_read_domains, old_write_domain;
  2718. int ret;
  2719. if (pipelined != obj->ring) {
  2720. ret = i915_gem_object_sync(obj, pipelined);
  2721. if (ret)
  2722. return ret;
  2723. }
  2724. /* The display engine is not coherent with the LLC cache on gen6. As
  2725. * a result, we make sure that the pinning that is about to occur is
  2726. * done with uncached PTEs. This is lowest common denominator for all
  2727. * chipsets.
  2728. *
  2729. * However for gen6+, we could do better by using the GFDT bit instead
  2730. * of uncaching, which would allow us to flush all the LLC-cached data
  2731. * with that bit in the PTE to main memory with just one PIPE_CONTROL.
  2732. */
  2733. ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
  2734. if (ret)
  2735. return ret;
  2736. /* As the user may map the buffer once pinned in the display plane
  2737. * (e.g. libkms for the bootup splash), we have to ensure that we
  2738. * always use map_and_fenceable for all scanout buffers.
  2739. */
  2740. ret = i915_gem_object_pin(obj, alignment, true, false);
  2741. if (ret)
  2742. return ret;
  2743. i915_gem_object_flush_cpu_write_domain(obj);
  2744. old_write_domain = obj->base.write_domain;
  2745. old_read_domains = obj->base.read_domains;
  2746. /* It should now be out of any other write domains, and we can update
  2747. * the domain values for our changes.
  2748. */
  2749. obj->base.write_domain = 0;
  2750. obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
  2751. trace_i915_gem_object_change_domain(obj,
  2752. old_read_domains,
  2753. old_write_domain);
  2754. return 0;
  2755. }
  2756. int
  2757. i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
  2758. {
  2759. int ret;
  2760. if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
  2761. return 0;
  2762. ret = i915_gem_object_wait_rendering(obj, false);
  2763. if (ret)
  2764. return ret;
  2765. /* Ensure that we invalidate the GPU's caches and TLBs. */
  2766. obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
  2767. return 0;
  2768. }
  2769. /**
  2770. * Moves a single object to the CPU read, and possibly write domain.
  2771. *
  2772. * This function returns when the move is complete, including waiting on
  2773. * flushes to occur.
  2774. */
  2775. int
  2776. i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
  2777. {
  2778. uint32_t old_write_domain, old_read_domains;
  2779. int ret;
  2780. if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
  2781. return 0;
  2782. ret = i915_gem_object_wait_rendering(obj, !write);
  2783. if (ret)
  2784. return ret;
  2785. i915_gem_object_flush_gtt_write_domain(obj);
  2786. old_write_domain = obj->base.write_domain;
  2787. old_read_domains = obj->base.read_domains;
  2788. /* Flush the CPU cache if it's still invalid. */
  2789. if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
  2790. i915_gem_clflush_object(obj);
  2791. obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
  2792. }
  2793. /* It should now be out of any other write domains, and we can update
  2794. * the domain values for our changes.
  2795. */
  2796. BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
  2797. /* If we're writing through the CPU, then the GPU read domains will
  2798. * need to be invalidated at next use.
  2799. */
  2800. if (write) {
  2801. obj->base.read_domains = I915_GEM_DOMAIN_CPU;
  2802. obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  2803. }
  2804. trace_i915_gem_object_change_domain(obj,
  2805. old_read_domains,
  2806. old_write_domain);
  2807. return 0;
  2808. }
  2809. /* Throttle our rendering by waiting until the ring has completed our requests
  2810. * emitted over 20 msec ago.
  2811. *
  2812. * Note that if we were to use the current jiffies each time around the loop,
  2813. * we wouldn't escape the function with any frames outstanding if the time to
  2814. * render a frame was over 20ms.
  2815. *
  2816. * This should get us reasonable parallelism between CPU and GPU but also
  2817. * relatively low latency when blocking on a particular request to finish.
  2818. */
  2819. static int
  2820. i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
  2821. {
  2822. struct drm_i915_private *dev_priv = dev->dev_private;
  2823. struct drm_i915_file_private *file_priv = file->driver_priv;
  2824. unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
  2825. struct drm_i915_gem_request *request;
  2826. struct intel_ring_buffer *ring = NULL;
  2827. u32 seqno = 0;
  2828. int ret;
  2829. if (atomic_read(&dev_priv->mm.wedged))
  2830. return -EIO;
  2831. spin_lock(&file_priv->mm.lock);
  2832. list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
  2833. if (time_after_eq(request->emitted_jiffies, recent_enough))
  2834. break;
  2835. ring = request->ring;
  2836. seqno = request->seqno;
  2837. }
  2838. spin_unlock(&file_priv->mm.lock);
  2839. if (seqno == 0)
  2840. return 0;
  2841. ret = __wait_seqno(ring, seqno, true, NULL);
  2842. if (ret == 0)
  2843. queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
  2844. return ret;
  2845. }
  2846. int
  2847. i915_gem_object_pin(struct drm_i915_gem_object *obj,
  2848. uint32_t alignment,
  2849. bool map_and_fenceable,
  2850. bool nonblocking)
  2851. {
  2852. int ret;
  2853. if (WARN_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
  2854. return -EBUSY;
  2855. if (obj->gtt_space != NULL) {
  2856. if ((alignment && obj->gtt_offset & (alignment - 1)) ||
  2857. (map_and_fenceable && !obj->map_and_fenceable)) {
  2858. WARN(obj->pin_count,
  2859. "bo is already pinned with incorrect alignment:"
  2860. " offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
  2861. " obj->map_and_fenceable=%d\n",
  2862. obj->gtt_offset, alignment,
  2863. map_and_fenceable,
  2864. obj->map_and_fenceable);
  2865. ret = i915_gem_object_unbind(obj);
  2866. if (ret)
  2867. return ret;
  2868. }
  2869. }
  2870. if (obj->gtt_space == NULL) {
  2871. struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  2872. ret = i915_gem_object_bind_to_gtt(obj, alignment,
  2873. map_and_fenceable,
  2874. nonblocking);
  2875. if (ret)
  2876. return ret;
  2877. if (!dev_priv->mm.aliasing_ppgtt)
  2878. i915_gem_gtt_bind_object(obj, obj->cache_level);
  2879. }
  2880. if (!obj->has_global_gtt_mapping && map_and_fenceable)
  2881. i915_gem_gtt_bind_object(obj, obj->cache_level);
  2882. obj->pin_count++;
  2883. obj->pin_mappable |= map_and_fenceable;
  2884. return 0;
  2885. }
  2886. void
  2887. i915_gem_object_unpin(struct drm_i915_gem_object *obj)
  2888. {
  2889. BUG_ON(obj->pin_count == 0);
  2890. BUG_ON(obj->gtt_space == NULL);
  2891. if (--obj->pin_count == 0)
  2892. obj->pin_mappable = false;
  2893. }
  2894. int
  2895. i915_gem_pin_ioctl(struct drm_device *dev, void *data,
  2896. struct drm_file *file)
  2897. {
  2898. struct drm_i915_gem_pin *args = data;
  2899. struct drm_i915_gem_object *obj;
  2900. int ret;
  2901. ret = i915_mutex_lock_interruptible(dev);
  2902. if (ret)
  2903. return ret;
  2904. obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  2905. if (&obj->base == NULL) {
  2906. ret = -ENOENT;
  2907. goto unlock;
  2908. }
  2909. if (obj->madv != I915_MADV_WILLNEED) {
  2910. DRM_ERROR("Attempting to pin a purgeable buffer\n");
  2911. ret = -EINVAL;
  2912. goto out;
  2913. }
  2914. if (obj->pin_filp != NULL && obj->pin_filp != file) {
  2915. DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
  2916. args->handle);
  2917. ret = -EINVAL;
  2918. goto out;
  2919. }
  2920. obj->user_pin_count++;
  2921. obj->pin_filp = file;
  2922. if (obj->user_pin_count == 1) {
  2923. ret = i915_gem_object_pin(obj, args->alignment, true, false);
  2924. if (ret)
  2925. goto out;
  2926. }
  2927. /* XXX - flush the CPU caches for pinned objects
  2928. * as the X server doesn't manage domains yet
  2929. */
  2930. i915_gem_object_flush_cpu_write_domain(obj);
  2931. args->offset = obj->gtt_offset;
  2932. out:
  2933. drm_gem_object_unreference(&obj->base);
  2934. unlock:
  2935. mutex_unlock(&dev->struct_mutex);
  2936. return ret;
  2937. }
  2938. int
  2939. i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
  2940. struct drm_file *file)
  2941. {
  2942. struct drm_i915_gem_pin *args = data;
  2943. struct drm_i915_gem_object *obj;
  2944. int ret;
  2945. ret = i915_mutex_lock_interruptible(dev);
  2946. if (ret)
  2947. return ret;
  2948. obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  2949. if (&obj->base == NULL) {
  2950. ret = -ENOENT;
  2951. goto unlock;
  2952. }
  2953. if (obj->pin_filp != file) {
  2954. DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
  2955. args->handle);
  2956. ret = -EINVAL;
  2957. goto out;
  2958. }
  2959. obj->user_pin_count--;
  2960. if (obj->user_pin_count == 0) {
  2961. obj->pin_filp = NULL;
  2962. i915_gem_object_unpin(obj);
  2963. }
  2964. out:
  2965. drm_gem_object_unreference(&obj->base);
  2966. unlock:
  2967. mutex_unlock(&dev->struct_mutex);
  2968. return ret;
  2969. }
  2970. int
  2971. i915_gem_busy_ioctl(struct drm_device *dev, void *data,
  2972. struct drm_file *file)
  2973. {
  2974. struct drm_i915_gem_busy *args = data;
  2975. struct drm_i915_gem_object *obj;
  2976. int ret;
  2977. ret = i915_mutex_lock_interruptible(dev);
  2978. if (ret)
  2979. return ret;
  2980. obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  2981. if (&obj->base == NULL) {
  2982. ret = -ENOENT;
  2983. goto unlock;
  2984. }
  2985. /* Count all active objects as busy, even if they are currently not used
  2986. * by the gpu. Users of this interface expect objects to eventually
  2987. * become non-busy without any further actions, therefore emit any
  2988. * necessary flushes here.
  2989. */
  2990. ret = i915_gem_object_flush_active(obj);
  2991. args->busy = obj->active;
  2992. if (obj->ring) {
  2993. BUILD_BUG_ON(I915_NUM_RINGS > 16);
  2994. args->busy |= intel_ring_flag(obj->ring) << 16;
  2995. }
  2996. drm_gem_object_unreference(&obj->base);
  2997. unlock:
  2998. mutex_unlock(&dev->struct_mutex);
  2999. return ret;
  3000. }
  3001. int
  3002. i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
  3003. struct drm_file *file_priv)
  3004. {
  3005. return i915_gem_ring_throttle(dev, file_priv);
  3006. }
  3007. int
  3008. i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
  3009. struct drm_file *file_priv)
  3010. {
  3011. struct drm_i915_gem_madvise *args = data;
  3012. struct drm_i915_gem_object *obj;
  3013. int ret;
  3014. switch (args->madv) {
  3015. case I915_MADV_DONTNEED:
  3016. case I915_MADV_WILLNEED:
  3017. break;
  3018. default:
  3019. return -EINVAL;
  3020. }
  3021. ret = i915_mutex_lock_interruptible(dev);
  3022. if (ret)
  3023. return ret;
  3024. obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
  3025. if (&obj->base == NULL) {
  3026. ret = -ENOENT;
  3027. goto unlock;
  3028. }
  3029. if (obj->pin_count) {
  3030. ret = -EINVAL;
  3031. goto out;
  3032. }
  3033. if (obj->madv != __I915_MADV_PURGED)
  3034. obj->madv = args->madv;
  3035. /* if the object is no longer attached, discard its backing storage */
  3036. if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL)
  3037. i915_gem_object_truncate(obj);
  3038. args->retained = obj->madv != __I915_MADV_PURGED;
  3039. out:
  3040. drm_gem_object_unreference(&obj->base);
  3041. unlock:
  3042. mutex_unlock(&dev->struct_mutex);
  3043. return ret;
  3044. }
  3045. void i915_gem_object_init(struct drm_i915_gem_object *obj,
  3046. const struct drm_i915_gem_object_ops *ops)
  3047. {
  3048. INIT_LIST_HEAD(&obj->mm_list);
  3049. INIT_LIST_HEAD(&obj->gtt_list);
  3050. INIT_LIST_HEAD(&obj->ring_list);
  3051. INIT_LIST_HEAD(&obj->exec_list);
  3052. obj->ops = ops;
  3053. obj->fence_reg = I915_FENCE_REG_NONE;
  3054. obj->madv = I915_MADV_WILLNEED;
  3055. /* Avoid an unnecessary call to unbind on the first bind. */
  3056. obj->map_and_fenceable = true;
  3057. i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
  3058. }
  3059. static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
  3060. .get_pages = i915_gem_object_get_pages_gtt,
  3061. .put_pages = i915_gem_object_put_pages_gtt,
  3062. };
  3063. struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
  3064. size_t size)
  3065. {
  3066. struct drm_i915_gem_object *obj;
  3067. struct address_space *mapping;
  3068. gfp_t mask;
  3069. obj = i915_gem_object_alloc(dev);
  3070. if (obj == NULL)
  3071. return NULL;
  3072. if (drm_gem_object_init(dev, &obj->base, size) != 0) {
  3073. i915_gem_object_free(obj);
  3074. return NULL;
  3075. }
  3076. mask = GFP_HIGHUSER | __GFP_RECLAIMABLE;
  3077. if (IS_CRESTLINE(dev) || IS_BROADWATER(dev)) {
  3078. /* 965gm cannot relocate objects above 4GiB. */
  3079. mask &= ~__GFP_HIGHMEM;
  3080. mask |= __GFP_DMA32;
  3081. }
  3082. mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
  3083. mapping_set_gfp_mask(mapping, mask);
  3084. i915_gem_object_init(obj, &i915_gem_object_ops);
  3085. obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  3086. obj->base.read_domains = I915_GEM_DOMAIN_CPU;
  3087. if (HAS_LLC(dev)) {
  3088. /* On some devices, we can have the GPU use the LLC (the CPU
  3089. * cache) for about a 10% performance improvement
  3090. * compared to uncached. Graphics requests other than
  3091. * display scanout are coherent with the CPU in
  3092. * accessing this cache. This means in this mode we
  3093. * don't need to clflush on the CPU side, and on the
  3094. * GPU side we only need to flush internal caches to
  3095. * get data visible to the CPU.
  3096. *
  3097. * However, we maintain the display planes as UC, and so
  3098. * need to rebind when first used as such.
  3099. */
  3100. obj->cache_level = I915_CACHE_LLC;
  3101. } else
  3102. obj->cache_level = I915_CACHE_NONE;
  3103. return obj;
  3104. }
  3105. int i915_gem_init_object(struct drm_gem_object *obj)
  3106. {
  3107. BUG();
  3108. return 0;
  3109. }
  3110. void i915_gem_free_object(struct drm_gem_object *gem_obj)
  3111. {
  3112. struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
  3113. struct drm_device *dev = obj->base.dev;
  3114. drm_i915_private_t *dev_priv = dev->dev_private;
  3115. trace_i915_gem_object_destroy(obj);
  3116. if (obj->phys_obj)
  3117. i915_gem_detach_phys_object(dev, obj);
  3118. obj->pin_count = 0;
  3119. if (WARN_ON(i915_gem_object_unbind(obj) == -ERESTARTSYS)) {
  3120. bool was_interruptible;
  3121. was_interruptible = dev_priv->mm.interruptible;
  3122. dev_priv->mm.interruptible = false;
  3123. WARN_ON(i915_gem_object_unbind(obj));
  3124. dev_priv->mm.interruptible = was_interruptible;
  3125. }
  3126. obj->pages_pin_count = 0;
  3127. i915_gem_object_put_pages(obj);
  3128. i915_gem_object_free_mmap_offset(obj);
  3129. i915_gem_object_release_stolen(obj);
  3130. BUG_ON(obj->pages);
  3131. if (obj->base.import_attach)
  3132. drm_prime_gem_destroy(&obj->base, NULL);
  3133. drm_gem_object_release(&obj->base);
  3134. i915_gem_info_remove_obj(dev_priv, obj->base.size);
  3135. kfree(obj->bit_17);
  3136. i915_gem_object_free(obj);
  3137. }
  3138. int
  3139. i915_gem_idle(struct drm_device *dev)
  3140. {
  3141. drm_i915_private_t *dev_priv = dev->dev_private;
  3142. int ret;
  3143. mutex_lock(&dev->struct_mutex);
  3144. if (dev_priv->mm.suspended) {
  3145. mutex_unlock(&dev->struct_mutex);
  3146. return 0;
  3147. }
  3148. ret = i915_gpu_idle(dev);
  3149. if (ret) {
  3150. mutex_unlock(&dev->struct_mutex);
  3151. return ret;
  3152. }
  3153. i915_gem_retire_requests(dev);
  3154. /* Under UMS, be paranoid and evict. */
  3155. if (!drm_core_check_feature(dev, DRIVER_MODESET))
  3156. i915_gem_evict_everything(dev);
  3157. i915_gem_reset_fences(dev);
  3158. /* Hack! Don't let anybody do execbuf while we don't control the chip.
  3159. * We need to replace this with a semaphore, or something.
  3160. * And not confound mm.suspended!
  3161. */
  3162. dev_priv->mm.suspended = 1;
  3163. del_timer_sync(&dev_priv->hangcheck_timer);
  3164. i915_kernel_lost_context(dev);
  3165. i915_gem_cleanup_ringbuffer(dev);
  3166. mutex_unlock(&dev->struct_mutex);
  3167. /* Cancel the retire work handler, which should be idle now. */
  3168. cancel_delayed_work_sync(&dev_priv->mm.retire_work);
  3169. return 0;
  3170. }
  3171. void i915_gem_l3_remap(struct drm_device *dev)
  3172. {
  3173. drm_i915_private_t *dev_priv = dev->dev_private;
  3174. u32 misccpctl;
  3175. int i;
  3176. if (!IS_IVYBRIDGE(dev))
  3177. return;
  3178. if (!dev_priv->l3_parity.remap_info)
  3179. return;
  3180. misccpctl = I915_READ(GEN7_MISCCPCTL);
  3181. I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
  3182. POSTING_READ(GEN7_MISCCPCTL);
  3183. for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
  3184. u32 remap = I915_READ(GEN7_L3LOG_BASE + i);
  3185. if (remap && remap != dev_priv->l3_parity.remap_info[i/4])
  3186. DRM_DEBUG("0x%x was already programmed to %x\n",
  3187. GEN7_L3LOG_BASE + i, remap);
  3188. if (remap && !dev_priv->l3_parity.remap_info[i/4])
  3189. DRM_DEBUG_DRIVER("Clearing remapped register\n");
  3190. I915_WRITE(GEN7_L3LOG_BASE + i, dev_priv->l3_parity.remap_info[i/4]);
  3191. }
  3192. /* Make sure all the writes land before disabling dop clock gating */
  3193. POSTING_READ(GEN7_L3LOG_BASE);
  3194. I915_WRITE(GEN7_MISCCPCTL, misccpctl);
  3195. }
  3196. void i915_gem_init_swizzling(struct drm_device *dev)
  3197. {
  3198. drm_i915_private_t *dev_priv = dev->dev_private;
  3199. if (INTEL_INFO(dev)->gen < 5 ||
  3200. dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
  3201. return;
  3202. I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
  3203. DISP_TILE_SURFACE_SWIZZLING);
  3204. if (IS_GEN5(dev))
  3205. return;
  3206. I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
  3207. if (IS_GEN6(dev))
  3208. I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
  3209. else
  3210. I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
  3211. }
  3212. static bool
  3213. intel_enable_blt(struct drm_device *dev)
  3214. {
  3215. if (!HAS_BLT(dev))
  3216. return false;
  3217. /* The blitter was dysfunctional on early prototypes */
  3218. if (IS_GEN6(dev) && dev->pdev->revision < 8) {
  3219. DRM_INFO("BLT not supported on this pre-production hardware;"
  3220. " graphics performance will be degraded.\n");
  3221. return false;
  3222. }
  3223. return true;
  3224. }
  3225. int
  3226. i915_gem_init_hw(struct drm_device *dev)
  3227. {
  3228. drm_i915_private_t *dev_priv = dev->dev_private;
  3229. int ret;
  3230. if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt())
  3231. return -EIO;
  3232. if (IS_HASWELL(dev) && (I915_READ(0x120010) == 1))
  3233. I915_WRITE(0x9008, I915_READ(0x9008) | 0xf0000);
  3234. i915_gem_l3_remap(dev);
  3235. i915_gem_init_swizzling(dev);
  3236. ret = intel_init_render_ring_buffer(dev);
  3237. if (ret)
  3238. return ret;
  3239. if (HAS_BSD(dev)) {
  3240. ret = intel_init_bsd_ring_buffer(dev);
  3241. if (ret)
  3242. goto cleanup_render_ring;
  3243. }
  3244. if (intel_enable_blt(dev)) {
  3245. ret = intel_init_blt_ring_buffer(dev);
  3246. if (ret)
  3247. goto cleanup_bsd_ring;
  3248. }
  3249. dev_priv->next_seqno = 1;
  3250. /*
  3251. * XXX: There was some w/a described somewhere suggesting loading
  3252. * contexts before PPGTT.
  3253. */
  3254. i915_gem_context_init(dev);
  3255. i915_gem_init_ppgtt(dev);
  3256. return 0;
  3257. cleanup_bsd_ring:
  3258. intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
  3259. cleanup_render_ring:
  3260. intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
  3261. return ret;
  3262. }
  3263. static bool
  3264. intel_enable_ppgtt(struct drm_device *dev)
  3265. {
  3266. if (i915_enable_ppgtt >= 0)
  3267. return i915_enable_ppgtt;
  3268. #ifdef CONFIG_INTEL_IOMMU
  3269. /* Disable ppgtt on SNB if VT-d is on. */
  3270. if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped)
  3271. return false;
  3272. #endif
  3273. return true;
  3274. }
  3275. int i915_gem_init(struct drm_device *dev)
  3276. {
  3277. struct drm_i915_private *dev_priv = dev->dev_private;
  3278. unsigned long gtt_size, mappable_size;
  3279. int ret;
  3280. gtt_size = dev_priv->mm.gtt->gtt_total_entries << PAGE_SHIFT;
  3281. mappable_size = dev_priv->mm.gtt->gtt_mappable_entries << PAGE_SHIFT;
  3282. mutex_lock(&dev->struct_mutex);
  3283. if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) {
  3284. /* PPGTT pdes are stolen from global gtt ptes, so shrink the
  3285. * aperture accordingly when using aliasing ppgtt. */
  3286. gtt_size -= I915_PPGTT_PD_ENTRIES*PAGE_SIZE;
  3287. i915_gem_init_global_gtt(dev, 0, mappable_size, gtt_size);
  3288. ret = i915_gem_init_aliasing_ppgtt(dev);
  3289. if (ret) {
  3290. mutex_unlock(&dev->struct_mutex);
  3291. return ret;
  3292. }
  3293. } else {
  3294. /* Let GEM Manage all of the aperture.
  3295. *
  3296. * However, leave one page at the end still bound to the scratch
  3297. * page. There are a number of places where the hardware
  3298. * apparently prefetches past the end of the object, and we've
  3299. * seen multiple hangs with the GPU head pointer stuck in a
  3300. * batchbuffer bound at the last page of the aperture. One page
  3301. * should be enough to keep any prefetching inside of the
  3302. * aperture.
  3303. */
  3304. i915_gem_init_global_gtt(dev, 0, mappable_size,
  3305. gtt_size);
  3306. }
  3307. ret = i915_gem_init_hw(dev);
  3308. mutex_unlock(&dev->struct_mutex);
  3309. if (ret) {
  3310. i915_gem_cleanup_aliasing_ppgtt(dev);
  3311. return ret;
  3312. }
  3313. /* Allow hardware batchbuffers unless told otherwise, but not for KMS. */
  3314. if (!drm_core_check_feature(dev, DRIVER_MODESET))
  3315. dev_priv->dri1.allow_batchbuffer = 1;
  3316. return 0;
  3317. }
  3318. void
  3319. i915_gem_cleanup_ringbuffer(struct drm_device *dev)
  3320. {
  3321. drm_i915_private_t *dev_priv = dev->dev_private;
  3322. struct intel_ring_buffer *ring;
  3323. int i;
  3324. for_each_ring(ring, dev_priv, i)
  3325. intel_cleanup_ring_buffer(ring);
  3326. }
  3327. int
  3328. i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
  3329. struct drm_file *file_priv)
  3330. {
  3331. drm_i915_private_t *dev_priv = dev->dev_private;
  3332. int ret;
  3333. if (drm_core_check_feature(dev, DRIVER_MODESET))
  3334. return 0;
  3335. if (atomic_read(&dev_priv->mm.wedged)) {
  3336. DRM_ERROR("Reenabling wedged hardware, good luck\n");
  3337. atomic_set(&dev_priv->mm.wedged, 0);
  3338. }
  3339. mutex_lock(&dev->struct_mutex);
  3340. dev_priv->mm.suspended = 0;
  3341. ret = i915_gem_init_hw(dev);
  3342. if (ret != 0) {
  3343. mutex_unlock(&dev->struct_mutex);
  3344. return ret;
  3345. }
  3346. BUG_ON(!list_empty(&dev_priv->mm.active_list));
  3347. mutex_unlock(&dev->struct_mutex);
  3348. ret = drm_irq_install(dev);
  3349. if (ret)
  3350. goto cleanup_ringbuffer;
  3351. return 0;
  3352. cleanup_ringbuffer:
  3353. mutex_lock(&dev->struct_mutex);
  3354. i915_gem_cleanup_ringbuffer(dev);
  3355. dev_priv->mm.suspended = 1;
  3356. mutex_unlock(&dev->struct_mutex);
  3357. return ret;
  3358. }
  3359. int
  3360. i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
  3361. struct drm_file *file_priv)
  3362. {
  3363. if (drm_core_check_feature(dev, DRIVER_MODESET))
  3364. return 0;
  3365. drm_irq_uninstall(dev);
  3366. return i915_gem_idle(dev);
  3367. }
  3368. void
  3369. i915_gem_lastclose(struct drm_device *dev)
  3370. {
  3371. int ret;
  3372. if (drm_core_check_feature(dev, DRIVER_MODESET))
  3373. return;
  3374. ret = i915_gem_idle(dev);
  3375. if (ret)
  3376. DRM_ERROR("failed to idle hardware: %d\n", ret);
  3377. }
  3378. static void
  3379. init_ring_lists(struct intel_ring_buffer *ring)
  3380. {
  3381. INIT_LIST_HEAD(&ring->active_list);
  3382. INIT_LIST_HEAD(&ring->request_list);
  3383. }
  3384. void
  3385. i915_gem_load(struct drm_device *dev)
  3386. {
  3387. drm_i915_private_t *dev_priv = dev->dev_private;
  3388. int i;
  3389. dev_priv->slab =
  3390. kmem_cache_create("i915_gem_object",
  3391. sizeof(struct drm_i915_gem_object), 0,
  3392. SLAB_HWCACHE_ALIGN,
  3393. NULL);
  3394. INIT_LIST_HEAD(&dev_priv->mm.active_list);
  3395. INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
  3396. INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
  3397. INIT_LIST_HEAD(&dev_priv->mm.bound_list);
  3398. INIT_LIST_HEAD(&dev_priv->mm.fence_list);
  3399. for (i = 0; i < I915_NUM_RINGS; i++)
  3400. init_ring_lists(&dev_priv->ring[i]);
  3401. for (i = 0; i < I915_MAX_NUM_FENCES; i++)
  3402. INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
  3403. INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
  3404. i915_gem_retire_work_handler);
  3405. init_completion(&dev_priv->error_completion);
  3406. /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
  3407. if (IS_GEN3(dev)) {
  3408. I915_WRITE(MI_ARB_STATE,
  3409. _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
  3410. }
  3411. dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
  3412. /* Old X drivers will take 0-2 for front, back, depth buffers */
  3413. if (!drm_core_check_feature(dev, DRIVER_MODESET))
  3414. dev_priv->fence_reg_start = 3;
  3415. if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
  3416. dev_priv->num_fence_regs = 16;
  3417. else
  3418. dev_priv->num_fence_regs = 8;
  3419. /* Initialize fence registers to zero */
  3420. i915_gem_reset_fences(dev);
  3421. i915_gem_detect_bit_6_swizzle(dev);
  3422. init_waitqueue_head(&dev_priv->pending_flip_queue);
  3423. dev_priv->mm.interruptible = true;
  3424. dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
  3425. dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
  3426. register_shrinker(&dev_priv->mm.inactive_shrinker);
  3427. }
  3428. /*
  3429. * Create a physically contiguous memory object for this object
  3430. * e.g. for cursor + overlay regs
  3431. */
  3432. static int i915_gem_init_phys_object(struct drm_device *dev,
  3433. int id, int size, int align)
  3434. {
  3435. drm_i915_private_t *dev_priv = dev->dev_private;
  3436. struct drm_i915_gem_phys_object *phys_obj;
  3437. int ret;
  3438. if (dev_priv->mm.phys_objs[id - 1] || !size)
  3439. return 0;
  3440. phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
  3441. if (!phys_obj)
  3442. return -ENOMEM;
  3443. phys_obj->id = id;
  3444. phys_obj->handle = drm_pci_alloc(dev, size, align);
  3445. if (!phys_obj->handle) {
  3446. ret = -ENOMEM;
  3447. goto kfree_obj;
  3448. }
  3449. #ifdef CONFIG_X86
  3450. set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
  3451. #endif
  3452. dev_priv->mm.phys_objs[id - 1] = phys_obj;
  3453. return 0;
  3454. kfree_obj:
  3455. kfree(phys_obj);
  3456. return ret;
  3457. }
  3458. static void i915_gem_free_phys_object(struct drm_device *dev, int id)
  3459. {
  3460. drm_i915_private_t *dev_priv = dev->dev_private;
  3461. struct drm_i915_gem_phys_object *phys_obj;
  3462. if (!dev_priv->mm.phys_objs[id - 1])
  3463. return;
  3464. phys_obj = dev_priv->mm.phys_objs[id - 1];
  3465. if (phys_obj->cur_obj) {
  3466. i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
  3467. }
  3468. #ifdef CONFIG_X86
  3469. set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
  3470. #endif
  3471. drm_pci_free(dev, phys_obj->handle);
  3472. kfree(phys_obj);
  3473. dev_priv->mm.phys_objs[id - 1] = NULL;
  3474. }
  3475. void i915_gem_free_all_phys_object(struct drm_device *dev)
  3476. {
  3477. int i;
  3478. for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
  3479. i915_gem_free_phys_object(dev, i);
  3480. }
  3481. void i915_gem_detach_phys_object(struct drm_device *dev,
  3482. struct drm_i915_gem_object *obj)
  3483. {
  3484. struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
  3485. char *vaddr;
  3486. int i;
  3487. int page_count;
  3488. if (!obj->phys_obj)
  3489. return;
  3490. vaddr = obj->phys_obj->handle->vaddr;
  3491. page_count = obj->base.size / PAGE_SIZE;
  3492. for (i = 0; i < page_count; i++) {
  3493. struct page *page = shmem_read_mapping_page(mapping, i);
  3494. if (!IS_ERR(page)) {
  3495. char *dst = kmap_atomic(page);
  3496. memcpy(dst, vaddr + i*PAGE_SIZE, PAGE_SIZE);
  3497. kunmap_atomic(dst);
  3498. drm_clflush_pages(&page, 1);
  3499. set_page_dirty(page);
  3500. mark_page_accessed(page);
  3501. page_cache_release(page);
  3502. }
  3503. }
  3504. i915_gem_chipset_flush(dev);
  3505. obj->phys_obj->cur_obj = NULL;
  3506. obj->phys_obj = NULL;
  3507. }
  3508. int
  3509. i915_gem_attach_phys_object(struct drm_device *dev,
  3510. struct drm_i915_gem_object *obj,
  3511. int id,
  3512. int align)
  3513. {
  3514. struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
  3515. drm_i915_private_t *dev_priv = dev->dev_private;
  3516. int ret = 0;
  3517. int page_count;
  3518. int i;
  3519. if (id > I915_MAX_PHYS_OBJECT)
  3520. return -EINVAL;
  3521. if (obj->phys_obj) {
  3522. if (obj->phys_obj->id == id)
  3523. return 0;
  3524. i915_gem_detach_phys_object(dev, obj);
  3525. }
  3526. /* create a new object */
  3527. if (!dev_priv->mm.phys_objs[id - 1]) {
  3528. ret = i915_gem_init_phys_object(dev, id,
  3529. obj->base.size, align);
  3530. if (ret) {
  3531. DRM_ERROR("failed to init phys object %d size: %zu\n",
  3532. id, obj->base.size);
  3533. return ret;
  3534. }
  3535. }
  3536. /* bind to the object */
  3537. obj->phys_obj = dev_priv->mm.phys_objs[id - 1];
  3538. obj->phys_obj->cur_obj = obj;
  3539. page_count = obj->base.size / PAGE_SIZE;
  3540. for (i = 0; i < page_count; i++) {
  3541. struct page *page;
  3542. char *dst, *src;
  3543. page = shmem_read_mapping_page(mapping, i);
  3544. if (IS_ERR(page))
  3545. return PTR_ERR(page);
  3546. src = kmap_atomic(page);
  3547. dst = obj->phys_obj->handle->vaddr + (i * PAGE_SIZE);
  3548. memcpy(dst, src, PAGE_SIZE);
  3549. kunmap_atomic(src);
  3550. mark_page_accessed(page);
  3551. page_cache_release(page);
  3552. }
  3553. return 0;
  3554. }
  3555. static int
  3556. i915_gem_phys_pwrite(struct drm_device *dev,
  3557. struct drm_i915_gem_object *obj,
  3558. struct drm_i915_gem_pwrite *args,
  3559. struct drm_file *file_priv)
  3560. {
  3561. void *vaddr = obj->phys_obj->handle->vaddr + args->offset;
  3562. char __user *user_data = (char __user *) (uintptr_t) args->data_ptr;
  3563. if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
  3564. unsigned long unwritten;
  3565. /* The physical object once assigned is fixed for the lifetime
  3566. * of the obj, so we can safely drop the lock and continue
  3567. * to access vaddr.
  3568. */
  3569. mutex_unlock(&dev->struct_mutex);
  3570. unwritten = copy_from_user(vaddr, user_data, args->size);
  3571. mutex_lock(&dev->struct_mutex);
  3572. if (unwritten)
  3573. return -EFAULT;
  3574. }
  3575. i915_gem_chipset_flush(dev);
  3576. return 0;
  3577. }
  3578. void i915_gem_release(struct drm_device *dev, struct drm_file *file)
  3579. {
  3580. struct drm_i915_file_private *file_priv = file->driver_priv;
  3581. /* Clean up our request list when the client is going away, so that
  3582. * later retire_requests won't dereference our soon-to-be-gone
  3583. * file_priv.
  3584. */
  3585. spin_lock(&file_priv->mm.lock);
  3586. while (!list_empty(&file_priv->mm.request_list)) {
  3587. struct drm_i915_gem_request *request;
  3588. request = list_first_entry(&file_priv->mm.request_list,
  3589. struct drm_i915_gem_request,
  3590. client_list);
  3591. list_del(&request->client_list);
  3592. request->file_priv = NULL;
  3593. }
  3594. spin_unlock(&file_priv->mm.lock);
  3595. }
  3596. static bool mutex_is_locked_by(struct mutex *mutex, struct task_struct *task)
  3597. {
  3598. if (!mutex_is_locked(mutex))
  3599. return false;
  3600. #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_MUTEXES)
  3601. return mutex->owner == task;
  3602. #else
  3603. /* Since UP may be pre-empted, we cannot assume that we own the lock */
  3604. return false;
  3605. #endif
  3606. }
  3607. static int
  3608. i915_gem_inactive_shrink(struct shrinker *shrinker, struct shrink_control *sc)
  3609. {
  3610. struct drm_i915_private *dev_priv =
  3611. container_of(shrinker,
  3612. struct drm_i915_private,
  3613. mm.inactive_shrinker);
  3614. struct drm_device *dev = dev_priv->dev;
  3615. struct drm_i915_gem_object *obj;
  3616. int nr_to_scan = sc->nr_to_scan;
  3617. bool unlock = true;
  3618. int cnt;
  3619. if (!mutex_trylock(&dev->struct_mutex)) {
  3620. if (!mutex_is_locked_by(&dev->struct_mutex, current))
  3621. return 0;
  3622. unlock = false;
  3623. }
  3624. if (nr_to_scan) {
  3625. nr_to_scan -= i915_gem_purge(dev_priv, nr_to_scan);
  3626. if (nr_to_scan > 0)
  3627. i915_gem_shrink_all(dev_priv);
  3628. }
  3629. cnt = 0;
  3630. list_for_each_entry(obj, &dev_priv->mm.unbound_list, gtt_list)
  3631. if (obj->pages_pin_count == 0)
  3632. cnt += obj->base.size >> PAGE_SHIFT;
  3633. list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list)
  3634. if (obj->pin_count == 0 && obj->pages_pin_count == 0)
  3635. cnt += obj->base.size >> PAGE_SHIFT;
  3636. if (unlock)
  3637. mutex_unlock(&dev->struct_mutex);
  3638. return cnt;
  3639. }