i915_gem.c 105 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188
  1. /*
  2. * Copyright © 2008 Intel Corporation
  3. *
  4. * Permission is hereby granted, free of charge, to any person obtaining a
  5. * copy of this software and associated documentation files (the "Software"),
  6. * to deal in the Software without restriction, including without limitation
  7. * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8. * and/or sell copies of the Software, and to permit persons to whom the
  9. * Software is furnished to do so, subject to the following conditions:
  10. *
  11. * The above copyright notice and this permission notice (including the next
  12. * paragraph) shall be included in all copies or substantial portions of the
  13. * Software.
  14. *
  15. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17. * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  18. * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19. * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20. * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21. * IN THE SOFTWARE.
  22. *
  23. * Authors:
  24. * Eric Anholt <eric@anholt.net>
  25. *
  26. */
  27. #include "drmP.h"
  28. #include "drm.h"
  29. #include "i915_drm.h"
  30. #include "i915_drv.h"
  31. #include "i915_trace.h"
  32. #include "intel_drv.h"
  33. #include <linux/shmem_fs.h>
  34. #include <linux/slab.h>
  35. #include <linux/swap.h>
  36. #include <linux/pci.h>
  37. static __must_check int i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj);
  38. static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
  39. static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
  40. static __must_check int i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,
  41. uint64_t offset,
  42. uint64_t size);
  43. static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj);
  44. static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
  45. unsigned alignment,
  46. bool map_and_fenceable);
  47. static void i915_gem_clear_fence_reg(struct drm_device *dev,
  48. struct drm_i915_fence_reg *reg);
  49. static int i915_gem_phys_pwrite(struct drm_device *dev,
  50. struct drm_i915_gem_object *obj,
  51. struct drm_i915_gem_pwrite *args,
  52. struct drm_file *file);
  53. static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj);
  54. static int i915_gem_inactive_shrink(struct shrinker *shrinker,
  55. struct shrink_control *sc);
  56. static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
  57. /* some bookkeeping */
  58. static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
  59. size_t size)
  60. {
  61. dev_priv->mm.object_count++;
  62. dev_priv->mm.object_memory += size;
  63. }
  64. static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
  65. size_t size)
  66. {
  67. dev_priv->mm.object_count--;
  68. dev_priv->mm.object_memory -= size;
  69. }
  70. static int
  71. i915_gem_wait_for_error(struct drm_device *dev)
  72. {
  73. struct drm_i915_private *dev_priv = dev->dev_private;
  74. struct completion *x = &dev_priv->error_completion;
  75. unsigned long flags;
  76. int ret;
  77. if (!atomic_read(&dev_priv->mm.wedged))
  78. return 0;
  79. ret = wait_for_completion_interruptible(x);
  80. if (ret)
  81. return ret;
  82. if (atomic_read(&dev_priv->mm.wedged)) {
  83. /* GPU is hung, bump the completion count to account for
  84. * the token we just consumed so that we never hit zero and
  85. * end up waiting upon a subsequent completion event that
  86. * will never happen.
  87. */
  88. spin_lock_irqsave(&x->wait.lock, flags);
  89. x->done++;
  90. spin_unlock_irqrestore(&x->wait.lock, flags);
  91. }
  92. return 0;
  93. }
  94. int i915_mutex_lock_interruptible(struct drm_device *dev)
  95. {
  96. int ret;
  97. ret = i915_gem_wait_for_error(dev);
  98. if (ret)
  99. return ret;
  100. ret = mutex_lock_interruptible(&dev->struct_mutex);
  101. if (ret)
  102. return ret;
  103. WARN_ON(i915_verify_lists(dev));
  104. return 0;
  105. }
  106. static inline bool
  107. i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
  108. {
  109. return obj->gtt_space && !obj->active && obj->pin_count == 0;
  110. }
  111. int
  112. i915_gem_init_ioctl(struct drm_device *dev, void *data,
  113. struct drm_file *file)
  114. {
  115. struct drm_i915_gem_init *args = data;
  116. if (args->gtt_start >= args->gtt_end ||
  117. (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
  118. return -EINVAL;
  119. mutex_lock(&dev->struct_mutex);
  120. i915_gem_init_global_gtt(dev, args->gtt_start,
  121. args->gtt_end, args->gtt_end);
  122. mutex_unlock(&dev->struct_mutex);
  123. return 0;
  124. }
  125. int
  126. i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
  127. struct drm_file *file)
  128. {
  129. struct drm_i915_private *dev_priv = dev->dev_private;
  130. struct drm_i915_gem_get_aperture *args = data;
  131. struct drm_i915_gem_object *obj;
  132. size_t pinned;
  133. if (!(dev->driver->driver_features & DRIVER_GEM))
  134. return -ENODEV;
  135. pinned = 0;
  136. mutex_lock(&dev->struct_mutex);
  137. list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list)
  138. pinned += obj->gtt_space->size;
  139. mutex_unlock(&dev->struct_mutex);
  140. args->aper_size = dev_priv->mm.gtt_total;
  141. args->aper_available_size = args->aper_size - pinned;
  142. return 0;
  143. }
  144. static int
  145. i915_gem_create(struct drm_file *file,
  146. struct drm_device *dev,
  147. uint64_t size,
  148. uint32_t *handle_p)
  149. {
  150. struct drm_i915_gem_object *obj;
  151. int ret;
  152. u32 handle;
  153. size = roundup(size, PAGE_SIZE);
  154. if (size == 0)
  155. return -EINVAL;
  156. /* Allocate the new object */
  157. obj = i915_gem_alloc_object(dev, size);
  158. if (obj == NULL)
  159. return -ENOMEM;
  160. ret = drm_gem_handle_create(file, &obj->base, &handle);
  161. if (ret) {
  162. drm_gem_object_release(&obj->base);
  163. i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
  164. kfree(obj);
  165. return ret;
  166. }
  167. /* drop reference from allocate - handle holds it now */
  168. drm_gem_object_unreference(&obj->base);
  169. trace_i915_gem_object_create(obj);
  170. *handle_p = handle;
  171. return 0;
  172. }
  173. int
  174. i915_gem_dumb_create(struct drm_file *file,
  175. struct drm_device *dev,
  176. struct drm_mode_create_dumb *args)
  177. {
  178. /* have to work out size/pitch and return them */
  179. args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
  180. args->size = args->pitch * args->height;
  181. return i915_gem_create(file, dev,
  182. args->size, &args->handle);
  183. }
  184. int i915_gem_dumb_destroy(struct drm_file *file,
  185. struct drm_device *dev,
  186. uint32_t handle)
  187. {
  188. return drm_gem_handle_delete(file, handle);
  189. }
  190. /**
  191. * Creates a new mm object and returns a handle to it.
  192. */
  193. int
  194. i915_gem_create_ioctl(struct drm_device *dev, void *data,
  195. struct drm_file *file)
  196. {
  197. struct drm_i915_gem_create *args = data;
  198. return i915_gem_create(file, dev,
  199. args->size, &args->handle);
  200. }
  201. static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
  202. {
  203. drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
  204. return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
  205. obj->tiling_mode != I915_TILING_NONE;
  206. }
  207. /**
  208. * This is the fast shmem pread path, which attempts to copy_from_user directly
  209. * from the backing pages of the object to the user's address space. On a
  210. * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
  211. */
  212. static int
  213. i915_gem_shmem_pread_fast(struct drm_device *dev,
  214. struct drm_i915_gem_object *obj,
  215. struct drm_i915_gem_pread *args,
  216. struct drm_file *file)
  217. {
  218. struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
  219. ssize_t remain;
  220. loff_t offset;
  221. char __user *user_data;
  222. int page_offset, page_length;
  223. user_data = (char __user *) (uintptr_t) args->data_ptr;
  224. remain = args->size;
  225. offset = args->offset;
  226. while (remain > 0) {
  227. struct page *page;
  228. char *vaddr;
  229. int ret;
  230. /* Operation in this page
  231. *
  232. * page_offset = offset within page
  233. * page_length = bytes to copy for this page
  234. */
  235. page_offset = offset_in_page(offset);
  236. page_length = remain;
  237. if ((page_offset + remain) > PAGE_SIZE)
  238. page_length = PAGE_SIZE - page_offset;
  239. page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
  240. if (IS_ERR(page))
  241. return PTR_ERR(page);
  242. vaddr = kmap_atomic(page);
  243. ret = __copy_to_user_inatomic(user_data,
  244. vaddr + page_offset,
  245. page_length);
  246. kunmap_atomic(vaddr);
  247. mark_page_accessed(page);
  248. page_cache_release(page);
  249. if (ret)
  250. return -EFAULT;
  251. remain -= page_length;
  252. user_data += page_length;
  253. offset += page_length;
  254. }
  255. return 0;
  256. }
  257. static inline int
  258. __copy_to_user_swizzled(char __user *cpu_vaddr,
  259. const char *gpu_vaddr, int gpu_offset,
  260. int length)
  261. {
  262. int ret, cpu_offset = 0;
  263. while (length > 0) {
  264. int cacheline_end = ALIGN(gpu_offset + 1, 64);
  265. int this_length = min(cacheline_end - gpu_offset, length);
  266. int swizzled_gpu_offset = gpu_offset ^ 64;
  267. ret = __copy_to_user(cpu_vaddr + cpu_offset,
  268. gpu_vaddr + swizzled_gpu_offset,
  269. this_length);
  270. if (ret)
  271. return ret + length;
  272. cpu_offset += this_length;
  273. gpu_offset += this_length;
  274. length -= this_length;
  275. }
  276. return 0;
  277. }
  278. static inline int
  279. __copy_from_user_swizzled(char __user *gpu_vaddr, int gpu_offset,
  280. const char *cpu_vaddr,
  281. int length)
  282. {
  283. int ret, cpu_offset = 0;
  284. while (length > 0) {
  285. int cacheline_end = ALIGN(gpu_offset + 1, 64);
  286. int this_length = min(cacheline_end - gpu_offset, length);
  287. int swizzled_gpu_offset = gpu_offset ^ 64;
  288. ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
  289. cpu_vaddr + cpu_offset,
  290. this_length);
  291. if (ret)
  292. return ret + length;
  293. cpu_offset += this_length;
  294. gpu_offset += this_length;
  295. length -= this_length;
  296. }
  297. return 0;
  298. }
  299. /**
  300. * This is the fallback shmem pread path, which allocates temporary storage
  301. * in kernel space to copy_to_user into outside of the struct_mutex, so we
  302. * can copy out of the object's backing pages while holding the struct mutex
  303. * and not take page faults.
  304. */
  305. static int
  306. i915_gem_shmem_pread_slow(struct drm_device *dev,
  307. struct drm_i915_gem_object *obj,
  308. struct drm_i915_gem_pread *args,
  309. struct drm_file *file)
  310. {
  311. struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
  312. char __user *user_data;
  313. ssize_t remain;
  314. loff_t offset;
  315. int shmem_page_offset, page_length, ret = 0;
  316. int obj_do_bit17_swizzling, page_do_bit17_swizzling;
  317. user_data = (char __user *) (uintptr_t) args->data_ptr;
  318. remain = args->size;
  319. obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
  320. offset = args->offset;
  321. mutex_unlock(&dev->struct_mutex);
  322. while (remain > 0) {
  323. struct page *page;
  324. char *vaddr;
  325. /* Operation in this page
  326. *
  327. * shmem_page_offset = offset within page in shmem file
  328. * page_length = bytes to copy for this page
  329. */
  330. shmem_page_offset = offset_in_page(offset);
  331. page_length = remain;
  332. if ((shmem_page_offset + page_length) > PAGE_SIZE)
  333. page_length = PAGE_SIZE - shmem_page_offset;
  334. page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
  335. if (IS_ERR(page)) {
  336. ret = PTR_ERR(page);
  337. goto out;
  338. }
  339. page_do_bit17_swizzling = obj_do_bit17_swizzling &&
  340. (page_to_phys(page) & (1 << 17)) != 0;
  341. vaddr = kmap(page);
  342. if (page_do_bit17_swizzling)
  343. ret = __copy_to_user_swizzled(user_data,
  344. vaddr, shmem_page_offset,
  345. page_length);
  346. else
  347. ret = __copy_to_user(user_data,
  348. vaddr + shmem_page_offset,
  349. page_length);
  350. kunmap(page);
  351. mark_page_accessed(page);
  352. page_cache_release(page);
  353. if (ret) {
  354. ret = -EFAULT;
  355. goto out;
  356. }
  357. remain -= page_length;
  358. user_data += page_length;
  359. offset += page_length;
  360. }
  361. out:
  362. mutex_lock(&dev->struct_mutex);
  363. /* Fixup: Kill any reinstated backing storage pages */
  364. if (obj->madv == __I915_MADV_PURGED)
  365. i915_gem_object_truncate(obj);
  366. return ret;
  367. }
  368. /**
  369. * Reads data from the object referenced by handle.
  370. *
  371. * On error, the contents of *data are undefined.
  372. */
  373. int
  374. i915_gem_pread_ioctl(struct drm_device *dev, void *data,
  375. struct drm_file *file)
  376. {
  377. struct drm_i915_gem_pread *args = data;
  378. struct drm_i915_gem_object *obj;
  379. int ret = 0;
  380. if (args->size == 0)
  381. return 0;
  382. if (!access_ok(VERIFY_WRITE,
  383. (char __user *)(uintptr_t)args->data_ptr,
  384. args->size))
  385. return -EFAULT;
  386. ret = fault_in_pages_writeable((char __user *)(uintptr_t)args->data_ptr,
  387. args->size);
  388. if (ret)
  389. return -EFAULT;
  390. ret = i915_mutex_lock_interruptible(dev);
  391. if (ret)
  392. return ret;
  393. obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  394. if (&obj->base == NULL) {
  395. ret = -ENOENT;
  396. goto unlock;
  397. }
  398. /* Bounds check source. */
  399. if (args->offset > obj->base.size ||
  400. args->size > obj->base.size - args->offset) {
  401. ret = -EINVAL;
  402. goto out;
  403. }
  404. trace_i915_gem_object_pread(obj, args->offset, args->size);
  405. ret = i915_gem_object_set_cpu_read_domain_range(obj,
  406. args->offset,
  407. args->size);
  408. if (ret)
  409. goto out;
  410. ret = -EFAULT;
  411. if (!i915_gem_object_needs_bit17_swizzle(obj))
  412. ret = i915_gem_shmem_pread_fast(dev, obj, args, file);
  413. if (ret == -EFAULT)
  414. ret = i915_gem_shmem_pread_slow(dev, obj, args, file);
  415. out:
  416. drm_gem_object_unreference(&obj->base);
  417. unlock:
  418. mutex_unlock(&dev->struct_mutex);
  419. return ret;
  420. }
  421. /* This is the fast write path which cannot handle
  422. * page faults in the source data
  423. */
  424. static inline int
  425. fast_user_write(struct io_mapping *mapping,
  426. loff_t page_base, int page_offset,
  427. char __user *user_data,
  428. int length)
  429. {
  430. char *vaddr_atomic;
  431. unsigned long unwritten;
  432. vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
  433. unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
  434. user_data, length);
  435. io_mapping_unmap_atomic(vaddr_atomic);
  436. return unwritten;
  437. }
  438. /* Here's the write path which can sleep for
  439. * page faults
  440. */
  441. static inline void
  442. slow_kernel_write(struct io_mapping *mapping,
  443. loff_t gtt_base, int gtt_offset,
  444. struct page *user_page, int user_offset,
  445. int length)
  446. {
  447. char __iomem *dst_vaddr;
  448. char *src_vaddr;
  449. dst_vaddr = io_mapping_map_wc(mapping, gtt_base);
  450. src_vaddr = kmap(user_page);
  451. memcpy_toio(dst_vaddr + gtt_offset,
  452. src_vaddr + user_offset,
  453. length);
  454. kunmap(user_page);
  455. io_mapping_unmap(dst_vaddr);
  456. }
  457. /**
  458. * This is the fast pwrite path, where we copy the data directly from the
  459. * user into the GTT, uncached.
  460. */
  461. static int
  462. i915_gem_gtt_pwrite_fast(struct drm_device *dev,
  463. struct drm_i915_gem_object *obj,
  464. struct drm_i915_gem_pwrite *args,
  465. struct drm_file *file)
  466. {
  467. drm_i915_private_t *dev_priv = dev->dev_private;
  468. ssize_t remain;
  469. loff_t offset, page_base;
  470. char __user *user_data;
  471. int page_offset, page_length;
  472. user_data = (char __user *) (uintptr_t) args->data_ptr;
  473. remain = args->size;
  474. offset = obj->gtt_offset + args->offset;
  475. while (remain > 0) {
  476. /* Operation in this page
  477. *
  478. * page_base = page offset within aperture
  479. * page_offset = offset within page
  480. * page_length = bytes to copy for this page
  481. */
  482. page_base = offset & PAGE_MASK;
  483. page_offset = offset_in_page(offset);
  484. page_length = remain;
  485. if ((page_offset + remain) > PAGE_SIZE)
  486. page_length = PAGE_SIZE - page_offset;
  487. /* If we get a fault while copying data, then (presumably) our
  488. * source page isn't available. Return the error and we'll
  489. * retry in the slow path.
  490. */
  491. if (fast_user_write(dev_priv->mm.gtt_mapping, page_base,
  492. page_offset, user_data, page_length))
  493. return -EFAULT;
  494. remain -= page_length;
  495. user_data += page_length;
  496. offset += page_length;
  497. }
  498. return 0;
  499. }
  500. /**
  501. * This is the fallback GTT pwrite path, which uses get_user_pages to pin
  502. * the memory and maps it using kmap_atomic for copying.
  503. *
  504. * This code resulted in x11perf -rgb10text consuming about 10% more CPU
  505. * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
  506. */
  507. static int
  508. i915_gem_gtt_pwrite_slow(struct drm_device *dev,
  509. struct drm_i915_gem_object *obj,
  510. struct drm_i915_gem_pwrite *args,
  511. struct drm_file *file)
  512. {
  513. drm_i915_private_t *dev_priv = dev->dev_private;
  514. ssize_t remain;
  515. loff_t gtt_page_base, offset;
  516. loff_t first_data_page, last_data_page, num_pages;
  517. loff_t pinned_pages, i;
  518. struct page **user_pages;
  519. struct mm_struct *mm = current->mm;
  520. int gtt_page_offset, data_page_offset, data_page_index, page_length;
  521. int ret;
  522. uint64_t data_ptr = args->data_ptr;
  523. remain = args->size;
  524. /* Pin the user pages containing the data. We can't fault while
  525. * holding the struct mutex, and all of the pwrite implementations
  526. * want to hold it while dereferencing the user data.
  527. */
  528. first_data_page = data_ptr / PAGE_SIZE;
  529. last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
  530. num_pages = last_data_page - first_data_page + 1;
  531. user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
  532. if (user_pages == NULL)
  533. return -ENOMEM;
  534. mutex_unlock(&dev->struct_mutex);
  535. down_read(&mm->mmap_sem);
  536. pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
  537. num_pages, 0, 0, user_pages, NULL);
  538. up_read(&mm->mmap_sem);
  539. mutex_lock(&dev->struct_mutex);
  540. if (pinned_pages < num_pages) {
  541. ret = -EFAULT;
  542. goto out_unpin_pages;
  543. }
  544. ret = i915_gem_object_set_to_gtt_domain(obj, true);
  545. if (ret)
  546. goto out_unpin_pages;
  547. ret = i915_gem_object_put_fence(obj);
  548. if (ret)
  549. goto out_unpin_pages;
  550. offset = obj->gtt_offset + args->offset;
  551. while (remain > 0) {
  552. /* Operation in this page
  553. *
  554. * gtt_page_base = page offset within aperture
  555. * gtt_page_offset = offset within page in aperture
  556. * data_page_index = page number in get_user_pages return
  557. * data_page_offset = offset with data_page_index page.
  558. * page_length = bytes to copy for this page
  559. */
  560. gtt_page_base = offset & PAGE_MASK;
  561. gtt_page_offset = offset_in_page(offset);
  562. data_page_index = data_ptr / PAGE_SIZE - first_data_page;
  563. data_page_offset = offset_in_page(data_ptr);
  564. page_length = remain;
  565. if ((gtt_page_offset + page_length) > PAGE_SIZE)
  566. page_length = PAGE_SIZE - gtt_page_offset;
  567. if ((data_page_offset + page_length) > PAGE_SIZE)
  568. page_length = PAGE_SIZE - data_page_offset;
  569. slow_kernel_write(dev_priv->mm.gtt_mapping,
  570. gtt_page_base, gtt_page_offset,
  571. user_pages[data_page_index],
  572. data_page_offset,
  573. page_length);
  574. remain -= page_length;
  575. offset += page_length;
  576. data_ptr += page_length;
  577. }
  578. out_unpin_pages:
  579. for (i = 0; i < pinned_pages; i++)
  580. page_cache_release(user_pages[i]);
  581. drm_free_large(user_pages);
  582. return ret;
  583. }
  584. static int
  585. i915_gem_shmem_pwrite(struct drm_device *dev,
  586. struct drm_i915_gem_object *obj,
  587. struct drm_i915_gem_pwrite *args,
  588. struct drm_file *file)
  589. {
  590. struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
  591. ssize_t remain;
  592. loff_t offset;
  593. char __user *user_data;
  594. int shmem_page_offset, page_length, ret = 0;
  595. int obj_do_bit17_swizzling, page_do_bit17_swizzling;
  596. int hit_slowpath = 0;
  597. user_data = (char __user *) (uintptr_t) args->data_ptr;
  598. remain = args->size;
  599. obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
  600. offset = args->offset;
  601. obj->dirty = 1;
  602. while (remain > 0) {
  603. struct page *page;
  604. char *vaddr;
  605. /* Operation in this page
  606. *
  607. * shmem_page_offset = offset within page in shmem file
  608. * page_length = bytes to copy for this page
  609. */
  610. shmem_page_offset = offset_in_page(offset);
  611. page_length = remain;
  612. if ((shmem_page_offset + page_length) > PAGE_SIZE)
  613. page_length = PAGE_SIZE - shmem_page_offset;
  614. page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
  615. if (IS_ERR(page)) {
  616. ret = PTR_ERR(page);
  617. goto out;
  618. }
  619. page_do_bit17_swizzling = obj_do_bit17_swizzling &&
  620. (page_to_phys(page) & (1 << 17)) != 0;
  621. if (!page_do_bit17_swizzling) {
  622. vaddr = kmap_atomic(page);
  623. ret = __copy_from_user_inatomic(vaddr + shmem_page_offset,
  624. user_data,
  625. page_length);
  626. kunmap_atomic(vaddr);
  627. if (ret == 0)
  628. goto next_page;
  629. }
  630. hit_slowpath = 1;
  631. mutex_unlock(&dev->struct_mutex);
  632. vaddr = kmap(page);
  633. if (page_do_bit17_swizzling)
  634. ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
  635. user_data,
  636. page_length);
  637. else
  638. ret = __copy_from_user(vaddr + shmem_page_offset,
  639. user_data,
  640. page_length);
  641. kunmap(page);
  642. mutex_lock(&dev->struct_mutex);
  643. next_page:
  644. set_page_dirty(page);
  645. mark_page_accessed(page);
  646. page_cache_release(page);
  647. if (ret) {
  648. ret = -EFAULT;
  649. goto out;
  650. }
  651. remain -= page_length;
  652. user_data += page_length;
  653. offset += page_length;
  654. }
  655. out:
  656. if (hit_slowpath) {
  657. /* Fixup: Kill any reinstated backing storage pages */
  658. if (obj->madv == __I915_MADV_PURGED)
  659. i915_gem_object_truncate(obj);
  660. /* and flush dirty cachelines in case the object isn't in the cpu write
  661. * domain anymore. */
  662. if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
  663. i915_gem_clflush_object(obj);
  664. intel_gtt_chipset_flush();
  665. }
  666. }
  667. return ret;
  668. }
  669. /**
  670. * Writes data to the object referenced by handle.
  671. *
  672. * On error, the contents of the buffer that were to be modified are undefined.
  673. */
  674. int
  675. i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
  676. struct drm_file *file)
  677. {
  678. struct drm_i915_gem_pwrite *args = data;
  679. struct drm_i915_gem_object *obj;
  680. int ret;
  681. if (args->size == 0)
  682. return 0;
  683. if (!access_ok(VERIFY_READ,
  684. (char __user *)(uintptr_t)args->data_ptr,
  685. args->size))
  686. return -EFAULT;
  687. ret = fault_in_pages_readable((char __user *)(uintptr_t)args->data_ptr,
  688. args->size);
  689. if (ret)
  690. return -EFAULT;
  691. ret = i915_mutex_lock_interruptible(dev);
  692. if (ret)
  693. return ret;
  694. obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  695. if (&obj->base == NULL) {
  696. ret = -ENOENT;
  697. goto unlock;
  698. }
  699. /* Bounds check destination. */
  700. if (args->offset > obj->base.size ||
  701. args->size > obj->base.size - args->offset) {
  702. ret = -EINVAL;
  703. goto out;
  704. }
  705. trace_i915_gem_object_pwrite(obj, args->offset, args->size);
  706. /* We can only do the GTT pwrite on untiled buffers, as otherwise
  707. * it would end up going through the fenced access, and we'll get
  708. * different detiling behavior between reading and writing.
  709. * pread/pwrite currently are reading and writing from the CPU
  710. * perspective, requiring manual detiling by the client.
  711. */
  712. if (obj->phys_obj) {
  713. ret = i915_gem_phys_pwrite(dev, obj, args, file);
  714. goto out;
  715. }
  716. if (obj->gtt_space &&
  717. obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
  718. ret = i915_gem_object_pin(obj, 0, true);
  719. if (ret)
  720. goto out;
  721. ret = i915_gem_object_set_to_gtt_domain(obj, true);
  722. if (ret)
  723. goto out_unpin;
  724. ret = i915_gem_object_put_fence(obj);
  725. if (ret)
  726. goto out_unpin;
  727. ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
  728. if (ret == -EFAULT)
  729. ret = i915_gem_gtt_pwrite_slow(dev, obj, args, file);
  730. out_unpin:
  731. i915_gem_object_unpin(obj);
  732. if (ret != -EFAULT)
  733. goto out;
  734. /* Fall through to the shmfs paths because the gtt paths might
  735. * fail with non-page-backed user pointers (e.g. gtt mappings
  736. * when moving data between textures). */
  737. }
  738. ret = i915_gem_object_set_to_cpu_domain(obj, 1);
  739. if (ret)
  740. goto out;
  741. ret = i915_gem_shmem_pwrite(dev, obj, args, file);
  742. out:
  743. drm_gem_object_unreference(&obj->base);
  744. unlock:
  745. mutex_unlock(&dev->struct_mutex);
  746. return ret;
  747. }
  748. /**
  749. * Called when user space prepares to use an object with the CPU, either
  750. * through the mmap ioctl's mapping or a GTT mapping.
  751. */
  752. int
  753. i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
  754. struct drm_file *file)
  755. {
  756. struct drm_i915_gem_set_domain *args = data;
  757. struct drm_i915_gem_object *obj;
  758. uint32_t read_domains = args->read_domains;
  759. uint32_t write_domain = args->write_domain;
  760. int ret;
  761. if (!(dev->driver->driver_features & DRIVER_GEM))
  762. return -ENODEV;
  763. /* Only handle setting domains to types used by the CPU. */
  764. if (write_domain & I915_GEM_GPU_DOMAINS)
  765. return -EINVAL;
  766. if (read_domains & I915_GEM_GPU_DOMAINS)
  767. return -EINVAL;
  768. /* Having something in the write domain implies it's in the read
  769. * domain, and only that read domain. Enforce that in the request.
  770. */
  771. if (write_domain != 0 && read_domains != write_domain)
  772. return -EINVAL;
  773. ret = i915_mutex_lock_interruptible(dev);
  774. if (ret)
  775. return ret;
  776. obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  777. if (&obj->base == NULL) {
  778. ret = -ENOENT;
  779. goto unlock;
  780. }
  781. if (read_domains & I915_GEM_DOMAIN_GTT) {
  782. ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
  783. /* Silently promote "you're not bound, there was nothing to do"
  784. * to success, since the client was just asking us to
  785. * make sure everything was done.
  786. */
  787. if (ret == -EINVAL)
  788. ret = 0;
  789. } else {
  790. ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
  791. }
  792. drm_gem_object_unreference(&obj->base);
  793. unlock:
  794. mutex_unlock(&dev->struct_mutex);
  795. return ret;
  796. }
  797. /**
  798. * Called when user space has done writes to this buffer
  799. */
  800. int
  801. i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
  802. struct drm_file *file)
  803. {
  804. struct drm_i915_gem_sw_finish *args = data;
  805. struct drm_i915_gem_object *obj;
  806. int ret = 0;
  807. if (!(dev->driver->driver_features & DRIVER_GEM))
  808. return -ENODEV;
  809. ret = i915_mutex_lock_interruptible(dev);
  810. if (ret)
  811. return ret;
  812. obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  813. if (&obj->base == NULL) {
  814. ret = -ENOENT;
  815. goto unlock;
  816. }
  817. /* Pinned buffers may be scanout, so flush the cache */
  818. if (obj->pin_count)
  819. i915_gem_object_flush_cpu_write_domain(obj);
  820. drm_gem_object_unreference(&obj->base);
  821. unlock:
  822. mutex_unlock(&dev->struct_mutex);
  823. return ret;
  824. }
  825. /**
  826. * Maps the contents of an object, returning the address it is mapped
  827. * into.
  828. *
  829. * While the mapping holds a reference on the contents of the object, it doesn't
  830. * imply a ref on the object itself.
  831. */
  832. int
  833. i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
  834. struct drm_file *file)
  835. {
  836. struct drm_i915_gem_mmap *args = data;
  837. struct drm_gem_object *obj;
  838. unsigned long addr;
  839. if (!(dev->driver->driver_features & DRIVER_GEM))
  840. return -ENODEV;
  841. obj = drm_gem_object_lookup(dev, file, args->handle);
  842. if (obj == NULL)
  843. return -ENOENT;
  844. down_write(&current->mm->mmap_sem);
  845. addr = do_mmap(obj->filp, 0, args->size,
  846. PROT_READ | PROT_WRITE, MAP_SHARED,
  847. args->offset);
  848. up_write(&current->mm->mmap_sem);
  849. drm_gem_object_unreference_unlocked(obj);
  850. if (IS_ERR((void *)addr))
  851. return addr;
  852. args->addr_ptr = (uint64_t) addr;
  853. return 0;
  854. }
  855. /**
  856. * i915_gem_fault - fault a page into the GTT
  857. * vma: VMA in question
  858. * vmf: fault info
  859. *
  860. * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
  861. * from userspace. The fault handler takes care of binding the object to
  862. * the GTT (if needed), allocating and programming a fence register (again,
  863. * only if needed based on whether the old reg is still valid or the object
  864. * is tiled) and inserting a new PTE into the faulting process.
  865. *
  866. * Note that the faulting process may involve evicting existing objects
  867. * from the GTT and/or fence registers to make room. So performance may
  868. * suffer if the GTT working set is large or there are few fence registers
  869. * left.
  870. */
  871. int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
  872. {
  873. struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
  874. struct drm_device *dev = obj->base.dev;
  875. drm_i915_private_t *dev_priv = dev->dev_private;
  876. pgoff_t page_offset;
  877. unsigned long pfn;
  878. int ret = 0;
  879. bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
  880. /* We don't use vmf->pgoff since that has the fake offset */
  881. page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
  882. PAGE_SHIFT;
  883. ret = i915_mutex_lock_interruptible(dev);
  884. if (ret)
  885. goto out;
  886. trace_i915_gem_object_fault(obj, page_offset, true, write);
  887. /* Now bind it into the GTT if needed */
  888. if (!obj->map_and_fenceable) {
  889. ret = i915_gem_object_unbind(obj);
  890. if (ret)
  891. goto unlock;
  892. }
  893. if (!obj->gtt_space) {
  894. ret = i915_gem_object_bind_to_gtt(obj, 0, true);
  895. if (ret)
  896. goto unlock;
  897. ret = i915_gem_object_set_to_gtt_domain(obj, write);
  898. if (ret)
  899. goto unlock;
  900. }
  901. if (!obj->has_global_gtt_mapping)
  902. i915_gem_gtt_bind_object(obj, obj->cache_level);
  903. if (obj->tiling_mode == I915_TILING_NONE)
  904. ret = i915_gem_object_put_fence(obj);
  905. else
  906. ret = i915_gem_object_get_fence(obj, NULL);
  907. if (ret)
  908. goto unlock;
  909. if (i915_gem_object_is_inactive(obj))
  910. list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
  911. obj->fault_mappable = true;
  912. pfn = ((dev->agp->base + obj->gtt_offset) >> PAGE_SHIFT) +
  913. page_offset;
  914. /* Finally, remap it using the new GTT offset */
  915. ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
  916. unlock:
  917. mutex_unlock(&dev->struct_mutex);
  918. out:
  919. switch (ret) {
  920. case -EIO:
  921. case -EAGAIN:
  922. /* Give the error handler a chance to run and move the
  923. * objects off the GPU active list. Next time we service the
  924. * fault, we should be able to transition the page into the
  925. * GTT without touching the GPU (and so avoid further
  926. * EIO/EGAIN). If the GPU is wedged, then there is no issue
  927. * with coherency, just lost writes.
  928. */
  929. set_need_resched();
  930. case 0:
  931. case -ERESTARTSYS:
  932. case -EINTR:
  933. return VM_FAULT_NOPAGE;
  934. case -ENOMEM:
  935. return VM_FAULT_OOM;
  936. default:
  937. return VM_FAULT_SIGBUS;
  938. }
  939. }
  940. /**
  941. * i915_gem_release_mmap - remove physical page mappings
  942. * @obj: obj in question
  943. *
  944. * Preserve the reservation of the mmapping with the DRM core code, but
  945. * relinquish ownership of the pages back to the system.
  946. *
  947. * It is vital that we remove the page mapping if we have mapped a tiled
  948. * object through the GTT and then lose the fence register due to
  949. * resource pressure. Similarly if the object has been moved out of the
  950. * aperture, than pages mapped into userspace must be revoked. Removing the
  951. * mapping will then trigger a page fault on the next user access, allowing
  952. * fixup by i915_gem_fault().
  953. */
  954. void
  955. i915_gem_release_mmap(struct drm_i915_gem_object *obj)
  956. {
  957. if (!obj->fault_mappable)
  958. return;
  959. if (obj->base.dev->dev_mapping)
  960. unmap_mapping_range(obj->base.dev->dev_mapping,
  961. (loff_t)obj->base.map_list.hash.key<<PAGE_SHIFT,
  962. obj->base.size, 1);
  963. obj->fault_mappable = false;
  964. }
  965. static uint32_t
  966. i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
  967. {
  968. uint32_t gtt_size;
  969. if (INTEL_INFO(dev)->gen >= 4 ||
  970. tiling_mode == I915_TILING_NONE)
  971. return size;
  972. /* Previous chips need a power-of-two fence region when tiling */
  973. if (INTEL_INFO(dev)->gen == 3)
  974. gtt_size = 1024*1024;
  975. else
  976. gtt_size = 512*1024;
  977. while (gtt_size < size)
  978. gtt_size <<= 1;
  979. return gtt_size;
  980. }
  981. /**
  982. * i915_gem_get_gtt_alignment - return required GTT alignment for an object
  983. * @obj: object to check
  984. *
  985. * Return the required GTT alignment for an object, taking into account
  986. * potential fence register mapping.
  987. */
  988. static uint32_t
  989. i915_gem_get_gtt_alignment(struct drm_device *dev,
  990. uint32_t size,
  991. int tiling_mode)
  992. {
  993. /*
  994. * Minimum alignment is 4k (GTT page size), but might be greater
  995. * if a fence register is needed for the object.
  996. */
  997. if (INTEL_INFO(dev)->gen >= 4 ||
  998. tiling_mode == I915_TILING_NONE)
  999. return 4096;
  1000. /*
  1001. * Previous chips need to be aligned to the size of the smallest
  1002. * fence register that can contain the object.
  1003. */
  1004. return i915_gem_get_gtt_size(dev, size, tiling_mode);
  1005. }
  1006. /**
  1007. * i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
  1008. * unfenced object
  1009. * @dev: the device
  1010. * @size: size of the object
  1011. * @tiling_mode: tiling mode of the object
  1012. *
  1013. * Return the required GTT alignment for an object, only taking into account
  1014. * unfenced tiled surface requirements.
  1015. */
  1016. uint32_t
  1017. i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,
  1018. uint32_t size,
  1019. int tiling_mode)
  1020. {
  1021. /*
  1022. * Minimum alignment is 4k (GTT page size) for sane hw.
  1023. */
  1024. if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev) ||
  1025. tiling_mode == I915_TILING_NONE)
  1026. return 4096;
  1027. /* Previous hardware however needs to be aligned to a power-of-two
  1028. * tile height. The simplest method for determining this is to reuse
  1029. * the power-of-tile object size.
  1030. */
  1031. return i915_gem_get_gtt_size(dev, size, tiling_mode);
  1032. }
  1033. int
  1034. i915_gem_mmap_gtt(struct drm_file *file,
  1035. struct drm_device *dev,
  1036. uint32_t handle,
  1037. uint64_t *offset)
  1038. {
  1039. struct drm_i915_private *dev_priv = dev->dev_private;
  1040. struct drm_i915_gem_object *obj;
  1041. int ret;
  1042. if (!(dev->driver->driver_features & DRIVER_GEM))
  1043. return -ENODEV;
  1044. ret = i915_mutex_lock_interruptible(dev);
  1045. if (ret)
  1046. return ret;
  1047. obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
  1048. if (&obj->base == NULL) {
  1049. ret = -ENOENT;
  1050. goto unlock;
  1051. }
  1052. if (obj->base.size > dev_priv->mm.gtt_mappable_end) {
  1053. ret = -E2BIG;
  1054. goto out;
  1055. }
  1056. if (obj->madv != I915_MADV_WILLNEED) {
  1057. DRM_ERROR("Attempting to mmap a purgeable buffer\n");
  1058. ret = -EINVAL;
  1059. goto out;
  1060. }
  1061. if (!obj->base.map_list.map) {
  1062. ret = drm_gem_create_mmap_offset(&obj->base);
  1063. if (ret)
  1064. goto out;
  1065. }
  1066. *offset = (u64)obj->base.map_list.hash.key << PAGE_SHIFT;
  1067. out:
  1068. drm_gem_object_unreference(&obj->base);
  1069. unlock:
  1070. mutex_unlock(&dev->struct_mutex);
  1071. return ret;
  1072. }
  1073. /**
  1074. * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
  1075. * @dev: DRM device
  1076. * @data: GTT mapping ioctl data
  1077. * @file: GEM object info
  1078. *
  1079. * Simply returns the fake offset to userspace so it can mmap it.
  1080. * The mmap call will end up in drm_gem_mmap(), which will set things
  1081. * up so we can get faults in the handler above.
  1082. *
  1083. * The fault handler will take care of binding the object into the GTT
  1084. * (since it may have been evicted to make room for something), allocating
  1085. * a fence register, and mapping the appropriate aperture address into
  1086. * userspace.
  1087. */
  1088. int
  1089. i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
  1090. struct drm_file *file)
  1091. {
  1092. struct drm_i915_gem_mmap_gtt *args = data;
  1093. if (!(dev->driver->driver_features & DRIVER_GEM))
  1094. return -ENODEV;
  1095. return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
  1096. }
  1097. static int
  1098. i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
  1099. gfp_t gfpmask)
  1100. {
  1101. int page_count, i;
  1102. struct address_space *mapping;
  1103. struct inode *inode;
  1104. struct page *page;
  1105. /* Get the list of pages out of our struct file. They'll be pinned
  1106. * at this point until we release them.
  1107. */
  1108. page_count = obj->base.size / PAGE_SIZE;
  1109. BUG_ON(obj->pages != NULL);
  1110. obj->pages = drm_malloc_ab(page_count, sizeof(struct page *));
  1111. if (obj->pages == NULL)
  1112. return -ENOMEM;
  1113. inode = obj->base.filp->f_path.dentry->d_inode;
  1114. mapping = inode->i_mapping;
  1115. gfpmask |= mapping_gfp_mask(mapping);
  1116. for (i = 0; i < page_count; i++) {
  1117. page = shmem_read_mapping_page_gfp(mapping, i, gfpmask);
  1118. if (IS_ERR(page))
  1119. goto err_pages;
  1120. obj->pages[i] = page;
  1121. }
  1122. if (i915_gem_object_needs_bit17_swizzle(obj))
  1123. i915_gem_object_do_bit_17_swizzle(obj);
  1124. return 0;
  1125. err_pages:
  1126. while (i--)
  1127. page_cache_release(obj->pages[i]);
  1128. drm_free_large(obj->pages);
  1129. obj->pages = NULL;
  1130. return PTR_ERR(page);
  1131. }
  1132. static void
  1133. i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
  1134. {
  1135. int page_count = obj->base.size / PAGE_SIZE;
  1136. int i;
  1137. BUG_ON(obj->madv == __I915_MADV_PURGED);
  1138. if (i915_gem_object_needs_bit17_swizzle(obj))
  1139. i915_gem_object_save_bit_17_swizzle(obj);
  1140. if (obj->madv == I915_MADV_DONTNEED)
  1141. obj->dirty = 0;
  1142. for (i = 0; i < page_count; i++) {
  1143. if (obj->dirty)
  1144. set_page_dirty(obj->pages[i]);
  1145. if (obj->madv == I915_MADV_WILLNEED)
  1146. mark_page_accessed(obj->pages[i]);
  1147. page_cache_release(obj->pages[i]);
  1148. }
  1149. obj->dirty = 0;
  1150. drm_free_large(obj->pages);
  1151. obj->pages = NULL;
  1152. }
  1153. void
  1154. i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
  1155. struct intel_ring_buffer *ring,
  1156. u32 seqno)
  1157. {
  1158. struct drm_device *dev = obj->base.dev;
  1159. struct drm_i915_private *dev_priv = dev->dev_private;
  1160. BUG_ON(ring == NULL);
  1161. obj->ring = ring;
  1162. /* Add a reference if we're newly entering the active list. */
  1163. if (!obj->active) {
  1164. drm_gem_object_reference(&obj->base);
  1165. obj->active = 1;
  1166. }
  1167. /* Move from whatever list we were on to the tail of execution. */
  1168. list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
  1169. list_move_tail(&obj->ring_list, &ring->active_list);
  1170. obj->last_rendering_seqno = seqno;
  1171. if (obj->fenced_gpu_access) {
  1172. struct drm_i915_fence_reg *reg;
  1173. BUG_ON(obj->fence_reg == I915_FENCE_REG_NONE);
  1174. obj->last_fenced_seqno = seqno;
  1175. obj->last_fenced_ring = ring;
  1176. reg = &dev_priv->fence_regs[obj->fence_reg];
  1177. list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
  1178. }
  1179. }
  1180. static void
  1181. i915_gem_object_move_off_active(struct drm_i915_gem_object *obj)
  1182. {
  1183. list_del_init(&obj->ring_list);
  1184. obj->last_rendering_seqno = 0;
  1185. }
  1186. static void
  1187. i915_gem_object_move_to_flushing(struct drm_i915_gem_object *obj)
  1188. {
  1189. struct drm_device *dev = obj->base.dev;
  1190. drm_i915_private_t *dev_priv = dev->dev_private;
  1191. BUG_ON(!obj->active);
  1192. list_move_tail(&obj->mm_list, &dev_priv->mm.flushing_list);
  1193. i915_gem_object_move_off_active(obj);
  1194. }
  1195. static void
  1196. i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
  1197. {
  1198. struct drm_device *dev = obj->base.dev;
  1199. struct drm_i915_private *dev_priv = dev->dev_private;
  1200. if (obj->pin_count != 0)
  1201. list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list);
  1202. else
  1203. list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
  1204. BUG_ON(!list_empty(&obj->gpu_write_list));
  1205. BUG_ON(!obj->active);
  1206. obj->ring = NULL;
  1207. i915_gem_object_move_off_active(obj);
  1208. obj->fenced_gpu_access = false;
  1209. obj->active = 0;
  1210. obj->pending_gpu_write = false;
  1211. drm_gem_object_unreference(&obj->base);
  1212. WARN_ON(i915_verify_lists(dev));
  1213. }
  1214. /* Immediately discard the backing storage */
  1215. static void
  1216. i915_gem_object_truncate(struct drm_i915_gem_object *obj)
  1217. {
  1218. struct inode *inode;
  1219. /* Our goal here is to return as much of the memory as
  1220. * is possible back to the system as we are called from OOM.
  1221. * To do this we must instruct the shmfs to drop all of its
  1222. * backing pages, *now*.
  1223. */
  1224. inode = obj->base.filp->f_path.dentry->d_inode;
  1225. shmem_truncate_range(inode, 0, (loff_t)-1);
  1226. if (obj->base.map_list.map)
  1227. drm_gem_free_mmap_offset(&obj->base);
  1228. obj->madv = __I915_MADV_PURGED;
  1229. }
  1230. static inline int
  1231. i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
  1232. {
  1233. return obj->madv == I915_MADV_DONTNEED;
  1234. }
  1235. static void
  1236. i915_gem_process_flushing_list(struct intel_ring_buffer *ring,
  1237. uint32_t flush_domains)
  1238. {
  1239. struct drm_i915_gem_object *obj, *next;
  1240. list_for_each_entry_safe(obj, next,
  1241. &ring->gpu_write_list,
  1242. gpu_write_list) {
  1243. if (obj->base.write_domain & flush_domains) {
  1244. uint32_t old_write_domain = obj->base.write_domain;
  1245. obj->base.write_domain = 0;
  1246. list_del_init(&obj->gpu_write_list);
  1247. i915_gem_object_move_to_active(obj, ring,
  1248. i915_gem_next_request_seqno(ring));
  1249. trace_i915_gem_object_change_domain(obj,
  1250. obj->base.read_domains,
  1251. old_write_domain);
  1252. }
  1253. }
  1254. }
  1255. static u32
  1256. i915_gem_get_seqno(struct drm_device *dev)
  1257. {
  1258. drm_i915_private_t *dev_priv = dev->dev_private;
  1259. u32 seqno = dev_priv->next_seqno;
  1260. /* reserve 0 for non-seqno */
  1261. if (++dev_priv->next_seqno == 0)
  1262. dev_priv->next_seqno = 1;
  1263. return seqno;
  1264. }
  1265. u32
  1266. i915_gem_next_request_seqno(struct intel_ring_buffer *ring)
  1267. {
  1268. if (ring->outstanding_lazy_request == 0)
  1269. ring->outstanding_lazy_request = i915_gem_get_seqno(ring->dev);
  1270. return ring->outstanding_lazy_request;
  1271. }
  1272. int
  1273. i915_add_request(struct intel_ring_buffer *ring,
  1274. struct drm_file *file,
  1275. struct drm_i915_gem_request *request)
  1276. {
  1277. drm_i915_private_t *dev_priv = ring->dev->dev_private;
  1278. uint32_t seqno;
  1279. u32 request_ring_position;
  1280. int was_empty;
  1281. int ret;
  1282. BUG_ON(request == NULL);
  1283. seqno = i915_gem_next_request_seqno(ring);
  1284. /* Record the position of the start of the request so that
  1285. * should we detect the updated seqno part-way through the
  1286. * GPU processing the request, we never over-estimate the
  1287. * position of the head.
  1288. */
  1289. request_ring_position = intel_ring_get_tail(ring);
  1290. ret = ring->add_request(ring, &seqno);
  1291. if (ret)
  1292. return ret;
  1293. trace_i915_gem_request_add(ring, seqno);
  1294. request->seqno = seqno;
  1295. request->ring = ring;
  1296. request->tail = request_ring_position;
  1297. request->emitted_jiffies = jiffies;
  1298. was_empty = list_empty(&ring->request_list);
  1299. list_add_tail(&request->list, &ring->request_list);
  1300. if (file) {
  1301. struct drm_i915_file_private *file_priv = file->driver_priv;
  1302. spin_lock(&file_priv->mm.lock);
  1303. request->file_priv = file_priv;
  1304. list_add_tail(&request->client_list,
  1305. &file_priv->mm.request_list);
  1306. spin_unlock(&file_priv->mm.lock);
  1307. }
  1308. ring->outstanding_lazy_request = 0;
  1309. if (!dev_priv->mm.suspended) {
  1310. if (i915_enable_hangcheck) {
  1311. mod_timer(&dev_priv->hangcheck_timer,
  1312. jiffies +
  1313. msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
  1314. }
  1315. if (was_empty)
  1316. queue_delayed_work(dev_priv->wq,
  1317. &dev_priv->mm.retire_work, HZ);
  1318. }
  1319. return 0;
  1320. }
  1321. static inline void
  1322. i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
  1323. {
  1324. struct drm_i915_file_private *file_priv = request->file_priv;
  1325. if (!file_priv)
  1326. return;
  1327. spin_lock(&file_priv->mm.lock);
  1328. if (request->file_priv) {
  1329. list_del(&request->client_list);
  1330. request->file_priv = NULL;
  1331. }
  1332. spin_unlock(&file_priv->mm.lock);
  1333. }
  1334. static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
  1335. struct intel_ring_buffer *ring)
  1336. {
  1337. while (!list_empty(&ring->request_list)) {
  1338. struct drm_i915_gem_request *request;
  1339. request = list_first_entry(&ring->request_list,
  1340. struct drm_i915_gem_request,
  1341. list);
  1342. list_del(&request->list);
  1343. i915_gem_request_remove_from_client(request);
  1344. kfree(request);
  1345. }
  1346. while (!list_empty(&ring->active_list)) {
  1347. struct drm_i915_gem_object *obj;
  1348. obj = list_first_entry(&ring->active_list,
  1349. struct drm_i915_gem_object,
  1350. ring_list);
  1351. obj->base.write_domain = 0;
  1352. list_del_init(&obj->gpu_write_list);
  1353. i915_gem_object_move_to_inactive(obj);
  1354. }
  1355. }
  1356. static void i915_gem_reset_fences(struct drm_device *dev)
  1357. {
  1358. struct drm_i915_private *dev_priv = dev->dev_private;
  1359. int i;
  1360. for (i = 0; i < dev_priv->num_fence_regs; i++) {
  1361. struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
  1362. struct drm_i915_gem_object *obj = reg->obj;
  1363. if (!obj)
  1364. continue;
  1365. if (obj->tiling_mode)
  1366. i915_gem_release_mmap(obj);
  1367. reg->obj->fence_reg = I915_FENCE_REG_NONE;
  1368. reg->obj->fenced_gpu_access = false;
  1369. reg->obj->last_fenced_seqno = 0;
  1370. reg->obj->last_fenced_ring = NULL;
  1371. i915_gem_clear_fence_reg(dev, reg);
  1372. }
  1373. }
  1374. void i915_gem_reset(struct drm_device *dev)
  1375. {
  1376. struct drm_i915_private *dev_priv = dev->dev_private;
  1377. struct drm_i915_gem_object *obj;
  1378. int i;
  1379. for (i = 0; i < I915_NUM_RINGS; i++)
  1380. i915_gem_reset_ring_lists(dev_priv, &dev_priv->ring[i]);
  1381. /* Remove anything from the flushing lists. The GPU cache is likely
  1382. * to be lost on reset along with the data, so simply move the
  1383. * lost bo to the inactive list.
  1384. */
  1385. while (!list_empty(&dev_priv->mm.flushing_list)) {
  1386. obj = list_first_entry(&dev_priv->mm.flushing_list,
  1387. struct drm_i915_gem_object,
  1388. mm_list);
  1389. obj->base.write_domain = 0;
  1390. list_del_init(&obj->gpu_write_list);
  1391. i915_gem_object_move_to_inactive(obj);
  1392. }
  1393. /* Move everything out of the GPU domains to ensure we do any
  1394. * necessary invalidation upon reuse.
  1395. */
  1396. list_for_each_entry(obj,
  1397. &dev_priv->mm.inactive_list,
  1398. mm_list)
  1399. {
  1400. obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
  1401. }
  1402. /* The fence registers are invalidated so clear them out */
  1403. i915_gem_reset_fences(dev);
  1404. }
  1405. /**
  1406. * This function clears the request list as sequence numbers are passed.
  1407. */
  1408. void
  1409. i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
  1410. {
  1411. uint32_t seqno;
  1412. int i;
  1413. if (list_empty(&ring->request_list))
  1414. return;
  1415. WARN_ON(i915_verify_lists(ring->dev));
  1416. seqno = ring->get_seqno(ring);
  1417. for (i = 0; i < ARRAY_SIZE(ring->sync_seqno); i++)
  1418. if (seqno >= ring->sync_seqno[i])
  1419. ring->sync_seqno[i] = 0;
  1420. while (!list_empty(&ring->request_list)) {
  1421. struct drm_i915_gem_request *request;
  1422. request = list_first_entry(&ring->request_list,
  1423. struct drm_i915_gem_request,
  1424. list);
  1425. if (!i915_seqno_passed(seqno, request->seqno))
  1426. break;
  1427. trace_i915_gem_request_retire(ring, request->seqno);
  1428. /* We know the GPU must have read the request to have
  1429. * sent us the seqno + interrupt, so use the position
  1430. * of tail of the request to update the last known position
  1431. * of the GPU head.
  1432. */
  1433. ring->last_retired_head = request->tail;
  1434. list_del(&request->list);
  1435. i915_gem_request_remove_from_client(request);
  1436. kfree(request);
  1437. }
  1438. /* Move any buffers on the active list that are no longer referenced
  1439. * by the ringbuffer to the flushing/inactive lists as appropriate.
  1440. */
  1441. while (!list_empty(&ring->active_list)) {
  1442. struct drm_i915_gem_object *obj;
  1443. obj = list_first_entry(&ring->active_list,
  1444. struct drm_i915_gem_object,
  1445. ring_list);
  1446. if (!i915_seqno_passed(seqno, obj->last_rendering_seqno))
  1447. break;
  1448. if (obj->base.write_domain != 0)
  1449. i915_gem_object_move_to_flushing(obj);
  1450. else
  1451. i915_gem_object_move_to_inactive(obj);
  1452. }
  1453. if (unlikely(ring->trace_irq_seqno &&
  1454. i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
  1455. ring->irq_put(ring);
  1456. ring->trace_irq_seqno = 0;
  1457. }
  1458. WARN_ON(i915_verify_lists(ring->dev));
  1459. }
  1460. void
  1461. i915_gem_retire_requests(struct drm_device *dev)
  1462. {
  1463. drm_i915_private_t *dev_priv = dev->dev_private;
  1464. int i;
  1465. if (!list_empty(&dev_priv->mm.deferred_free_list)) {
  1466. struct drm_i915_gem_object *obj, *next;
  1467. /* We must be careful that during unbind() we do not
  1468. * accidentally infinitely recurse into retire requests.
  1469. * Currently:
  1470. * retire -> free -> unbind -> wait -> retire_ring
  1471. */
  1472. list_for_each_entry_safe(obj, next,
  1473. &dev_priv->mm.deferred_free_list,
  1474. mm_list)
  1475. i915_gem_free_object_tail(obj);
  1476. }
  1477. for (i = 0; i < I915_NUM_RINGS; i++)
  1478. i915_gem_retire_requests_ring(&dev_priv->ring[i]);
  1479. }
  1480. static void
  1481. i915_gem_retire_work_handler(struct work_struct *work)
  1482. {
  1483. drm_i915_private_t *dev_priv;
  1484. struct drm_device *dev;
  1485. bool idle;
  1486. int i;
  1487. dev_priv = container_of(work, drm_i915_private_t,
  1488. mm.retire_work.work);
  1489. dev = dev_priv->dev;
  1490. /* Come back later if the device is busy... */
  1491. if (!mutex_trylock(&dev->struct_mutex)) {
  1492. queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
  1493. return;
  1494. }
  1495. i915_gem_retire_requests(dev);
  1496. /* Send a periodic flush down the ring so we don't hold onto GEM
  1497. * objects indefinitely.
  1498. */
  1499. idle = true;
  1500. for (i = 0; i < I915_NUM_RINGS; i++) {
  1501. struct intel_ring_buffer *ring = &dev_priv->ring[i];
  1502. if (!list_empty(&ring->gpu_write_list)) {
  1503. struct drm_i915_gem_request *request;
  1504. int ret;
  1505. ret = i915_gem_flush_ring(ring,
  1506. 0, I915_GEM_GPU_DOMAINS);
  1507. request = kzalloc(sizeof(*request), GFP_KERNEL);
  1508. if (ret || request == NULL ||
  1509. i915_add_request(ring, NULL, request))
  1510. kfree(request);
  1511. }
  1512. idle &= list_empty(&ring->request_list);
  1513. }
  1514. if (!dev_priv->mm.suspended && !idle)
  1515. queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
  1516. mutex_unlock(&dev->struct_mutex);
  1517. }
  1518. /**
  1519. * Waits for a sequence number to be signaled, and cleans up the
  1520. * request and object lists appropriately for that event.
  1521. */
  1522. int
  1523. i915_wait_request(struct intel_ring_buffer *ring,
  1524. uint32_t seqno,
  1525. bool do_retire)
  1526. {
  1527. drm_i915_private_t *dev_priv = ring->dev->dev_private;
  1528. u32 ier;
  1529. int ret = 0;
  1530. BUG_ON(seqno == 0);
  1531. if (atomic_read(&dev_priv->mm.wedged)) {
  1532. struct completion *x = &dev_priv->error_completion;
  1533. bool recovery_complete;
  1534. unsigned long flags;
  1535. /* Give the error handler a chance to run. */
  1536. spin_lock_irqsave(&x->wait.lock, flags);
  1537. recovery_complete = x->done > 0;
  1538. spin_unlock_irqrestore(&x->wait.lock, flags);
  1539. return recovery_complete ? -EIO : -EAGAIN;
  1540. }
  1541. if (seqno == ring->outstanding_lazy_request) {
  1542. struct drm_i915_gem_request *request;
  1543. request = kzalloc(sizeof(*request), GFP_KERNEL);
  1544. if (request == NULL)
  1545. return -ENOMEM;
  1546. ret = i915_add_request(ring, NULL, request);
  1547. if (ret) {
  1548. kfree(request);
  1549. return ret;
  1550. }
  1551. seqno = request->seqno;
  1552. }
  1553. if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {
  1554. if (HAS_PCH_SPLIT(ring->dev))
  1555. ier = I915_READ(DEIER) | I915_READ(GTIER);
  1556. else
  1557. ier = I915_READ(IER);
  1558. if (!ier) {
  1559. DRM_ERROR("something (likely vbetool) disabled "
  1560. "interrupts, re-enabling\n");
  1561. ring->dev->driver->irq_preinstall(ring->dev);
  1562. ring->dev->driver->irq_postinstall(ring->dev);
  1563. }
  1564. trace_i915_gem_request_wait_begin(ring, seqno);
  1565. ring->waiting_seqno = seqno;
  1566. if (ring->irq_get(ring)) {
  1567. if (dev_priv->mm.interruptible)
  1568. ret = wait_event_interruptible(ring->irq_queue,
  1569. i915_seqno_passed(ring->get_seqno(ring), seqno)
  1570. || atomic_read(&dev_priv->mm.wedged));
  1571. else
  1572. wait_event(ring->irq_queue,
  1573. i915_seqno_passed(ring->get_seqno(ring), seqno)
  1574. || atomic_read(&dev_priv->mm.wedged));
  1575. ring->irq_put(ring);
  1576. } else if (wait_for_atomic(i915_seqno_passed(ring->get_seqno(ring),
  1577. seqno) ||
  1578. atomic_read(&dev_priv->mm.wedged), 3000))
  1579. ret = -EBUSY;
  1580. ring->waiting_seqno = 0;
  1581. trace_i915_gem_request_wait_end(ring, seqno);
  1582. }
  1583. if (atomic_read(&dev_priv->mm.wedged))
  1584. ret = -EAGAIN;
  1585. /* Directly dispatch request retiring. While we have the work queue
  1586. * to handle this, the waiter on a request often wants an associated
  1587. * buffer to have made it to the inactive list, and we would need
  1588. * a separate wait queue to handle that.
  1589. */
  1590. if (ret == 0 && do_retire)
  1591. i915_gem_retire_requests_ring(ring);
  1592. return ret;
  1593. }
  1594. /**
  1595. * Ensures that all rendering to the object has completed and the object is
  1596. * safe to unbind from the GTT or access from the CPU.
  1597. */
  1598. int
  1599. i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj)
  1600. {
  1601. int ret;
  1602. /* This function only exists to support waiting for existing rendering,
  1603. * not for emitting required flushes.
  1604. */
  1605. BUG_ON((obj->base.write_domain & I915_GEM_GPU_DOMAINS) != 0);
  1606. /* If there is rendering queued on the buffer being evicted, wait for
  1607. * it.
  1608. */
  1609. if (obj->active) {
  1610. ret = i915_wait_request(obj->ring, obj->last_rendering_seqno,
  1611. true);
  1612. if (ret)
  1613. return ret;
  1614. }
  1615. return 0;
  1616. }
  1617. static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
  1618. {
  1619. u32 old_write_domain, old_read_domains;
  1620. /* Act a barrier for all accesses through the GTT */
  1621. mb();
  1622. /* Force a pagefault for domain tracking on next user access */
  1623. i915_gem_release_mmap(obj);
  1624. if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
  1625. return;
  1626. old_read_domains = obj->base.read_domains;
  1627. old_write_domain = obj->base.write_domain;
  1628. obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
  1629. obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
  1630. trace_i915_gem_object_change_domain(obj,
  1631. old_read_domains,
  1632. old_write_domain);
  1633. }
  1634. /**
  1635. * Unbinds an object from the GTT aperture.
  1636. */
  1637. int
  1638. i915_gem_object_unbind(struct drm_i915_gem_object *obj)
  1639. {
  1640. drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
  1641. int ret = 0;
  1642. if (obj->gtt_space == NULL)
  1643. return 0;
  1644. if (obj->pin_count != 0) {
  1645. DRM_ERROR("Attempting to unbind pinned buffer\n");
  1646. return -EINVAL;
  1647. }
  1648. ret = i915_gem_object_finish_gpu(obj);
  1649. if (ret == -ERESTARTSYS)
  1650. return ret;
  1651. /* Continue on if we fail due to EIO, the GPU is hung so we
  1652. * should be safe and we need to cleanup or else we might
  1653. * cause memory corruption through use-after-free.
  1654. */
  1655. i915_gem_object_finish_gtt(obj);
  1656. /* Move the object to the CPU domain to ensure that
  1657. * any possible CPU writes while it's not in the GTT
  1658. * are flushed when we go to remap it.
  1659. */
  1660. if (ret == 0)
  1661. ret = i915_gem_object_set_to_cpu_domain(obj, 1);
  1662. if (ret == -ERESTARTSYS)
  1663. return ret;
  1664. if (ret) {
  1665. /* In the event of a disaster, abandon all caches and
  1666. * hope for the best.
  1667. */
  1668. i915_gem_clflush_object(obj);
  1669. obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  1670. }
  1671. /* release the fence reg _after_ flushing */
  1672. ret = i915_gem_object_put_fence(obj);
  1673. if (ret == -ERESTARTSYS)
  1674. return ret;
  1675. trace_i915_gem_object_unbind(obj);
  1676. if (obj->has_global_gtt_mapping)
  1677. i915_gem_gtt_unbind_object(obj);
  1678. if (obj->has_aliasing_ppgtt_mapping) {
  1679. i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
  1680. obj->has_aliasing_ppgtt_mapping = 0;
  1681. }
  1682. i915_gem_gtt_finish_object(obj);
  1683. i915_gem_object_put_pages_gtt(obj);
  1684. list_del_init(&obj->gtt_list);
  1685. list_del_init(&obj->mm_list);
  1686. /* Avoid an unnecessary call to unbind on rebind. */
  1687. obj->map_and_fenceable = true;
  1688. drm_mm_put_block(obj->gtt_space);
  1689. obj->gtt_space = NULL;
  1690. obj->gtt_offset = 0;
  1691. if (i915_gem_object_is_purgeable(obj))
  1692. i915_gem_object_truncate(obj);
  1693. return ret;
  1694. }
  1695. int
  1696. i915_gem_flush_ring(struct intel_ring_buffer *ring,
  1697. uint32_t invalidate_domains,
  1698. uint32_t flush_domains)
  1699. {
  1700. int ret;
  1701. if (((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) == 0)
  1702. return 0;
  1703. trace_i915_gem_ring_flush(ring, invalidate_domains, flush_domains);
  1704. ret = ring->flush(ring, invalidate_domains, flush_domains);
  1705. if (ret)
  1706. return ret;
  1707. if (flush_domains & I915_GEM_GPU_DOMAINS)
  1708. i915_gem_process_flushing_list(ring, flush_domains);
  1709. return 0;
  1710. }
  1711. static int i915_ring_idle(struct intel_ring_buffer *ring, bool do_retire)
  1712. {
  1713. int ret;
  1714. if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list))
  1715. return 0;
  1716. if (!list_empty(&ring->gpu_write_list)) {
  1717. ret = i915_gem_flush_ring(ring,
  1718. I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
  1719. if (ret)
  1720. return ret;
  1721. }
  1722. return i915_wait_request(ring, i915_gem_next_request_seqno(ring),
  1723. do_retire);
  1724. }
  1725. int i915_gpu_idle(struct drm_device *dev, bool do_retire)
  1726. {
  1727. drm_i915_private_t *dev_priv = dev->dev_private;
  1728. int ret, i;
  1729. /* Flush everything onto the inactive list. */
  1730. for (i = 0; i < I915_NUM_RINGS; i++) {
  1731. ret = i915_ring_idle(&dev_priv->ring[i], do_retire);
  1732. if (ret)
  1733. return ret;
  1734. }
  1735. return 0;
  1736. }
  1737. static int sandybridge_write_fence_reg(struct drm_i915_gem_object *obj,
  1738. struct intel_ring_buffer *pipelined)
  1739. {
  1740. struct drm_device *dev = obj->base.dev;
  1741. drm_i915_private_t *dev_priv = dev->dev_private;
  1742. u32 size = obj->gtt_space->size;
  1743. int regnum = obj->fence_reg;
  1744. uint64_t val;
  1745. val = (uint64_t)((obj->gtt_offset + size - 4096) &
  1746. 0xfffff000) << 32;
  1747. val |= obj->gtt_offset & 0xfffff000;
  1748. val |= (uint64_t)((obj->stride / 128) - 1) <<
  1749. SANDYBRIDGE_FENCE_PITCH_SHIFT;
  1750. if (obj->tiling_mode == I915_TILING_Y)
  1751. val |= 1 << I965_FENCE_TILING_Y_SHIFT;
  1752. val |= I965_FENCE_REG_VALID;
  1753. if (pipelined) {
  1754. int ret = intel_ring_begin(pipelined, 6);
  1755. if (ret)
  1756. return ret;
  1757. intel_ring_emit(pipelined, MI_NOOP);
  1758. intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));
  1759. intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8);
  1760. intel_ring_emit(pipelined, (u32)val);
  1761. intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8 + 4);
  1762. intel_ring_emit(pipelined, (u32)(val >> 32));
  1763. intel_ring_advance(pipelined);
  1764. } else
  1765. I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + regnum * 8, val);
  1766. return 0;
  1767. }
  1768. static int i965_write_fence_reg(struct drm_i915_gem_object *obj,
  1769. struct intel_ring_buffer *pipelined)
  1770. {
  1771. struct drm_device *dev = obj->base.dev;
  1772. drm_i915_private_t *dev_priv = dev->dev_private;
  1773. u32 size = obj->gtt_space->size;
  1774. int regnum = obj->fence_reg;
  1775. uint64_t val;
  1776. val = (uint64_t)((obj->gtt_offset + size - 4096) &
  1777. 0xfffff000) << 32;
  1778. val |= obj->gtt_offset & 0xfffff000;
  1779. val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
  1780. if (obj->tiling_mode == I915_TILING_Y)
  1781. val |= 1 << I965_FENCE_TILING_Y_SHIFT;
  1782. val |= I965_FENCE_REG_VALID;
  1783. if (pipelined) {
  1784. int ret = intel_ring_begin(pipelined, 6);
  1785. if (ret)
  1786. return ret;
  1787. intel_ring_emit(pipelined, MI_NOOP);
  1788. intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));
  1789. intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8);
  1790. intel_ring_emit(pipelined, (u32)val);
  1791. intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8 + 4);
  1792. intel_ring_emit(pipelined, (u32)(val >> 32));
  1793. intel_ring_advance(pipelined);
  1794. } else
  1795. I915_WRITE64(FENCE_REG_965_0 + regnum * 8, val);
  1796. return 0;
  1797. }
  1798. static int i915_write_fence_reg(struct drm_i915_gem_object *obj,
  1799. struct intel_ring_buffer *pipelined)
  1800. {
  1801. struct drm_device *dev = obj->base.dev;
  1802. drm_i915_private_t *dev_priv = dev->dev_private;
  1803. u32 size = obj->gtt_space->size;
  1804. u32 fence_reg, val, pitch_val;
  1805. int tile_width;
  1806. if (WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) ||
  1807. (size & -size) != size ||
  1808. (obj->gtt_offset & (size - 1)),
  1809. "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
  1810. obj->gtt_offset, obj->map_and_fenceable, size))
  1811. return -EINVAL;
  1812. if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
  1813. tile_width = 128;
  1814. else
  1815. tile_width = 512;
  1816. /* Note: pitch better be a power of two tile widths */
  1817. pitch_val = obj->stride / tile_width;
  1818. pitch_val = ffs(pitch_val) - 1;
  1819. val = obj->gtt_offset;
  1820. if (obj->tiling_mode == I915_TILING_Y)
  1821. val |= 1 << I830_FENCE_TILING_Y_SHIFT;
  1822. val |= I915_FENCE_SIZE_BITS(size);
  1823. val |= pitch_val << I830_FENCE_PITCH_SHIFT;
  1824. val |= I830_FENCE_REG_VALID;
  1825. fence_reg = obj->fence_reg;
  1826. if (fence_reg < 8)
  1827. fence_reg = FENCE_REG_830_0 + fence_reg * 4;
  1828. else
  1829. fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
  1830. if (pipelined) {
  1831. int ret = intel_ring_begin(pipelined, 4);
  1832. if (ret)
  1833. return ret;
  1834. intel_ring_emit(pipelined, MI_NOOP);
  1835. intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));
  1836. intel_ring_emit(pipelined, fence_reg);
  1837. intel_ring_emit(pipelined, val);
  1838. intel_ring_advance(pipelined);
  1839. } else
  1840. I915_WRITE(fence_reg, val);
  1841. return 0;
  1842. }
  1843. static int i830_write_fence_reg(struct drm_i915_gem_object *obj,
  1844. struct intel_ring_buffer *pipelined)
  1845. {
  1846. struct drm_device *dev = obj->base.dev;
  1847. drm_i915_private_t *dev_priv = dev->dev_private;
  1848. u32 size = obj->gtt_space->size;
  1849. int regnum = obj->fence_reg;
  1850. uint32_t val;
  1851. uint32_t pitch_val;
  1852. if (WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) ||
  1853. (size & -size) != size ||
  1854. (obj->gtt_offset & (size - 1)),
  1855. "object 0x%08x not 512K or pot-size 0x%08x aligned\n",
  1856. obj->gtt_offset, size))
  1857. return -EINVAL;
  1858. pitch_val = obj->stride / 128;
  1859. pitch_val = ffs(pitch_val) - 1;
  1860. val = obj->gtt_offset;
  1861. if (obj->tiling_mode == I915_TILING_Y)
  1862. val |= 1 << I830_FENCE_TILING_Y_SHIFT;
  1863. val |= I830_FENCE_SIZE_BITS(size);
  1864. val |= pitch_val << I830_FENCE_PITCH_SHIFT;
  1865. val |= I830_FENCE_REG_VALID;
  1866. if (pipelined) {
  1867. int ret = intel_ring_begin(pipelined, 4);
  1868. if (ret)
  1869. return ret;
  1870. intel_ring_emit(pipelined, MI_NOOP);
  1871. intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));
  1872. intel_ring_emit(pipelined, FENCE_REG_830_0 + regnum*4);
  1873. intel_ring_emit(pipelined, val);
  1874. intel_ring_advance(pipelined);
  1875. } else
  1876. I915_WRITE(FENCE_REG_830_0 + regnum * 4, val);
  1877. return 0;
  1878. }
  1879. static bool ring_passed_seqno(struct intel_ring_buffer *ring, u32 seqno)
  1880. {
  1881. return i915_seqno_passed(ring->get_seqno(ring), seqno);
  1882. }
  1883. static int
  1884. i915_gem_object_flush_fence(struct drm_i915_gem_object *obj,
  1885. struct intel_ring_buffer *pipelined)
  1886. {
  1887. int ret;
  1888. if (obj->fenced_gpu_access) {
  1889. if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
  1890. ret = i915_gem_flush_ring(obj->last_fenced_ring,
  1891. 0, obj->base.write_domain);
  1892. if (ret)
  1893. return ret;
  1894. }
  1895. obj->fenced_gpu_access = false;
  1896. }
  1897. if (obj->last_fenced_seqno && pipelined != obj->last_fenced_ring) {
  1898. if (!ring_passed_seqno(obj->last_fenced_ring,
  1899. obj->last_fenced_seqno)) {
  1900. ret = i915_wait_request(obj->last_fenced_ring,
  1901. obj->last_fenced_seqno,
  1902. true);
  1903. if (ret)
  1904. return ret;
  1905. }
  1906. obj->last_fenced_seqno = 0;
  1907. obj->last_fenced_ring = NULL;
  1908. }
  1909. /* Ensure that all CPU reads are completed before installing a fence
  1910. * and all writes before removing the fence.
  1911. */
  1912. if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
  1913. mb();
  1914. return 0;
  1915. }
  1916. int
  1917. i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
  1918. {
  1919. int ret;
  1920. if (obj->tiling_mode)
  1921. i915_gem_release_mmap(obj);
  1922. ret = i915_gem_object_flush_fence(obj, NULL);
  1923. if (ret)
  1924. return ret;
  1925. if (obj->fence_reg != I915_FENCE_REG_NONE) {
  1926. struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  1927. WARN_ON(dev_priv->fence_regs[obj->fence_reg].pin_count);
  1928. i915_gem_clear_fence_reg(obj->base.dev,
  1929. &dev_priv->fence_regs[obj->fence_reg]);
  1930. obj->fence_reg = I915_FENCE_REG_NONE;
  1931. }
  1932. return 0;
  1933. }
  1934. static struct drm_i915_fence_reg *
  1935. i915_find_fence_reg(struct drm_device *dev,
  1936. struct intel_ring_buffer *pipelined)
  1937. {
  1938. struct drm_i915_private *dev_priv = dev->dev_private;
  1939. struct drm_i915_fence_reg *reg, *first, *avail;
  1940. int i;
  1941. /* First try to find a free reg */
  1942. avail = NULL;
  1943. for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
  1944. reg = &dev_priv->fence_regs[i];
  1945. if (!reg->obj)
  1946. return reg;
  1947. if (!reg->pin_count)
  1948. avail = reg;
  1949. }
  1950. if (avail == NULL)
  1951. return NULL;
  1952. /* None available, try to steal one or wait for a user to finish */
  1953. avail = first = NULL;
  1954. list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
  1955. if (reg->pin_count)
  1956. continue;
  1957. if (first == NULL)
  1958. first = reg;
  1959. if (!pipelined ||
  1960. !reg->obj->last_fenced_ring ||
  1961. reg->obj->last_fenced_ring == pipelined) {
  1962. avail = reg;
  1963. break;
  1964. }
  1965. }
  1966. if (avail == NULL)
  1967. avail = first;
  1968. return avail;
  1969. }
  1970. /**
  1971. * i915_gem_object_get_fence - set up a fence reg for an object
  1972. * @obj: object to map through a fence reg
  1973. * @pipelined: ring on which to queue the change, or NULL for CPU access
  1974. * @interruptible: must we wait uninterruptibly for the register to retire?
  1975. *
  1976. * When mapping objects through the GTT, userspace wants to be able to write
  1977. * to them without having to worry about swizzling if the object is tiled.
  1978. *
  1979. * This function walks the fence regs looking for a free one for @obj,
  1980. * stealing one if it can't find any.
  1981. *
  1982. * It then sets up the reg based on the object's properties: address, pitch
  1983. * and tiling format.
  1984. */
  1985. int
  1986. i915_gem_object_get_fence(struct drm_i915_gem_object *obj,
  1987. struct intel_ring_buffer *pipelined)
  1988. {
  1989. struct drm_device *dev = obj->base.dev;
  1990. struct drm_i915_private *dev_priv = dev->dev_private;
  1991. struct drm_i915_fence_reg *reg;
  1992. int ret;
  1993. /* XXX disable pipelining. There are bugs. Shocking. */
  1994. pipelined = NULL;
  1995. /* Just update our place in the LRU if our fence is getting reused. */
  1996. if (obj->fence_reg != I915_FENCE_REG_NONE) {
  1997. reg = &dev_priv->fence_regs[obj->fence_reg];
  1998. list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
  1999. if (obj->tiling_changed) {
  2000. ret = i915_gem_object_flush_fence(obj, pipelined);
  2001. if (ret)
  2002. return ret;
  2003. if (!obj->fenced_gpu_access && !obj->last_fenced_seqno)
  2004. pipelined = NULL;
  2005. if (pipelined) {
  2006. reg->setup_seqno =
  2007. i915_gem_next_request_seqno(pipelined);
  2008. obj->last_fenced_seqno = reg->setup_seqno;
  2009. obj->last_fenced_ring = pipelined;
  2010. }
  2011. goto update;
  2012. }
  2013. if (!pipelined) {
  2014. if (reg->setup_seqno) {
  2015. if (!ring_passed_seqno(obj->last_fenced_ring,
  2016. reg->setup_seqno)) {
  2017. ret = i915_wait_request(obj->last_fenced_ring,
  2018. reg->setup_seqno,
  2019. true);
  2020. if (ret)
  2021. return ret;
  2022. }
  2023. reg->setup_seqno = 0;
  2024. }
  2025. } else if (obj->last_fenced_ring &&
  2026. obj->last_fenced_ring != pipelined) {
  2027. ret = i915_gem_object_flush_fence(obj, pipelined);
  2028. if (ret)
  2029. return ret;
  2030. }
  2031. return 0;
  2032. }
  2033. reg = i915_find_fence_reg(dev, pipelined);
  2034. if (reg == NULL)
  2035. return -EDEADLK;
  2036. ret = i915_gem_object_flush_fence(obj, pipelined);
  2037. if (ret)
  2038. return ret;
  2039. if (reg->obj) {
  2040. struct drm_i915_gem_object *old = reg->obj;
  2041. drm_gem_object_reference(&old->base);
  2042. if (old->tiling_mode)
  2043. i915_gem_release_mmap(old);
  2044. ret = i915_gem_object_flush_fence(old, pipelined);
  2045. if (ret) {
  2046. drm_gem_object_unreference(&old->base);
  2047. return ret;
  2048. }
  2049. if (old->last_fenced_seqno == 0 && obj->last_fenced_seqno == 0)
  2050. pipelined = NULL;
  2051. old->fence_reg = I915_FENCE_REG_NONE;
  2052. old->last_fenced_ring = pipelined;
  2053. old->last_fenced_seqno =
  2054. pipelined ? i915_gem_next_request_seqno(pipelined) : 0;
  2055. drm_gem_object_unreference(&old->base);
  2056. } else if (obj->last_fenced_seqno == 0)
  2057. pipelined = NULL;
  2058. reg->obj = obj;
  2059. list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
  2060. obj->fence_reg = reg - dev_priv->fence_regs;
  2061. obj->last_fenced_ring = pipelined;
  2062. reg->setup_seqno =
  2063. pipelined ? i915_gem_next_request_seqno(pipelined) : 0;
  2064. obj->last_fenced_seqno = reg->setup_seqno;
  2065. update:
  2066. obj->tiling_changed = false;
  2067. switch (INTEL_INFO(dev)->gen) {
  2068. case 7:
  2069. case 6:
  2070. ret = sandybridge_write_fence_reg(obj, pipelined);
  2071. break;
  2072. case 5:
  2073. case 4:
  2074. ret = i965_write_fence_reg(obj, pipelined);
  2075. break;
  2076. case 3:
  2077. ret = i915_write_fence_reg(obj, pipelined);
  2078. break;
  2079. case 2:
  2080. ret = i830_write_fence_reg(obj, pipelined);
  2081. break;
  2082. }
  2083. return ret;
  2084. }
  2085. /**
  2086. * i915_gem_clear_fence_reg - clear out fence register info
  2087. * @obj: object to clear
  2088. *
  2089. * Zeroes out the fence register itself and clears out the associated
  2090. * data structures in dev_priv and obj.
  2091. */
  2092. static void
  2093. i915_gem_clear_fence_reg(struct drm_device *dev,
  2094. struct drm_i915_fence_reg *reg)
  2095. {
  2096. drm_i915_private_t *dev_priv = dev->dev_private;
  2097. uint32_t fence_reg = reg - dev_priv->fence_regs;
  2098. switch (INTEL_INFO(dev)->gen) {
  2099. case 7:
  2100. case 6:
  2101. I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + fence_reg*8, 0);
  2102. break;
  2103. case 5:
  2104. case 4:
  2105. I915_WRITE64(FENCE_REG_965_0 + fence_reg*8, 0);
  2106. break;
  2107. case 3:
  2108. if (fence_reg >= 8)
  2109. fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
  2110. else
  2111. case 2:
  2112. fence_reg = FENCE_REG_830_0 + fence_reg * 4;
  2113. I915_WRITE(fence_reg, 0);
  2114. break;
  2115. }
  2116. list_del_init(&reg->lru_list);
  2117. reg->obj = NULL;
  2118. reg->setup_seqno = 0;
  2119. reg->pin_count = 0;
  2120. }
  2121. /**
  2122. * Finds free space in the GTT aperture and binds the object there.
  2123. */
  2124. static int
  2125. i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
  2126. unsigned alignment,
  2127. bool map_and_fenceable)
  2128. {
  2129. struct drm_device *dev = obj->base.dev;
  2130. drm_i915_private_t *dev_priv = dev->dev_private;
  2131. struct drm_mm_node *free_space;
  2132. gfp_t gfpmask = __GFP_NORETRY | __GFP_NOWARN;
  2133. u32 size, fence_size, fence_alignment, unfenced_alignment;
  2134. bool mappable, fenceable;
  2135. int ret;
  2136. if (obj->madv != I915_MADV_WILLNEED) {
  2137. DRM_ERROR("Attempting to bind a purgeable object\n");
  2138. return -EINVAL;
  2139. }
  2140. fence_size = i915_gem_get_gtt_size(dev,
  2141. obj->base.size,
  2142. obj->tiling_mode);
  2143. fence_alignment = i915_gem_get_gtt_alignment(dev,
  2144. obj->base.size,
  2145. obj->tiling_mode);
  2146. unfenced_alignment =
  2147. i915_gem_get_unfenced_gtt_alignment(dev,
  2148. obj->base.size,
  2149. obj->tiling_mode);
  2150. if (alignment == 0)
  2151. alignment = map_and_fenceable ? fence_alignment :
  2152. unfenced_alignment;
  2153. if (map_and_fenceable && alignment & (fence_alignment - 1)) {
  2154. DRM_ERROR("Invalid object alignment requested %u\n", alignment);
  2155. return -EINVAL;
  2156. }
  2157. size = map_and_fenceable ? fence_size : obj->base.size;
  2158. /* If the object is bigger than the entire aperture, reject it early
  2159. * before evicting everything in a vain attempt to find space.
  2160. */
  2161. if (obj->base.size >
  2162. (map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
  2163. DRM_ERROR("Attempting to bind an object larger than the aperture\n");
  2164. return -E2BIG;
  2165. }
  2166. search_free:
  2167. if (map_and_fenceable)
  2168. free_space =
  2169. drm_mm_search_free_in_range(&dev_priv->mm.gtt_space,
  2170. size, alignment, 0,
  2171. dev_priv->mm.gtt_mappable_end,
  2172. 0);
  2173. else
  2174. free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
  2175. size, alignment, 0);
  2176. if (free_space != NULL) {
  2177. if (map_and_fenceable)
  2178. obj->gtt_space =
  2179. drm_mm_get_block_range_generic(free_space,
  2180. size, alignment, 0,
  2181. dev_priv->mm.gtt_mappable_end,
  2182. 0);
  2183. else
  2184. obj->gtt_space =
  2185. drm_mm_get_block(free_space, size, alignment);
  2186. }
  2187. if (obj->gtt_space == NULL) {
  2188. /* If the gtt is empty and we're still having trouble
  2189. * fitting our object in, we're out of memory.
  2190. */
  2191. ret = i915_gem_evict_something(dev, size, alignment,
  2192. map_and_fenceable);
  2193. if (ret)
  2194. return ret;
  2195. goto search_free;
  2196. }
  2197. ret = i915_gem_object_get_pages_gtt(obj, gfpmask);
  2198. if (ret) {
  2199. drm_mm_put_block(obj->gtt_space);
  2200. obj->gtt_space = NULL;
  2201. if (ret == -ENOMEM) {
  2202. /* first try to reclaim some memory by clearing the GTT */
  2203. ret = i915_gem_evict_everything(dev, false);
  2204. if (ret) {
  2205. /* now try to shrink everyone else */
  2206. if (gfpmask) {
  2207. gfpmask = 0;
  2208. goto search_free;
  2209. }
  2210. return -ENOMEM;
  2211. }
  2212. goto search_free;
  2213. }
  2214. return ret;
  2215. }
  2216. ret = i915_gem_gtt_prepare_object(obj);
  2217. if (ret) {
  2218. i915_gem_object_put_pages_gtt(obj);
  2219. drm_mm_put_block(obj->gtt_space);
  2220. obj->gtt_space = NULL;
  2221. if (i915_gem_evict_everything(dev, false))
  2222. return ret;
  2223. goto search_free;
  2224. }
  2225. if (!dev_priv->mm.aliasing_ppgtt)
  2226. i915_gem_gtt_bind_object(obj, obj->cache_level);
  2227. list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list);
  2228. list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
  2229. /* Assert that the object is not currently in any GPU domain. As it
  2230. * wasn't in the GTT, there shouldn't be any way it could have been in
  2231. * a GPU cache
  2232. */
  2233. BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
  2234. BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
  2235. obj->gtt_offset = obj->gtt_space->start;
  2236. fenceable =
  2237. obj->gtt_space->size == fence_size &&
  2238. (obj->gtt_space->start & (fence_alignment - 1)) == 0;
  2239. mappable =
  2240. obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
  2241. obj->map_and_fenceable = mappable && fenceable;
  2242. trace_i915_gem_object_bind(obj, map_and_fenceable);
  2243. return 0;
  2244. }
  2245. void
  2246. i915_gem_clflush_object(struct drm_i915_gem_object *obj)
  2247. {
  2248. /* If we don't have a page list set up, then we're not pinned
  2249. * to GPU, and we can ignore the cache flush because it'll happen
  2250. * again at bind time.
  2251. */
  2252. if (obj->pages == NULL)
  2253. return;
  2254. /* If the GPU is snooping the contents of the CPU cache,
  2255. * we do not need to manually clear the CPU cache lines. However,
  2256. * the caches are only snooped when the render cache is
  2257. * flushed/invalidated. As we always have to emit invalidations
  2258. * and flushes when moving into and out of the RENDER domain, correct
  2259. * snooping behaviour occurs naturally as the result of our domain
  2260. * tracking.
  2261. */
  2262. if (obj->cache_level != I915_CACHE_NONE)
  2263. return;
  2264. trace_i915_gem_object_clflush(obj);
  2265. drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE);
  2266. }
  2267. /** Flushes any GPU write domain for the object if it's dirty. */
  2268. static int
  2269. i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj)
  2270. {
  2271. if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0)
  2272. return 0;
  2273. /* Queue the GPU write cache flushing we need. */
  2274. return i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain);
  2275. }
  2276. /** Flushes the GTT write domain for the object if it's dirty. */
  2277. static void
  2278. i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
  2279. {
  2280. uint32_t old_write_domain;
  2281. if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
  2282. return;
  2283. /* No actual flushing is required for the GTT write domain. Writes
  2284. * to it immediately go to main memory as far as we know, so there's
  2285. * no chipset flush. It also doesn't land in render cache.
  2286. *
  2287. * However, we do have to enforce the order so that all writes through
  2288. * the GTT land before any writes to the device, such as updates to
  2289. * the GATT itself.
  2290. */
  2291. wmb();
  2292. old_write_domain = obj->base.write_domain;
  2293. obj->base.write_domain = 0;
  2294. trace_i915_gem_object_change_domain(obj,
  2295. obj->base.read_domains,
  2296. old_write_domain);
  2297. }
  2298. /** Flushes the CPU write domain for the object if it's dirty. */
  2299. static void
  2300. i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
  2301. {
  2302. uint32_t old_write_domain;
  2303. if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
  2304. return;
  2305. i915_gem_clflush_object(obj);
  2306. intel_gtt_chipset_flush();
  2307. old_write_domain = obj->base.write_domain;
  2308. obj->base.write_domain = 0;
  2309. trace_i915_gem_object_change_domain(obj,
  2310. obj->base.read_domains,
  2311. old_write_domain);
  2312. }
  2313. /**
  2314. * Moves a single object to the GTT read, and possibly write domain.
  2315. *
  2316. * This function returns when the move is complete, including waiting on
  2317. * flushes to occur.
  2318. */
  2319. int
  2320. i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
  2321. {
  2322. uint32_t old_write_domain, old_read_domains;
  2323. int ret;
  2324. /* Not valid to be called on unbound objects. */
  2325. if (obj->gtt_space == NULL)
  2326. return -EINVAL;
  2327. if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
  2328. return 0;
  2329. ret = i915_gem_object_flush_gpu_write_domain(obj);
  2330. if (ret)
  2331. return ret;
  2332. if (obj->pending_gpu_write || write) {
  2333. ret = i915_gem_object_wait_rendering(obj);
  2334. if (ret)
  2335. return ret;
  2336. }
  2337. i915_gem_object_flush_cpu_write_domain(obj);
  2338. old_write_domain = obj->base.write_domain;
  2339. old_read_domains = obj->base.read_domains;
  2340. /* It should now be out of any other write domains, and we can update
  2341. * the domain values for our changes.
  2342. */
  2343. BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
  2344. obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
  2345. if (write) {
  2346. obj->base.read_domains = I915_GEM_DOMAIN_GTT;
  2347. obj->base.write_domain = I915_GEM_DOMAIN_GTT;
  2348. obj->dirty = 1;
  2349. }
  2350. trace_i915_gem_object_change_domain(obj,
  2351. old_read_domains,
  2352. old_write_domain);
  2353. return 0;
  2354. }
  2355. int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
  2356. enum i915_cache_level cache_level)
  2357. {
  2358. struct drm_device *dev = obj->base.dev;
  2359. drm_i915_private_t *dev_priv = dev->dev_private;
  2360. int ret;
  2361. if (obj->cache_level == cache_level)
  2362. return 0;
  2363. if (obj->pin_count) {
  2364. DRM_DEBUG("can not change the cache level of pinned objects\n");
  2365. return -EBUSY;
  2366. }
  2367. if (obj->gtt_space) {
  2368. ret = i915_gem_object_finish_gpu(obj);
  2369. if (ret)
  2370. return ret;
  2371. i915_gem_object_finish_gtt(obj);
  2372. /* Before SandyBridge, you could not use tiling or fence
  2373. * registers with snooped memory, so relinquish any fences
  2374. * currently pointing to our region in the aperture.
  2375. */
  2376. if (INTEL_INFO(obj->base.dev)->gen < 6) {
  2377. ret = i915_gem_object_put_fence(obj);
  2378. if (ret)
  2379. return ret;
  2380. }
  2381. if (obj->has_global_gtt_mapping)
  2382. i915_gem_gtt_bind_object(obj, cache_level);
  2383. if (obj->has_aliasing_ppgtt_mapping)
  2384. i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
  2385. obj, cache_level);
  2386. }
  2387. if (cache_level == I915_CACHE_NONE) {
  2388. u32 old_read_domains, old_write_domain;
  2389. /* If we're coming from LLC cached, then we haven't
  2390. * actually been tracking whether the data is in the
  2391. * CPU cache or not, since we only allow one bit set
  2392. * in obj->write_domain and have been skipping the clflushes.
  2393. * Just set it to the CPU cache for now.
  2394. */
  2395. WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
  2396. WARN_ON(obj->base.read_domains & ~I915_GEM_DOMAIN_CPU);
  2397. old_read_domains = obj->base.read_domains;
  2398. old_write_domain = obj->base.write_domain;
  2399. obj->base.read_domains = I915_GEM_DOMAIN_CPU;
  2400. obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  2401. trace_i915_gem_object_change_domain(obj,
  2402. old_read_domains,
  2403. old_write_domain);
  2404. }
  2405. obj->cache_level = cache_level;
  2406. return 0;
  2407. }
  2408. /*
  2409. * Prepare buffer for display plane (scanout, cursors, etc).
  2410. * Can be called from an uninterruptible phase (modesetting) and allows
  2411. * any flushes to be pipelined (for pageflips).
  2412. *
  2413. * For the display plane, we want to be in the GTT but out of any write
  2414. * domains. So in many ways this looks like set_to_gtt_domain() apart from the
  2415. * ability to pipeline the waits, pinning and any additional subtleties
  2416. * that may differentiate the display plane from ordinary buffers.
  2417. */
  2418. int
  2419. i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
  2420. u32 alignment,
  2421. struct intel_ring_buffer *pipelined)
  2422. {
  2423. u32 old_read_domains, old_write_domain;
  2424. int ret;
  2425. ret = i915_gem_object_flush_gpu_write_domain(obj);
  2426. if (ret)
  2427. return ret;
  2428. if (pipelined != obj->ring) {
  2429. ret = i915_gem_object_wait_rendering(obj);
  2430. if (ret == -ERESTARTSYS)
  2431. return ret;
  2432. }
  2433. /* The display engine is not coherent with the LLC cache on gen6. As
  2434. * a result, we make sure that the pinning that is about to occur is
  2435. * done with uncached PTEs. This is lowest common denominator for all
  2436. * chipsets.
  2437. *
  2438. * However for gen6+, we could do better by using the GFDT bit instead
  2439. * of uncaching, which would allow us to flush all the LLC-cached data
  2440. * with that bit in the PTE to main memory with just one PIPE_CONTROL.
  2441. */
  2442. ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
  2443. if (ret)
  2444. return ret;
  2445. /* As the user may map the buffer once pinned in the display plane
  2446. * (e.g. libkms for the bootup splash), we have to ensure that we
  2447. * always use map_and_fenceable for all scanout buffers.
  2448. */
  2449. ret = i915_gem_object_pin(obj, alignment, true);
  2450. if (ret)
  2451. return ret;
  2452. i915_gem_object_flush_cpu_write_domain(obj);
  2453. old_write_domain = obj->base.write_domain;
  2454. old_read_domains = obj->base.read_domains;
  2455. /* It should now be out of any other write domains, and we can update
  2456. * the domain values for our changes.
  2457. */
  2458. BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
  2459. obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
  2460. trace_i915_gem_object_change_domain(obj,
  2461. old_read_domains,
  2462. old_write_domain);
  2463. return 0;
  2464. }
  2465. int
  2466. i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
  2467. {
  2468. int ret;
  2469. if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
  2470. return 0;
  2471. if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
  2472. ret = i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain);
  2473. if (ret)
  2474. return ret;
  2475. }
  2476. ret = i915_gem_object_wait_rendering(obj);
  2477. if (ret)
  2478. return ret;
  2479. /* Ensure that we invalidate the GPU's caches and TLBs. */
  2480. obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
  2481. return 0;
  2482. }
  2483. /**
  2484. * Moves a single object to the CPU read, and possibly write domain.
  2485. *
  2486. * This function returns when the move is complete, including waiting on
  2487. * flushes to occur.
  2488. */
  2489. int
  2490. i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
  2491. {
  2492. uint32_t old_write_domain, old_read_domains;
  2493. int ret;
  2494. if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
  2495. return 0;
  2496. ret = i915_gem_object_flush_gpu_write_domain(obj);
  2497. if (ret)
  2498. return ret;
  2499. ret = i915_gem_object_wait_rendering(obj);
  2500. if (ret)
  2501. return ret;
  2502. i915_gem_object_flush_gtt_write_domain(obj);
  2503. /* If we have a partially-valid cache of the object in the CPU,
  2504. * finish invalidating it and free the per-page flags.
  2505. */
  2506. i915_gem_object_set_to_full_cpu_read_domain(obj);
  2507. old_write_domain = obj->base.write_domain;
  2508. old_read_domains = obj->base.read_domains;
  2509. /* Flush the CPU cache if it's still invalid. */
  2510. if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
  2511. i915_gem_clflush_object(obj);
  2512. obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
  2513. }
  2514. /* It should now be out of any other write domains, and we can update
  2515. * the domain values for our changes.
  2516. */
  2517. BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
  2518. /* If we're writing through the CPU, then the GPU read domains will
  2519. * need to be invalidated at next use.
  2520. */
  2521. if (write) {
  2522. obj->base.read_domains = I915_GEM_DOMAIN_CPU;
  2523. obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  2524. }
  2525. trace_i915_gem_object_change_domain(obj,
  2526. old_read_domains,
  2527. old_write_domain);
  2528. return 0;
  2529. }
  2530. /**
  2531. * Moves the object from a partially CPU read to a full one.
  2532. *
  2533. * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
  2534. * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
  2535. */
  2536. static void
  2537. i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj)
  2538. {
  2539. if (!obj->page_cpu_valid)
  2540. return;
  2541. /* If we're partially in the CPU read domain, finish moving it in.
  2542. */
  2543. if (obj->base.read_domains & I915_GEM_DOMAIN_CPU) {
  2544. int i;
  2545. for (i = 0; i <= (obj->base.size - 1) / PAGE_SIZE; i++) {
  2546. if (obj->page_cpu_valid[i])
  2547. continue;
  2548. drm_clflush_pages(obj->pages + i, 1);
  2549. }
  2550. }
  2551. /* Free the page_cpu_valid mappings which are now stale, whether
  2552. * or not we've got I915_GEM_DOMAIN_CPU.
  2553. */
  2554. kfree(obj->page_cpu_valid);
  2555. obj->page_cpu_valid = NULL;
  2556. }
  2557. /**
  2558. * Set the CPU read domain on a range of the object.
  2559. *
  2560. * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
  2561. * not entirely valid. The page_cpu_valid member of the object flags which
  2562. * pages have been flushed, and will be respected by
  2563. * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
  2564. * of the whole object.
  2565. *
  2566. * This function returns when the move is complete, including waiting on
  2567. * flushes to occur.
  2568. */
  2569. static int
  2570. i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,
  2571. uint64_t offset, uint64_t size)
  2572. {
  2573. uint32_t old_read_domains;
  2574. int i, ret;
  2575. if (offset == 0 && size == obj->base.size)
  2576. return i915_gem_object_set_to_cpu_domain(obj, 0);
  2577. ret = i915_gem_object_flush_gpu_write_domain(obj);
  2578. if (ret)
  2579. return ret;
  2580. ret = i915_gem_object_wait_rendering(obj);
  2581. if (ret)
  2582. return ret;
  2583. i915_gem_object_flush_gtt_write_domain(obj);
  2584. /* If we're already fully in the CPU read domain, we're done. */
  2585. if (obj->page_cpu_valid == NULL &&
  2586. (obj->base.read_domains & I915_GEM_DOMAIN_CPU) != 0)
  2587. return 0;
  2588. /* Otherwise, create/clear the per-page CPU read domain flag if we're
  2589. * newly adding I915_GEM_DOMAIN_CPU
  2590. */
  2591. if (obj->page_cpu_valid == NULL) {
  2592. obj->page_cpu_valid = kzalloc(obj->base.size / PAGE_SIZE,
  2593. GFP_KERNEL);
  2594. if (obj->page_cpu_valid == NULL)
  2595. return -ENOMEM;
  2596. } else if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
  2597. memset(obj->page_cpu_valid, 0, obj->base.size / PAGE_SIZE);
  2598. /* Flush the cache on any pages that are still invalid from the CPU's
  2599. * perspective.
  2600. */
  2601. for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
  2602. i++) {
  2603. if (obj->page_cpu_valid[i])
  2604. continue;
  2605. drm_clflush_pages(obj->pages + i, 1);
  2606. obj->page_cpu_valid[i] = 1;
  2607. }
  2608. /* It should now be out of any other write domains, and we can update
  2609. * the domain values for our changes.
  2610. */
  2611. BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
  2612. old_read_domains = obj->base.read_domains;
  2613. obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
  2614. trace_i915_gem_object_change_domain(obj,
  2615. old_read_domains,
  2616. obj->base.write_domain);
  2617. return 0;
  2618. }
  2619. /* Throttle our rendering by waiting until the ring has completed our requests
  2620. * emitted over 20 msec ago.
  2621. *
  2622. * Note that if we were to use the current jiffies each time around the loop,
  2623. * we wouldn't escape the function with any frames outstanding if the time to
  2624. * render a frame was over 20ms.
  2625. *
  2626. * This should get us reasonable parallelism between CPU and GPU but also
  2627. * relatively low latency when blocking on a particular request to finish.
  2628. */
  2629. static int
  2630. i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
  2631. {
  2632. struct drm_i915_private *dev_priv = dev->dev_private;
  2633. struct drm_i915_file_private *file_priv = file->driver_priv;
  2634. unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
  2635. struct drm_i915_gem_request *request;
  2636. struct intel_ring_buffer *ring = NULL;
  2637. u32 seqno = 0;
  2638. int ret;
  2639. if (atomic_read(&dev_priv->mm.wedged))
  2640. return -EIO;
  2641. spin_lock(&file_priv->mm.lock);
  2642. list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
  2643. if (time_after_eq(request->emitted_jiffies, recent_enough))
  2644. break;
  2645. ring = request->ring;
  2646. seqno = request->seqno;
  2647. }
  2648. spin_unlock(&file_priv->mm.lock);
  2649. if (seqno == 0)
  2650. return 0;
  2651. ret = 0;
  2652. if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {
  2653. /* And wait for the seqno passing without holding any locks and
  2654. * causing extra latency for others. This is safe as the irq
  2655. * generation is designed to be run atomically and so is
  2656. * lockless.
  2657. */
  2658. if (ring->irq_get(ring)) {
  2659. ret = wait_event_interruptible(ring->irq_queue,
  2660. i915_seqno_passed(ring->get_seqno(ring), seqno)
  2661. || atomic_read(&dev_priv->mm.wedged));
  2662. ring->irq_put(ring);
  2663. if (ret == 0 && atomic_read(&dev_priv->mm.wedged))
  2664. ret = -EIO;
  2665. } else if (wait_for_atomic(i915_seqno_passed(ring->get_seqno(ring),
  2666. seqno) ||
  2667. atomic_read(&dev_priv->mm.wedged), 3000)) {
  2668. ret = -EBUSY;
  2669. }
  2670. }
  2671. if (ret == 0)
  2672. queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
  2673. return ret;
  2674. }
  2675. int
  2676. i915_gem_object_pin(struct drm_i915_gem_object *obj,
  2677. uint32_t alignment,
  2678. bool map_and_fenceable)
  2679. {
  2680. struct drm_device *dev = obj->base.dev;
  2681. struct drm_i915_private *dev_priv = dev->dev_private;
  2682. int ret;
  2683. BUG_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
  2684. WARN_ON(i915_verify_lists(dev));
  2685. if (obj->gtt_space != NULL) {
  2686. if ((alignment && obj->gtt_offset & (alignment - 1)) ||
  2687. (map_and_fenceable && !obj->map_and_fenceable)) {
  2688. WARN(obj->pin_count,
  2689. "bo is already pinned with incorrect alignment:"
  2690. " offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
  2691. " obj->map_and_fenceable=%d\n",
  2692. obj->gtt_offset, alignment,
  2693. map_and_fenceable,
  2694. obj->map_and_fenceable);
  2695. ret = i915_gem_object_unbind(obj);
  2696. if (ret)
  2697. return ret;
  2698. }
  2699. }
  2700. if (obj->gtt_space == NULL) {
  2701. ret = i915_gem_object_bind_to_gtt(obj, alignment,
  2702. map_and_fenceable);
  2703. if (ret)
  2704. return ret;
  2705. }
  2706. if (!obj->has_global_gtt_mapping && map_and_fenceable)
  2707. i915_gem_gtt_bind_object(obj, obj->cache_level);
  2708. if (obj->pin_count++ == 0) {
  2709. if (!obj->active)
  2710. list_move_tail(&obj->mm_list,
  2711. &dev_priv->mm.pinned_list);
  2712. }
  2713. obj->pin_mappable |= map_and_fenceable;
  2714. WARN_ON(i915_verify_lists(dev));
  2715. return 0;
  2716. }
  2717. void
  2718. i915_gem_object_unpin(struct drm_i915_gem_object *obj)
  2719. {
  2720. struct drm_device *dev = obj->base.dev;
  2721. drm_i915_private_t *dev_priv = dev->dev_private;
  2722. WARN_ON(i915_verify_lists(dev));
  2723. BUG_ON(obj->pin_count == 0);
  2724. BUG_ON(obj->gtt_space == NULL);
  2725. if (--obj->pin_count == 0) {
  2726. if (!obj->active)
  2727. list_move_tail(&obj->mm_list,
  2728. &dev_priv->mm.inactive_list);
  2729. obj->pin_mappable = false;
  2730. }
  2731. WARN_ON(i915_verify_lists(dev));
  2732. }
  2733. int
  2734. i915_gem_pin_ioctl(struct drm_device *dev, void *data,
  2735. struct drm_file *file)
  2736. {
  2737. struct drm_i915_gem_pin *args = data;
  2738. struct drm_i915_gem_object *obj;
  2739. int ret;
  2740. ret = i915_mutex_lock_interruptible(dev);
  2741. if (ret)
  2742. return ret;
  2743. obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  2744. if (&obj->base == NULL) {
  2745. ret = -ENOENT;
  2746. goto unlock;
  2747. }
  2748. if (obj->madv != I915_MADV_WILLNEED) {
  2749. DRM_ERROR("Attempting to pin a purgeable buffer\n");
  2750. ret = -EINVAL;
  2751. goto out;
  2752. }
  2753. if (obj->pin_filp != NULL && obj->pin_filp != file) {
  2754. DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
  2755. args->handle);
  2756. ret = -EINVAL;
  2757. goto out;
  2758. }
  2759. obj->user_pin_count++;
  2760. obj->pin_filp = file;
  2761. if (obj->user_pin_count == 1) {
  2762. ret = i915_gem_object_pin(obj, args->alignment, true);
  2763. if (ret)
  2764. goto out;
  2765. }
  2766. /* XXX - flush the CPU caches for pinned objects
  2767. * as the X server doesn't manage domains yet
  2768. */
  2769. i915_gem_object_flush_cpu_write_domain(obj);
  2770. args->offset = obj->gtt_offset;
  2771. out:
  2772. drm_gem_object_unreference(&obj->base);
  2773. unlock:
  2774. mutex_unlock(&dev->struct_mutex);
  2775. return ret;
  2776. }
  2777. int
  2778. i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
  2779. struct drm_file *file)
  2780. {
  2781. struct drm_i915_gem_pin *args = data;
  2782. struct drm_i915_gem_object *obj;
  2783. int ret;
  2784. ret = i915_mutex_lock_interruptible(dev);
  2785. if (ret)
  2786. return ret;
  2787. obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  2788. if (&obj->base == NULL) {
  2789. ret = -ENOENT;
  2790. goto unlock;
  2791. }
  2792. if (obj->pin_filp != file) {
  2793. DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
  2794. args->handle);
  2795. ret = -EINVAL;
  2796. goto out;
  2797. }
  2798. obj->user_pin_count--;
  2799. if (obj->user_pin_count == 0) {
  2800. obj->pin_filp = NULL;
  2801. i915_gem_object_unpin(obj);
  2802. }
  2803. out:
  2804. drm_gem_object_unreference(&obj->base);
  2805. unlock:
  2806. mutex_unlock(&dev->struct_mutex);
  2807. return ret;
  2808. }
  2809. int
  2810. i915_gem_busy_ioctl(struct drm_device *dev, void *data,
  2811. struct drm_file *file)
  2812. {
  2813. struct drm_i915_gem_busy *args = data;
  2814. struct drm_i915_gem_object *obj;
  2815. int ret;
  2816. ret = i915_mutex_lock_interruptible(dev);
  2817. if (ret)
  2818. return ret;
  2819. obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  2820. if (&obj->base == NULL) {
  2821. ret = -ENOENT;
  2822. goto unlock;
  2823. }
  2824. /* Count all active objects as busy, even if they are currently not used
  2825. * by the gpu. Users of this interface expect objects to eventually
  2826. * become non-busy without any further actions, therefore emit any
  2827. * necessary flushes here.
  2828. */
  2829. args->busy = obj->active;
  2830. if (args->busy) {
  2831. /* Unconditionally flush objects, even when the gpu still uses this
  2832. * object. Userspace calling this function indicates that it wants to
  2833. * use this buffer rather sooner than later, so issuing the required
  2834. * flush earlier is beneficial.
  2835. */
  2836. if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
  2837. ret = i915_gem_flush_ring(obj->ring,
  2838. 0, obj->base.write_domain);
  2839. } else if (obj->ring->outstanding_lazy_request ==
  2840. obj->last_rendering_seqno) {
  2841. struct drm_i915_gem_request *request;
  2842. /* This ring is not being cleared by active usage,
  2843. * so emit a request to do so.
  2844. */
  2845. request = kzalloc(sizeof(*request), GFP_KERNEL);
  2846. if (request) {
  2847. ret = i915_add_request(obj->ring, NULL, request);
  2848. if (ret)
  2849. kfree(request);
  2850. } else
  2851. ret = -ENOMEM;
  2852. }
  2853. /* Update the active list for the hardware's current position.
  2854. * Otherwise this only updates on a delayed timer or when irqs
  2855. * are actually unmasked, and our working set ends up being
  2856. * larger than required.
  2857. */
  2858. i915_gem_retire_requests_ring(obj->ring);
  2859. args->busy = obj->active;
  2860. }
  2861. drm_gem_object_unreference(&obj->base);
  2862. unlock:
  2863. mutex_unlock(&dev->struct_mutex);
  2864. return ret;
  2865. }
  2866. int
  2867. i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
  2868. struct drm_file *file_priv)
  2869. {
  2870. return i915_gem_ring_throttle(dev, file_priv);
  2871. }
  2872. int
  2873. i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
  2874. struct drm_file *file_priv)
  2875. {
  2876. struct drm_i915_gem_madvise *args = data;
  2877. struct drm_i915_gem_object *obj;
  2878. int ret;
  2879. switch (args->madv) {
  2880. case I915_MADV_DONTNEED:
  2881. case I915_MADV_WILLNEED:
  2882. break;
  2883. default:
  2884. return -EINVAL;
  2885. }
  2886. ret = i915_mutex_lock_interruptible(dev);
  2887. if (ret)
  2888. return ret;
  2889. obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
  2890. if (&obj->base == NULL) {
  2891. ret = -ENOENT;
  2892. goto unlock;
  2893. }
  2894. if (obj->pin_count) {
  2895. ret = -EINVAL;
  2896. goto out;
  2897. }
  2898. if (obj->madv != __I915_MADV_PURGED)
  2899. obj->madv = args->madv;
  2900. /* if the object is no longer bound, discard its backing storage */
  2901. if (i915_gem_object_is_purgeable(obj) &&
  2902. obj->gtt_space == NULL)
  2903. i915_gem_object_truncate(obj);
  2904. args->retained = obj->madv != __I915_MADV_PURGED;
  2905. out:
  2906. drm_gem_object_unreference(&obj->base);
  2907. unlock:
  2908. mutex_unlock(&dev->struct_mutex);
  2909. return ret;
  2910. }
  2911. struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
  2912. size_t size)
  2913. {
  2914. struct drm_i915_private *dev_priv = dev->dev_private;
  2915. struct drm_i915_gem_object *obj;
  2916. struct address_space *mapping;
  2917. obj = kzalloc(sizeof(*obj), GFP_KERNEL);
  2918. if (obj == NULL)
  2919. return NULL;
  2920. if (drm_gem_object_init(dev, &obj->base, size) != 0) {
  2921. kfree(obj);
  2922. return NULL;
  2923. }
  2924. mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
  2925. mapping_set_gfp_mask(mapping, GFP_HIGHUSER | __GFP_RECLAIMABLE);
  2926. i915_gem_info_add_obj(dev_priv, size);
  2927. obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  2928. obj->base.read_domains = I915_GEM_DOMAIN_CPU;
  2929. if (HAS_LLC(dev)) {
  2930. /* On some devices, we can have the GPU use the LLC (the CPU
  2931. * cache) for about a 10% performance improvement
  2932. * compared to uncached. Graphics requests other than
  2933. * display scanout are coherent with the CPU in
  2934. * accessing this cache. This means in this mode we
  2935. * don't need to clflush on the CPU side, and on the
  2936. * GPU side we only need to flush internal caches to
  2937. * get data visible to the CPU.
  2938. *
  2939. * However, we maintain the display planes as UC, and so
  2940. * need to rebind when first used as such.
  2941. */
  2942. obj->cache_level = I915_CACHE_LLC;
  2943. } else
  2944. obj->cache_level = I915_CACHE_NONE;
  2945. obj->base.driver_private = NULL;
  2946. obj->fence_reg = I915_FENCE_REG_NONE;
  2947. INIT_LIST_HEAD(&obj->mm_list);
  2948. INIT_LIST_HEAD(&obj->gtt_list);
  2949. INIT_LIST_HEAD(&obj->ring_list);
  2950. INIT_LIST_HEAD(&obj->exec_list);
  2951. INIT_LIST_HEAD(&obj->gpu_write_list);
  2952. obj->madv = I915_MADV_WILLNEED;
  2953. /* Avoid an unnecessary call to unbind on the first bind. */
  2954. obj->map_and_fenceable = true;
  2955. return obj;
  2956. }
  2957. int i915_gem_init_object(struct drm_gem_object *obj)
  2958. {
  2959. BUG();
  2960. return 0;
  2961. }
  2962. static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj)
  2963. {
  2964. struct drm_device *dev = obj->base.dev;
  2965. drm_i915_private_t *dev_priv = dev->dev_private;
  2966. int ret;
  2967. ret = i915_gem_object_unbind(obj);
  2968. if (ret == -ERESTARTSYS) {
  2969. list_move(&obj->mm_list,
  2970. &dev_priv->mm.deferred_free_list);
  2971. return;
  2972. }
  2973. trace_i915_gem_object_destroy(obj);
  2974. if (obj->base.map_list.map)
  2975. drm_gem_free_mmap_offset(&obj->base);
  2976. drm_gem_object_release(&obj->base);
  2977. i915_gem_info_remove_obj(dev_priv, obj->base.size);
  2978. kfree(obj->page_cpu_valid);
  2979. kfree(obj->bit_17);
  2980. kfree(obj);
  2981. }
  2982. void i915_gem_free_object(struct drm_gem_object *gem_obj)
  2983. {
  2984. struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
  2985. struct drm_device *dev = obj->base.dev;
  2986. while (obj->pin_count > 0)
  2987. i915_gem_object_unpin(obj);
  2988. if (obj->phys_obj)
  2989. i915_gem_detach_phys_object(dev, obj);
  2990. i915_gem_free_object_tail(obj);
  2991. }
  2992. int
  2993. i915_gem_idle(struct drm_device *dev)
  2994. {
  2995. drm_i915_private_t *dev_priv = dev->dev_private;
  2996. int ret;
  2997. mutex_lock(&dev->struct_mutex);
  2998. if (dev_priv->mm.suspended) {
  2999. mutex_unlock(&dev->struct_mutex);
  3000. return 0;
  3001. }
  3002. ret = i915_gpu_idle(dev, true);
  3003. if (ret) {
  3004. mutex_unlock(&dev->struct_mutex);
  3005. return ret;
  3006. }
  3007. /* Under UMS, be paranoid and evict. */
  3008. if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
  3009. ret = i915_gem_evict_inactive(dev, false);
  3010. if (ret) {
  3011. mutex_unlock(&dev->struct_mutex);
  3012. return ret;
  3013. }
  3014. }
  3015. i915_gem_reset_fences(dev);
  3016. /* Hack! Don't let anybody do execbuf while we don't control the chip.
  3017. * We need to replace this with a semaphore, or something.
  3018. * And not confound mm.suspended!
  3019. */
  3020. dev_priv->mm.suspended = 1;
  3021. del_timer_sync(&dev_priv->hangcheck_timer);
  3022. i915_kernel_lost_context(dev);
  3023. i915_gem_cleanup_ringbuffer(dev);
  3024. mutex_unlock(&dev->struct_mutex);
  3025. /* Cancel the retire work handler, which should be idle now. */
  3026. cancel_delayed_work_sync(&dev_priv->mm.retire_work);
  3027. return 0;
  3028. }
  3029. void i915_gem_init_swizzling(struct drm_device *dev)
  3030. {
  3031. drm_i915_private_t *dev_priv = dev->dev_private;
  3032. if (INTEL_INFO(dev)->gen < 5 ||
  3033. dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
  3034. return;
  3035. I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
  3036. DISP_TILE_SURFACE_SWIZZLING);
  3037. if (IS_GEN5(dev))
  3038. return;
  3039. I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
  3040. if (IS_GEN6(dev))
  3041. I915_WRITE(ARB_MODE, ARB_MODE_ENABLE(ARB_MODE_SWIZZLE_SNB));
  3042. else
  3043. I915_WRITE(ARB_MODE, ARB_MODE_ENABLE(ARB_MODE_SWIZZLE_IVB));
  3044. }
  3045. void i915_gem_init_ppgtt(struct drm_device *dev)
  3046. {
  3047. drm_i915_private_t *dev_priv = dev->dev_private;
  3048. uint32_t pd_offset;
  3049. struct intel_ring_buffer *ring;
  3050. int i;
  3051. if (!dev_priv->mm.aliasing_ppgtt)
  3052. return;
  3053. pd_offset = dev_priv->mm.aliasing_ppgtt->pd_offset;
  3054. pd_offset /= 64; /* in cachelines, */
  3055. pd_offset <<= 16;
  3056. if (INTEL_INFO(dev)->gen == 6) {
  3057. uint32_t ecochk = I915_READ(GAM_ECOCHK);
  3058. I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT |
  3059. ECOCHK_PPGTT_CACHE64B);
  3060. I915_WRITE(GFX_MODE, GFX_MODE_ENABLE(GFX_PPGTT_ENABLE));
  3061. } else if (INTEL_INFO(dev)->gen >= 7) {
  3062. I915_WRITE(GAM_ECOCHK, ECOCHK_PPGTT_CACHE64B);
  3063. /* GFX_MODE is per-ring on gen7+ */
  3064. }
  3065. for (i = 0; i < I915_NUM_RINGS; i++) {
  3066. ring = &dev_priv->ring[i];
  3067. if (INTEL_INFO(dev)->gen >= 7)
  3068. I915_WRITE(RING_MODE_GEN7(ring),
  3069. GFX_MODE_ENABLE(GFX_PPGTT_ENABLE));
  3070. I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
  3071. I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset);
  3072. }
  3073. }
  3074. int
  3075. i915_gem_init_hw(struct drm_device *dev)
  3076. {
  3077. drm_i915_private_t *dev_priv = dev->dev_private;
  3078. int ret;
  3079. i915_gem_init_swizzling(dev);
  3080. ret = intel_init_render_ring_buffer(dev);
  3081. if (ret)
  3082. return ret;
  3083. if (HAS_BSD(dev)) {
  3084. ret = intel_init_bsd_ring_buffer(dev);
  3085. if (ret)
  3086. goto cleanup_render_ring;
  3087. }
  3088. if (HAS_BLT(dev)) {
  3089. ret = intel_init_blt_ring_buffer(dev);
  3090. if (ret)
  3091. goto cleanup_bsd_ring;
  3092. }
  3093. dev_priv->next_seqno = 1;
  3094. i915_gem_init_ppgtt(dev);
  3095. return 0;
  3096. cleanup_bsd_ring:
  3097. intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
  3098. cleanup_render_ring:
  3099. intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
  3100. return ret;
  3101. }
  3102. void
  3103. i915_gem_cleanup_ringbuffer(struct drm_device *dev)
  3104. {
  3105. drm_i915_private_t *dev_priv = dev->dev_private;
  3106. int i;
  3107. for (i = 0; i < I915_NUM_RINGS; i++)
  3108. intel_cleanup_ring_buffer(&dev_priv->ring[i]);
  3109. }
  3110. int
  3111. i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
  3112. struct drm_file *file_priv)
  3113. {
  3114. drm_i915_private_t *dev_priv = dev->dev_private;
  3115. int ret, i;
  3116. if (drm_core_check_feature(dev, DRIVER_MODESET))
  3117. return 0;
  3118. if (atomic_read(&dev_priv->mm.wedged)) {
  3119. DRM_ERROR("Reenabling wedged hardware, good luck\n");
  3120. atomic_set(&dev_priv->mm.wedged, 0);
  3121. }
  3122. mutex_lock(&dev->struct_mutex);
  3123. dev_priv->mm.suspended = 0;
  3124. ret = i915_gem_init_hw(dev);
  3125. if (ret != 0) {
  3126. mutex_unlock(&dev->struct_mutex);
  3127. return ret;
  3128. }
  3129. BUG_ON(!list_empty(&dev_priv->mm.active_list));
  3130. BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
  3131. BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
  3132. for (i = 0; i < I915_NUM_RINGS; i++) {
  3133. BUG_ON(!list_empty(&dev_priv->ring[i].active_list));
  3134. BUG_ON(!list_empty(&dev_priv->ring[i].request_list));
  3135. }
  3136. mutex_unlock(&dev->struct_mutex);
  3137. ret = drm_irq_install(dev);
  3138. if (ret)
  3139. goto cleanup_ringbuffer;
  3140. return 0;
  3141. cleanup_ringbuffer:
  3142. mutex_lock(&dev->struct_mutex);
  3143. i915_gem_cleanup_ringbuffer(dev);
  3144. dev_priv->mm.suspended = 1;
  3145. mutex_unlock(&dev->struct_mutex);
  3146. return ret;
  3147. }
  3148. int
  3149. i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
  3150. struct drm_file *file_priv)
  3151. {
  3152. if (drm_core_check_feature(dev, DRIVER_MODESET))
  3153. return 0;
  3154. drm_irq_uninstall(dev);
  3155. return i915_gem_idle(dev);
  3156. }
  3157. void
  3158. i915_gem_lastclose(struct drm_device *dev)
  3159. {
  3160. int ret;
  3161. if (drm_core_check_feature(dev, DRIVER_MODESET))
  3162. return;
  3163. ret = i915_gem_idle(dev);
  3164. if (ret)
  3165. DRM_ERROR("failed to idle hardware: %d\n", ret);
  3166. }
  3167. static void
  3168. init_ring_lists(struct intel_ring_buffer *ring)
  3169. {
  3170. INIT_LIST_HEAD(&ring->active_list);
  3171. INIT_LIST_HEAD(&ring->request_list);
  3172. INIT_LIST_HEAD(&ring->gpu_write_list);
  3173. }
  3174. void
  3175. i915_gem_load(struct drm_device *dev)
  3176. {
  3177. int i;
  3178. drm_i915_private_t *dev_priv = dev->dev_private;
  3179. INIT_LIST_HEAD(&dev_priv->mm.active_list);
  3180. INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
  3181. INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
  3182. INIT_LIST_HEAD(&dev_priv->mm.pinned_list);
  3183. INIT_LIST_HEAD(&dev_priv->mm.fence_list);
  3184. INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
  3185. INIT_LIST_HEAD(&dev_priv->mm.gtt_list);
  3186. for (i = 0; i < I915_NUM_RINGS; i++)
  3187. init_ring_lists(&dev_priv->ring[i]);
  3188. for (i = 0; i < I915_MAX_NUM_FENCES; i++)
  3189. INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
  3190. INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
  3191. i915_gem_retire_work_handler);
  3192. init_completion(&dev_priv->error_completion);
  3193. /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
  3194. if (IS_GEN3(dev)) {
  3195. u32 tmp = I915_READ(MI_ARB_STATE);
  3196. if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) {
  3197. /* arb state is a masked write, so set bit + bit in mask */
  3198. tmp = MI_ARB_C3_LP_WRITE_ENABLE | (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT);
  3199. I915_WRITE(MI_ARB_STATE, tmp);
  3200. }
  3201. }
  3202. dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
  3203. /* Old X drivers will take 0-2 for front, back, depth buffers */
  3204. if (!drm_core_check_feature(dev, DRIVER_MODESET))
  3205. dev_priv->fence_reg_start = 3;
  3206. if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
  3207. dev_priv->num_fence_regs = 16;
  3208. else
  3209. dev_priv->num_fence_regs = 8;
  3210. /* Initialize fence registers to zero */
  3211. for (i = 0; i < dev_priv->num_fence_regs; i++) {
  3212. i915_gem_clear_fence_reg(dev, &dev_priv->fence_regs[i]);
  3213. }
  3214. i915_gem_detect_bit_6_swizzle(dev);
  3215. init_waitqueue_head(&dev_priv->pending_flip_queue);
  3216. dev_priv->mm.interruptible = true;
  3217. dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
  3218. dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
  3219. register_shrinker(&dev_priv->mm.inactive_shrinker);
  3220. }
  3221. /*
  3222. * Create a physically contiguous memory object for this object
  3223. * e.g. for cursor + overlay regs
  3224. */
  3225. static int i915_gem_init_phys_object(struct drm_device *dev,
  3226. int id, int size, int align)
  3227. {
  3228. drm_i915_private_t *dev_priv = dev->dev_private;
  3229. struct drm_i915_gem_phys_object *phys_obj;
  3230. int ret;
  3231. if (dev_priv->mm.phys_objs[id - 1] || !size)
  3232. return 0;
  3233. phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
  3234. if (!phys_obj)
  3235. return -ENOMEM;
  3236. phys_obj->id = id;
  3237. phys_obj->handle = drm_pci_alloc(dev, size, align);
  3238. if (!phys_obj->handle) {
  3239. ret = -ENOMEM;
  3240. goto kfree_obj;
  3241. }
  3242. #ifdef CONFIG_X86
  3243. set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
  3244. #endif
  3245. dev_priv->mm.phys_objs[id - 1] = phys_obj;
  3246. return 0;
  3247. kfree_obj:
  3248. kfree(phys_obj);
  3249. return ret;
  3250. }
  3251. static void i915_gem_free_phys_object(struct drm_device *dev, int id)
  3252. {
  3253. drm_i915_private_t *dev_priv = dev->dev_private;
  3254. struct drm_i915_gem_phys_object *phys_obj;
  3255. if (!dev_priv->mm.phys_objs[id - 1])
  3256. return;
  3257. phys_obj = dev_priv->mm.phys_objs[id - 1];
  3258. if (phys_obj->cur_obj) {
  3259. i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
  3260. }
  3261. #ifdef CONFIG_X86
  3262. set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
  3263. #endif
  3264. drm_pci_free(dev, phys_obj->handle);
  3265. kfree(phys_obj);
  3266. dev_priv->mm.phys_objs[id - 1] = NULL;
  3267. }
  3268. void i915_gem_free_all_phys_object(struct drm_device *dev)
  3269. {
  3270. int i;
  3271. for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
  3272. i915_gem_free_phys_object(dev, i);
  3273. }
  3274. void i915_gem_detach_phys_object(struct drm_device *dev,
  3275. struct drm_i915_gem_object *obj)
  3276. {
  3277. struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
  3278. char *vaddr;
  3279. int i;
  3280. int page_count;
  3281. if (!obj->phys_obj)
  3282. return;
  3283. vaddr = obj->phys_obj->handle->vaddr;
  3284. page_count = obj->base.size / PAGE_SIZE;
  3285. for (i = 0; i < page_count; i++) {
  3286. struct page *page = shmem_read_mapping_page(mapping, i);
  3287. if (!IS_ERR(page)) {
  3288. char *dst = kmap_atomic(page);
  3289. memcpy(dst, vaddr + i*PAGE_SIZE, PAGE_SIZE);
  3290. kunmap_atomic(dst);
  3291. drm_clflush_pages(&page, 1);
  3292. set_page_dirty(page);
  3293. mark_page_accessed(page);
  3294. page_cache_release(page);
  3295. }
  3296. }
  3297. intel_gtt_chipset_flush();
  3298. obj->phys_obj->cur_obj = NULL;
  3299. obj->phys_obj = NULL;
  3300. }
  3301. int
  3302. i915_gem_attach_phys_object(struct drm_device *dev,
  3303. struct drm_i915_gem_object *obj,
  3304. int id,
  3305. int align)
  3306. {
  3307. struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
  3308. drm_i915_private_t *dev_priv = dev->dev_private;
  3309. int ret = 0;
  3310. int page_count;
  3311. int i;
  3312. if (id > I915_MAX_PHYS_OBJECT)
  3313. return -EINVAL;
  3314. if (obj->phys_obj) {
  3315. if (obj->phys_obj->id == id)
  3316. return 0;
  3317. i915_gem_detach_phys_object(dev, obj);
  3318. }
  3319. /* create a new object */
  3320. if (!dev_priv->mm.phys_objs[id - 1]) {
  3321. ret = i915_gem_init_phys_object(dev, id,
  3322. obj->base.size, align);
  3323. if (ret) {
  3324. DRM_ERROR("failed to init phys object %d size: %zu\n",
  3325. id, obj->base.size);
  3326. return ret;
  3327. }
  3328. }
  3329. /* bind to the object */
  3330. obj->phys_obj = dev_priv->mm.phys_objs[id - 1];
  3331. obj->phys_obj->cur_obj = obj;
  3332. page_count = obj->base.size / PAGE_SIZE;
  3333. for (i = 0; i < page_count; i++) {
  3334. struct page *page;
  3335. char *dst, *src;
  3336. page = shmem_read_mapping_page(mapping, i);
  3337. if (IS_ERR(page))
  3338. return PTR_ERR(page);
  3339. src = kmap_atomic(page);
  3340. dst = obj->phys_obj->handle->vaddr + (i * PAGE_SIZE);
  3341. memcpy(dst, src, PAGE_SIZE);
  3342. kunmap_atomic(src);
  3343. mark_page_accessed(page);
  3344. page_cache_release(page);
  3345. }
  3346. return 0;
  3347. }
  3348. static int
  3349. i915_gem_phys_pwrite(struct drm_device *dev,
  3350. struct drm_i915_gem_object *obj,
  3351. struct drm_i915_gem_pwrite *args,
  3352. struct drm_file *file_priv)
  3353. {
  3354. void *vaddr = obj->phys_obj->handle->vaddr + args->offset;
  3355. char __user *user_data = (char __user *) (uintptr_t) args->data_ptr;
  3356. if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
  3357. unsigned long unwritten;
  3358. /* The physical object once assigned is fixed for the lifetime
  3359. * of the obj, so we can safely drop the lock and continue
  3360. * to access vaddr.
  3361. */
  3362. mutex_unlock(&dev->struct_mutex);
  3363. unwritten = copy_from_user(vaddr, user_data, args->size);
  3364. mutex_lock(&dev->struct_mutex);
  3365. if (unwritten)
  3366. return -EFAULT;
  3367. }
  3368. intel_gtt_chipset_flush();
  3369. return 0;
  3370. }
  3371. void i915_gem_release(struct drm_device *dev, struct drm_file *file)
  3372. {
  3373. struct drm_i915_file_private *file_priv = file->driver_priv;
  3374. /* Clean up our request list when the client is going away, so that
  3375. * later retire_requests won't dereference our soon-to-be-gone
  3376. * file_priv.
  3377. */
  3378. spin_lock(&file_priv->mm.lock);
  3379. while (!list_empty(&file_priv->mm.request_list)) {
  3380. struct drm_i915_gem_request *request;
  3381. request = list_first_entry(&file_priv->mm.request_list,
  3382. struct drm_i915_gem_request,
  3383. client_list);
  3384. list_del(&request->client_list);
  3385. request->file_priv = NULL;
  3386. }
  3387. spin_unlock(&file_priv->mm.lock);
  3388. }
  3389. static int
  3390. i915_gpu_is_active(struct drm_device *dev)
  3391. {
  3392. drm_i915_private_t *dev_priv = dev->dev_private;
  3393. int lists_empty;
  3394. lists_empty = list_empty(&dev_priv->mm.flushing_list) &&
  3395. list_empty(&dev_priv->mm.active_list);
  3396. return !lists_empty;
  3397. }
  3398. static int
  3399. i915_gem_inactive_shrink(struct shrinker *shrinker, struct shrink_control *sc)
  3400. {
  3401. struct drm_i915_private *dev_priv =
  3402. container_of(shrinker,
  3403. struct drm_i915_private,
  3404. mm.inactive_shrinker);
  3405. struct drm_device *dev = dev_priv->dev;
  3406. struct drm_i915_gem_object *obj, *next;
  3407. int nr_to_scan = sc->nr_to_scan;
  3408. int cnt;
  3409. if (!mutex_trylock(&dev->struct_mutex))
  3410. return 0;
  3411. /* "fast-path" to count number of available objects */
  3412. if (nr_to_scan == 0) {
  3413. cnt = 0;
  3414. list_for_each_entry(obj,
  3415. &dev_priv->mm.inactive_list,
  3416. mm_list)
  3417. cnt++;
  3418. mutex_unlock(&dev->struct_mutex);
  3419. return cnt / 100 * sysctl_vfs_cache_pressure;
  3420. }
  3421. rescan:
  3422. /* first scan for clean buffers */
  3423. i915_gem_retire_requests(dev);
  3424. list_for_each_entry_safe(obj, next,
  3425. &dev_priv->mm.inactive_list,
  3426. mm_list) {
  3427. if (i915_gem_object_is_purgeable(obj)) {
  3428. if (i915_gem_object_unbind(obj) == 0 &&
  3429. --nr_to_scan == 0)
  3430. break;
  3431. }
  3432. }
  3433. /* second pass, evict/count anything still on the inactive list */
  3434. cnt = 0;
  3435. list_for_each_entry_safe(obj, next,
  3436. &dev_priv->mm.inactive_list,
  3437. mm_list) {
  3438. if (nr_to_scan &&
  3439. i915_gem_object_unbind(obj) == 0)
  3440. nr_to_scan--;
  3441. else
  3442. cnt++;
  3443. }
  3444. if (nr_to_scan && i915_gpu_is_active(dev)) {
  3445. /*
  3446. * We are desperate for pages, so as a last resort, wait
  3447. * for the GPU to finish and discard whatever we can.
  3448. * This has a dramatic impact to reduce the number of
  3449. * OOM-killer events whilst running the GPU aggressively.
  3450. */
  3451. if (i915_gpu_idle(dev, true) == 0)
  3452. goto rescan;
  3453. }
  3454. mutex_unlock(&dev->struct_mutex);
  3455. return cnt / 100 * sysctl_vfs_cache_pressure;
  3456. }