i915_gem.c 135 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615461646174618461946204621462246234624462546264627462846294630463146324633463446354636463746384639464046414642464346444645464646474648464946504651465246534654465546564657465846594660466146624663466446654666466746684669467046714672467346744675467646774678467946804681468246834684468546864687468846894690469146924693469446954696469746984699470047014702470347044705470647074708470947104711471247134714471547164717471847194720472147224723472447254726472747284729473047314732473347344735473647374738473947404741474247434744474547464747474847494750475147524753475447554756475747584759476047614762476347644765476647674768476947704771477247734774477547764777477847794780478147824783478447854786478747884789479047914792479347944795479647974798479948004801480248034804480548064807480848094810481148124813481448154816481748184819482048214822482348244825482648274828482948304831483248334834483548364837483848394840484148424843484448454846484748484849485048514852485348544855485648574858485948604861486248634864486548664867486848694870487148724873487448754876487748784879488048814882488348844885488648874888488948904891489248934894489548964897489848994900490149024903490449054906490749084909491049114912491349144915491649174918491949204921492249234924492549264927492849294930493149324933493449354936493749384939494049414942494349444945494649474948494949504951495249534954495549564957495849594960496149624963496449654966496749684969497049714972497349744975497649774978497949804981498249834984498549864987498849894990499149924993499449954996499749984999500050015002500350045005500650075008500950105011501250135014501550165017501850195020502150225023502450255026502750285029503050315032503350345035503650375038503950405041504250435044504550465047504850495050505150525053505450555056505750585059506050615062506350645065506650675068506950705071507250735074507550765077507850795080508150825083508450855086508750885089509050915092509350945095509650975098509951005101510251035104510551065107510851095110511151125113511451155116511751185119512051215122512351245125512651275128512951305131513251335134513551365137513851395140514151425143514451455146514751485149515051515152515351545155515651575158515951605161516251635164516551665167516851695170517151725173517451755176517751785179518051815182518351845185518651875188518951905191
  1. /*
  2. * Copyright © 2008 Intel Corporation
  3. *
  4. * Permission is hereby granted, free of charge, to any person obtaining a
  5. * copy of this software and associated documentation files (the "Software"),
  6. * to deal in the Software without restriction, including without limitation
  7. * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8. * and/or sell copies of the Software, and to permit persons to whom the
  9. * Software is furnished to do so, subject to the following conditions:
  10. *
  11. * The above copyright notice and this permission notice (including the next
  12. * paragraph) shall be included in all copies or substantial portions of the
  13. * Software.
  14. *
  15. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17. * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  18. * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19. * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20. * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21. * IN THE SOFTWARE.
  22. *
  23. * Authors:
  24. * Eric Anholt <eric@anholt.net>
  25. *
  26. */
  27. #include "drmP.h"
  28. #include "drm.h"
  29. #include "i915_drm.h"
  30. #include "i915_drv.h"
  31. #include "i915_trace.h"
  32. #include "intel_drv.h"
  33. #include <linux/slab.h>
  34. #include <linux/swap.h>
  35. #include <linux/pci.h>
  36. static int i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj);
  37. static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
  38. static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
  39. static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj,
  40. int write);
  41. static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
  42. uint64_t offset,
  43. uint64_t size);
  44. static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj);
  45. static int i915_gem_object_wait_rendering(struct drm_gem_object *obj);
  46. static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
  47. unsigned alignment);
  48. static void i915_gem_clear_fence_reg(struct drm_gem_object *obj);
  49. static int i915_gem_evict_something(struct drm_device *dev, int min_size);
  50. static int i915_gem_evict_from_inactive_list(struct drm_device *dev);
  51. static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
  52. struct drm_i915_gem_pwrite *args,
  53. struct drm_file *file_priv);
  54. static void i915_gem_free_object_tail(struct drm_gem_object *obj);
  55. static LIST_HEAD(shrink_list);
  56. static DEFINE_SPINLOCK(shrink_list_lock);
  57. int i915_gem_do_init(struct drm_device *dev, unsigned long start,
  58. unsigned long end)
  59. {
  60. drm_i915_private_t *dev_priv = dev->dev_private;
  61. if (start >= end ||
  62. (start & (PAGE_SIZE - 1)) != 0 ||
  63. (end & (PAGE_SIZE - 1)) != 0) {
  64. return -EINVAL;
  65. }
  66. drm_mm_init(&dev_priv->mm.gtt_space, start,
  67. end - start);
  68. dev->gtt_total = (uint32_t) (end - start);
  69. return 0;
  70. }
  71. int
  72. i915_gem_init_ioctl(struct drm_device *dev, void *data,
  73. struct drm_file *file_priv)
  74. {
  75. struct drm_i915_gem_init *args = data;
  76. int ret;
  77. mutex_lock(&dev->struct_mutex);
  78. ret = i915_gem_do_init(dev, args->gtt_start, args->gtt_end);
  79. mutex_unlock(&dev->struct_mutex);
  80. return ret;
  81. }
  82. int
  83. i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
  84. struct drm_file *file_priv)
  85. {
  86. struct drm_i915_gem_get_aperture *args = data;
  87. if (!(dev->driver->driver_features & DRIVER_GEM))
  88. return -ENODEV;
  89. args->aper_size = dev->gtt_total;
  90. args->aper_available_size = (args->aper_size -
  91. atomic_read(&dev->pin_memory));
  92. return 0;
  93. }
  94. /**
  95. * Creates a new mm object and returns a handle to it.
  96. */
  97. int
  98. i915_gem_create_ioctl(struct drm_device *dev, void *data,
  99. struct drm_file *file_priv)
  100. {
  101. struct drm_i915_gem_create *args = data;
  102. struct drm_gem_object *obj;
  103. int ret;
  104. u32 handle;
  105. args->size = roundup(args->size, PAGE_SIZE);
  106. /* Allocate the new object */
  107. obj = i915_gem_alloc_object(dev, args->size);
  108. if (obj == NULL)
  109. return -ENOMEM;
  110. ret = drm_gem_handle_create(file_priv, obj, &handle);
  111. drm_gem_object_unreference_unlocked(obj);
  112. if (ret)
  113. return ret;
  114. args->handle = handle;
  115. return 0;
  116. }
  117. static inline int
  118. fast_shmem_read(struct page **pages,
  119. loff_t page_base, int page_offset,
  120. char __user *data,
  121. int length)
  122. {
  123. char __iomem *vaddr;
  124. int unwritten;
  125. vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
  126. if (vaddr == NULL)
  127. return -ENOMEM;
  128. unwritten = __copy_to_user_inatomic(data, vaddr + page_offset, length);
  129. kunmap_atomic(vaddr, KM_USER0);
  130. if (unwritten)
  131. return -EFAULT;
  132. return 0;
  133. }
  134. static int i915_gem_object_needs_bit17_swizzle(struct drm_gem_object *obj)
  135. {
  136. drm_i915_private_t *dev_priv = obj->dev->dev_private;
  137. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  138. return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
  139. obj_priv->tiling_mode != I915_TILING_NONE;
  140. }
  141. static inline void
  142. slow_shmem_copy(struct page *dst_page,
  143. int dst_offset,
  144. struct page *src_page,
  145. int src_offset,
  146. int length)
  147. {
  148. char *dst_vaddr, *src_vaddr;
  149. dst_vaddr = kmap(dst_page);
  150. src_vaddr = kmap(src_page);
  151. memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);
  152. kunmap(src_page);
  153. kunmap(dst_page);
  154. }
  155. static inline void
  156. slow_shmem_bit17_copy(struct page *gpu_page,
  157. int gpu_offset,
  158. struct page *cpu_page,
  159. int cpu_offset,
  160. int length,
  161. int is_read)
  162. {
  163. char *gpu_vaddr, *cpu_vaddr;
  164. /* Use the unswizzled path if this page isn't affected. */
  165. if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
  166. if (is_read)
  167. return slow_shmem_copy(cpu_page, cpu_offset,
  168. gpu_page, gpu_offset, length);
  169. else
  170. return slow_shmem_copy(gpu_page, gpu_offset,
  171. cpu_page, cpu_offset, length);
  172. }
  173. gpu_vaddr = kmap(gpu_page);
  174. cpu_vaddr = kmap(cpu_page);
  175. /* Copy the data, XORing A6 with A17 (1). The user already knows he's
  176. * XORing with the other bits (A9 for Y, A9 and A10 for X)
  177. */
  178. while (length > 0) {
  179. int cacheline_end = ALIGN(gpu_offset + 1, 64);
  180. int this_length = min(cacheline_end - gpu_offset, length);
  181. int swizzled_gpu_offset = gpu_offset ^ 64;
  182. if (is_read) {
  183. memcpy(cpu_vaddr + cpu_offset,
  184. gpu_vaddr + swizzled_gpu_offset,
  185. this_length);
  186. } else {
  187. memcpy(gpu_vaddr + swizzled_gpu_offset,
  188. cpu_vaddr + cpu_offset,
  189. this_length);
  190. }
  191. cpu_offset += this_length;
  192. gpu_offset += this_length;
  193. length -= this_length;
  194. }
  195. kunmap(cpu_page);
  196. kunmap(gpu_page);
  197. }
  198. /**
  199. * This is the fast shmem pread path, which attempts to copy_from_user directly
  200. * from the backing pages of the object to the user's address space. On a
  201. * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
  202. */
  203. static int
  204. i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj,
  205. struct drm_i915_gem_pread *args,
  206. struct drm_file *file_priv)
  207. {
  208. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  209. ssize_t remain;
  210. loff_t offset, page_base;
  211. char __user *user_data;
  212. int page_offset, page_length;
  213. int ret;
  214. user_data = (char __user *) (uintptr_t) args->data_ptr;
  215. remain = args->size;
  216. mutex_lock(&dev->struct_mutex);
  217. ret = i915_gem_object_get_pages(obj, 0);
  218. if (ret != 0)
  219. goto fail_unlock;
  220. ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
  221. args->size);
  222. if (ret != 0)
  223. goto fail_put_pages;
  224. obj_priv = to_intel_bo(obj);
  225. offset = args->offset;
  226. while (remain > 0) {
  227. /* Operation in this page
  228. *
  229. * page_base = page offset within aperture
  230. * page_offset = offset within page
  231. * page_length = bytes to copy for this page
  232. */
  233. page_base = (offset & ~(PAGE_SIZE-1));
  234. page_offset = offset & (PAGE_SIZE-1);
  235. page_length = remain;
  236. if ((page_offset + remain) > PAGE_SIZE)
  237. page_length = PAGE_SIZE - page_offset;
  238. ret = fast_shmem_read(obj_priv->pages,
  239. page_base, page_offset,
  240. user_data, page_length);
  241. if (ret)
  242. goto fail_put_pages;
  243. remain -= page_length;
  244. user_data += page_length;
  245. offset += page_length;
  246. }
  247. fail_put_pages:
  248. i915_gem_object_put_pages(obj);
  249. fail_unlock:
  250. mutex_unlock(&dev->struct_mutex);
  251. return ret;
  252. }
  253. static int
  254. i915_gem_object_get_pages_or_evict(struct drm_gem_object *obj)
  255. {
  256. int ret;
  257. ret = i915_gem_object_get_pages(obj, __GFP_NORETRY | __GFP_NOWARN);
  258. /* If we've insufficient memory to map in the pages, attempt
  259. * to make some space by throwing out some old buffers.
  260. */
  261. if (ret == -ENOMEM) {
  262. struct drm_device *dev = obj->dev;
  263. ret = i915_gem_evict_something(dev, obj->size);
  264. if (ret)
  265. return ret;
  266. ret = i915_gem_object_get_pages(obj, 0);
  267. }
  268. return ret;
  269. }
  270. /**
  271. * This is the fallback shmem pread path, which allocates temporary storage
  272. * in kernel space to copy_to_user into outside of the struct_mutex, so we
  273. * can copy out of the object's backing pages while holding the struct mutex
  274. * and not take page faults.
  275. */
  276. static int
  277. i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
  278. struct drm_i915_gem_pread *args,
  279. struct drm_file *file_priv)
  280. {
  281. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  282. struct mm_struct *mm = current->mm;
  283. struct page **user_pages;
  284. ssize_t remain;
  285. loff_t offset, pinned_pages, i;
  286. loff_t first_data_page, last_data_page, num_pages;
  287. int shmem_page_index, shmem_page_offset;
  288. int data_page_index, data_page_offset;
  289. int page_length;
  290. int ret;
  291. uint64_t data_ptr = args->data_ptr;
  292. int do_bit17_swizzling;
  293. remain = args->size;
  294. /* Pin the user pages containing the data. We can't fault while
  295. * holding the struct mutex, yet we want to hold it while
  296. * dereferencing the user data.
  297. */
  298. first_data_page = data_ptr / PAGE_SIZE;
  299. last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
  300. num_pages = last_data_page - first_data_page + 1;
  301. user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
  302. if (user_pages == NULL)
  303. return -ENOMEM;
  304. down_read(&mm->mmap_sem);
  305. pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
  306. num_pages, 1, 0, user_pages, NULL);
  307. up_read(&mm->mmap_sem);
  308. if (pinned_pages < num_pages) {
  309. ret = -EFAULT;
  310. goto fail_put_user_pages;
  311. }
  312. do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
  313. mutex_lock(&dev->struct_mutex);
  314. ret = i915_gem_object_get_pages_or_evict(obj);
  315. if (ret)
  316. goto fail_unlock;
  317. ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
  318. args->size);
  319. if (ret != 0)
  320. goto fail_put_pages;
  321. obj_priv = to_intel_bo(obj);
  322. offset = args->offset;
  323. while (remain > 0) {
  324. /* Operation in this page
  325. *
  326. * shmem_page_index = page number within shmem file
  327. * shmem_page_offset = offset within page in shmem file
  328. * data_page_index = page number in get_user_pages return
  329. * data_page_offset = offset with data_page_index page.
  330. * page_length = bytes to copy for this page
  331. */
  332. shmem_page_index = offset / PAGE_SIZE;
  333. shmem_page_offset = offset & ~PAGE_MASK;
  334. data_page_index = data_ptr / PAGE_SIZE - first_data_page;
  335. data_page_offset = data_ptr & ~PAGE_MASK;
  336. page_length = remain;
  337. if ((shmem_page_offset + page_length) > PAGE_SIZE)
  338. page_length = PAGE_SIZE - shmem_page_offset;
  339. if ((data_page_offset + page_length) > PAGE_SIZE)
  340. page_length = PAGE_SIZE - data_page_offset;
  341. if (do_bit17_swizzling) {
  342. slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
  343. shmem_page_offset,
  344. user_pages[data_page_index],
  345. data_page_offset,
  346. page_length,
  347. 1);
  348. } else {
  349. slow_shmem_copy(user_pages[data_page_index],
  350. data_page_offset,
  351. obj_priv->pages[shmem_page_index],
  352. shmem_page_offset,
  353. page_length);
  354. }
  355. remain -= page_length;
  356. data_ptr += page_length;
  357. offset += page_length;
  358. }
  359. fail_put_pages:
  360. i915_gem_object_put_pages(obj);
  361. fail_unlock:
  362. mutex_unlock(&dev->struct_mutex);
  363. fail_put_user_pages:
  364. for (i = 0; i < pinned_pages; i++) {
  365. SetPageDirty(user_pages[i]);
  366. page_cache_release(user_pages[i]);
  367. }
  368. drm_free_large(user_pages);
  369. return ret;
  370. }
  371. /**
  372. * Reads data from the object referenced by handle.
  373. *
  374. * On error, the contents of *data are undefined.
  375. */
  376. int
  377. i915_gem_pread_ioctl(struct drm_device *dev, void *data,
  378. struct drm_file *file_priv)
  379. {
  380. struct drm_i915_gem_pread *args = data;
  381. struct drm_gem_object *obj;
  382. struct drm_i915_gem_object *obj_priv;
  383. int ret;
  384. obj = drm_gem_object_lookup(dev, file_priv, args->handle);
  385. if (obj == NULL)
  386. return -ENOENT;
  387. obj_priv = to_intel_bo(obj);
  388. /* Bounds check source.
  389. *
  390. * XXX: This could use review for overflow issues...
  391. */
  392. if (args->offset > obj->size || args->size > obj->size ||
  393. args->offset + args->size > obj->size) {
  394. drm_gem_object_unreference_unlocked(obj);
  395. return -EINVAL;
  396. }
  397. if (i915_gem_object_needs_bit17_swizzle(obj)) {
  398. ret = i915_gem_shmem_pread_slow(dev, obj, args, file_priv);
  399. } else {
  400. ret = i915_gem_shmem_pread_fast(dev, obj, args, file_priv);
  401. if (ret != 0)
  402. ret = i915_gem_shmem_pread_slow(dev, obj, args,
  403. file_priv);
  404. }
  405. drm_gem_object_unreference_unlocked(obj);
  406. return ret;
  407. }
  408. /* This is the fast write path which cannot handle
  409. * page faults in the source data
  410. */
  411. static inline int
  412. fast_user_write(struct io_mapping *mapping,
  413. loff_t page_base, int page_offset,
  414. char __user *user_data,
  415. int length)
  416. {
  417. char *vaddr_atomic;
  418. unsigned long unwritten;
  419. vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base, KM_USER0);
  420. unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
  421. user_data, length);
  422. io_mapping_unmap_atomic(vaddr_atomic, KM_USER0);
  423. if (unwritten)
  424. return -EFAULT;
  425. return 0;
  426. }
  427. /* Here's the write path which can sleep for
  428. * page faults
  429. */
  430. static inline void
  431. slow_kernel_write(struct io_mapping *mapping,
  432. loff_t gtt_base, int gtt_offset,
  433. struct page *user_page, int user_offset,
  434. int length)
  435. {
  436. char __iomem *dst_vaddr;
  437. char *src_vaddr;
  438. dst_vaddr = io_mapping_map_wc(mapping, gtt_base);
  439. src_vaddr = kmap(user_page);
  440. memcpy_toio(dst_vaddr + gtt_offset,
  441. src_vaddr + user_offset,
  442. length);
  443. kunmap(user_page);
  444. io_mapping_unmap(dst_vaddr);
  445. }
  446. static inline int
  447. fast_shmem_write(struct page **pages,
  448. loff_t page_base, int page_offset,
  449. char __user *data,
  450. int length)
  451. {
  452. char __iomem *vaddr;
  453. unsigned long unwritten;
  454. vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
  455. if (vaddr == NULL)
  456. return -ENOMEM;
  457. unwritten = __copy_from_user_inatomic(vaddr + page_offset, data, length);
  458. kunmap_atomic(vaddr, KM_USER0);
  459. if (unwritten)
  460. return -EFAULT;
  461. return 0;
  462. }
  463. /**
  464. * This is the fast pwrite path, where we copy the data directly from the
  465. * user into the GTT, uncached.
  466. */
  467. static int
  468. i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
  469. struct drm_i915_gem_pwrite *args,
  470. struct drm_file *file_priv)
  471. {
  472. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  473. drm_i915_private_t *dev_priv = dev->dev_private;
  474. ssize_t remain;
  475. loff_t offset, page_base;
  476. char __user *user_data;
  477. int page_offset, page_length;
  478. int ret;
  479. user_data = (char __user *) (uintptr_t) args->data_ptr;
  480. remain = args->size;
  481. if (!access_ok(VERIFY_READ, user_data, remain))
  482. return -EFAULT;
  483. mutex_lock(&dev->struct_mutex);
  484. ret = i915_gem_object_pin(obj, 0);
  485. if (ret) {
  486. mutex_unlock(&dev->struct_mutex);
  487. return ret;
  488. }
  489. ret = i915_gem_object_set_to_gtt_domain(obj, 1);
  490. if (ret)
  491. goto fail;
  492. obj_priv = to_intel_bo(obj);
  493. offset = obj_priv->gtt_offset + args->offset;
  494. while (remain > 0) {
  495. /* Operation in this page
  496. *
  497. * page_base = page offset within aperture
  498. * page_offset = offset within page
  499. * page_length = bytes to copy for this page
  500. */
  501. page_base = (offset & ~(PAGE_SIZE-1));
  502. page_offset = offset & (PAGE_SIZE-1);
  503. page_length = remain;
  504. if ((page_offset + remain) > PAGE_SIZE)
  505. page_length = PAGE_SIZE - page_offset;
  506. ret = fast_user_write (dev_priv->mm.gtt_mapping, page_base,
  507. page_offset, user_data, page_length);
  508. /* If we get a fault while copying data, then (presumably) our
  509. * source page isn't available. Return the error and we'll
  510. * retry in the slow path.
  511. */
  512. if (ret)
  513. goto fail;
  514. remain -= page_length;
  515. user_data += page_length;
  516. offset += page_length;
  517. }
  518. fail:
  519. i915_gem_object_unpin(obj);
  520. mutex_unlock(&dev->struct_mutex);
  521. return ret;
  522. }
  523. /**
  524. * This is the fallback GTT pwrite path, which uses get_user_pages to pin
  525. * the memory and maps it using kmap_atomic for copying.
  526. *
  527. * This code resulted in x11perf -rgb10text consuming about 10% more CPU
  528. * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
  529. */
  530. static int
  531. i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
  532. struct drm_i915_gem_pwrite *args,
  533. struct drm_file *file_priv)
  534. {
  535. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  536. drm_i915_private_t *dev_priv = dev->dev_private;
  537. ssize_t remain;
  538. loff_t gtt_page_base, offset;
  539. loff_t first_data_page, last_data_page, num_pages;
  540. loff_t pinned_pages, i;
  541. struct page **user_pages;
  542. struct mm_struct *mm = current->mm;
  543. int gtt_page_offset, data_page_offset, data_page_index, page_length;
  544. int ret;
  545. uint64_t data_ptr = args->data_ptr;
  546. remain = args->size;
  547. /* Pin the user pages containing the data. We can't fault while
  548. * holding the struct mutex, and all of the pwrite implementations
  549. * want to hold it while dereferencing the user data.
  550. */
  551. first_data_page = data_ptr / PAGE_SIZE;
  552. last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
  553. num_pages = last_data_page - first_data_page + 1;
  554. user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
  555. if (user_pages == NULL)
  556. return -ENOMEM;
  557. down_read(&mm->mmap_sem);
  558. pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
  559. num_pages, 0, 0, user_pages, NULL);
  560. up_read(&mm->mmap_sem);
  561. if (pinned_pages < num_pages) {
  562. ret = -EFAULT;
  563. goto out_unpin_pages;
  564. }
  565. mutex_lock(&dev->struct_mutex);
  566. ret = i915_gem_object_pin(obj, 0);
  567. if (ret)
  568. goto out_unlock;
  569. ret = i915_gem_object_set_to_gtt_domain(obj, 1);
  570. if (ret)
  571. goto out_unpin_object;
  572. obj_priv = to_intel_bo(obj);
  573. offset = obj_priv->gtt_offset + args->offset;
  574. while (remain > 0) {
  575. /* Operation in this page
  576. *
  577. * gtt_page_base = page offset within aperture
  578. * gtt_page_offset = offset within page in aperture
  579. * data_page_index = page number in get_user_pages return
  580. * data_page_offset = offset with data_page_index page.
  581. * page_length = bytes to copy for this page
  582. */
  583. gtt_page_base = offset & PAGE_MASK;
  584. gtt_page_offset = offset & ~PAGE_MASK;
  585. data_page_index = data_ptr / PAGE_SIZE - first_data_page;
  586. data_page_offset = data_ptr & ~PAGE_MASK;
  587. page_length = remain;
  588. if ((gtt_page_offset + page_length) > PAGE_SIZE)
  589. page_length = PAGE_SIZE - gtt_page_offset;
  590. if ((data_page_offset + page_length) > PAGE_SIZE)
  591. page_length = PAGE_SIZE - data_page_offset;
  592. slow_kernel_write(dev_priv->mm.gtt_mapping,
  593. gtt_page_base, gtt_page_offset,
  594. user_pages[data_page_index],
  595. data_page_offset,
  596. page_length);
  597. remain -= page_length;
  598. offset += page_length;
  599. data_ptr += page_length;
  600. }
  601. out_unpin_object:
  602. i915_gem_object_unpin(obj);
  603. out_unlock:
  604. mutex_unlock(&dev->struct_mutex);
  605. out_unpin_pages:
  606. for (i = 0; i < pinned_pages; i++)
  607. page_cache_release(user_pages[i]);
  608. drm_free_large(user_pages);
  609. return ret;
  610. }
  611. /**
  612. * This is the fast shmem pwrite path, which attempts to directly
  613. * copy_from_user into the kmapped pages backing the object.
  614. */
  615. static int
  616. i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
  617. struct drm_i915_gem_pwrite *args,
  618. struct drm_file *file_priv)
  619. {
  620. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  621. ssize_t remain;
  622. loff_t offset, page_base;
  623. char __user *user_data;
  624. int page_offset, page_length;
  625. int ret;
  626. user_data = (char __user *) (uintptr_t) args->data_ptr;
  627. remain = args->size;
  628. mutex_lock(&dev->struct_mutex);
  629. ret = i915_gem_object_get_pages(obj, 0);
  630. if (ret != 0)
  631. goto fail_unlock;
  632. ret = i915_gem_object_set_to_cpu_domain(obj, 1);
  633. if (ret != 0)
  634. goto fail_put_pages;
  635. obj_priv = to_intel_bo(obj);
  636. offset = args->offset;
  637. obj_priv->dirty = 1;
  638. while (remain > 0) {
  639. /* Operation in this page
  640. *
  641. * page_base = page offset within aperture
  642. * page_offset = offset within page
  643. * page_length = bytes to copy for this page
  644. */
  645. page_base = (offset & ~(PAGE_SIZE-1));
  646. page_offset = offset & (PAGE_SIZE-1);
  647. page_length = remain;
  648. if ((page_offset + remain) > PAGE_SIZE)
  649. page_length = PAGE_SIZE - page_offset;
  650. ret = fast_shmem_write(obj_priv->pages,
  651. page_base, page_offset,
  652. user_data, page_length);
  653. if (ret)
  654. goto fail_put_pages;
  655. remain -= page_length;
  656. user_data += page_length;
  657. offset += page_length;
  658. }
  659. fail_put_pages:
  660. i915_gem_object_put_pages(obj);
  661. fail_unlock:
  662. mutex_unlock(&dev->struct_mutex);
  663. return ret;
  664. }
  665. /**
  666. * This is the fallback shmem pwrite path, which uses get_user_pages to pin
  667. * the memory and maps it using kmap_atomic for copying.
  668. *
  669. * This avoids taking mmap_sem for faulting on the user's address while the
  670. * struct_mutex is held.
  671. */
  672. static int
  673. i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
  674. struct drm_i915_gem_pwrite *args,
  675. struct drm_file *file_priv)
  676. {
  677. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  678. struct mm_struct *mm = current->mm;
  679. struct page **user_pages;
  680. ssize_t remain;
  681. loff_t offset, pinned_pages, i;
  682. loff_t first_data_page, last_data_page, num_pages;
  683. int shmem_page_index, shmem_page_offset;
  684. int data_page_index, data_page_offset;
  685. int page_length;
  686. int ret;
  687. uint64_t data_ptr = args->data_ptr;
  688. int do_bit17_swizzling;
  689. remain = args->size;
  690. /* Pin the user pages containing the data. We can't fault while
  691. * holding the struct mutex, and all of the pwrite implementations
  692. * want to hold it while dereferencing the user data.
  693. */
  694. first_data_page = data_ptr / PAGE_SIZE;
  695. last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
  696. num_pages = last_data_page - first_data_page + 1;
  697. user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
  698. if (user_pages == NULL)
  699. return -ENOMEM;
  700. down_read(&mm->mmap_sem);
  701. pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
  702. num_pages, 0, 0, user_pages, NULL);
  703. up_read(&mm->mmap_sem);
  704. if (pinned_pages < num_pages) {
  705. ret = -EFAULT;
  706. goto fail_put_user_pages;
  707. }
  708. do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
  709. mutex_lock(&dev->struct_mutex);
  710. ret = i915_gem_object_get_pages_or_evict(obj);
  711. if (ret)
  712. goto fail_unlock;
  713. ret = i915_gem_object_set_to_cpu_domain(obj, 1);
  714. if (ret != 0)
  715. goto fail_put_pages;
  716. obj_priv = to_intel_bo(obj);
  717. offset = args->offset;
  718. obj_priv->dirty = 1;
  719. while (remain > 0) {
  720. /* Operation in this page
  721. *
  722. * shmem_page_index = page number within shmem file
  723. * shmem_page_offset = offset within page in shmem file
  724. * data_page_index = page number in get_user_pages return
  725. * data_page_offset = offset with data_page_index page.
  726. * page_length = bytes to copy for this page
  727. */
  728. shmem_page_index = offset / PAGE_SIZE;
  729. shmem_page_offset = offset & ~PAGE_MASK;
  730. data_page_index = data_ptr / PAGE_SIZE - first_data_page;
  731. data_page_offset = data_ptr & ~PAGE_MASK;
  732. page_length = remain;
  733. if ((shmem_page_offset + page_length) > PAGE_SIZE)
  734. page_length = PAGE_SIZE - shmem_page_offset;
  735. if ((data_page_offset + page_length) > PAGE_SIZE)
  736. page_length = PAGE_SIZE - data_page_offset;
  737. if (do_bit17_swizzling) {
  738. slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
  739. shmem_page_offset,
  740. user_pages[data_page_index],
  741. data_page_offset,
  742. page_length,
  743. 0);
  744. } else {
  745. slow_shmem_copy(obj_priv->pages[shmem_page_index],
  746. shmem_page_offset,
  747. user_pages[data_page_index],
  748. data_page_offset,
  749. page_length);
  750. }
  751. remain -= page_length;
  752. data_ptr += page_length;
  753. offset += page_length;
  754. }
  755. fail_put_pages:
  756. i915_gem_object_put_pages(obj);
  757. fail_unlock:
  758. mutex_unlock(&dev->struct_mutex);
  759. fail_put_user_pages:
  760. for (i = 0; i < pinned_pages; i++)
  761. page_cache_release(user_pages[i]);
  762. drm_free_large(user_pages);
  763. return ret;
  764. }
  765. /**
  766. * Writes data to the object referenced by handle.
  767. *
  768. * On error, the contents of the buffer that were to be modified are undefined.
  769. */
  770. int
  771. i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
  772. struct drm_file *file_priv)
  773. {
  774. struct drm_i915_gem_pwrite *args = data;
  775. struct drm_gem_object *obj;
  776. struct drm_i915_gem_object *obj_priv;
  777. int ret = 0;
  778. obj = drm_gem_object_lookup(dev, file_priv, args->handle);
  779. if (obj == NULL)
  780. return -ENOENT;
  781. obj_priv = to_intel_bo(obj);
  782. /* Bounds check destination.
  783. *
  784. * XXX: This could use review for overflow issues...
  785. */
  786. if (args->offset > obj->size || args->size > obj->size ||
  787. args->offset + args->size > obj->size) {
  788. drm_gem_object_unreference_unlocked(obj);
  789. return -EINVAL;
  790. }
  791. /* We can only do the GTT pwrite on untiled buffers, as otherwise
  792. * it would end up going through the fenced access, and we'll get
  793. * different detiling behavior between reading and writing.
  794. * pread/pwrite currently are reading and writing from the CPU
  795. * perspective, requiring manual detiling by the client.
  796. */
  797. if (obj_priv->phys_obj)
  798. ret = i915_gem_phys_pwrite(dev, obj, args, file_priv);
  799. else if (obj_priv->tiling_mode == I915_TILING_NONE &&
  800. dev->gtt_total != 0 &&
  801. obj->write_domain != I915_GEM_DOMAIN_CPU) {
  802. ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file_priv);
  803. if (ret == -EFAULT) {
  804. ret = i915_gem_gtt_pwrite_slow(dev, obj, args,
  805. file_priv);
  806. }
  807. } else if (i915_gem_object_needs_bit17_swizzle(obj)) {
  808. ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file_priv);
  809. } else {
  810. ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file_priv);
  811. if (ret == -EFAULT) {
  812. ret = i915_gem_shmem_pwrite_slow(dev, obj, args,
  813. file_priv);
  814. }
  815. }
  816. #if WATCH_PWRITE
  817. if (ret)
  818. DRM_INFO("pwrite failed %d\n", ret);
  819. #endif
  820. drm_gem_object_unreference_unlocked(obj);
  821. return ret;
  822. }
  823. /**
  824. * Called when user space prepares to use an object with the CPU, either
  825. * through the mmap ioctl's mapping or a GTT mapping.
  826. */
  827. int
  828. i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
  829. struct drm_file *file_priv)
  830. {
  831. struct drm_i915_private *dev_priv = dev->dev_private;
  832. struct drm_i915_gem_set_domain *args = data;
  833. struct drm_gem_object *obj;
  834. struct drm_i915_gem_object *obj_priv;
  835. uint32_t read_domains = args->read_domains;
  836. uint32_t write_domain = args->write_domain;
  837. int ret;
  838. if (!(dev->driver->driver_features & DRIVER_GEM))
  839. return -ENODEV;
  840. /* Only handle setting domains to types used by the CPU. */
  841. if (write_domain & I915_GEM_GPU_DOMAINS)
  842. return -EINVAL;
  843. if (read_domains & I915_GEM_GPU_DOMAINS)
  844. return -EINVAL;
  845. /* Having something in the write domain implies it's in the read
  846. * domain, and only that read domain. Enforce that in the request.
  847. */
  848. if (write_domain != 0 && read_domains != write_domain)
  849. return -EINVAL;
  850. obj = drm_gem_object_lookup(dev, file_priv, args->handle);
  851. if (obj == NULL)
  852. return -ENOENT;
  853. obj_priv = to_intel_bo(obj);
  854. mutex_lock(&dev->struct_mutex);
  855. intel_mark_busy(dev, obj);
  856. #if WATCH_BUF
  857. DRM_INFO("set_domain_ioctl %p(%zd), %08x %08x\n",
  858. obj, obj->size, read_domains, write_domain);
  859. #endif
  860. if (read_domains & I915_GEM_DOMAIN_GTT) {
  861. ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
  862. /* Update the LRU on the fence for the CPU access that's
  863. * about to occur.
  864. */
  865. if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
  866. struct drm_i915_fence_reg *reg =
  867. &dev_priv->fence_regs[obj_priv->fence_reg];
  868. list_move_tail(&reg->lru_list,
  869. &dev_priv->mm.fence_list);
  870. }
  871. /* Silently promote "you're not bound, there was nothing to do"
  872. * to success, since the client was just asking us to
  873. * make sure everything was done.
  874. */
  875. if (ret == -EINVAL)
  876. ret = 0;
  877. } else {
  878. ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
  879. }
  880. drm_gem_object_unreference(obj);
  881. mutex_unlock(&dev->struct_mutex);
  882. return ret;
  883. }
  884. /**
  885. * Called when user space has done writes to this buffer
  886. */
  887. int
  888. i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
  889. struct drm_file *file_priv)
  890. {
  891. struct drm_i915_gem_sw_finish *args = data;
  892. struct drm_gem_object *obj;
  893. struct drm_i915_gem_object *obj_priv;
  894. int ret = 0;
  895. if (!(dev->driver->driver_features & DRIVER_GEM))
  896. return -ENODEV;
  897. mutex_lock(&dev->struct_mutex);
  898. obj = drm_gem_object_lookup(dev, file_priv, args->handle);
  899. if (obj == NULL) {
  900. mutex_unlock(&dev->struct_mutex);
  901. return -ENOENT;
  902. }
  903. #if WATCH_BUF
  904. DRM_INFO("%s: sw_finish %d (%p %zd)\n",
  905. __func__, args->handle, obj, obj->size);
  906. #endif
  907. obj_priv = to_intel_bo(obj);
  908. /* Pinned buffers may be scanout, so flush the cache */
  909. if (obj_priv->pin_count)
  910. i915_gem_object_flush_cpu_write_domain(obj);
  911. drm_gem_object_unreference(obj);
  912. mutex_unlock(&dev->struct_mutex);
  913. return ret;
  914. }
  915. /**
  916. * Maps the contents of an object, returning the address it is mapped
  917. * into.
  918. *
  919. * While the mapping holds a reference on the contents of the object, it doesn't
  920. * imply a ref on the object itself.
  921. */
  922. int
  923. i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
  924. struct drm_file *file_priv)
  925. {
  926. struct drm_i915_gem_mmap *args = data;
  927. struct drm_gem_object *obj;
  928. loff_t offset;
  929. unsigned long addr;
  930. if (!(dev->driver->driver_features & DRIVER_GEM))
  931. return -ENODEV;
  932. obj = drm_gem_object_lookup(dev, file_priv, args->handle);
  933. if (obj == NULL)
  934. return -ENOENT;
  935. offset = args->offset;
  936. down_write(&current->mm->mmap_sem);
  937. addr = do_mmap(obj->filp, 0, args->size,
  938. PROT_READ | PROT_WRITE, MAP_SHARED,
  939. args->offset);
  940. up_write(&current->mm->mmap_sem);
  941. drm_gem_object_unreference_unlocked(obj);
  942. if (IS_ERR((void *)addr))
  943. return addr;
  944. args->addr_ptr = (uint64_t) addr;
  945. return 0;
  946. }
  947. /**
  948. * i915_gem_fault - fault a page into the GTT
  949. * vma: VMA in question
  950. * vmf: fault info
  951. *
  952. * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
  953. * from userspace. The fault handler takes care of binding the object to
  954. * the GTT (if needed), allocating and programming a fence register (again,
  955. * only if needed based on whether the old reg is still valid or the object
  956. * is tiled) and inserting a new PTE into the faulting process.
  957. *
  958. * Note that the faulting process may involve evicting existing objects
  959. * from the GTT and/or fence registers to make room. So performance may
  960. * suffer if the GTT working set is large or there are few fence registers
  961. * left.
  962. */
  963. int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
  964. {
  965. struct drm_gem_object *obj = vma->vm_private_data;
  966. struct drm_device *dev = obj->dev;
  967. struct drm_i915_private *dev_priv = dev->dev_private;
  968. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  969. pgoff_t page_offset;
  970. unsigned long pfn;
  971. int ret = 0;
  972. bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
  973. /* We don't use vmf->pgoff since that has the fake offset */
  974. page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
  975. PAGE_SHIFT;
  976. /* Now bind it into the GTT if needed */
  977. mutex_lock(&dev->struct_mutex);
  978. if (!obj_priv->gtt_space) {
  979. ret = i915_gem_object_bind_to_gtt(obj, 0);
  980. if (ret)
  981. goto unlock;
  982. list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
  983. ret = i915_gem_object_set_to_gtt_domain(obj, write);
  984. if (ret)
  985. goto unlock;
  986. }
  987. /* Need a new fence register? */
  988. if (obj_priv->tiling_mode != I915_TILING_NONE) {
  989. ret = i915_gem_object_get_fence_reg(obj);
  990. if (ret)
  991. goto unlock;
  992. }
  993. pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) +
  994. page_offset;
  995. /* Finally, remap it using the new GTT offset */
  996. ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
  997. unlock:
  998. mutex_unlock(&dev->struct_mutex);
  999. switch (ret) {
  1000. case 0:
  1001. case -ERESTARTSYS:
  1002. return VM_FAULT_NOPAGE;
  1003. case -ENOMEM:
  1004. case -EAGAIN:
  1005. return VM_FAULT_OOM;
  1006. default:
  1007. return VM_FAULT_SIGBUS;
  1008. }
  1009. }
  1010. /**
  1011. * i915_gem_create_mmap_offset - create a fake mmap offset for an object
  1012. * @obj: obj in question
  1013. *
  1014. * GEM memory mapping works by handing back to userspace a fake mmap offset
  1015. * it can use in a subsequent mmap(2) call. The DRM core code then looks
  1016. * up the object based on the offset and sets up the various memory mapping
  1017. * structures.
  1018. *
  1019. * This routine allocates and attaches a fake offset for @obj.
  1020. */
  1021. static int
  1022. i915_gem_create_mmap_offset(struct drm_gem_object *obj)
  1023. {
  1024. struct drm_device *dev = obj->dev;
  1025. struct drm_gem_mm *mm = dev->mm_private;
  1026. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  1027. struct drm_map_list *list;
  1028. struct drm_local_map *map;
  1029. int ret = 0;
  1030. /* Set the object up for mmap'ing */
  1031. list = &obj->map_list;
  1032. list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL);
  1033. if (!list->map)
  1034. return -ENOMEM;
  1035. map = list->map;
  1036. map->type = _DRM_GEM;
  1037. map->size = obj->size;
  1038. map->handle = obj;
  1039. /* Get a DRM GEM mmap offset allocated... */
  1040. list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
  1041. obj->size / PAGE_SIZE, 0, 0);
  1042. if (!list->file_offset_node) {
  1043. DRM_ERROR("failed to allocate offset for bo %d\n", obj->name);
  1044. ret = -ENOMEM;
  1045. goto out_free_list;
  1046. }
  1047. list->file_offset_node = drm_mm_get_block(list->file_offset_node,
  1048. obj->size / PAGE_SIZE, 0);
  1049. if (!list->file_offset_node) {
  1050. ret = -ENOMEM;
  1051. goto out_free_list;
  1052. }
  1053. list->hash.key = list->file_offset_node->start;
  1054. if (drm_ht_insert_item(&mm->offset_hash, &list->hash)) {
  1055. DRM_ERROR("failed to add to map hash\n");
  1056. ret = -ENOMEM;
  1057. goto out_free_mm;
  1058. }
  1059. /* By now we should be all set, any drm_mmap request on the offset
  1060. * below will get to our mmap & fault handler */
  1061. obj_priv->mmap_offset = ((uint64_t) list->hash.key) << PAGE_SHIFT;
  1062. return 0;
  1063. out_free_mm:
  1064. drm_mm_put_block(list->file_offset_node);
  1065. out_free_list:
  1066. kfree(list->map);
  1067. return ret;
  1068. }
  1069. /**
  1070. * i915_gem_release_mmap - remove physical page mappings
  1071. * @obj: obj in question
  1072. *
  1073. * Preserve the reservation of the mmapping with the DRM core code, but
  1074. * relinquish ownership of the pages back to the system.
  1075. *
  1076. * It is vital that we remove the page mapping if we have mapped a tiled
  1077. * object through the GTT and then lose the fence register due to
  1078. * resource pressure. Similarly if the object has been moved out of the
  1079. * aperture, than pages mapped into userspace must be revoked. Removing the
  1080. * mapping will then trigger a page fault on the next user access, allowing
  1081. * fixup by i915_gem_fault().
  1082. */
  1083. void
  1084. i915_gem_release_mmap(struct drm_gem_object *obj)
  1085. {
  1086. struct drm_device *dev = obj->dev;
  1087. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  1088. if (dev->dev_mapping)
  1089. unmap_mapping_range(dev->dev_mapping,
  1090. obj_priv->mmap_offset, obj->size, 1);
  1091. }
  1092. static void
  1093. i915_gem_free_mmap_offset(struct drm_gem_object *obj)
  1094. {
  1095. struct drm_device *dev = obj->dev;
  1096. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  1097. struct drm_gem_mm *mm = dev->mm_private;
  1098. struct drm_map_list *list;
  1099. list = &obj->map_list;
  1100. drm_ht_remove_item(&mm->offset_hash, &list->hash);
  1101. if (list->file_offset_node) {
  1102. drm_mm_put_block(list->file_offset_node);
  1103. list->file_offset_node = NULL;
  1104. }
  1105. if (list->map) {
  1106. kfree(list->map);
  1107. list->map = NULL;
  1108. }
  1109. obj_priv->mmap_offset = 0;
  1110. }
  1111. /**
  1112. * i915_gem_get_gtt_alignment - return required GTT alignment for an object
  1113. * @obj: object to check
  1114. *
  1115. * Return the required GTT alignment for an object, taking into account
  1116. * potential fence register mapping if needed.
  1117. */
  1118. static uint32_t
  1119. i915_gem_get_gtt_alignment(struct drm_gem_object *obj)
  1120. {
  1121. struct drm_device *dev = obj->dev;
  1122. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  1123. int start, i;
  1124. /*
  1125. * Minimum alignment is 4k (GTT page size), but might be greater
  1126. * if a fence register is needed for the object.
  1127. */
  1128. if (IS_I965G(dev) || obj_priv->tiling_mode == I915_TILING_NONE)
  1129. return 4096;
  1130. /*
  1131. * Previous chips need to be aligned to the size of the smallest
  1132. * fence register that can contain the object.
  1133. */
  1134. if (IS_I9XX(dev))
  1135. start = 1024*1024;
  1136. else
  1137. start = 512*1024;
  1138. for (i = start; i < obj->size; i <<= 1)
  1139. ;
  1140. return i;
  1141. }
  1142. /**
  1143. * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
  1144. * @dev: DRM device
  1145. * @data: GTT mapping ioctl data
  1146. * @file_priv: GEM object info
  1147. *
  1148. * Simply returns the fake offset to userspace so it can mmap it.
  1149. * The mmap call will end up in drm_gem_mmap(), which will set things
  1150. * up so we can get faults in the handler above.
  1151. *
  1152. * The fault handler will take care of binding the object into the GTT
  1153. * (since it may have been evicted to make room for something), allocating
  1154. * a fence register, and mapping the appropriate aperture address into
  1155. * userspace.
  1156. */
  1157. int
  1158. i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
  1159. struct drm_file *file_priv)
  1160. {
  1161. struct drm_i915_gem_mmap_gtt *args = data;
  1162. struct drm_i915_private *dev_priv = dev->dev_private;
  1163. struct drm_gem_object *obj;
  1164. struct drm_i915_gem_object *obj_priv;
  1165. int ret;
  1166. if (!(dev->driver->driver_features & DRIVER_GEM))
  1167. return -ENODEV;
  1168. obj = drm_gem_object_lookup(dev, file_priv, args->handle);
  1169. if (obj == NULL)
  1170. return -ENOENT;
  1171. mutex_lock(&dev->struct_mutex);
  1172. obj_priv = to_intel_bo(obj);
  1173. if (obj_priv->madv != I915_MADV_WILLNEED) {
  1174. DRM_ERROR("Attempting to mmap a purgeable buffer\n");
  1175. drm_gem_object_unreference(obj);
  1176. mutex_unlock(&dev->struct_mutex);
  1177. return -EINVAL;
  1178. }
  1179. if (!obj_priv->mmap_offset) {
  1180. ret = i915_gem_create_mmap_offset(obj);
  1181. if (ret) {
  1182. drm_gem_object_unreference(obj);
  1183. mutex_unlock(&dev->struct_mutex);
  1184. return ret;
  1185. }
  1186. }
  1187. args->offset = obj_priv->mmap_offset;
  1188. /*
  1189. * Pull it into the GTT so that we have a page list (makes the
  1190. * initial fault faster and any subsequent flushing possible).
  1191. */
  1192. if (!obj_priv->agp_mem) {
  1193. ret = i915_gem_object_bind_to_gtt(obj, 0);
  1194. if (ret) {
  1195. drm_gem_object_unreference(obj);
  1196. mutex_unlock(&dev->struct_mutex);
  1197. return ret;
  1198. }
  1199. list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
  1200. }
  1201. drm_gem_object_unreference(obj);
  1202. mutex_unlock(&dev->struct_mutex);
  1203. return 0;
  1204. }
  1205. void
  1206. i915_gem_object_put_pages(struct drm_gem_object *obj)
  1207. {
  1208. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  1209. int page_count = obj->size / PAGE_SIZE;
  1210. int i;
  1211. BUG_ON(obj_priv->pages_refcount == 0);
  1212. BUG_ON(obj_priv->madv == __I915_MADV_PURGED);
  1213. if (--obj_priv->pages_refcount != 0)
  1214. return;
  1215. if (obj_priv->tiling_mode != I915_TILING_NONE)
  1216. i915_gem_object_save_bit_17_swizzle(obj);
  1217. if (obj_priv->madv == I915_MADV_DONTNEED)
  1218. obj_priv->dirty = 0;
  1219. for (i = 0; i < page_count; i++) {
  1220. if (obj_priv->dirty)
  1221. set_page_dirty(obj_priv->pages[i]);
  1222. if (obj_priv->madv == I915_MADV_WILLNEED)
  1223. mark_page_accessed(obj_priv->pages[i]);
  1224. page_cache_release(obj_priv->pages[i]);
  1225. }
  1226. obj_priv->dirty = 0;
  1227. drm_free_large(obj_priv->pages);
  1228. obj_priv->pages = NULL;
  1229. }
  1230. static void
  1231. i915_gem_object_move_to_active(struct drm_gem_object *obj, uint32_t seqno,
  1232. struct intel_ring_buffer *ring)
  1233. {
  1234. struct drm_device *dev = obj->dev;
  1235. drm_i915_private_t *dev_priv = dev->dev_private;
  1236. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  1237. BUG_ON(ring == NULL);
  1238. obj_priv->ring = ring;
  1239. /* Add a reference if we're newly entering the active list. */
  1240. if (!obj_priv->active) {
  1241. drm_gem_object_reference(obj);
  1242. obj_priv->active = 1;
  1243. }
  1244. /* Move from whatever list we were on to the tail of execution. */
  1245. spin_lock(&dev_priv->mm.active_list_lock);
  1246. list_move_tail(&obj_priv->list, &ring->active_list);
  1247. spin_unlock(&dev_priv->mm.active_list_lock);
  1248. obj_priv->last_rendering_seqno = seqno;
  1249. }
  1250. static void
  1251. i915_gem_object_move_to_flushing(struct drm_gem_object *obj)
  1252. {
  1253. struct drm_device *dev = obj->dev;
  1254. drm_i915_private_t *dev_priv = dev->dev_private;
  1255. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  1256. BUG_ON(!obj_priv->active);
  1257. list_move_tail(&obj_priv->list, &dev_priv->mm.flushing_list);
  1258. obj_priv->last_rendering_seqno = 0;
  1259. }
  1260. /* Immediately discard the backing storage */
  1261. static void
  1262. i915_gem_object_truncate(struct drm_gem_object *obj)
  1263. {
  1264. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  1265. struct inode *inode;
  1266. inode = obj->filp->f_path.dentry->d_inode;
  1267. if (inode->i_op->truncate)
  1268. inode->i_op->truncate (inode);
  1269. obj_priv->madv = __I915_MADV_PURGED;
  1270. }
  1271. static inline int
  1272. i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj_priv)
  1273. {
  1274. return obj_priv->madv == I915_MADV_DONTNEED;
  1275. }
  1276. static void
  1277. i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
  1278. {
  1279. struct drm_device *dev = obj->dev;
  1280. drm_i915_private_t *dev_priv = dev->dev_private;
  1281. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  1282. i915_verify_inactive(dev, __FILE__, __LINE__);
  1283. if (obj_priv->pin_count != 0)
  1284. list_del_init(&obj_priv->list);
  1285. else
  1286. list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
  1287. BUG_ON(!list_empty(&obj_priv->gpu_write_list));
  1288. obj_priv->last_rendering_seqno = 0;
  1289. obj_priv->ring = NULL;
  1290. if (obj_priv->active) {
  1291. obj_priv->active = 0;
  1292. drm_gem_object_unreference(obj);
  1293. }
  1294. i915_verify_inactive(dev, __FILE__, __LINE__);
  1295. }
  1296. static void
  1297. i915_gem_process_flushing_list(struct drm_device *dev,
  1298. uint32_t flush_domains, uint32_t seqno,
  1299. struct intel_ring_buffer *ring)
  1300. {
  1301. drm_i915_private_t *dev_priv = dev->dev_private;
  1302. struct drm_i915_gem_object *obj_priv, *next;
  1303. list_for_each_entry_safe(obj_priv, next,
  1304. &dev_priv->mm.gpu_write_list,
  1305. gpu_write_list) {
  1306. struct drm_gem_object *obj = &obj_priv->base;
  1307. if ((obj->write_domain & flush_domains) ==
  1308. obj->write_domain &&
  1309. obj_priv->ring->ring_flag == ring->ring_flag) {
  1310. uint32_t old_write_domain = obj->write_domain;
  1311. obj->write_domain = 0;
  1312. list_del_init(&obj_priv->gpu_write_list);
  1313. i915_gem_object_move_to_active(obj, seqno, ring);
  1314. /* update the fence lru list */
  1315. if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
  1316. struct drm_i915_fence_reg *reg =
  1317. &dev_priv->fence_regs[obj_priv->fence_reg];
  1318. list_move_tail(&reg->lru_list,
  1319. &dev_priv->mm.fence_list);
  1320. }
  1321. trace_i915_gem_object_change_domain(obj,
  1322. obj->read_domains,
  1323. old_write_domain);
  1324. }
  1325. }
  1326. }
  1327. uint32_t
  1328. i915_add_request(struct drm_device *dev, struct drm_file *file_priv,
  1329. uint32_t flush_domains, struct intel_ring_buffer *ring)
  1330. {
  1331. drm_i915_private_t *dev_priv = dev->dev_private;
  1332. struct drm_i915_file_private *i915_file_priv = NULL;
  1333. struct drm_i915_gem_request *request;
  1334. uint32_t seqno;
  1335. int was_empty;
  1336. if (file_priv != NULL)
  1337. i915_file_priv = file_priv->driver_priv;
  1338. request = kzalloc(sizeof(*request), GFP_KERNEL);
  1339. if (request == NULL)
  1340. return 0;
  1341. seqno = ring->add_request(dev, ring, file_priv, flush_domains);
  1342. request->seqno = seqno;
  1343. request->ring = ring;
  1344. request->emitted_jiffies = jiffies;
  1345. was_empty = list_empty(&ring->request_list);
  1346. list_add_tail(&request->list, &ring->request_list);
  1347. if (i915_file_priv) {
  1348. list_add_tail(&request->client_list,
  1349. &i915_file_priv->mm.request_list);
  1350. } else {
  1351. INIT_LIST_HEAD(&request->client_list);
  1352. }
  1353. /* Associate any objects on the flushing list matching the write
  1354. * domain we're flushing with our flush.
  1355. */
  1356. if (flush_domains != 0)
  1357. i915_gem_process_flushing_list(dev, flush_domains, seqno, ring);
  1358. if (!dev_priv->mm.suspended) {
  1359. mod_timer(&dev_priv->hangcheck_timer, jiffies + DRM_I915_HANGCHECK_PERIOD);
  1360. if (was_empty)
  1361. queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
  1362. }
  1363. return seqno;
  1364. }
  1365. /**
  1366. * Command execution barrier
  1367. *
  1368. * Ensures that all commands in the ring are finished
  1369. * before signalling the CPU
  1370. */
  1371. static uint32_t
  1372. i915_retire_commands(struct drm_device *dev, struct intel_ring_buffer *ring)
  1373. {
  1374. uint32_t flush_domains = 0;
  1375. /* The sampler always gets flushed on i965 (sigh) */
  1376. if (IS_I965G(dev))
  1377. flush_domains |= I915_GEM_DOMAIN_SAMPLER;
  1378. ring->flush(dev, ring,
  1379. I915_GEM_DOMAIN_COMMAND, flush_domains);
  1380. return flush_domains;
  1381. }
  1382. /**
  1383. * Moves buffers associated only with the given active seqno from the active
  1384. * to inactive list, potentially freeing them.
  1385. */
  1386. static void
  1387. i915_gem_retire_request(struct drm_device *dev,
  1388. struct drm_i915_gem_request *request)
  1389. {
  1390. drm_i915_private_t *dev_priv = dev->dev_private;
  1391. trace_i915_gem_request_retire(dev, request->seqno);
  1392. /* Move any buffers on the active list that are no longer referenced
  1393. * by the ringbuffer to the flushing/inactive lists as appropriate.
  1394. */
  1395. spin_lock(&dev_priv->mm.active_list_lock);
  1396. while (!list_empty(&request->ring->active_list)) {
  1397. struct drm_gem_object *obj;
  1398. struct drm_i915_gem_object *obj_priv;
  1399. obj_priv = list_first_entry(&request->ring->active_list,
  1400. struct drm_i915_gem_object,
  1401. list);
  1402. obj = &obj_priv->base;
  1403. /* If the seqno being retired doesn't match the oldest in the
  1404. * list, then the oldest in the list must still be newer than
  1405. * this seqno.
  1406. */
  1407. if (obj_priv->last_rendering_seqno != request->seqno)
  1408. goto out;
  1409. #if WATCH_LRU
  1410. DRM_INFO("%s: retire %d moves to inactive list %p\n",
  1411. __func__, request->seqno, obj);
  1412. #endif
  1413. if (obj->write_domain != 0)
  1414. i915_gem_object_move_to_flushing(obj);
  1415. else {
  1416. /* Take a reference on the object so it won't be
  1417. * freed while the spinlock is held. The list
  1418. * protection for this spinlock is safe when breaking
  1419. * the lock like this since the next thing we do
  1420. * is just get the head of the list again.
  1421. */
  1422. drm_gem_object_reference(obj);
  1423. i915_gem_object_move_to_inactive(obj);
  1424. spin_unlock(&dev_priv->mm.active_list_lock);
  1425. drm_gem_object_unreference(obj);
  1426. spin_lock(&dev_priv->mm.active_list_lock);
  1427. }
  1428. }
  1429. out:
  1430. spin_unlock(&dev_priv->mm.active_list_lock);
  1431. }
  1432. /**
  1433. * Returns true if seq1 is later than seq2.
  1434. */
  1435. bool
  1436. i915_seqno_passed(uint32_t seq1, uint32_t seq2)
  1437. {
  1438. return (int32_t)(seq1 - seq2) >= 0;
  1439. }
  1440. uint32_t
  1441. i915_get_gem_seqno(struct drm_device *dev,
  1442. struct intel_ring_buffer *ring)
  1443. {
  1444. return ring->get_gem_seqno(dev, ring);
  1445. }
  1446. /**
  1447. * This function clears the request list as sequence numbers are passed.
  1448. */
  1449. static void
  1450. i915_gem_retire_requests_ring(struct drm_device *dev,
  1451. struct intel_ring_buffer *ring)
  1452. {
  1453. drm_i915_private_t *dev_priv = dev->dev_private;
  1454. uint32_t seqno;
  1455. if (!ring->status_page.page_addr
  1456. || list_empty(&ring->request_list))
  1457. return;
  1458. seqno = i915_get_gem_seqno(dev, ring);
  1459. while (!list_empty(&ring->request_list)) {
  1460. struct drm_i915_gem_request *request;
  1461. uint32_t retiring_seqno;
  1462. request = list_first_entry(&ring->request_list,
  1463. struct drm_i915_gem_request,
  1464. list);
  1465. retiring_seqno = request->seqno;
  1466. if (i915_seqno_passed(seqno, retiring_seqno) ||
  1467. atomic_read(&dev_priv->mm.wedged)) {
  1468. i915_gem_retire_request(dev, request);
  1469. list_del(&request->list);
  1470. list_del(&request->client_list);
  1471. kfree(request);
  1472. } else
  1473. break;
  1474. }
  1475. if (unlikely (dev_priv->trace_irq_seqno &&
  1476. i915_seqno_passed(dev_priv->trace_irq_seqno, seqno))) {
  1477. ring->user_irq_put(dev, ring);
  1478. dev_priv->trace_irq_seqno = 0;
  1479. }
  1480. }
  1481. void
  1482. i915_gem_retire_requests(struct drm_device *dev)
  1483. {
  1484. drm_i915_private_t *dev_priv = dev->dev_private;
  1485. if (!list_empty(&dev_priv->mm.deferred_free_list)) {
  1486. struct drm_i915_gem_object *obj_priv, *tmp;
  1487. /* We must be careful that during unbind() we do not
  1488. * accidentally infinitely recurse into retire requests.
  1489. * Currently:
  1490. * retire -> free -> unbind -> wait -> retire_ring
  1491. */
  1492. list_for_each_entry_safe(obj_priv, tmp,
  1493. &dev_priv->mm.deferred_free_list,
  1494. list)
  1495. i915_gem_free_object_tail(&obj_priv->base);
  1496. }
  1497. i915_gem_retire_requests_ring(dev, &dev_priv->render_ring);
  1498. if (HAS_BSD(dev))
  1499. i915_gem_retire_requests_ring(dev, &dev_priv->bsd_ring);
  1500. }
  1501. void
  1502. i915_gem_retire_work_handler(struct work_struct *work)
  1503. {
  1504. drm_i915_private_t *dev_priv;
  1505. struct drm_device *dev;
  1506. dev_priv = container_of(work, drm_i915_private_t,
  1507. mm.retire_work.work);
  1508. dev = dev_priv->dev;
  1509. mutex_lock(&dev->struct_mutex);
  1510. i915_gem_retire_requests(dev);
  1511. if (!dev_priv->mm.suspended &&
  1512. (!list_empty(&dev_priv->render_ring.request_list) ||
  1513. (HAS_BSD(dev) &&
  1514. !list_empty(&dev_priv->bsd_ring.request_list))))
  1515. queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
  1516. mutex_unlock(&dev->struct_mutex);
  1517. }
  1518. int
  1519. i915_do_wait_request(struct drm_device *dev, uint32_t seqno,
  1520. int interruptible, struct intel_ring_buffer *ring)
  1521. {
  1522. drm_i915_private_t *dev_priv = dev->dev_private;
  1523. u32 ier;
  1524. int ret = 0;
  1525. BUG_ON(seqno == 0);
  1526. if (atomic_read(&dev_priv->mm.wedged))
  1527. return -EIO;
  1528. if (!i915_seqno_passed(ring->get_gem_seqno(dev, ring), seqno)) {
  1529. if (HAS_PCH_SPLIT(dev))
  1530. ier = I915_READ(DEIER) | I915_READ(GTIER);
  1531. else
  1532. ier = I915_READ(IER);
  1533. if (!ier) {
  1534. DRM_ERROR("something (likely vbetool) disabled "
  1535. "interrupts, re-enabling\n");
  1536. i915_driver_irq_preinstall(dev);
  1537. i915_driver_irq_postinstall(dev);
  1538. }
  1539. trace_i915_gem_request_wait_begin(dev, seqno);
  1540. ring->waiting_gem_seqno = seqno;
  1541. ring->user_irq_get(dev, ring);
  1542. if (interruptible)
  1543. ret = wait_event_interruptible(ring->irq_queue,
  1544. i915_seqno_passed(
  1545. ring->get_gem_seqno(dev, ring), seqno)
  1546. || atomic_read(&dev_priv->mm.wedged));
  1547. else
  1548. wait_event(ring->irq_queue,
  1549. i915_seqno_passed(
  1550. ring->get_gem_seqno(dev, ring), seqno)
  1551. || atomic_read(&dev_priv->mm.wedged));
  1552. ring->user_irq_put(dev, ring);
  1553. ring->waiting_gem_seqno = 0;
  1554. trace_i915_gem_request_wait_end(dev, seqno);
  1555. }
  1556. if (atomic_read(&dev_priv->mm.wedged))
  1557. ret = -EIO;
  1558. if (ret && ret != -ERESTARTSYS)
  1559. DRM_ERROR("%s returns %d (awaiting %d at %d)\n",
  1560. __func__, ret, seqno, ring->get_gem_seqno(dev, ring));
  1561. /* Directly dispatch request retiring. While we have the work queue
  1562. * to handle this, the waiter on a request often wants an associated
  1563. * buffer to have made it to the inactive list, and we would need
  1564. * a separate wait queue to handle that.
  1565. */
  1566. if (ret == 0)
  1567. i915_gem_retire_requests_ring(dev, ring);
  1568. return ret;
  1569. }
  1570. /**
  1571. * Waits for a sequence number to be signaled, and cleans up the
  1572. * request and object lists appropriately for that event.
  1573. */
  1574. static int
  1575. i915_wait_request(struct drm_device *dev, uint32_t seqno,
  1576. struct intel_ring_buffer *ring)
  1577. {
  1578. return i915_do_wait_request(dev, seqno, 1, ring);
  1579. }
  1580. static void
  1581. i915_gem_flush(struct drm_device *dev,
  1582. uint32_t invalidate_domains,
  1583. uint32_t flush_domains)
  1584. {
  1585. drm_i915_private_t *dev_priv = dev->dev_private;
  1586. if (flush_domains & I915_GEM_DOMAIN_CPU)
  1587. drm_agp_chipset_flush(dev);
  1588. dev_priv->render_ring.flush(dev, &dev_priv->render_ring,
  1589. invalidate_domains,
  1590. flush_domains);
  1591. if (HAS_BSD(dev))
  1592. dev_priv->bsd_ring.flush(dev, &dev_priv->bsd_ring,
  1593. invalidate_domains,
  1594. flush_domains);
  1595. }
  1596. static void
  1597. i915_gem_flush_ring(struct drm_device *dev,
  1598. uint32_t invalidate_domains,
  1599. uint32_t flush_domains,
  1600. struct intel_ring_buffer *ring)
  1601. {
  1602. if (flush_domains & I915_GEM_DOMAIN_CPU)
  1603. drm_agp_chipset_flush(dev);
  1604. ring->flush(dev, ring,
  1605. invalidate_domains,
  1606. flush_domains);
  1607. }
  1608. /**
  1609. * Ensures that all rendering to the object has completed and the object is
  1610. * safe to unbind from the GTT or access from the CPU.
  1611. */
  1612. static int
  1613. i915_gem_object_wait_rendering(struct drm_gem_object *obj)
  1614. {
  1615. struct drm_device *dev = obj->dev;
  1616. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  1617. int ret;
  1618. /* This function only exists to support waiting for existing rendering,
  1619. * not for emitting required flushes.
  1620. */
  1621. BUG_ON((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0);
  1622. /* If there is rendering queued on the buffer being evicted, wait for
  1623. * it.
  1624. */
  1625. if (obj_priv->active) {
  1626. #if WATCH_BUF
  1627. DRM_INFO("%s: object %p wait for seqno %08x\n",
  1628. __func__, obj, obj_priv->last_rendering_seqno);
  1629. #endif
  1630. ret = i915_wait_request(dev,
  1631. obj_priv->last_rendering_seqno, obj_priv->ring);
  1632. if (ret != 0)
  1633. return ret;
  1634. }
  1635. return 0;
  1636. }
  1637. /**
  1638. * Unbinds an object from the GTT aperture.
  1639. */
  1640. int
  1641. i915_gem_object_unbind(struct drm_gem_object *obj)
  1642. {
  1643. struct drm_device *dev = obj->dev;
  1644. drm_i915_private_t *dev_priv = dev->dev_private;
  1645. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  1646. int ret = 0;
  1647. #if WATCH_BUF
  1648. DRM_INFO("%s:%d %p\n", __func__, __LINE__, obj);
  1649. DRM_INFO("gtt_space %p\n", obj_priv->gtt_space);
  1650. #endif
  1651. if (obj_priv->gtt_space == NULL)
  1652. return 0;
  1653. if (obj_priv->pin_count != 0) {
  1654. DRM_ERROR("Attempting to unbind pinned buffer\n");
  1655. return -EINVAL;
  1656. }
  1657. /* blow away mappings if mapped through GTT */
  1658. i915_gem_release_mmap(obj);
  1659. /* Move the object to the CPU domain to ensure that
  1660. * any possible CPU writes while it's not in the GTT
  1661. * are flushed when we go to remap it. This will
  1662. * also ensure that all pending GPU writes are finished
  1663. * before we unbind.
  1664. */
  1665. ret = i915_gem_object_set_to_cpu_domain(obj, 1);
  1666. if (ret == -ERESTARTSYS)
  1667. return ret;
  1668. /* Continue on if we fail due to EIO, the GPU is hung so we
  1669. * should be safe and we need to cleanup or else we might
  1670. * cause memory corruption through use-after-free.
  1671. */
  1672. BUG_ON(obj_priv->active);
  1673. /* release the fence reg _after_ flushing */
  1674. if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
  1675. i915_gem_clear_fence_reg(obj);
  1676. if (obj_priv->agp_mem != NULL) {
  1677. drm_unbind_agp(obj_priv->agp_mem);
  1678. drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE);
  1679. obj_priv->agp_mem = NULL;
  1680. }
  1681. i915_gem_object_put_pages(obj);
  1682. BUG_ON(obj_priv->pages_refcount);
  1683. if (obj_priv->gtt_space) {
  1684. atomic_dec(&dev->gtt_count);
  1685. atomic_sub(obj->size, &dev->gtt_memory);
  1686. drm_mm_put_block(obj_priv->gtt_space);
  1687. obj_priv->gtt_space = NULL;
  1688. }
  1689. /* Remove ourselves from the LRU list if present. */
  1690. spin_lock(&dev_priv->mm.active_list_lock);
  1691. if (!list_empty(&obj_priv->list))
  1692. list_del_init(&obj_priv->list);
  1693. spin_unlock(&dev_priv->mm.active_list_lock);
  1694. if (i915_gem_object_is_purgeable(obj_priv))
  1695. i915_gem_object_truncate(obj);
  1696. trace_i915_gem_object_unbind(obj);
  1697. return ret;
  1698. }
  1699. static struct drm_gem_object *
  1700. i915_gem_find_inactive_object(struct drm_device *dev, int min_size)
  1701. {
  1702. drm_i915_private_t *dev_priv = dev->dev_private;
  1703. struct drm_i915_gem_object *obj_priv;
  1704. struct drm_gem_object *best = NULL;
  1705. struct drm_gem_object *first = NULL;
  1706. /* Try to find the smallest clean object */
  1707. list_for_each_entry(obj_priv, &dev_priv->mm.inactive_list, list) {
  1708. struct drm_gem_object *obj = &obj_priv->base;
  1709. if (obj->size >= min_size) {
  1710. if ((!obj_priv->dirty ||
  1711. i915_gem_object_is_purgeable(obj_priv)) &&
  1712. (!best || obj->size < best->size)) {
  1713. best = obj;
  1714. if (best->size == min_size)
  1715. return best;
  1716. }
  1717. if (!first)
  1718. first = obj;
  1719. }
  1720. }
  1721. return best ? best : first;
  1722. }
  1723. static int
  1724. i915_gpu_idle(struct drm_device *dev)
  1725. {
  1726. drm_i915_private_t *dev_priv = dev->dev_private;
  1727. bool lists_empty;
  1728. uint32_t seqno1, seqno2;
  1729. int ret;
  1730. spin_lock(&dev_priv->mm.active_list_lock);
  1731. lists_empty = (list_empty(&dev_priv->mm.flushing_list) &&
  1732. list_empty(&dev_priv->render_ring.active_list) &&
  1733. (!HAS_BSD(dev) ||
  1734. list_empty(&dev_priv->bsd_ring.active_list)));
  1735. spin_unlock(&dev_priv->mm.active_list_lock);
  1736. if (lists_empty)
  1737. return 0;
  1738. /* Flush everything onto the inactive list. */
  1739. i915_gem_flush(dev, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
  1740. seqno1 = i915_add_request(dev, NULL, I915_GEM_GPU_DOMAINS,
  1741. &dev_priv->render_ring);
  1742. if (seqno1 == 0)
  1743. return -ENOMEM;
  1744. ret = i915_wait_request(dev, seqno1, &dev_priv->render_ring);
  1745. if (HAS_BSD(dev)) {
  1746. seqno2 = i915_add_request(dev, NULL, I915_GEM_GPU_DOMAINS,
  1747. &dev_priv->bsd_ring);
  1748. if (seqno2 == 0)
  1749. return -ENOMEM;
  1750. ret = i915_wait_request(dev, seqno2, &dev_priv->bsd_ring);
  1751. if (ret)
  1752. return ret;
  1753. }
  1754. return ret;
  1755. }
  1756. static int
  1757. i915_gem_evict_everything(struct drm_device *dev)
  1758. {
  1759. drm_i915_private_t *dev_priv = dev->dev_private;
  1760. int ret;
  1761. bool lists_empty;
  1762. spin_lock(&dev_priv->mm.active_list_lock);
  1763. lists_empty = (list_empty(&dev_priv->mm.inactive_list) &&
  1764. list_empty(&dev_priv->mm.flushing_list) &&
  1765. list_empty(&dev_priv->render_ring.active_list) &&
  1766. (!HAS_BSD(dev)
  1767. || list_empty(&dev_priv->bsd_ring.active_list)));
  1768. spin_unlock(&dev_priv->mm.active_list_lock);
  1769. if (lists_empty)
  1770. return -ENOSPC;
  1771. /* Flush everything (on to the inactive lists) and evict */
  1772. ret = i915_gpu_idle(dev);
  1773. if (ret)
  1774. return ret;
  1775. BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
  1776. ret = i915_gem_evict_from_inactive_list(dev);
  1777. if (ret)
  1778. return ret;
  1779. spin_lock(&dev_priv->mm.active_list_lock);
  1780. lists_empty = (list_empty(&dev_priv->mm.inactive_list) &&
  1781. list_empty(&dev_priv->mm.flushing_list) &&
  1782. list_empty(&dev_priv->render_ring.active_list) &&
  1783. (!HAS_BSD(dev)
  1784. || list_empty(&dev_priv->bsd_ring.active_list)));
  1785. spin_unlock(&dev_priv->mm.active_list_lock);
  1786. BUG_ON(!lists_empty);
  1787. return 0;
  1788. }
  1789. static int
  1790. i915_gem_evict_something(struct drm_device *dev, int min_size)
  1791. {
  1792. drm_i915_private_t *dev_priv = dev->dev_private;
  1793. struct drm_gem_object *obj;
  1794. int ret;
  1795. struct intel_ring_buffer *render_ring = &dev_priv->render_ring;
  1796. struct intel_ring_buffer *bsd_ring = &dev_priv->bsd_ring;
  1797. for (;;) {
  1798. i915_gem_retire_requests(dev);
  1799. /* If there's an inactive buffer available now, grab it
  1800. * and be done.
  1801. */
  1802. obj = i915_gem_find_inactive_object(dev, min_size);
  1803. if (obj) {
  1804. struct drm_i915_gem_object *obj_priv;
  1805. #if WATCH_LRU
  1806. DRM_INFO("%s: evicting %p\n", __func__, obj);
  1807. #endif
  1808. obj_priv = to_intel_bo(obj);
  1809. BUG_ON(obj_priv->pin_count != 0);
  1810. BUG_ON(obj_priv->active);
  1811. /* Wait on the rendering and unbind the buffer. */
  1812. return i915_gem_object_unbind(obj);
  1813. }
  1814. /* If we didn't get anything, but the ring is still processing
  1815. * things, wait for the next to finish and hopefully leave us
  1816. * a buffer to evict.
  1817. */
  1818. if (!list_empty(&render_ring->request_list)) {
  1819. struct drm_i915_gem_request *request;
  1820. request = list_first_entry(&render_ring->request_list,
  1821. struct drm_i915_gem_request,
  1822. list);
  1823. ret = i915_wait_request(dev,
  1824. request->seqno, request->ring);
  1825. if (ret)
  1826. return ret;
  1827. continue;
  1828. }
  1829. if (HAS_BSD(dev) && !list_empty(&bsd_ring->request_list)) {
  1830. struct drm_i915_gem_request *request;
  1831. request = list_first_entry(&bsd_ring->request_list,
  1832. struct drm_i915_gem_request,
  1833. list);
  1834. ret = i915_wait_request(dev,
  1835. request->seqno, request->ring);
  1836. if (ret)
  1837. return ret;
  1838. continue;
  1839. }
  1840. /* If we didn't have anything on the request list but there
  1841. * are buffers awaiting a flush, emit one and try again.
  1842. * When we wait on it, those buffers waiting for that flush
  1843. * will get moved to inactive.
  1844. */
  1845. if (!list_empty(&dev_priv->mm.flushing_list)) {
  1846. struct drm_i915_gem_object *obj_priv;
  1847. /* Find an object that we can immediately reuse */
  1848. list_for_each_entry(obj_priv, &dev_priv->mm.flushing_list, list) {
  1849. obj = &obj_priv->base;
  1850. if (obj->size >= min_size)
  1851. break;
  1852. obj = NULL;
  1853. }
  1854. if (obj != NULL) {
  1855. uint32_t seqno;
  1856. i915_gem_flush_ring(dev,
  1857. obj->write_domain,
  1858. obj->write_domain,
  1859. obj_priv->ring);
  1860. seqno = i915_add_request(dev, NULL,
  1861. obj->write_domain,
  1862. obj_priv->ring);
  1863. if (seqno == 0)
  1864. return -ENOMEM;
  1865. continue;
  1866. }
  1867. }
  1868. /* If we didn't do any of the above, there's no single buffer
  1869. * large enough to swap out for the new one, so just evict
  1870. * everything and start again. (This should be rare.)
  1871. */
  1872. if (!list_empty (&dev_priv->mm.inactive_list))
  1873. return i915_gem_evict_from_inactive_list(dev);
  1874. else
  1875. return i915_gem_evict_everything(dev);
  1876. }
  1877. }
  1878. int
  1879. i915_gem_object_get_pages(struct drm_gem_object *obj,
  1880. gfp_t gfpmask)
  1881. {
  1882. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  1883. int page_count, i;
  1884. struct address_space *mapping;
  1885. struct inode *inode;
  1886. struct page *page;
  1887. BUG_ON(obj_priv->pages_refcount
  1888. == DRM_I915_GEM_OBJECT_MAX_PAGES_REFCOUNT);
  1889. if (obj_priv->pages_refcount++ != 0)
  1890. return 0;
  1891. /* Get the list of pages out of our struct file. They'll be pinned
  1892. * at this point until we release them.
  1893. */
  1894. page_count = obj->size / PAGE_SIZE;
  1895. BUG_ON(obj_priv->pages != NULL);
  1896. obj_priv->pages = drm_calloc_large(page_count, sizeof(struct page *));
  1897. if (obj_priv->pages == NULL) {
  1898. obj_priv->pages_refcount--;
  1899. return -ENOMEM;
  1900. }
  1901. inode = obj->filp->f_path.dentry->d_inode;
  1902. mapping = inode->i_mapping;
  1903. for (i = 0; i < page_count; i++) {
  1904. page = read_cache_page_gfp(mapping, i,
  1905. GFP_HIGHUSER |
  1906. __GFP_COLD |
  1907. __GFP_RECLAIMABLE |
  1908. gfpmask);
  1909. if (IS_ERR(page))
  1910. goto err_pages;
  1911. obj_priv->pages[i] = page;
  1912. }
  1913. if (obj_priv->tiling_mode != I915_TILING_NONE)
  1914. i915_gem_object_do_bit_17_swizzle(obj);
  1915. return 0;
  1916. err_pages:
  1917. while (i--)
  1918. page_cache_release(obj_priv->pages[i]);
  1919. drm_free_large(obj_priv->pages);
  1920. obj_priv->pages = NULL;
  1921. obj_priv->pages_refcount--;
  1922. return PTR_ERR(page);
  1923. }
  1924. static void sandybridge_write_fence_reg(struct drm_i915_fence_reg *reg)
  1925. {
  1926. struct drm_gem_object *obj = reg->obj;
  1927. struct drm_device *dev = obj->dev;
  1928. drm_i915_private_t *dev_priv = dev->dev_private;
  1929. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  1930. int regnum = obj_priv->fence_reg;
  1931. uint64_t val;
  1932. val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
  1933. 0xfffff000) << 32;
  1934. val |= obj_priv->gtt_offset & 0xfffff000;
  1935. val |= (uint64_t)((obj_priv->stride / 128) - 1) <<
  1936. SANDYBRIDGE_FENCE_PITCH_SHIFT;
  1937. if (obj_priv->tiling_mode == I915_TILING_Y)
  1938. val |= 1 << I965_FENCE_TILING_Y_SHIFT;
  1939. val |= I965_FENCE_REG_VALID;
  1940. I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (regnum * 8), val);
  1941. }
  1942. static void i965_write_fence_reg(struct drm_i915_fence_reg *reg)
  1943. {
  1944. struct drm_gem_object *obj = reg->obj;
  1945. struct drm_device *dev = obj->dev;
  1946. drm_i915_private_t *dev_priv = dev->dev_private;
  1947. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  1948. int regnum = obj_priv->fence_reg;
  1949. uint64_t val;
  1950. val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
  1951. 0xfffff000) << 32;
  1952. val |= obj_priv->gtt_offset & 0xfffff000;
  1953. val |= ((obj_priv->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
  1954. if (obj_priv->tiling_mode == I915_TILING_Y)
  1955. val |= 1 << I965_FENCE_TILING_Y_SHIFT;
  1956. val |= I965_FENCE_REG_VALID;
  1957. I915_WRITE64(FENCE_REG_965_0 + (regnum * 8), val);
  1958. }
  1959. static void i915_write_fence_reg(struct drm_i915_fence_reg *reg)
  1960. {
  1961. struct drm_gem_object *obj = reg->obj;
  1962. struct drm_device *dev = obj->dev;
  1963. drm_i915_private_t *dev_priv = dev->dev_private;
  1964. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  1965. int regnum = obj_priv->fence_reg;
  1966. int tile_width;
  1967. uint32_t fence_reg, val;
  1968. uint32_t pitch_val;
  1969. if ((obj_priv->gtt_offset & ~I915_FENCE_START_MASK) ||
  1970. (obj_priv->gtt_offset & (obj->size - 1))) {
  1971. WARN(1, "%s: object 0x%08x not 1M or size (0x%zx) aligned\n",
  1972. __func__, obj_priv->gtt_offset, obj->size);
  1973. return;
  1974. }
  1975. if (obj_priv->tiling_mode == I915_TILING_Y &&
  1976. HAS_128_BYTE_Y_TILING(dev))
  1977. tile_width = 128;
  1978. else
  1979. tile_width = 512;
  1980. /* Note: pitch better be a power of two tile widths */
  1981. pitch_val = obj_priv->stride / tile_width;
  1982. pitch_val = ffs(pitch_val) - 1;
  1983. if (obj_priv->tiling_mode == I915_TILING_Y &&
  1984. HAS_128_BYTE_Y_TILING(dev))
  1985. WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
  1986. else
  1987. WARN_ON(pitch_val > I915_FENCE_MAX_PITCH_VAL);
  1988. val = obj_priv->gtt_offset;
  1989. if (obj_priv->tiling_mode == I915_TILING_Y)
  1990. val |= 1 << I830_FENCE_TILING_Y_SHIFT;
  1991. val |= I915_FENCE_SIZE_BITS(obj->size);
  1992. val |= pitch_val << I830_FENCE_PITCH_SHIFT;
  1993. val |= I830_FENCE_REG_VALID;
  1994. if (regnum < 8)
  1995. fence_reg = FENCE_REG_830_0 + (regnum * 4);
  1996. else
  1997. fence_reg = FENCE_REG_945_8 + ((regnum - 8) * 4);
  1998. I915_WRITE(fence_reg, val);
  1999. }
  2000. static void i830_write_fence_reg(struct drm_i915_fence_reg *reg)
  2001. {
  2002. struct drm_gem_object *obj = reg->obj;
  2003. struct drm_device *dev = obj->dev;
  2004. drm_i915_private_t *dev_priv = dev->dev_private;
  2005. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  2006. int regnum = obj_priv->fence_reg;
  2007. uint32_t val;
  2008. uint32_t pitch_val;
  2009. uint32_t fence_size_bits;
  2010. if ((obj_priv->gtt_offset & ~I830_FENCE_START_MASK) ||
  2011. (obj_priv->gtt_offset & (obj->size - 1))) {
  2012. WARN(1, "%s: object 0x%08x not 512K or size aligned\n",
  2013. __func__, obj_priv->gtt_offset);
  2014. return;
  2015. }
  2016. pitch_val = obj_priv->stride / 128;
  2017. pitch_val = ffs(pitch_val) - 1;
  2018. WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
  2019. val = obj_priv->gtt_offset;
  2020. if (obj_priv->tiling_mode == I915_TILING_Y)
  2021. val |= 1 << I830_FENCE_TILING_Y_SHIFT;
  2022. fence_size_bits = I830_FENCE_SIZE_BITS(obj->size);
  2023. WARN_ON(fence_size_bits & ~0x00000f00);
  2024. val |= fence_size_bits;
  2025. val |= pitch_val << I830_FENCE_PITCH_SHIFT;
  2026. val |= I830_FENCE_REG_VALID;
  2027. I915_WRITE(FENCE_REG_830_0 + (regnum * 4), val);
  2028. }
  2029. static int i915_find_fence_reg(struct drm_device *dev)
  2030. {
  2031. struct drm_i915_fence_reg *reg = NULL;
  2032. struct drm_i915_gem_object *obj_priv = NULL;
  2033. struct drm_i915_private *dev_priv = dev->dev_private;
  2034. struct drm_gem_object *obj = NULL;
  2035. int i, avail, ret;
  2036. /* First try to find a free reg */
  2037. avail = 0;
  2038. for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
  2039. reg = &dev_priv->fence_regs[i];
  2040. if (!reg->obj)
  2041. return i;
  2042. obj_priv = to_intel_bo(reg->obj);
  2043. if (!obj_priv->pin_count)
  2044. avail++;
  2045. }
  2046. if (avail == 0)
  2047. return -ENOSPC;
  2048. /* None available, try to steal one or wait for a user to finish */
  2049. i = I915_FENCE_REG_NONE;
  2050. list_for_each_entry(reg, &dev_priv->mm.fence_list,
  2051. lru_list) {
  2052. obj = reg->obj;
  2053. obj_priv = to_intel_bo(obj);
  2054. if (obj_priv->pin_count)
  2055. continue;
  2056. /* found one! */
  2057. i = obj_priv->fence_reg;
  2058. break;
  2059. }
  2060. BUG_ON(i == I915_FENCE_REG_NONE);
  2061. /* We only have a reference on obj from the active list. put_fence_reg
  2062. * might drop that one, causing a use-after-free in it. So hold a
  2063. * private reference to obj like the other callers of put_fence_reg
  2064. * (set_tiling ioctl) do. */
  2065. drm_gem_object_reference(obj);
  2066. ret = i915_gem_object_put_fence_reg(obj);
  2067. drm_gem_object_unreference(obj);
  2068. if (ret != 0)
  2069. return ret;
  2070. return i;
  2071. }
  2072. /**
  2073. * i915_gem_object_get_fence_reg - set up a fence reg for an object
  2074. * @obj: object to map through a fence reg
  2075. *
  2076. * When mapping objects through the GTT, userspace wants to be able to write
  2077. * to them without having to worry about swizzling if the object is tiled.
  2078. *
  2079. * This function walks the fence regs looking for a free one for @obj,
  2080. * stealing one if it can't find any.
  2081. *
  2082. * It then sets up the reg based on the object's properties: address, pitch
  2083. * and tiling format.
  2084. */
  2085. int
  2086. i915_gem_object_get_fence_reg(struct drm_gem_object *obj)
  2087. {
  2088. struct drm_device *dev = obj->dev;
  2089. struct drm_i915_private *dev_priv = dev->dev_private;
  2090. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  2091. struct drm_i915_fence_reg *reg = NULL;
  2092. int ret;
  2093. /* Just update our place in the LRU if our fence is getting used. */
  2094. if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
  2095. reg = &dev_priv->fence_regs[obj_priv->fence_reg];
  2096. list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
  2097. return 0;
  2098. }
  2099. switch (obj_priv->tiling_mode) {
  2100. case I915_TILING_NONE:
  2101. WARN(1, "allocating a fence for non-tiled object?\n");
  2102. break;
  2103. case I915_TILING_X:
  2104. if (!obj_priv->stride)
  2105. return -EINVAL;
  2106. WARN((obj_priv->stride & (512 - 1)),
  2107. "object 0x%08x is X tiled but has non-512B pitch\n",
  2108. obj_priv->gtt_offset);
  2109. break;
  2110. case I915_TILING_Y:
  2111. if (!obj_priv->stride)
  2112. return -EINVAL;
  2113. WARN((obj_priv->stride & (128 - 1)),
  2114. "object 0x%08x is Y tiled but has non-128B pitch\n",
  2115. obj_priv->gtt_offset);
  2116. break;
  2117. }
  2118. ret = i915_find_fence_reg(dev);
  2119. if (ret < 0)
  2120. return ret;
  2121. obj_priv->fence_reg = ret;
  2122. reg = &dev_priv->fence_regs[obj_priv->fence_reg];
  2123. list_add_tail(&reg->lru_list, &dev_priv->mm.fence_list);
  2124. reg->obj = obj;
  2125. if (IS_GEN6(dev))
  2126. sandybridge_write_fence_reg(reg);
  2127. else if (IS_I965G(dev))
  2128. i965_write_fence_reg(reg);
  2129. else if (IS_I9XX(dev))
  2130. i915_write_fence_reg(reg);
  2131. else
  2132. i830_write_fence_reg(reg);
  2133. trace_i915_gem_object_get_fence(obj, obj_priv->fence_reg,
  2134. obj_priv->tiling_mode);
  2135. return 0;
  2136. }
  2137. /**
  2138. * i915_gem_clear_fence_reg - clear out fence register info
  2139. * @obj: object to clear
  2140. *
  2141. * Zeroes out the fence register itself and clears out the associated
  2142. * data structures in dev_priv and obj_priv.
  2143. */
  2144. static void
  2145. i915_gem_clear_fence_reg(struct drm_gem_object *obj)
  2146. {
  2147. struct drm_device *dev = obj->dev;
  2148. drm_i915_private_t *dev_priv = dev->dev_private;
  2149. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  2150. struct drm_i915_fence_reg *reg =
  2151. &dev_priv->fence_regs[obj_priv->fence_reg];
  2152. if (IS_GEN6(dev)) {
  2153. I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 +
  2154. (obj_priv->fence_reg * 8), 0);
  2155. } else if (IS_I965G(dev)) {
  2156. I915_WRITE64(FENCE_REG_965_0 + (obj_priv->fence_reg * 8), 0);
  2157. } else {
  2158. uint32_t fence_reg;
  2159. if (obj_priv->fence_reg < 8)
  2160. fence_reg = FENCE_REG_830_0 + obj_priv->fence_reg * 4;
  2161. else
  2162. fence_reg = FENCE_REG_945_8 + (obj_priv->fence_reg -
  2163. 8) * 4;
  2164. I915_WRITE(fence_reg, 0);
  2165. }
  2166. reg->obj = NULL;
  2167. obj_priv->fence_reg = I915_FENCE_REG_NONE;
  2168. list_del_init(&reg->lru_list);
  2169. }
  2170. /**
  2171. * i915_gem_object_put_fence_reg - waits on outstanding fenced access
  2172. * to the buffer to finish, and then resets the fence register.
  2173. * @obj: tiled object holding a fence register.
  2174. *
  2175. * Zeroes out the fence register itself and clears out the associated
  2176. * data structures in dev_priv and obj_priv.
  2177. */
  2178. int
  2179. i915_gem_object_put_fence_reg(struct drm_gem_object *obj)
  2180. {
  2181. struct drm_device *dev = obj->dev;
  2182. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  2183. if (obj_priv->fence_reg == I915_FENCE_REG_NONE)
  2184. return 0;
  2185. /* If we've changed tiling, GTT-mappings of the object
  2186. * need to re-fault to ensure that the correct fence register
  2187. * setup is in place.
  2188. */
  2189. i915_gem_release_mmap(obj);
  2190. /* On the i915, GPU access to tiled buffers is via a fence,
  2191. * therefore we must wait for any outstanding access to complete
  2192. * before clearing the fence.
  2193. */
  2194. if (!IS_I965G(dev)) {
  2195. int ret;
  2196. ret = i915_gem_object_flush_gpu_write_domain(obj);
  2197. if (ret != 0)
  2198. return ret;
  2199. ret = i915_gem_object_wait_rendering(obj);
  2200. if (ret != 0)
  2201. return ret;
  2202. }
  2203. i915_gem_object_flush_gtt_write_domain(obj);
  2204. i915_gem_clear_fence_reg (obj);
  2205. return 0;
  2206. }
  2207. /**
  2208. * Finds free space in the GTT aperture and binds the object there.
  2209. */
  2210. static int
  2211. i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
  2212. {
  2213. struct drm_device *dev = obj->dev;
  2214. drm_i915_private_t *dev_priv = dev->dev_private;
  2215. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  2216. struct drm_mm_node *free_space;
  2217. gfp_t gfpmask = __GFP_NORETRY | __GFP_NOWARN;
  2218. int ret;
  2219. if (obj_priv->madv != I915_MADV_WILLNEED) {
  2220. DRM_ERROR("Attempting to bind a purgeable object\n");
  2221. return -EINVAL;
  2222. }
  2223. if (alignment == 0)
  2224. alignment = i915_gem_get_gtt_alignment(obj);
  2225. if (alignment & (i915_gem_get_gtt_alignment(obj) - 1)) {
  2226. DRM_ERROR("Invalid object alignment requested %u\n", alignment);
  2227. return -EINVAL;
  2228. }
  2229. /* If the object is bigger than the entire aperture, reject it early
  2230. * before evicting everything in a vain attempt to find space.
  2231. */
  2232. if (obj->size > dev->gtt_total) {
  2233. DRM_ERROR("Attempting to bind an object larger than the aperture\n");
  2234. return -E2BIG;
  2235. }
  2236. search_free:
  2237. free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
  2238. obj->size, alignment, 0);
  2239. if (free_space != NULL) {
  2240. obj_priv->gtt_space = drm_mm_get_block(free_space, obj->size,
  2241. alignment);
  2242. if (obj_priv->gtt_space != NULL)
  2243. obj_priv->gtt_offset = obj_priv->gtt_space->start;
  2244. }
  2245. if (obj_priv->gtt_space == NULL) {
  2246. /* If the gtt is empty and we're still having trouble
  2247. * fitting our object in, we're out of memory.
  2248. */
  2249. #if WATCH_LRU
  2250. DRM_INFO("%s: GTT full, evicting something\n", __func__);
  2251. #endif
  2252. ret = i915_gem_evict_something(dev, obj->size);
  2253. if (ret)
  2254. return ret;
  2255. goto search_free;
  2256. }
  2257. #if WATCH_BUF
  2258. DRM_INFO("Binding object of size %zd at 0x%08x\n",
  2259. obj->size, obj_priv->gtt_offset);
  2260. #endif
  2261. ret = i915_gem_object_get_pages(obj, gfpmask);
  2262. if (ret) {
  2263. drm_mm_put_block(obj_priv->gtt_space);
  2264. obj_priv->gtt_space = NULL;
  2265. if (ret == -ENOMEM) {
  2266. /* first try to clear up some space from the GTT */
  2267. ret = i915_gem_evict_something(dev, obj->size);
  2268. if (ret) {
  2269. /* now try to shrink everyone else */
  2270. if (gfpmask) {
  2271. gfpmask = 0;
  2272. goto search_free;
  2273. }
  2274. return ret;
  2275. }
  2276. goto search_free;
  2277. }
  2278. return ret;
  2279. }
  2280. /* Create an AGP memory structure pointing at our pages, and bind it
  2281. * into the GTT.
  2282. */
  2283. obj_priv->agp_mem = drm_agp_bind_pages(dev,
  2284. obj_priv->pages,
  2285. obj->size >> PAGE_SHIFT,
  2286. obj_priv->gtt_offset,
  2287. obj_priv->agp_type);
  2288. if (obj_priv->agp_mem == NULL) {
  2289. i915_gem_object_put_pages(obj);
  2290. drm_mm_put_block(obj_priv->gtt_space);
  2291. obj_priv->gtt_space = NULL;
  2292. ret = i915_gem_evict_something(dev, obj->size);
  2293. if (ret)
  2294. return ret;
  2295. goto search_free;
  2296. }
  2297. atomic_inc(&dev->gtt_count);
  2298. atomic_add(obj->size, &dev->gtt_memory);
  2299. /* Assert that the object is not currently in any GPU domain. As it
  2300. * wasn't in the GTT, there shouldn't be any way it could have been in
  2301. * a GPU cache
  2302. */
  2303. BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS);
  2304. BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);
  2305. trace_i915_gem_object_bind(obj, obj_priv->gtt_offset);
  2306. return 0;
  2307. }
  2308. void
  2309. i915_gem_clflush_object(struct drm_gem_object *obj)
  2310. {
  2311. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  2312. /* If we don't have a page list set up, then we're not pinned
  2313. * to GPU, and we can ignore the cache flush because it'll happen
  2314. * again at bind time.
  2315. */
  2316. if (obj_priv->pages == NULL)
  2317. return;
  2318. trace_i915_gem_object_clflush(obj);
  2319. drm_clflush_pages(obj_priv->pages, obj->size / PAGE_SIZE);
  2320. }
  2321. /** Flushes any GPU write domain for the object if it's dirty. */
  2322. static int
  2323. i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj)
  2324. {
  2325. struct drm_device *dev = obj->dev;
  2326. uint32_t old_write_domain;
  2327. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  2328. if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
  2329. return 0;
  2330. /* Queue the GPU write cache flushing we need. */
  2331. old_write_domain = obj->write_domain;
  2332. i915_gem_flush(dev, 0, obj->write_domain);
  2333. if (i915_add_request(dev, NULL, obj->write_domain, obj_priv->ring) == 0)
  2334. return -ENOMEM;
  2335. trace_i915_gem_object_change_domain(obj,
  2336. obj->read_domains,
  2337. old_write_domain);
  2338. return 0;
  2339. }
  2340. /** Flushes the GTT write domain for the object if it's dirty. */
  2341. static void
  2342. i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj)
  2343. {
  2344. uint32_t old_write_domain;
  2345. if (obj->write_domain != I915_GEM_DOMAIN_GTT)
  2346. return;
  2347. /* No actual flushing is required for the GTT write domain. Writes
  2348. * to it immediately go to main memory as far as we know, so there's
  2349. * no chipset flush. It also doesn't land in render cache.
  2350. */
  2351. old_write_domain = obj->write_domain;
  2352. obj->write_domain = 0;
  2353. trace_i915_gem_object_change_domain(obj,
  2354. obj->read_domains,
  2355. old_write_domain);
  2356. }
  2357. /** Flushes the CPU write domain for the object if it's dirty. */
  2358. static void
  2359. i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj)
  2360. {
  2361. struct drm_device *dev = obj->dev;
  2362. uint32_t old_write_domain;
  2363. if (obj->write_domain != I915_GEM_DOMAIN_CPU)
  2364. return;
  2365. i915_gem_clflush_object(obj);
  2366. drm_agp_chipset_flush(dev);
  2367. old_write_domain = obj->write_domain;
  2368. obj->write_domain = 0;
  2369. trace_i915_gem_object_change_domain(obj,
  2370. obj->read_domains,
  2371. old_write_domain);
  2372. }
  2373. int
  2374. i915_gem_object_flush_write_domain(struct drm_gem_object *obj)
  2375. {
  2376. int ret = 0;
  2377. switch (obj->write_domain) {
  2378. case I915_GEM_DOMAIN_GTT:
  2379. i915_gem_object_flush_gtt_write_domain(obj);
  2380. break;
  2381. case I915_GEM_DOMAIN_CPU:
  2382. i915_gem_object_flush_cpu_write_domain(obj);
  2383. break;
  2384. default:
  2385. ret = i915_gem_object_flush_gpu_write_domain(obj);
  2386. break;
  2387. }
  2388. return ret;
  2389. }
  2390. /**
  2391. * Moves a single object to the GTT read, and possibly write domain.
  2392. *
  2393. * This function returns when the move is complete, including waiting on
  2394. * flushes to occur.
  2395. */
  2396. int
  2397. i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
  2398. {
  2399. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  2400. uint32_t old_write_domain, old_read_domains;
  2401. int ret;
  2402. /* Not valid to be called on unbound objects. */
  2403. if (obj_priv->gtt_space == NULL)
  2404. return -EINVAL;
  2405. ret = i915_gem_object_flush_gpu_write_domain(obj);
  2406. if (ret != 0)
  2407. return ret;
  2408. /* Wait on any GPU rendering and flushing to occur. */
  2409. ret = i915_gem_object_wait_rendering(obj);
  2410. if (ret != 0)
  2411. return ret;
  2412. old_write_domain = obj->write_domain;
  2413. old_read_domains = obj->read_domains;
  2414. /* If we're writing through the GTT domain, then CPU and GPU caches
  2415. * will need to be invalidated at next use.
  2416. */
  2417. if (write)
  2418. obj->read_domains &= I915_GEM_DOMAIN_GTT;
  2419. i915_gem_object_flush_cpu_write_domain(obj);
  2420. /* It should now be out of any other write domains, and we can update
  2421. * the domain values for our changes.
  2422. */
  2423. BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
  2424. obj->read_domains |= I915_GEM_DOMAIN_GTT;
  2425. if (write) {
  2426. obj->write_domain = I915_GEM_DOMAIN_GTT;
  2427. obj_priv->dirty = 1;
  2428. }
  2429. trace_i915_gem_object_change_domain(obj,
  2430. old_read_domains,
  2431. old_write_domain);
  2432. return 0;
  2433. }
  2434. /*
  2435. * Prepare buffer for display plane. Use uninterruptible for possible flush
  2436. * wait, as in modesetting process we're not supposed to be interrupted.
  2437. */
  2438. int
  2439. i915_gem_object_set_to_display_plane(struct drm_gem_object *obj)
  2440. {
  2441. struct drm_device *dev = obj->dev;
  2442. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  2443. uint32_t old_write_domain, old_read_domains;
  2444. int ret;
  2445. /* Not valid to be called on unbound objects. */
  2446. if (obj_priv->gtt_space == NULL)
  2447. return -EINVAL;
  2448. ret = i915_gem_object_flush_gpu_write_domain(obj);
  2449. if (ret)
  2450. return ret;
  2451. /* Wait on any GPU rendering and flushing to occur. */
  2452. if (obj_priv->active) {
  2453. #if WATCH_BUF
  2454. DRM_INFO("%s: object %p wait for seqno %08x\n",
  2455. __func__, obj, obj_priv->last_rendering_seqno);
  2456. #endif
  2457. ret = i915_do_wait_request(dev,
  2458. obj_priv->last_rendering_seqno,
  2459. 0,
  2460. obj_priv->ring);
  2461. if (ret != 0)
  2462. return ret;
  2463. }
  2464. i915_gem_object_flush_cpu_write_domain(obj);
  2465. old_write_domain = obj->write_domain;
  2466. old_read_domains = obj->read_domains;
  2467. /* It should now be out of any other write domains, and we can update
  2468. * the domain values for our changes.
  2469. */
  2470. BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
  2471. obj->read_domains = I915_GEM_DOMAIN_GTT;
  2472. obj->write_domain = I915_GEM_DOMAIN_GTT;
  2473. obj_priv->dirty = 1;
  2474. trace_i915_gem_object_change_domain(obj,
  2475. old_read_domains,
  2476. old_write_domain);
  2477. return 0;
  2478. }
  2479. /**
  2480. * Moves a single object to the CPU read, and possibly write domain.
  2481. *
  2482. * This function returns when the move is complete, including waiting on
  2483. * flushes to occur.
  2484. */
  2485. static int
  2486. i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
  2487. {
  2488. uint32_t old_write_domain, old_read_domains;
  2489. int ret;
  2490. ret = i915_gem_object_flush_gpu_write_domain(obj);
  2491. if (ret)
  2492. return ret;
  2493. /* Wait on any GPU rendering and flushing to occur. */
  2494. ret = i915_gem_object_wait_rendering(obj);
  2495. if (ret != 0)
  2496. return ret;
  2497. i915_gem_object_flush_gtt_write_domain(obj);
  2498. /* If we have a partially-valid cache of the object in the CPU,
  2499. * finish invalidating it and free the per-page flags.
  2500. */
  2501. i915_gem_object_set_to_full_cpu_read_domain(obj);
  2502. old_write_domain = obj->write_domain;
  2503. old_read_domains = obj->read_domains;
  2504. /* Flush the CPU cache if it's still invalid. */
  2505. if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
  2506. i915_gem_clflush_object(obj);
  2507. obj->read_domains |= I915_GEM_DOMAIN_CPU;
  2508. }
  2509. /* It should now be out of any other write domains, and we can update
  2510. * the domain values for our changes.
  2511. */
  2512. BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
  2513. /* If we're writing through the CPU, then the GPU read domains will
  2514. * need to be invalidated at next use.
  2515. */
  2516. if (write) {
  2517. obj->read_domains &= I915_GEM_DOMAIN_CPU;
  2518. obj->write_domain = I915_GEM_DOMAIN_CPU;
  2519. }
  2520. trace_i915_gem_object_change_domain(obj,
  2521. old_read_domains,
  2522. old_write_domain);
  2523. return 0;
  2524. }
  2525. /*
  2526. * Set the next domain for the specified object. This
  2527. * may not actually perform the necessary flushing/invaliding though,
  2528. * as that may want to be batched with other set_domain operations
  2529. *
  2530. * This is (we hope) the only really tricky part of gem. The goal
  2531. * is fairly simple -- track which caches hold bits of the object
  2532. * and make sure they remain coherent. A few concrete examples may
  2533. * help to explain how it works. For shorthand, we use the notation
  2534. * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
  2535. * a pair of read and write domain masks.
  2536. *
  2537. * Case 1: the batch buffer
  2538. *
  2539. * 1. Allocated
  2540. * 2. Written by CPU
  2541. * 3. Mapped to GTT
  2542. * 4. Read by GPU
  2543. * 5. Unmapped from GTT
  2544. * 6. Freed
  2545. *
  2546. * Let's take these a step at a time
  2547. *
  2548. * 1. Allocated
  2549. * Pages allocated from the kernel may still have
  2550. * cache contents, so we set them to (CPU, CPU) always.
  2551. * 2. Written by CPU (using pwrite)
  2552. * The pwrite function calls set_domain (CPU, CPU) and
  2553. * this function does nothing (as nothing changes)
  2554. * 3. Mapped by GTT
  2555. * This function asserts that the object is not
  2556. * currently in any GPU-based read or write domains
  2557. * 4. Read by GPU
  2558. * i915_gem_execbuffer calls set_domain (COMMAND, 0).
  2559. * As write_domain is zero, this function adds in the
  2560. * current read domains (CPU+COMMAND, 0).
  2561. * flush_domains is set to CPU.
  2562. * invalidate_domains is set to COMMAND
  2563. * clflush is run to get data out of the CPU caches
  2564. * then i915_dev_set_domain calls i915_gem_flush to
  2565. * emit an MI_FLUSH and drm_agp_chipset_flush
  2566. * 5. Unmapped from GTT
  2567. * i915_gem_object_unbind calls set_domain (CPU, CPU)
  2568. * flush_domains and invalidate_domains end up both zero
  2569. * so no flushing/invalidating happens
  2570. * 6. Freed
  2571. * yay, done
  2572. *
  2573. * Case 2: The shared render buffer
  2574. *
  2575. * 1. Allocated
  2576. * 2. Mapped to GTT
  2577. * 3. Read/written by GPU
  2578. * 4. set_domain to (CPU,CPU)
  2579. * 5. Read/written by CPU
  2580. * 6. Read/written by GPU
  2581. *
  2582. * 1. Allocated
  2583. * Same as last example, (CPU, CPU)
  2584. * 2. Mapped to GTT
  2585. * Nothing changes (assertions find that it is not in the GPU)
  2586. * 3. Read/written by GPU
  2587. * execbuffer calls set_domain (RENDER, RENDER)
  2588. * flush_domains gets CPU
  2589. * invalidate_domains gets GPU
  2590. * clflush (obj)
  2591. * MI_FLUSH and drm_agp_chipset_flush
  2592. * 4. set_domain (CPU, CPU)
  2593. * flush_domains gets GPU
  2594. * invalidate_domains gets CPU
  2595. * wait_rendering (obj) to make sure all drawing is complete.
  2596. * This will include an MI_FLUSH to get the data from GPU
  2597. * to memory
  2598. * clflush (obj) to invalidate the CPU cache
  2599. * Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
  2600. * 5. Read/written by CPU
  2601. * cache lines are loaded and dirtied
  2602. * 6. Read written by GPU
  2603. * Same as last GPU access
  2604. *
  2605. * Case 3: The constant buffer
  2606. *
  2607. * 1. Allocated
  2608. * 2. Written by CPU
  2609. * 3. Read by GPU
  2610. * 4. Updated (written) by CPU again
  2611. * 5. Read by GPU
  2612. *
  2613. * 1. Allocated
  2614. * (CPU, CPU)
  2615. * 2. Written by CPU
  2616. * (CPU, CPU)
  2617. * 3. Read by GPU
  2618. * (CPU+RENDER, 0)
  2619. * flush_domains = CPU
  2620. * invalidate_domains = RENDER
  2621. * clflush (obj)
  2622. * MI_FLUSH
  2623. * drm_agp_chipset_flush
  2624. * 4. Updated (written) by CPU again
  2625. * (CPU, CPU)
  2626. * flush_domains = 0 (no previous write domain)
  2627. * invalidate_domains = 0 (no new read domains)
  2628. * 5. Read by GPU
  2629. * (CPU+RENDER, 0)
  2630. * flush_domains = CPU
  2631. * invalidate_domains = RENDER
  2632. * clflush (obj)
  2633. * MI_FLUSH
  2634. * drm_agp_chipset_flush
  2635. */
  2636. static void
  2637. i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj)
  2638. {
  2639. struct drm_device *dev = obj->dev;
  2640. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  2641. uint32_t invalidate_domains = 0;
  2642. uint32_t flush_domains = 0;
  2643. uint32_t old_read_domains;
  2644. BUG_ON(obj->pending_read_domains & I915_GEM_DOMAIN_CPU);
  2645. BUG_ON(obj->pending_write_domain == I915_GEM_DOMAIN_CPU);
  2646. intel_mark_busy(dev, obj);
  2647. #if WATCH_BUF
  2648. DRM_INFO("%s: object %p read %08x -> %08x write %08x -> %08x\n",
  2649. __func__, obj,
  2650. obj->read_domains, obj->pending_read_domains,
  2651. obj->write_domain, obj->pending_write_domain);
  2652. #endif
  2653. /*
  2654. * If the object isn't moving to a new write domain,
  2655. * let the object stay in multiple read domains
  2656. */
  2657. if (obj->pending_write_domain == 0)
  2658. obj->pending_read_domains |= obj->read_domains;
  2659. else
  2660. obj_priv->dirty = 1;
  2661. /*
  2662. * Flush the current write domain if
  2663. * the new read domains don't match. Invalidate
  2664. * any read domains which differ from the old
  2665. * write domain
  2666. */
  2667. if (obj->write_domain &&
  2668. obj->write_domain != obj->pending_read_domains) {
  2669. flush_domains |= obj->write_domain;
  2670. invalidate_domains |=
  2671. obj->pending_read_domains & ~obj->write_domain;
  2672. }
  2673. /*
  2674. * Invalidate any read caches which may have
  2675. * stale data. That is, any new read domains.
  2676. */
  2677. invalidate_domains |= obj->pending_read_domains & ~obj->read_domains;
  2678. if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU) {
  2679. #if WATCH_BUF
  2680. DRM_INFO("%s: CPU domain flush %08x invalidate %08x\n",
  2681. __func__, flush_domains, invalidate_domains);
  2682. #endif
  2683. i915_gem_clflush_object(obj);
  2684. }
  2685. old_read_domains = obj->read_domains;
  2686. /* The actual obj->write_domain will be updated with
  2687. * pending_write_domain after we emit the accumulated flush for all
  2688. * of our domain changes in execbuffers (which clears objects'
  2689. * write_domains). So if we have a current write domain that we
  2690. * aren't changing, set pending_write_domain to that.
  2691. */
  2692. if (flush_domains == 0 && obj->pending_write_domain == 0)
  2693. obj->pending_write_domain = obj->write_domain;
  2694. obj->read_domains = obj->pending_read_domains;
  2695. dev->invalidate_domains |= invalidate_domains;
  2696. dev->flush_domains |= flush_domains;
  2697. #if WATCH_BUF
  2698. DRM_INFO("%s: read %08x write %08x invalidate %08x flush %08x\n",
  2699. __func__,
  2700. obj->read_domains, obj->write_domain,
  2701. dev->invalidate_domains, dev->flush_domains);
  2702. #endif
  2703. trace_i915_gem_object_change_domain(obj,
  2704. old_read_domains,
  2705. obj->write_domain);
  2706. }
  2707. /**
  2708. * Moves the object from a partially CPU read to a full one.
  2709. *
  2710. * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
  2711. * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
  2712. */
  2713. static void
  2714. i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj)
  2715. {
  2716. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  2717. if (!obj_priv->page_cpu_valid)
  2718. return;
  2719. /* If we're partially in the CPU read domain, finish moving it in.
  2720. */
  2721. if (obj->read_domains & I915_GEM_DOMAIN_CPU) {
  2722. int i;
  2723. for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) {
  2724. if (obj_priv->page_cpu_valid[i])
  2725. continue;
  2726. drm_clflush_pages(obj_priv->pages + i, 1);
  2727. }
  2728. }
  2729. /* Free the page_cpu_valid mappings which are now stale, whether
  2730. * or not we've got I915_GEM_DOMAIN_CPU.
  2731. */
  2732. kfree(obj_priv->page_cpu_valid);
  2733. obj_priv->page_cpu_valid = NULL;
  2734. }
  2735. /**
  2736. * Set the CPU read domain on a range of the object.
  2737. *
  2738. * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
  2739. * not entirely valid. The page_cpu_valid member of the object flags which
  2740. * pages have been flushed, and will be respected by
  2741. * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
  2742. * of the whole object.
  2743. *
  2744. * This function returns when the move is complete, including waiting on
  2745. * flushes to occur.
  2746. */
  2747. static int
  2748. i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
  2749. uint64_t offset, uint64_t size)
  2750. {
  2751. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  2752. uint32_t old_read_domains;
  2753. int i, ret;
  2754. if (offset == 0 && size == obj->size)
  2755. return i915_gem_object_set_to_cpu_domain(obj, 0);
  2756. ret = i915_gem_object_flush_gpu_write_domain(obj);
  2757. if (ret)
  2758. return ret;
  2759. /* Wait on any GPU rendering and flushing to occur. */
  2760. ret = i915_gem_object_wait_rendering(obj);
  2761. if (ret != 0)
  2762. return ret;
  2763. i915_gem_object_flush_gtt_write_domain(obj);
  2764. /* If we're already fully in the CPU read domain, we're done. */
  2765. if (obj_priv->page_cpu_valid == NULL &&
  2766. (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0)
  2767. return 0;
  2768. /* Otherwise, create/clear the per-page CPU read domain flag if we're
  2769. * newly adding I915_GEM_DOMAIN_CPU
  2770. */
  2771. if (obj_priv->page_cpu_valid == NULL) {
  2772. obj_priv->page_cpu_valid = kzalloc(obj->size / PAGE_SIZE,
  2773. GFP_KERNEL);
  2774. if (obj_priv->page_cpu_valid == NULL)
  2775. return -ENOMEM;
  2776. } else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0)
  2777. memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE);
  2778. /* Flush the cache on any pages that are still invalid from the CPU's
  2779. * perspective.
  2780. */
  2781. for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
  2782. i++) {
  2783. if (obj_priv->page_cpu_valid[i])
  2784. continue;
  2785. drm_clflush_pages(obj_priv->pages + i, 1);
  2786. obj_priv->page_cpu_valid[i] = 1;
  2787. }
  2788. /* It should now be out of any other write domains, and we can update
  2789. * the domain values for our changes.
  2790. */
  2791. BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
  2792. old_read_domains = obj->read_domains;
  2793. obj->read_domains |= I915_GEM_DOMAIN_CPU;
  2794. trace_i915_gem_object_change_domain(obj,
  2795. old_read_domains,
  2796. obj->write_domain);
  2797. return 0;
  2798. }
  2799. /**
  2800. * Pin an object to the GTT and evaluate the relocations landing in it.
  2801. */
  2802. static int
  2803. i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
  2804. struct drm_file *file_priv,
  2805. struct drm_i915_gem_exec_object2 *entry,
  2806. struct drm_i915_gem_relocation_entry *relocs)
  2807. {
  2808. struct drm_device *dev = obj->dev;
  2809. drm_i915_private_t *dev_priv = dev->dev_private;
  2810. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  2811. int i, ret;
  2812. void __iomem *reloc_page;
  2813. bool need_fence;
  2814. need_fence = entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
  2815. obj_priv->tiling_mode != I915_TILING_NONE;
  2816. /* Check fence reg constraints and rebind if necessary */
  2817. if (need_fence &&
  2818. !i915_gem_object_fence_offset_ok(obj,
  2819. obj_priv->tiling_mode)) {
  2820. ret = i915_gem_object_unbind(obj);
  2821. if (ret)
  2822. return ret;
  2823. }
  2824. /* Choose the GTT offset for our buffer and put it there. */
  2825. ret = i915_gem_object_pin(obj, (uint32_t) entry->alignment);
  2826. if (ret)
  2827. return ret;
  2828. /*
  2829. * Pre-965 chips need a fence register set up in order to
  2830. * properly handle blits to/from tiled surfaces.
  2831. */
  2832. if (need_fence) {
  2833. ret = i915_gem_object_get_fence_reg(obj);
  2834. if (ret != 0) {
  2835. i915_gem_object_unpin(obj);
  2836. return ret;
  2837. }
  2838. }
  2839. entry->offset = obj_priv->gtt_offset;
  2840. /* Apply the relocations, using the GTT aperture to avoid cache
  2841. * flushing requirements.
  2842. */
  2843. for (i = 0; i < entry->relocation_count; i++) {
  2844. struct drm_i915_gem_relocation_entry *reloc= &relocs[i];
  2845. struct drm_gem_object *target_obj;
  2846. struct drm_i915_gem_object *target_obj_priv;
  2847. uint32_t reloc_val, reloc_offset;
  2848. uint32_t __iomem *reloc_entry;
  2849. target_obj = drm_gem_object_lookup(obj->dev, file_priv,
  2850. reloc->target_handle);
  2851. if (target_obj == NULL) {
  2852. i915_gem_object_unpin(obj);
  2853. return -ENOENT;
  2854. }
  2855. target_obj_priv = to_intel_bo(target_obj);
  2856. #if WATCH_RELOC
  2857. DRM_INFO("%s: obj %p offset %08x target %d "
  2858. "read %08x write %08x gtt %08x "
  2859. "presumed %08x delta %08x\n",
  2860. __func__,
  2861. obj,
  2862. (int) reloc->offset,
  2863. (int) reloc->target_handle,
  2864. (int) reloc->read_domains,
  2865. (int) reloc->write_domain,
  2866. (int) target_obj_priv->gtt_offset,
  2867. (int) reloc->presumed_offset,
  2868. reloc->delta);
  2869. #endif
  2870. /* The target buffer should have appeared before us in the
  2871. * exec_object list, so it should have a GTT space bound by now.
  2872. */
  2873. if (target_obj_priv->gtt_space == NULL) {
  2874. DRM_ERROR("No GTT space found for object %d\n",
  2875. reloc->target_handle);
  2876. drm_gem_object_unreference(target_obj);
  2877. i915_gem_object_unpin(obj);
  2878. return -EINVAL;
  2879. }
  2880. /* Validate that the target is in a valid r/w GPU domain */
  2881. if (reloc->write_domain & (reloc->write_domain - 1)) {
  2882. DRM_ERROR("reloc with multiple write domains: "
  2883. "obj %p target %d offset %d "
  2884. "read %08x write %08x",
  2885. obj, reloc->target_handle,
  2886. (int) reloc->offset,
  2887. reloc->read_domains,
  2888. reloc->write_domain);
  2889. return -EINVAL;
  2890. }
  2891. if (reloc->write_domain & I915_GEM_DOMAIN_CPU ||
  2892. reloc->read_domains & I915_GEM_DOMAIN_CPU) {
  2893. DRM_ERROR("reloc with read/write CPU domains: "
  2894. "obj %p target %d offset %d "
  2895. "read %08x write %08x",
  2896. obj, reloc->target_handle,
  2897. (int) reloc->offset,
  2898. reloc->read_domains,
  2899. reloc->write_domain);
  2900. drm_gem_object_unreference(target_obj);
  2901. i915_gem_object_unpin(obj);
  2902. return -EINVAL;
  2903. }
  2904. if (reloc->write_domain && target_obj->pending_write_domain &&
  2905. reloc->write_domain != target_obj->pending_write_domain) {
  2906. DRM_ERROR("Write domain conflict: "
  2907. "obj %p target %d offset %d "
  2908. "new %08x old %08x\n",
  2909. obj, reloc->target_handle,
  2910. (int) reloc->offset,
  2911. reloc->write_domain,
  2912. target_obj->pending_write_domain);
  2913. drm_gem_object_unreference(target_obj);
  2914. i915_gem_object_unpin(obj);
  2915. return -EINVAL;
  2916. }
  2917. target_obj->pending_read_domains |= reloc->read_domains;
  2918. target_obj->pending_write_domain |= reloc->write_domain;
  2919. /* If the relocation already has the right value in it, no
  2920. * more work needs to be done.
  2921. */
  2922. if (target_obj_priv->gtt_offset == reloc->presumed_offset) {
  2923. drm_gem_object_unreference(target_obj);
  2924. continue;
  2925. }
  2926. /* Check that the relocation address is valid... */
  2927. if (reloc->offset > obj->size - 4) {
  2928. DRM_ERROR("Relocation beyond object bounds: "
  2929. "obj %p target %d offset %d size %d.\n",
  2930. obj, reloc->target_handle,
  2931. (int) reloc->offset, (int) obj->size);
  2932. drm_gem_object_unreference(target_obj);
  2933. i915_gem_object_unpin(obj);
  2934. return -EINVAL;
  2935. }
  2936. if (reloc->offset & 3) {
  2937. DRM_ERROR("Relocation not 4-byte aligned: "
  2938. "obj %p target %d offset %d.\n",
  2939. obj, reloc->target_handle,
  2940. (int) reloc->offset);
  2941. drm_gem_object_unreference(target_obj);
  2942. i915_gem_object_unpin(obj);
  2943. return -EINVAL;
  2944. }
  2945. /* and points to somewhere within the target object. */
  2946. if (reloc->delta >= target_obj->size) {
  2947. DRM_ERROR("Relocation beyond target object bounds: "
  2948. "obj %p target %d delta %d size %d.\n",
  2949. obj, reloc->target_handle,
  2950. (int) reloc->delta, (int) target_obj->size);
  2951. drm_gem_object_unreference(target_obj);
  2952. i915_gem_object_unpin(obj);
  2953. return -EINVAL;
  2954. }
  2955. ret = i915_gem_object_set_to_gtt_domain(obj, 1);
  2956. if (ret != 0) {
  2957. drm_gem_object_unreference(target_obj);
  2958. i915_gem_object_unpin(obj);
  2959. return -EINVAL;
  2960. }
  2961. /* Map the page containing the relocation we're going to
  2962. * perform.
  2963. */
  2964. reloc_offset = obj_priv->gtt_offset + reloc->offset;
  2965. reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
  2966. (reloc_offset &
  2967. ~(PAGE_SIZE - 1)),
  2968. KM_USER0);
  2969. reloc_entry = (uint32_t __iomem *)(reloc_page +
  2970. (reloc_offset & (PAGE_SIZE - 1)));
  2971. reloc_val = target_obj_priv->gtt_offset + reloc->delta;
  2972. #if WATCH_BUF
  2973. DRM_INFO("Applied relocation: %p@0x%08x %08x -> %08x\n",
  2974. obj, (unsigned int) reloc->offset,
  2975. readl(reloc_entry), reloc_val);
  2976. #endif
  2977. writel(reloc_val, reloc_entry);
  2978. io_mapping_unmap_atomic(reloc_page, KM_USER0);
  2979. /* The updated presumed offset for this entry will be
  2980. * copied back out to the user.
  2981. */
  2982. reloc->presumed_offset = target_obj_priv->gtt_offset;
  2983. drm_gem_object_unreference(target_obj);
  2984. }
  2985. #if WATCH_BUF
  2986. if (0)
  2987. i915_gem_dump_object(obj, 128, __func__, ~0);
  2988. #endif
  2989. return 0;
  2990. }
  2991. /* Throttle our rendering by waiting until the ring has completed our requests
  2992. * emitted over 20 msec ago.
  2993. *
  2994. * Note that if we were to use the current jiffies each time around the loop,
  2995. * we wouldn't escape the function with any frames outstanding if the time to
  2996. * render a frame was over 20ms.
  2997. *
  2998. * This should get us reasonable parallelism between CPU and GPU but also
  2999. * relatively low latency when blocking on a particular request to finish.
  3000. */
  3001. static int
  3002. i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file_priv)
  3003. {
  3004. struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
  3005. int ret = 0;
  3006. unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
  3007. mutex_lock(&dev->struct_mutex);
  3008. while (!list_empty(&i915_file_priv->mm.request_list)) {
  3009. struct drm_i915_gem_request *request;
  3010. request = list_first_entry(&i915_file_priv->mm.request_list,
  3011. struct drm_i915_gem_request,
  3012. client_list);
  3013. if (time_after_eq(request->emitted_jiffies, recent_enough))
  3014. break;
  3015. ret = i915_wait_request(dev, request->seqno, request->ring);
  3016. if (ret != 0)
  3017. break;
  3018. }
  3019. mutex_unlock(&dev->struct_mutex);
  3020. return ret;
  3021. }
  3022. static int
  3023. i915_gem_get_relocs_from_user(struct drm_i915_gem_exec_object2 *exec_list,
  3024. uint32_t buffer_count,
  3025. struct drm_i915_gem_relocation_entry **relocs)
  3026. {
  3027. uint32_t reloc_count = 0, reloc_index = 0, i;
  3028. int ret;
  3029. *relocs = NULL;
  3030. for (i = 0; i < buffer_count; i++) {
  3031. if (reloc_count + exec_list[i].relocation_count < reloc_count)
  3032. return -EINVAL;
  3033. reloc_count += exec_list[i].relocation_count;
  3034. }
  3035. *relocs = drm_calloc_large(reloc_count, sizeof(**relocs));
  3036. if (*relocs == NULL) {
  3037. DRM_ERROR("failed to alloc relocs, count %d\n", reloc_count);
  3038. return -ENOMEM;
  3039. }
  3040. for (i = 0; i < buffer_count; i++) {
  3041. struct drm_i915_gem_relocation_entry __user *user_relocs;
  3042. user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
  3043. ret = copy_from_user(&(*relocs)[reloc_index],
  3044. user_relocs,
  3045. exec_list[i].relocation_count *
  3046. sizeof(**relocs));
  3047. if (ret != 0) {
  3048. drm_free_large(*relocs);
  3049. *relocs = NULL;
  3050. return -EFAULT;
  3051. }
  3052. reloc_index += exec_list[i].relocation_count;
  3053. }
  3054. return 0;
  3055. }
  3056. static int
  3057. i915_gem_put_relocs_to_user(struct drm_i915_gem_exec_object2 *exec_list,
  3058. uint32_t buffer_count,
  3059. struct drm_i915_gem_relocation_entry *relocs)
  3060. {
  3061. uint32_t reloc_count = 0, i;
  3062. int ret = 0;
  3063. if (relocs == NULL)
  3064. return 0;
  3065. for (i = 0; i < buffer_count; i++) {
  3066. struct drm_i915_gem_relocation_entry __user *user_relocs;
  3067. int unwritten;
  3068. user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
  3069. unwritten = copy_to_user(user_relocs,
  3070. &relocs[reloc_count],
  3071. exec_list[i].relocation_count *
  3072. sizeof(*relocs));
  3073. if (unwritten) {
  3074. ret = -EFAULT;
  3075. goto err;
  3076. }
  3077. reloc_count += exec_list[i].relocation_count;
  3078. }
  3079. err:
  3080. drm_free_large(relocs);
  3081. return ret;
  3082. }
  3083. static int
  3084. i915_gem_check_execbuffer (struct drm_i915_gem_execbuffer2 *exec,
  3085. uint64_t exec_offset)
  3086. {
  3087. uint32_t exec_start, exec_len;
  3088. exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
  3089. exec_len = (uint32_t) exec->batch_len;
  3090. if ((exec_start | exec_len) & 0x7)
  3091. return -EINVAL;
  3092. if (!exec_start)
  3093. return -EINVAL;
  3094. return 0;
  3095. }
  3096. static int
  3097. i915_gem_wait_for_pending_flip(struct drm_device *dev,
  3098. struct drm_gem_object **object_list,
  3099. int count)
  3100. {
  3101. drm_i915_private_t *dev_priv = dev->dev_private;
  3102. struct drm_i915_gem_object *obj_priv;
  3103. DEFINE_WAIT(wait);
  3104. int i, ret = 0;
  3105. for (;;) {
  3106. prepare_to_wait(&dev_priv->pending_flip_queue,
  3107. &wait, TASK_INTERRUPTIBLE);
  3108. for (i = 0; i < count; i++) {
  3109. obj_priv = to_intel_bo(object_list[i]);
  3110. if (atomic_read(&obj_priv->pending_flip) > 0)
  3111. break;
  3112. }
  3113. if (i == count)
  3114. break;
  3115. if (!signal_pending(current)) {
  3116. mutex_unlock(&dev->struct_mutex);
  3117. schedule();
  3118. mutex_lock(&dev->struct_mutex);
  3119. continue;
  3120. }
  3121. ret = -ERESTARTSYS;
  3122. break;
  3123. }
  3124. finish_wait(&dev_priv->pending_flip_queue, &wait);
  3125. return ret;
  3126. }
  3127. int
  3128. i915_gem_do_execbuffer(struct drm_device *dev, void *data,
  3129. struct drm_file *file_priv,
  3130. struct drm_i915_gem_execbuffer2 *args,
  3131. struct drm_i915_gem_exec_object2 *exec_list)
  3132. {
  3133. drm_i915_private_t *dev_priv = dev->dev_private;
  3134. struct drm_gem_object **object_list = NULL;
  3135. struct drm_gem_object *batch_obj;
  3136. struct drm_i915_gem_object *obj_priv;
  3137. struct drm_clip_rect *cliprects = NULL;
  3138. struct drm_i915_gem_relocation_entry *relocs = NULL;
  3139. int ret = 0, ret2, i, pinned = 0;
  3140. uint64_t exec_offset;
  3141. uint32_t seqno, flush_domains, reloc_index;
  3142. int pin_tries, flips;
  3143. struct intel_ring_buffer *ring = NULL;
  3144. #if WATCH_EXEC
  3145. DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
  3146. (int) args->buffers_ptr, args->buffer_count, args->batch_len);
  3147. #endif
  3148. if (args->flags & I915_EXEC_BSD) {
  3149. if (!HAS_BSD(dev)) {
  3150. DRM_ERROR("execbuf with wrong flag\n");
  3151. return -EINVAL;
  3152. }
  3153. ring = &dev_priv->bsd_ring;
  3154. } else {
  3155. ring = &dev_priv->render_ring;
  3156. }
  3157. if (args->buffer_count < 1) {
  3158. DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
  3159. return -EINVAL;
  3160. }
  3161. object_list = drm_malloc_ab(sizeof(*object_list), args->buffer_count);
  3162. if (object_list == NULL) {
  3163. DRM_ERROR("Failed to allocate object list for %d buffers\n",
  3164. args->buffer_count);
  3165. ret = -ENOMEM;
  3166. goto pre_mutex_err;
  3167. }
  3168. if (args->num_cliprects != 0) {
  3169. cliprects = kcalloc(args->num_cliprects, sizeof(*cliprects),
  3170. GFP_KERNEL);
  3171. if (cliprects == NULL) {
  3172. ret = -ENOMEM;
  3173. goto pre_mutex_err;
  3174. }
  3175. ret = copy_from_user(cliprects,
  3176. (struct drm_clip_rect __user *)
  3177. (uintptr_t) args->cliprects_ptr,
  3178. sizeof(*cliprects) * args->num_cliprects);
  3179. if (ret != 0) {
  3180. DRM_ERROR("copy %d cliprects failed: %d\n",
  3181. args->num_cliprects, ret);
  3182. goto pre_mutex_err;
  3183. }
  3184. }
  3185. ret = i915_gem_get_relocs_from_user(exec_list, args->buffer_count,
  3186. &relocs);
  3187. if (ret != 0)
  3188. goto pre_mutex_err;
  3189. mutex_lock(&dev->struct_mutex);
  3190. i915_verify_inactive(dev, __FILE__, __LINE__);
  3191. if (atomic_read(&dev_priv->mm.wedged)) {
  3192. mutex_unlock(&dev->struct_mutex);
  3193. ret = -EIO;
  3194. goto pre_mutex_err;
  3195. }
  3196. if (dev_priv->mm.suspended) {
  3197. mutex_unlock(&dev->struct_mutex);
  3198. ret = -EBUSY;
  3199. goto pre_mutex_err;
  3200. }
  3201. /* Look up object handles */
  3202. flips = 0;
  3203. for (i = 0; i < args->buffer_count; i++) {
  3204. object_list[i] = drm_gem_object_lookup(dev, file_priv,
  3205. exec_list[i].handle);
  3206. if (object_list[i] == NULL) {
  3207. DRM_ERROR("Invalid object handle %d at index %d\n",
  3208. exec_list[i].handle, i);
  3209. /* prevent error path from reading uninitialized data */
  3210. args->buffer_count = i + 1;
  3211. ret = -ENOENT;
  3212. goto err;
  3213. }
  3214. obj_priv = to_intel_bo(object_list[i]);
  3215. if (obj_priv->in_execbuffer) {
  3216. DRM_ERROR("Object %p appears more than once in object list\n",
  3217. object_list[i]);
  3218. /* prevent error path from reading uninitialized data */
  3219. args->buffer_count = i + 1;
  3220. ret = -EINVAL;
  3221. goto err;
  3222. }
  3223. obj_priv->in_execbuffer = true;
  3224. flips += atomic_read(&obj_priv->pending_flip);
  3225. }
  3226. if (flips > 0) {
  3227. ret = i915_gem_wait_for_pending_flip(dev, object_list,
  3228. args->buffer_count);
  3229. if (ret)
  3230. goto err;
  3231. }
  3232. /* Pin and relocate */
  3233. for (pin_tries = 0; ; pin_tries++) {
  3234. ret = 0;
  3235. reloc_index = 0;
  3236. for (i = 0; i < args->buffer_count; i++) {
  3237. object_list[i]->pending_read_domains = 0;
  3238. object_list[i]->pending_write_domain = 0;
  3239. ret = i915_gem_object_pin_and_relocate(object_list[i],
  3240. file_priv,
  3241. &exec_list[i],
  3242. &relocs[reloc_index]);
  3243. if (ret)
  3244. break;
  3245. pinned = i + 1;
  3246. reloc_index += exec_list[i].relocation_count;
  3247. }
  3248. /* success */
  3249. if (ret == 0)
  3250. break;
  3251. /* error other than GTT full, or we've already tried again */
  3252. if (ret != -ENOSPC || pin_tries >= 1) {
  3253. if (ret != -ERESTARTSYS) {
  3254. unsigned long long total_size = 0;
  3255. int num_fences = 0;
  3256. for (i = 0; i < args->buffer_count; i++) {
  3257. obj_priv = to_intel_bo(object_list[i]);
  3258. total_size += object_list[i]->size;
  3259. num_fences +=
  3260. exec_list[i].flags & EXEC_OBJECT_NEEDS_FENCE &&
  3261. obj_priv->tiling_mode != I915_TILING_NONE;
  3262. }
  3263. DRM_ERROR("Failed to pin buffer %d of %d, total %llu bytes, %d fences: %d\n",
  3264. pinned+1, args->buffer_count,
  3265. total_size, num_fences,
  3266. ret);
  3267. DRM_ERROR("%d objects [%d pinned], "
  3268. "%d object bytes [%d pinned], "
  3269. "%d/%d gtt bytes\n",
  3270. atomic_read(&dev->object_count),
  3271. atomic_read(&dev->pin_count),
  3272. atomic_read(&dev->object_memory),
  3273. atomic_read(&dev->pin_memory),
  3274. atomic_read(&dev->gtt_memory),
  3275. dev->gtt_total);
  3276. }
  3277. goto err;
  3278. }
  3279. /* unpin all of our buffers */
  3280. for (i = 0; i < pinned; i++)
  3281. i915_gem_object_unpin(object_list[i]);
  3282. pinned = 0;
  3283. /* evict everyone we can from the aperture */
  3284. ret = i915_gem_evict_everything(dev);
  3285. if (ret && ret != -ENOSPC)
  3286. goto err;
  3287. }
  3288. /* Set the pending read domains for the batch buffer to COMMAND */
  3289. batch_obj = object_list[args->buffer_count-1];
  3290. if (batch_obj->pending_write_domain) {
  3291. DRM_ERROR("Attempting to use self-modifying batch buffer\n");
  3292. ret = -EINVAL;
  3293. goto err;
  3294. }
  3295. batch_obj->pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
  3296. /* Sanity check the batch buffer, prior to moving objects */
  3297. exec_offset = exec_list[args->buffer_count - 1].offset;
  3298. ret = i915_gem_check_execbuffer (args, exec_offset);
  3299. if (ret != 0) {
  3300. DRM_ERROR("execbuf with invalid offset/length\n");
  3301. goto err;
  3302. }
  3303. i915_verify_inactive(dev, __FILE__, __LINE__);
  3304. /* Zero the global flush/invalidate flags. These
  3305. * will be modified as new domains are computed
  3306. * for each object
  3307. */
  3308. dev->invalidate_domains = 0;
  3309. dev->flush_domains = 0;
  3310. for (i = 0; i < args->buffer_count; i++) {
  3311. struct drm_gem_object *obj = object_list[i];
  3312. /* Compute new gpu domains and update invalidate/flush */
  3313. i915_gem_object_set_to_gpu_domain(obj);
  3314. }
  3315. i915_verify_inactive(dev, __FILE__, __LINE__);
  3316. if (dev->invalidate_domains | dev->flush_domains) {
  3317. #if WATCH_EXEC
  3318. DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
  3319. __func__,
  3320. dev->invalidate_domains,
  3321. dev->flush_domains);
  3322. #endif
  3323. i915_gem_flush(dev,
  3324. dev->invalidate_domains,
  3325. dev->flush_domains);
  3326. if (dev->flush_domains & I915_GEM_GPU_DOMAINS) {
  3327. (void)i915_add_request(dev, file_priv,
  3328. dev->flush_domains,
  3329. &dev_priv->render_ring);
  3330. if (HAS_BSD(dev))
  3331. (void)i915_add_request(dev, file_priv,
  3332. dev->flush_domains,
  3333. &dev_priv->bsd_ring);
  3334. }
  3335. }
  3336. for (i = 0; i < args->buffer_count; i++) {
  3337. struct drm_gem_object *obj = object_list[i];
  3338. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  3339. uint32_t old_write_domain = obj->write_domain;
  3340. obj->write_domain = obj->pending_write_domain;
  3341. if (obj->write_domain)
  3342. list_move_tail(&obj_priv->gpu_write_list,
  3343. &dev_priv->mm.gpu_write_list);
  3344. else
  3345. list_del_init(&obj_priv->gpu_write_list);
  3346. trace_i915_gem_object_change_domain(obj,
  3347. obj->read_domains,
  3348. old_write_domain);
  3349. }
  3350. i915_verify_inactive(dev, __FILE__, __LINE__);
  3351. #if WATCH_COHERENCY
  3352. for (i = 0; i < args->buffer_count; i++) {
  3353. i915_gem_object_check_coherency(object_list[i],
  3354. exec_list[i].handle);
  3355. }
  3356. #endif
  3357. #if WATCH_EXEC
  3358. i915_gem_dump_object(batch_obj,
  3359. args->batch_len,
  3360. __func__,
  3361. ~0);
  3362. #endif
  3363. /* Exec the batchbuffer */
  3364. ret = ring->dispatch_gem_execbuffer(dev, ring, args,
  3365. cliprects, exec_offset);
  3366. if (ret) {
  3367. DRM_ERROR("dispatch failed %d\n", ret);
  3368. goto err;
  3369. }
  3370. /*
  3371. * Ensure that the commands in the batch buffer are
  3372. * finished before the interrupt fires
  3373. */
  3374. flush_domains = i915_retire_commands(dev, ring);
  3375. i915_verify_inactive(dev, __FILE__, __LINE__);
  3376. /*
  3377. * Get a seqno representing the execution of the current buffer,
  3378. * which we can wait on. We would like to mitigate these interrupts,
  3379. * likely by only creating seqnos occasionally (so that we have
  3380. * *some* interrupts representing completion of buffers that we can
  3381. * wait on when trying to clear up gtt space).
  3382. */
  3383. seqno = i915_add_request(dev, file_priv, flush_domains, ring);
  3384. BUG_ON(seqno == 0);
  3385. for (i = 0; i < args->buffer_count; i++) {
  3386. struct drm_gem_object *obj = object_list[i];
  3387. obj_priv = to_intel_bo(obj);
  3388. i915_gem_object_move_to_active(obj, seqno, ring);
  3389. #if WATCH_LRU
  3390. DRM_INFO("%s: move to exec list %p\n", __func__, obj);
  3391. #endif
  3392. }
  3393. #if WATCH_LRU
  3394. i915_dump_lru(dev, __func__);
  3395. #endif
  3396. i915_verify_inactive(dev, __FILE__, __LINE__);
  3397. err:
  3398. for (i = 0; i < pinned; i++)
  3399. i915_gem_object_unpin(object_list[i]);
  3400. for (i = 0; i < args->buffer_count; i++) {
  3401. if (object_list[i]) {
  3402. obj_priv = to_intel_bo(object_list[i]);
  3403. obj_priv->in_execbuffer = false;
  3404. }
  3405. drm_gem_object_unreference(object_list[i]);
  3406. }
  3407. mutex_unlock(&dev->struct_mutex);
  3408. pre_mutex_err:
  3409. /* Copy the updated relocations out regardless of current error
  3410. * state. Failure to update the relocs would mean that the next
  3411. * time userland calls execbuf, it would do so with presumed offset
  3412. * state that didn't match the actual object state.
  3413. */
  3414. ret2 = i915_gem_put_relocs_to_user(exec_list, args->buffer_count,
  3415. relocs);
  3416. if (ret2 != 0) {
  3417. DRM_ERROR("Failed to copy relocations back out: %d\n", ret2);
  3418. if (ret == 0)
  3419. ret = ret2;
  3420. }
  3421. drm_free_large(object_list);
  3422. kfree(cliprects);
  3423. return ret;
  3424. }
  3425. /*
  3426. * Legacy execbuffer just creates an exec2 list from the original exec object
  3427. * list array and passes it to the real function.
  3428. */
  3429. int
  3430. i915_gem_execbuffer(struct drm_device *dev, void *data,
  3431. struct drm_file *file_priv)
  3432. {
  3433. struct drm_i915_gem_execbuffer *args = data;
  3434. struct drm_i915_gem_execbuffer2 exec2;
  3435. struct drm_i915_gem_exec_object *exec_list = NULL;
  3436. struct drm_i915_gem_exec_object2 *exec2_list = NULL;
  3437. int ret, i;
  3438. #if WATCH_EXEC
  3439. DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
  3440. (int) args->buffers_ptr, args->buffer_count, args->batch_len);
  3441. #endif
  3442. if (args->buffer_count < 1) {
  3443. DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
  3444. return -EINVAL;
  3445. }
  3446. /* Copy in the exec list from userland */
  3447. exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
  3448. exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
  3449. if (exec_list == NULL || exec2_list == NULL) {
  3450. DRM_ERROR("Failed to allocate exec list for %d buffers\n",
  3451. args->buffer_count);
  3452. drm_free_large(exec_list);
  3453. drm_free_large(exec2_list);
  3454. return -ENOMEM;
  3455. }
  3456. ret = copy_from_user(exec_list,
  3457. (struct drm_i915_relocation_entry __user *)
  3458. (uintptr_t) args->buffers_ptr,
  3459. sizeof(*exec_list) * args->buffer_count);
  3460. if (ret != 0) {
  3461. DRM_ERROR("copy %d exec entries failed %d\n",
  3462. args->buffer_count, ret);
  3463. drm_free_large(exec_list);
  3464. drm_free_large(exec2_list);
  3465. return -EFAULT;
  3466. }
  3467. for (i = 0; i < args->buffer_count; i++) {
  3468. exec2_list[i].handle = exec_list[i].handle;
  3469. exec2_list[i].relocation_count = exec_list[i].relocation_count;
  3470. exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
  3471. exec2_list[i].alignment = exec_list[i].alignment;
  3472. exec2_list[i].offset = exec_list[i].offset;
  3473. if (!IS_I965G(dev))
  3474. exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
  3475. else
  3476. exec2_list[i].flags = 0;
  3477. }
  3478. exec2.buffers_ptr = args->buffers_ptr;
  3479. exec2.buffer_count = args->buffer_count;
  3480. exec2.batch_start_offset = args->batch_start_offset;
  3481. exec2.batch_len = args->batch_len;
  3482. exec2.DR1 = args->DR1;
  3483. exec2.DR4 = args->DR4;
  3484. exec2.num_cliprects = args->num_cliprects;
  3485. exec2.cliprects_ptr = args->cliprects_ptr;
  3486. exec2.flags = I915_EXEC_RENDER;
  3487. ret = i915_gem_do_execbuffer(dev, data, file_priv, &exec2, exec2_list);
  3488. if (!ret) {
  3489. /* Copy the new buffer offsets back to the user's exec list. */
  3490. for (i = 0; i < args->buffer_count; i++)
  3491. exec_list[i].offset = exec2_list[i].offset;
  3492. /* ... and back out to userspace */
  3493. ret = copy_to_user((struct drm_i915_relocation_entry __user *)
  3494. (uintptr_t) args->buffers_ptr,
  3495. exec_list,
  3496. sizeof(*exec_list) * args->buffer_count);
  3497. if (ret) {
  3498. ret = -EFAULT;
  3499. DRM_ERROR("failed to copy %d exec entries "
  3500. "back to user (%d)\n",
  3501. args->buffer_count, ret);
  3502. }
  3503. }
  3504. drm_free_large(exec_list);
  3505. drm_free_large(exec2_list);
  3506. return ret;
  3507. }
  3508. int
  3509. i915_gem_execbuffer2(struct drm_device *dev, void *data,
  3510. struct drm_file *file_priv)
  3511. {
  3512. struct drm_i915_gem_execbuffer2 *args = data;
  3513. struct drm_i915_gem_exec_object2 *exec2_list = NULL;
  3514. int ret;
  3515. #if WATCH_EXEC
  3516. DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
  3517. (int) args->buffers_ptr, args->buffer_count, args->batch_len);
  3518. #endif
  3519. if (args->buffer_count < 1) {
  3520. DRM_ERROR("execbuf2 with %d buffers\n", args->buffer_count);
  3521. return -EINVAL;
  3522. }
  3523. exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
  3524. if (exec2_list == NULL) {
  3525. DRM_ERROR("Failed to allocate exec list for %d buffers\n",
  3526. args->buffer_count);
  3527. return -ENOMEM;
  3528. }
  3529. ret = copy_from_user(exec2_list,
  3530. (struct drm_i915_relocation_entry __user *)
  3531. (uintptr_t) args->buffers_ptr,
  3532. sizeof(*exec2_list) * args->buffer_count);
  3533. if (ret != 0) {
  3534. DRM_ERROR("copy %d exec entries failed %d\n",
  3535. args->buffer_count, ret);
  3536. drm_free_large(exec2_list);
  3537. return -EFAULT;
  3538. }
  3539. ret = i915_gem_do_execbuffer(dev, data, file_priv, args, exec2_list);
  3540. if (!ret) {
  3541. /* Copy the new buffer offsets back to the user's exec list. */
  3542. ret = copy_to_user((struct drm_i915_relocation_entry __user *)
  3543. (uintptr_t) args->buffers_ptr,
  3544. exec2_list,
  3545. sizeof(*exec2_list) * args->buffer_count);
  3546. if (ret) {
  3547. ret = -EFAULT;
  3548. DRM_ERROR("failed to copy %d exec entries "
  3549. "back to user (%d)\n",
  3550. args->buffer_count, ret);
  3551. }
  3552. }
  3553. drm_free_large(exec2_list);
  3554. return ret;
  3555. }
  3556. int
  3557. i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment)
  3558. {
  3559. struct drm_device *dev = obj->dev;
  3560. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  3561. int ret;
  3562. BUG_ON(obj_priv->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
  3563. i915_verify_inactive(dev, __FILE__, __LINE__);
  3564. if (obj_priv->gtt_space != NULL) {
  3565. if (alignment == 0)
  3566. alignment = i915_gem_get_gtt_alignment(obj);
  3567. if (obj_priv->gtt_offset & (alignment - 1)) {
  3568. ret = i915_gem_object_unbind(obj);
  3569. if (ret)
  3570. return ret;
  3571. }
  3572. }
  3573. if (obj_priv->gtt_space == NULL) {
  3574. ret = i915_gem_object_bind_to_gtt(obj, alignment);
  3575. if (ret)
  3576. return ret;
  3577. }
  3578. obj_priv->pin_count++;
  3579. /* If the object is not active and not pending a flush,
  3580. * remove it from the inactive list
  3581. */
  3582. if (obj_priv->pin_count == 1) {
  3583. atomic_inc(&dev->pin_count);
  3584. atomic_add(obj->size, &dev->pin_memory);
  3585. if (!obj_priv->active &&
  3586. (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0 &&
  3587. !list_empty(&obj_priv->list))
  3588. list_del_init(&obj_priv->list);
  3589. }
  3590. i915_verify_inactive(dev, __FILE__, __LINE__);
  3591. return 0;
  3592. }
  3593. void
  3594. i915_gem_object_unpin(struct drm_gem_object *obj)
  3595. {
  3596. struct drm_device *dev = obj->dev;
  3597. drm_i915_private_t *dev_priv = dev->dev_private;
  3598. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  3599. i915_verify_inactive(dev, __FILE__, __LINE__);
  3600. obj_priv->pin_count--;
  3601. BUG_ON(obj_priv->pin_count < 0);
  3602. BUG_ON(obj_priv->gtt_space == NULL);
  3603. /* If the object is no longer pinned, and is
  3604. * neither active nor being flushed, then stick it on
  3605. * the inactive list
  3606. */
  3607. if (obj_priv->pin_count == 0) {
  3608. if (!obj_priv->active &&
  3609. (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
  3610. list_move_tail(&obj_priv->list,
  3611. &dev_priv->mm.inactive_list);
  3612. atomic_dec(&dev->pin_count);
  3613. atomic_sub(obj->size, &dev->pin_memory);
  3614. }
  3615. i915_verify_inactive(dev, __FILE__, __LINE__);
  3616. }
  3617. int
  3618. i915_gem_pin_ioctl(struct drm_device *dev, void *data,
  3619. struct drm_file *file_priv)
  3620. {
  3621. struct drm_i915_gem_pin *args = data;
  3622. struct drm_gem_object *obj;
  3623. struct drm_i915_gem_object *obj_priv;
  3624. int ret;
  3625. mutex_lock(&dev->struct_mutex);
  3626. obj = drm_gem_object_lookup(dev, file_priv, args->handle);
  3627. if (obj == NULL) {
  3628. DRM_ERROR("Bad handle in i915_gem_pin_ioctl(): %d\n",
  3629. args->handle);
  3630. mutex_unlock(&dev->struct_mutex);
  3631. return -ENOENT;
  3632. }
  3633. obj_priv = to_intel_bo(obj);
  3634. if (obj_priv->madv != I915_MADV_WILLNEED) {
  3635. DRM_ERROR("Attempting to pin a purgeable buffer\n");
  3636. drm_gem_object_unreference(obj);
  3637. mutex_unlock(&dev->struct_mutex);
  3638. return -EINVAL;
  3639. }
  3640. if (obj_priv->pin_filp != NULL && obj_priv->pin_filp != file_priv) {
  3641. DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
  3642. args->handle);
  3643. drm_gem_object_unreference(obj);
  3644. mutex_unlock(&dev->struct_mutex);
  3645. return -EINVAL;
  3646. }
  3647. obj_priv->user_pin_count++;
  3648. obj_priv->pin_filp = file_priv;
  3649. if (obj_priv->user_pin_count == 1) {
  3650. ret = i915_gem_object_pin(obj, args->alignment);
  3651. if (ret != 0) {
  3652. drm_gem_object_unreference(obj);
  3653. mutex_unlock(&dev->struct_mutex);
  3654. return ret;
  3655. }
  3656. }
  3657. /* XXX - flush the CPU caches for pinned objects
  3658. * as the X server doesn't manage domains yet
  3659. */
  3660. i915_gem_object_flush_cpu_write_domain(obj);
  3661. args->offset = obj_priv->gtt_offset;
  3662. drm_gem_object_unreference(obj);
  3663. mutex_unlock(&dev->struct_mutex);
  3664. return 0;
  3665. }
  3666. int
  3667. i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
  3668. struct drm_file *file_priv)
  3669. {
  3670. struct drm_i915_gem_pin *args = data;
  3671. struct drm_gem_object *obj;
  3672. struct drm_i915_gem_object *obj_priv;
  3673. mutex_lock(&dev->struct_mutex);
  3674. obj = drm_gem_object_lookup(dev, file_priv, args->handle);
  3675. if (obj == NULL) {
  3676. DRM_ERROR("Bad handle in i915_gem_unpin_ioctl(): %d\n",
  3677. args->handle);
  3678. mutex_unlock(&dev->struct_mutex);
  3679. return -ENOENT;
  3680. }
  3681. obj_priv = to_intel_bo(obj);
  3682. if (obj_priv->pin_filp != file_priv) {
  3683. DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
  3684. args->handle);
  3685. drm_gem_object_unreference(obj);
  3686. mutex_unlock(&dev->struct_mutex);
  3687. return -EINVAL;
  3688. }
  3689. obj_priv->user_pin_count--;
  3690. if (obj_priv->user_pin_count == 0) {
  3691. obj_priv->pin_filp = NULL;
  3692. i915_gem_object_unpin(obj);
  3693. }
  3694. drm_gem_object_unreference(obj);
  3695. mutex_unlock(&dev->struct_mutex);
  3696. return 0;
  3697. }
  3698. int
  3699. i915_gem_busy_ioctl(struct drm_device *dev, void *data,
  3700. struct drm_file *file_priv)
  3701. {
  3702. struct drm_i915_gem_busy *args = data;
  3703. struct drm_gem_object *obj;
  3704. struct drm_i915_gem_object *obj_priv;
  3705. obj = drm_gem_object_lookup(dev, file_priv, args->handle);
  3706. if (obj == NULL) {
  3707. DRM_ERROR("Bad handle in i915_gem_busy_ioctl(): %d\n",
  3708. args->handle);
  3709. return -ENOENT;
  3710. }
  3711. mutex_lock(&dev->struct_mutex);
  3712. /* Update the active list for the hardware's current position.
  3713. * Otherwise this only updates on a delayed timer or when irqs are
  3714. * actually unmasked, and our working set ends up being larger than
  3715. * required.
  3716. */
  3717. i915_gem_retire_requests(dev);
  3718. obj_priv = to_intel_bo(obj);
  3719. /* Don't count being on the flushing list against the object being
  3720. * done. Otherwise, a buffer left on the flushing list but not getting
  3721. * flushed (because nobody's flushing that domain) won't ever return
  3722. * unbusy and get reused by libdrm's bo cache. The other expected
  3723. * consumer of this interface, OpenGL's occlusion queries, also specs
  3724. * that the objects get unbusy "eventually" without any interference.
  3725. */
  3726. args->busy = obj_priv->active && obj_priv->last_rendering_seqno != 0;
  3727. drm_gem_object_unreference(obj);
  3728. mutex_unlock(&dev->struct_mutex);
  3729. return 0;
  3730. }
  3731. int
  3732. i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
  3733. struct drm_file *file_priv)
  3734. {
  3735. return i915_gem_ring_throttle(dev, file_priv);
  3736. }
  3737. int
  3738. i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
  3739. struct drm_file *file_priv)
  3740. {
  3741. struct drm_i915_gem_madvise *args = data;
  3742. struct drm_gem_object *obj;
  3743. struct drm_i915_gem_object *obj_priv;
  3744. switch (args->madv) {
  3745. case I915_MADV_DONTNEED:
  3746. case I915_MADV_WILLNEED:
  3747. break;
  3748. default:
  3749. return -EINVAL;
  3750. }
  3751. obj = drm_gem_object_lookup(dev, file_priv, args->handle);
  3752. if (obj == NULL) {
  3753. DRM_ERROR("Bad handle in i915_gem_madvise_ioctl(): %d\n",
  3754. args->handle);
  3755. return -ENOENT;
  3756. }
  3757. mutex_lock(&dev->struct_mutex);
  3758. obj_priv = to_intel_bo(obj);
  3759. if (obj_priv->pin_count) {
  3760. drm_gem_object_unreference(obj);
  3761. mutex_unlock(&dev->struct_mutex);
  3762. DRM_ERROR("Attempted i915_gem_madvise_ioctl() on a pinned object\n");
  3763. return -EINVAL;
  3764. }
  3765. if (obj_priv->madv != __I915_MADV_PURGED)
  3766. obj_priv->madv = args->madv;
  3767. /* if the object is no longer bound, discard its backing storage */
  3768. if (i915_gem_object_is_purgeable(obj_priv) &&
  3769. obj_priv->gtt_space == NULL)
  3770. i915_gem_object_truncate(obj);
  3771. args->retained = obj_priv->madv != __I915_MADV_PURGED;
  3772. drm_gem_object_unreference(obj);
  3773. mutex_unlock(&dev->struct_mutex);
  3774. return 0;
  3775. }
  3776. struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev,
  3777. size_t size)
  3778. {
  3779. struct drm_i915_gem_object *obj;
  3780. obj = kzalloc(sizeof(*obj), GFP_KERNEL);
  3781. if (obj == NULL)
  3782. return NULL;
  3783. if (drm_gem_object_init(dev, &obj->base, size) != 0) {
  3784. kfree(obj);
  3785. return NULL;
  3786. }
  3787. obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  3788. obj->base.read_domains = I915_GEM_DOMAIN_CPU;
  3789. obj->agp_type = AGP_USER_MEMORY;
  3790. obj->base.driver_private = NULL;
  3791. obj->fence_reg = I915_FENCE_REG_NONE;
  3792. INIT_LIST_HEAD(&obj->list);
  3793. INIT_LIST_HEAD(&obj->gpu_write_list);
  3794. obj->madv = I915_MADV_WILLNEED;
  3795. trace_i915_gem_object_create(&obj->base);
  3796. return &obj->base;
  3797. }
  3798. int i915_gem_init_object(struct drm_gem_object *obj)
  3799. {
  3800. BUG();
  3801. return 0;
  3802. }
  3803. static void i915_gem_free_object_tail(struct drm_gem_object *obj)
  3804. {
  3805. struct drm_device *dev = obj->dev;
  3806. drm_i915_private_t *dev_priv = dev->dev_private;
  3807. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  3808. int ret;
  3809. ret = i915_gem_object_unbind(obj);
  3810. if (ret == -ERESTARTSYS) {
  3811. list_move(&obj_priv->list,
  3812. &dev_priv->mm.deferred_free_list);
  3813. return;
  3814. }
  3815. if (obj_priv->mmap_offset)
  3816. i915_gem_free_mmap_offset(obj);
  3817. drm_gem_object_release(obj);
  3818. kfree(obj_priv->page_cpu_valid);
  3819. kfree(obj_priv->bit_17);
  3820. kfree(obj_priv);
  3821. }
  3822. void i915_gem_free_object(struct drm_gem_object *obj)
  3823. {
  3824. struct drm_device *dev = obj->dev;
  3825. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  3826. trace_i915_gem_object_destroy(obj);
  3827. while (obj_priv->pin_count > 0)
  3828. i915_gem_object_unpin(obj);
  3829. if (obj_priv->phys_obj)
  3830. i915_gem_detach_phys_object(dev, obj);
  3831. i915_gem_free_object_tail(obj);
  3832. }
  3833. /** Unbinds all inactive objects. */
  3834. static int
  3835. i915_gem_evict_from_inactive_list(struct drm_device *dev)
  3836. {
  3837. drm_i915_private_t *dev_priv = dev->dev_private;
  3838. while (!list_empty(&dev_priv->mm.inactive_list)) {
  3839. struct drm_gem_object *obj;
  3840. int ret;
  3841. obj = &list_first_entry(&dev_priv->mm.inactive_list,
  3842. struct drm_i915_gem_object,
  3843. list)->base;
  3844. ret = i915_gem_object_unbind(obj);
  3845. if (ret != 0) {
  3846. DRM_ERROR("Error unbinding object: %d\n", ret);
  3847. return ret;
  3848. }
  3849. }
  3850. return 0;
  3851. }
  3852. int
  3853. i915_gem_idle(struct drm_device *dev)
  3854. {
  3855. drm_i915_private_t *dev_priv = dev->dev_private;
  3856. int ret;
  3857. mutex_lock(&dev->struct_mutex);
  3858. if (dev_priv->mm.suspended ||
  3859. (dev_priv->render_ring.gem_object == NULL) ||
  3860. (HAS_BSD(dev) &&
  3861. dev_priv->bsd_ring.gem_object == NULL)) {
  3862. mutex_unlock(&dev->struct_mutex);
  3863. return 0;
  3864. }
  3865. ret = i915_gpu_idle(dev);
  3866. if (ret) {
  3867. mutex_unlock(&dev->struct_mutex);
  3868. return ret;
  3869. }
  3870. /* Under UMS, be paranoid and evict. */
  3871. if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
  3872. ret = i915_gem_evict_from_inactive_list(dev);
  3873. if (ret) {
  3874. mutex_unlock(&dev->struct_mutex);
  3875. return ret;
  3876. }
  3877. }
  3878. /* Hack! Don't let anybody do execbuf while we don't control the chip.
  3879. * We need to replace this with a semaphore, or something.
  3880. * And not confound mm.suspended!
  3881. */
  3882. dev_priv->mm.suspended = 1;
  3883. del_timer(&dev_priv->hangcheck_timer);
  3884. i915_kernel_lost_context(dev);
  3885. i915_gem_cleanup_ringbuffer(dev);
  3886. mutex_unlock(&dev->struct_mutex);
  3887. /* Cancel the retire work handler, which should be idle now. */
  3888. cancel_delayed_work_sync(&dev_priv->mm.retire_work);
  3889. return 0;
  3890. }
  3891. /*
  3892. * 965+ support PIPE_CONTROL commands, which provide finer grained control
  3893. * over cache flushing.
  3894. */
  3895. static int
  3896. i915_gem_init_pipe_control(struct drm_device *dev)
  3897. {
  3898. drm_i915_private_t *dev_priv = dev->dev_private;
  3899. struct drm_gem_object *obj;
  3900. struct drm_i915_gem_object *obj_priv;
  3901. int ret;
  3902. obj = i915_gem_alloc_object(dev, 4096);
  3903. if (obj == NULL) {
  3904. DRM_ERROR("Failed to allocate seqno page\n");
  3905. ret = -ENOMEM;
  3906. goto err;
  3907. }
  3908. obj_priv = to_intel_bo(obj);
  3909. obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
  3910. ret = i915_gem_object_pin(obj, 4096);
  3911. if (ret)
  3912. goto err_unref;
  3913. dev_priv->seqno_gfx_addr = obj_priv->gtt_offset;
  3914. dev_priv->seqno_page = kmap(obj_priv->pages[0]);
  3915. if (dev_priv->seqno_page == NULL)
  3916. goto err_unpin;
  3917. dev_priv->seqno_obj = obj;
  3918. memset(dev_priv->seqno_page, 0, PAGE_SIZE);
  3919. return 0;
  3920. err_unpin:
  3921. i915_gem_object_unpin(obj);
  3922. err_unref:
  3923. drm_gem_object_unreference(obj);
  3924. err:
  3925. return ret;
  3926. }
  3927. static void
  3928. i915_gem_cleanup_pipe_control(struct drm_device *dev)
  3929. {
  3930. drm_i915_private_t *dev_priv = dev->dev_private;
  3931. struct drm_gem_object *obj;
  3932. struct drm_i915_gem_object *obj_priv;
  3933. obj = dev_priv->seqno_obj;
  3934. obj_priv = to_intel_bo(obj);
  3935. kunmap(obj_priv->pages[0]);
  3936. i915_gem_object_unpin(obj);
  3937. drm_gem_object_unreference(obj);
  3938. dev_priv->seqno_obj = NULL;
  3939. dev_priv->seqno_page = NULL;
  3940. }
  3941. int
  3942. i915_gem_init_ringbuffer(struct drm_device *dev)
  3943. {
  3944. drm_i915_private_t *dev_priv = dev->dev_private;
  3945. int ret;
  3946. dev_priv->render_ring = render_ring;
  3947. if (!I915_NEED_GFX_HWS(dev)) {
  3948. dev_priv->render_ring.status_page.page_addr
  3949. = dev_priv->status_page_dmah->vaddr;
  3950. memset(dev_priv->render_ring.status_page.page_addr,
  3951. 0, PAGE_SIZE);
  3952. }
  3953. if (HAS_PIPE_CONTROL(dev)) {
  3954. ret = i915_gem_init_pipe_control(dev);
  3955. if (ret)
  3956. return ret;
  3957. }
  3958. ret = intel_init_ring_buffer(dev, &dev_priv->render_ring);
  3959. if (ret)
  3960. goto cleanup_pipe_control;
  3961. if (HAS_BSD(dev)) {
  3962. dev_priv->bsd_ring = bsd_ring;
  3963. ret = intel_init_ring_buffer(dev, &dev_priv->bsd_ring);
  3964. if (ret)
  3965. goto cleanup_render_ring;
  3966. }
  3967. return 0;
  3968. cleanup_render_ring:
  3969. intel_cleanup_ring_buffer(dev, &dev_priv->render_ring);
  3970. cleanup_pipe_control:
  3971. if (HAS_PIPE_CONTROL(dev))
  3972. i915_gem_cleanup_pipe_control(dev);
  3973. return ret;
  3974. }
  3975. void
  3976. i915_gem_cleanup_ringbuffer(struct drm_device *dev)
  3977. {
  3978. drm_i915_private_t *dev_priv = dev->dev_private;
  3979. intel_cleanup_ring_buffer(dev, &dev_priv->render_ring);
  3980. if (HAS_BSD(dev))
  3981. intel_cleanup_ring_buffer(dev, &dev_priv->bsd_ring);
  3982. if (HAS_PIPE_CONTROL(dev))
  3983. i915_gem_cleanup_pipe_control(dev);
  3984. }
  3985. int
  3986. i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
  3987. struct drm_file *file_priv)
  3988. {
  3989. drm_i915_private_t *dev_priv = dev->dev_private;
  3990. int ret;
  3991. if (drm_core_check_feature(dev, DRIVER_MODESET))
  3992. return 0;
  3993. if (atomic_read(&dev_priv->mm.wedged)) {
  3994. DRM_ERROR("Reenabling wedged hardware, good luck\n");
  3995. atomic_set(&dev_priv->mm.wedged, 0);
  3996. }
  3997. mutex_lock(&dev->struct_mutex);
  3998. dev_priv->mm.suspended = 0;
  3999. ret = i915_gem_init_ringbuffer(dev);
  4000. if (ret != 0) {
  4001. mutex_unlock(&dev->struct_mutex);
  4002. return ret;
  4003. }
  4004. spin_lock(&dev_priv->mm.active_list_lock);
  4005. BUG_ON(!list_empty(&dev_priv->render_ring.active_list));
  4006. BUG_ON(HAS_BSD(dev) && !list_empty(&dev_priv->bsd_ring.active_list));
  4007. spin_unlock(&dev_priv->mm.active_list_lock);
  4008. BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
  4009. BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
  4010. BUG_ON(!list_empty(&dev_priv->render_ring.request_list));
  4011. BUG_ON(HAS_BSD(dev) && !list_empty(&dev_priv->bsd_ring.request_list));
  4012. mutex_unlock(&dev->struct_mutex);
  4013. ret = drm_irq_install(dev);
  4014. if (ret)
  4015. goto cleanup_ringbuffer;
  4016. return 0;
  4017. cleanup_ringbuffer:
  4018. mutex_lock(&dev->struct_mutex);
  4019. i915_gem_cleanup_ringbuffer(dev);
  4020. dev_priv->mm.suspended = 1;
  4021. mutex_unlock(&dev->struct_mutex);
  4022. return ret;
  4023. }
  4024. int
  4025. i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
  4026. struct drm_file *file_priv)
  4027. {
  4028. if (drm_core_check_feature(dev, DRIVER_MODESET))
  4029. return 0;
  4030. drm_irq_uninstall(dev);
  4031. return i915_gem_idle(dev);
  4032. }
  4033. void
  4034. i915_gem_lastclose(struct drm_device *dev)
  4035. {
  4036. int ret;
  4037. if (drm_core_check_feature(dev, DRIVER_MODESET))
  4038. return;
  4039. ret = i915_gem_idle(dev);
  4040. if (ret)
  4041. DRM_ERROR("failed to idle hardware: %d\n", ret);
  4042. }
  4043. void
  4044. i915_gem_load(struct drm_device *dev)
  4045. {
  4046. int i;
  4047. drm_i915_private_t *dev_priv = dev->dev_private;
  4048. spin_lock_init(&dev_priv->mm.active_list_lock);
  4049. INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
  4050. INIT_LIST_HEAD(&dev_priv->mm.gpu_write_list);
  4051. INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
  4052. INIT_LIST_HEAD(&dev_priv->mm.fence_list);
  4053. INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
  4054. INIT_LIST_HEAD(&dev_priv->render_ring.active_list);
  4055. INIT_LIST_HEAD(&dev_priv->render_ring.request_list);
  4056. if (HAS_BSD(dev)) {
  4057. INIT_LIST_HEAD(&dev_priv->bsd_ring.active_list);
  4058. INIT_LIST_HEAD(&dev_priv->bsd_ring.request_list);
  4059. }
  4060. for (i = 0; i < 16; i++)
  4061. INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
  4062. INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
  4063. i915_gem_retire_work_handler);
  4064. spin_lock(&shrink_list_lock);
  4065. list_add(&dev_priv->mm.shrink_list, &shrink_list);
  4066. spin_unlock(&shrink_list_lock);
  4067. /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
  4068. if (IS_GEN3(dev)) {
  4069. u32 tmp = I915_READ(MI_ARB_STATE);
  4070. if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) {
  4071. /* arb state is a masked write, so set bit + bit in mask */
  4072. tmp = MI_ARB_C3_LP_WRITE_ENABLE | (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT);
  4073. I915_WRITE(MI_ARB_STATE, tmp);
  4074. }
  4075. }
  4076. /* Old X drivers will take 0-2 for front, back, depth buffers */
  4077. if (!drm_core_check_feature(dev, DRIVER_MODESET))
  4078. dev_priv->fence_reg_start = 3;
  4079. if (IS_I965G(dev) || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
  4080. dev_priv->num_fence_regs = 16;
  4081. else
  4082. dev_priv->num_fence_regs = 8;
  4083. /* Initialize fence registers to zero */
  4084. if (IS_I965G(dev)) {
  4085. for (i = 0; i < 16; i++)
  4086. I915_WRITE64(FENCE_REG_965_0 + (i * 8), 0);
  4087. } else {
  4088. for (i = 0; i < 8; i++)
  4089. I915_WRITE(FENCE_REG_830_0 + (i * 4), 0);
  4090. if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
  4091. for (i = 0; i < 8; i++)
  4092. I915_WRITE(FENCE_REG_945_8 + (i * 4), 0);
  4093. }
  4094. i915_gem_detect_bit_6_swizzle(dev);
  4095. init_waitqueue_head(&dev_priv->pending_flip_queue);
  4096. }
  4097. /*
  4098. * Create a physically contiguous memory object for this object
  4099. * e.g. for cursor + overlay regs
  4100. */
  4101. int i915_gem_init_phys_object(struct drm_device *dev,
  4102. int id, int size)
  4103. {
  4104. drm_i915_private_t *dev_priv = dev->dev_private;
  4105. struct drm_i915_gem_phys_object *phys_obj;
  4106. int ret;
  4107. if (dev_priv->mm.phys_objs[id - 1] || !size)
  4108. return 0;
  4109. phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
  4110. if (!phys_obj)
  4111. return -ENOMEM;
  4112. phys_obj->id = id;
  4113. phys_obj->handle = drm_pci_alloc(dev, size, 0);
  4114. if (!phys_obj->handle) {
  4115. ret = -ENOMEM;
  4116. goto kfree_obj;
  4117. }
  4118. #ifdef CONFIG_X86
  4119. set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
  4120. #endif
  4121. dev_priv->mm.phys_objs[id - 1] = phys_obj;
  4122. return 0;
  4123. kfree_obj:
  4124. kfree(phys_obj);
  4125. return ret;
  4126. }
  4127. void i915_gem_free_phys_object(struct drm_device *dev, int id)
  4128. {
  4129. drm_i915_private_t *dev_priv = dev->dev_private;
  4130. struct drm_i915_gem_phys_object *phys_obj;
  4131. if (!dev_priv->mm.phys_objs[id - 1])
  4132. return;
  4133. phys_obj = dev_priv->mm.phys_objs[id - 1];
  4134. if (phys_obj->cur_obj) {
  4135. i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
  4136. }
  4137. #ifdef CONFIG_X86
  4138. set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
  4139. #endif
  4140. drm_pci_free(dev, phys_obj->handle);
  4141. kfree(phys_obj);
  4142. dev_priv->mm.phys_objs[id - 1] = NULL;
  4143. }
  4144. void i915_gem_free_all_phys_object(struct drm_device *dev)
  4145. {
  4146. int i;
  4147. for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
  4148. i915_gem_free_phys_object(dev, i);
  4149. }
  4150. void i915_gem_detach_phys_object(struct drm_device *dev,
  4151. struct drm_gem_object *obj)
  4152. {
  4153. struct drm_i915_gem_object *obj_priv;
  4154. int i;
  4155. int ret;
  4156. int page_count;
  4157. obj_priv = to_intel_bo(obj);
  4158. if (!obj_priv->phys_obj)
  4159. return;
  4160. ret = i915_gem_object_get_pages(obj, 0);
  4161. if (ret)
  4162. goto out;
  4163. page_count = obj->size / PAGE_SIZE;
  4164. for (i = 0; i < page_count; i++) {
  4165. char *dst = kmap_atomic(obj_priv->pages[i], KM_USER0);
  4166. char *src = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
  4167. memcpy(dst, src, PAGE_SIZE);
  4168. kunmap_atomic(dst, KM_USER0);
  4169. }
  4170. drm_clflush_pages(obj_priv->pages, page_count);
  4171. drm_agp_chipset_flush(dev);
  4172. i915_gem_object_put_pages(obj);
  4173. out:
  4174. obj_priv->phys_obj->cur_obj = NULL;
  4175. obj_priv->phys_obj = NULL;
  4176. }
  4177. int
  4178. i915_gem_attach_phys_object(struct drm_device *dev,
  4179. struct drm_gem_object *obj, int id)
  4180. {
  4181. drm_i915_private_t *dev_priv = dev->dev_private;
  4182. struct drm_i915_gem_object *obj_priv;
  4183. int ret = 0;
  4184. int page_count;
  4185. int i;
  4186. if (id > I915_MAX_PHYS_OBJECT)
  4187. return -EINVAL;
  4188. obj_priv = to_intel_bo(obj);
  4189. if (obj_priv->phys_obj) {
  4190. if (obj_priv->phys_obj->id == id)
  4191. return 0;
  4192. i915_gem_detach_phys_object(dev, obj);
  4193. }
  4194. /* create a new object */
  4195. if (!dev_priv->mm.phys_objs[id - 1]) {
  4196. ret = i915_gem_init_phys_object(dev, id,
  4197. obj->size);
  4198. if (ret) {
  4199. DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size);
  4200. goto out;
  4201. }
  4202. }
  4203. /* bind to the object */
  4204. obj_priv->phys_obj = dev_priv->mm.phys_objs[id - 1];
  4205. obj_priv->phys_obj->cur_obj = obj;
  4206. ret = i915_gem_object_get_pages(obj, 0);
  4207. if (ret) {
  4208. DRM_ERROR("failed to get page list\n");
  4209. goto out;
  4210. }
  4211. page_count = obj->size / PAGE_SIZE;
  4212. for (i = 0; i < page_count; i++) {
  4213. char *src = kmap_atomic(obj_priv->pages[i], KM_USER0);
  4214. char *dst = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
  4215. memcpy(dst, src, PAGE_SIZE);
  4216. kunmap_atomic(src, KM_USER0);
  4217. }
  4218. i915_gem_object_put_pages(obj);
  4219. return 0;
  4220. out:
  4221. return ret;
  4222. }
  4223. static int
  4224. i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
  4225. struct drm_i915_gem_pwrite *args,
  4226. struct drm_file *file_priv)
  4227. {
  4228. struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  4229. void *obj_addr;
  4230. int ret;
  4231. char __user *user_data;
  4232. user_data = (char __user *) (uintptr_t) args->data_ptr;
  4233. obj_addr = obj_priv->phys_obj->handle->vaddr + args->offset;
  4234. DRM_DEBUG_DRIVER("obj_addr %p, %lld\n", obj_addr, args->size);
  4235. ret = copy_from_user(obj_addr, user_data, args->size);
  4236. if (ret)
  4237. return -EFAULT;
  4238. drm_agp_chipset_flush(dev);
  4239. return 0;
  4240. }
  4241. void i915_gem_release(struct drm_device * dev, struct drm_file *file_priv)
  4242. {
  4243. struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
  4244. /* Clean up our request list when the client is going away, so that
  4245. * later retire_requests won't dereference our soon-to-be-gone
  4246. * file_priv.
  4247. */
  4248. mutex_lock(&dev->struct_mutex);
  4249. while (!list_empty(&i915_file_priv->mm.request_list))
  4250. list_del_init(i915_file_priv->mm.request_list.next);
  4251. mutex_unlock(&dev->struct_mutex);
  4252. }
  4253. static int
  4254. i915_gpu_is_active(struct drm_device *dev)
  4255. {
  4256. drm_i915_private_t *dev_priv = dev->dev_private;
  4257. int lists_empty;
  4258. spin_lock(&dev_priv->mm.active_list_lock);
  4259. lists_empty = list_empty(&dev_priv->mm.flushing_list) &&
  4260. list_empty(&dev_priv->render_ring.active_list);
  4261. if (HAS_BSD(dev))
  4262. lists_empty &= list_empty(&dev_priv->bsd_ring.active_list);
  4263. spin_unlock(&dev_priv->mm.active_list_lock);
  4264. return !lists_empty;
  4265. }
  4266. static int
  4267. i915_gem_shrink(struct shrinker *shrink, int nr_to_scan, gfp_t gfp_mask)
  4268. {
  4269. drm_i915_private_t *dev_priv, *next_dev;
  4270. struct drm_i915_gem_object *obj_priv, *next_obj;
  4271. int cnt = 0;
  4272. int would_deadlock = 1;
  4273. /* "fast-path" to count number of available objects */
  4274. if (nr_to_scan == 0) {
  4275. spin_lock(&shrink_list_lock);
  4276. list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
  4277. struct drm_device *dev = dev_priv->dev;
  4278. if (mutex_trylock(&dev->struct_mutex)) {
  4279. list_for_each_entry(obj_priv,
  4280. &dev_priv->mm.inactive_list,
  4281. list)
  4282. cnt++;
  4283. mutex_unlock(&dev->struct_mutex);
  4284. }
  4285. }
  4286. spin_unlock(&shrink_list_lock);
  4287. return (cnt / 100) * sysctl_vfs_cache_pressure;
  4288. }
  4289. spin_lock(&shrink_list_lock);
  4290. rescan:
  4291. /* first scan for clean buffers */
  4292. list_for_each_entry_safe(dev_priv, next_dev,
  4293. &shrink_list, mm.shrink_list) {
  4294. struct drm_device *dev = dev_priv->dev;
  4295. if (! mutex_trylock(&dev->struct_mutex))
  4296. continue;
  4297. spin_unlock(&shrink_list_lock);
  4298. i915_gem_retire_requests(dev);
  4299. list_for_each_entry_safe(obj_priv, next_obj,
  4300. &dev_priv->mm.inactive_list,
  4301. list) {
  4302. if (i915_gem_object_is_purgeable(obj_priv)) {
  4303. i915_gem_object_unbind(&obj_priv->base);
  4304. if (--nr_to_scan <= 0)
  4305. break;
  4306. }
  4307. }
  4308. spin_lock(&shrink_list_lock);
  4309. mutex_unlock(&dev->struct_mutex);
  4310. would_deadlock = 0;
  4311. if (nr_to_scan <= 0)
  4312. break;
  4313. }
  4314. /* second pass, evict/count anything still on the inactive list */
  4315. list_for_each_entry_safe(dev_priv, next_dev,
  4316. &shrink_list, mm.shrink_list) {
  4317. struct drm_device *dev = dev_priv->dev;
  4318. if (! mutex_trylock(&dev->struct_mutex))
  4319. continue;
  4320. spin_unlock(&shrink_list_lock);
  4321. list_for_each_entry_safe(obj_priv, next_obj,
  4322. &dev_priv->mm.inactive_list,
  4323. list) {
  4324. if (nr_to_scan > 0) {
  4325. i915_gem_object_unbind(&obj_priv->base);
  4326. nr_to_scan--;
  4327. } else
  4328. cnt++;
  4329. }
  4330. spin_lock(&shrink_list_lock);
  4331. mutex_unlock(&dev->struct_mutex);
  4332. would_deadlock = 0;
  4333. }
  4334. if (nr_to_scan) {
  4335. int active = 0;
  4336. /*
  4337. * We are desperate for pages, so as a last resort, wait
  4338. * for the GPU to finish and discard whatever we can.
  4339. * This has a dramatic impact to reduce the number of
  4340. * OOM-killer events whilst running the GPU aggressively.
  4341. */
  4342. list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
  4343. struct drm_device *dev = dev_priv->dev;
  4344. if (!mutex_trylock(&dev->struct_mutex))
  4345. continue;
  4346. spin_unlock(&shrink_list_lock);
  4347. if (i915_gpu_is_active(dev)) {
  4348. i915_gpu_idle(dev);
  4349. active++;
  4350. }
  4351. spin_lock(&shrink_list_lock);
  4352. mutex_unlock(&dev->struct_mutex);
  4353. }
  4354. if (active)
  4355. goto rescan;
  4356. }
  4357. spin_unlock(&shrink_list_lock);
  4358. if (would_deadlock)
  4359. return -1;
  4360. else if (cnt > 0)
  4361. return (cnt / 100) * sysctl_vfs_cache_pressure;
  4362. else
  4363. return 0;
  4364. }
  4365. static struct shrinker shrinker = {
  4366. .shrink = i915_gem_shrink,
  4367. .seeks = DEFAULT_SEEKS,
  4368. };
  4369. __init void
  4370. i915_gem_shrinker_init(void)
  4371. {
  4372. register_shrinker(&shrinker);
  4373. }
  4374. __exit void
  4375. i915_gem_shrinker_exit(void)
  4376. {
  4377. unregister_shrinker(&shrinker);
  4378. }