intel_ringbuffer.c 53 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053
  1. /*
  2. * Copyright © 2008-2010 Intel Corporation
  3. *
  4. * Permission is hereby granted, free of charge, to any person obtaining a
  5. * copy of this software and associated documentation files (the "Software"),
  6. * to deal in the Software without restriction, including without limitation
  7. * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8. * and/or sell copies of the Software, and to permit persons to whom the
  9. * Software is furnished to do so, subject to the following conditions:
  10. *
  11. * The above copyright notice and this permission notice (including the next
  12. * paragraph) shall be included in all copies or substantial portions of the
  13. * Software.
  14. *
  15. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17. * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  18. * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19. * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20. * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21. * IN THE SOFTWARE.
  22. *
  23. * Authors:
  24. * Eric Anholt <eric@anholt.net>
  25. * Zou Nan hai <nanhai.zou@intel.com>
  26. * Xiang Hai hao<haihao.xiang@intel.com>
  27. *
  28. */
  29. #include <drm/drmP.h>
  30. #include "i915_drv.h"
  31. #include <drm/i915_drm.h>
  32. #include "i915_trace.h"
  33. #include "intel_drv.h"
  34. /*
  35. * 965+ support PIPE_CONTROL commands, which provide finer grained control
  36. * over cache flushing.
  37. */
  38. struct pipe_control {
  39. struct drm_i915_gem_object *obj;
  40. volatile u32 *cpu_page;
  41. u32 gtt_offset;
  42. };
  43. static inline int ring_space(struct intel_ring_buffer *ring)
  44. {
  45. int space = (ring->head & HEAD_ADDR) - (ring->tail + I915_RING_FREE_SPACE);
  46. if (space < 0)
  47. space += ring->size;
  48. return space;
  49. }
  50. static int
  51. gen2_render_ring_flush(struct intel_ring_buffer *ring,
  52. u32 invalidate_domains,
  53. u32 flush_domains)
  54. {
  55. u32 cmd;
  56. int ret;
  57. cmd = MI_FLUSH;
  58. if (((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER) == 0)
  59. cmd |= MI_NO_WRITE_FLUSH;
  60. if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
  61. cmd |= MI_READ_FLUSH;
  62. ret = intel_ring_begin(ring, 2);
  63. if (ret)
  64. return ret;
  65. intel_ring_emit(ring, cmd);
  66. intel_ring_emit(ring, MI_NOOP);
  67. intel_ring_advance(ring);
  68. return 0;
  69. }
  70. static int
  71. gen4_render_ring_flush(struct intel_ring_buffer *ring,
  72. u32 invalidate_domains,
  73. u32 flush_domains)
  74. {
  75. struct drm_device *dev = ring->dev;
  76. u32 cmd;
  77. int ret;
  78. /*
  79. * read/write caches:
  80. *
  81. * I915_GEM_DOMAIN_RENDER is always invalidated, but is
  82. * only flushed if MI_NO_WRITE_FLUSH is unset. On 965, it is
  83. * also flushed at 2d versus 3d pipeline switches.
  84. *
  85. * read-only caches:
  86. *
  87. * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
  88. * MI_READ_FLUSH is set, and is always flushed on 965.
  89. *
  90. * I915_GEM_DOMAIN_COMMAND may not exist?
  91. *
  92. * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
  93. * invalidated when MI_EXE_FLUSH is set.
  94. *
  95. * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
  96. * invalidated with every MI_FLUSH.
  97. *
  98. * TLBs:
  99. *
  100. * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
  101. * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
  102. * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
  103. * are flushed at any MI_FLUSH.
  104. */
  105. cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
  106. if ((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER)
  107. cmd &= ~MI_NO_WRITE_FLUSH;
  108. if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
  109. cmd |= MI_EXE_FLUSH;
  110. if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&
  111. (IS_G4X(dev) || IS_GEN5(dev)))
  112. cmd |= MI_INVALIDATE_ISP;
  113. ret = intel_ring_begin(ring, 2);
  114. if (ret)
  115. return ret;
  116. intel_ring_emit(ring, cmd);
  117. intel_ring_emit(ring, MI_NOOP);
  118. intel_ring_advance(ring);
  119. return 0;
  120. }
  121. /**
  122. * Emits a PIPE_CONTROL with a non-zero post-sync operation, for
  123. * implementing two workarounds on gen6. From section 1.4.7.1
  124. * "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1:
  125. *
  126. * [DevSNB-C+{W/A}] Before any depth stall flush (including those
  127. * produced by non-pipelined state commands), software needs to first
  128. * send a PIPE_CONTROL with no bits set except Post-Sync Operation !=
  129. * 0.
  130. *
  131. * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable
  132. * =1, a PIPE_CONTROL with any non-zero post-sync-op is required.
  133. *
  134. * And the workaround for these two requires this workaround first:
  135. *
  136. * [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent
  137. * BEFORE the pipe-control with a post-sync op and no write-cache
  138. * flushes.
  139. *
  140. * And this last workaround is tricky because of the requirements on
  141. * that bit. From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM
  142. * volume 2 part 1:
  143. *
  144. * "1 of the following must also be set:
  145. * - Render Target Cache Flush Enable ([12] of DW1)
  146. * - Depth Cache Flush Enable ([0] of DW1)
  147. * - Stall at Pixel Scoreboard ([1] of DW1)
  148. * - Depth Stall ([13] of DW1)
  149. * - Post-Sync Operation ([13] of DW1)
  150. * - Notify Enable ([8] of DW1)"
  151. *
  152. * The cache flushes require the workaround flush that triggered this
  153. * one, so we can't use it. Depth stall would trigger the same.
  154. * Post-sync nonzero is what triggered this second workaround, so we
  155. * can't use that one either. Notify enable is IRQs, which aren't
  156. * really our business. That leaves only stall at scoreboard.
  157. */
  158. static int
  159. intel_emit_post_sync_nonzero_flush(struct intel_ring_buffer *ring)
  160. {
  161. struct pipe_control *pc = ring->private;
  162. u32 scratch_addr = pc->gtt_offset + 128;
  163. int ret;
  164. ret = intel_ring_begin(ring, 6);
  165. if (ret)
  166. return ret;
  167. intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
  168. intel_ring_emit(ring, PIPE_CONTROL_CS_STALL |
  169. PIPE_CONTROL_STALL_AT_SCOREBOARD);
  170. intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */
  171. intel_ring_emit(ring, 0); /* low dword */
  172. intel_ring_emit(ring, 0); /* high dword */
  173. intel_ring_emit(ring, MI_NOOP);
  174. intel_ring_advance(ring);
  175. ret = intel_ring_begin(ring, 6);
  176. if (ret)
  177. return ret;
  178. intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
  179. intel_ring_emit(ring, PIPE_CONTROL_QW_WRITE);
  180. intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */
  181. intel_ring_emit(ring, 0);
  182. intel_ring_emit(ring, 0);
  183. intel_ring_emit(ring, MI_NOOP);
  184. intel_ring_advance(ring);
  185. return 0;
  186. }
  187. static int
  188. gen6_render_ring_flush(struct intel_ring_buffer *ring,
  189. u32 invalidate_domains, u32 flush_domains)
  190. {
  191. u32 flags = 0;
  192. struct pipe_control *pc = ring->private;
  193. u32 scratch_addr = pc->gtt_offset + 128;
  194. int ret;
  195. /* Force SNB workarounds for PIPE_CONTROL flushes */
  196. ret = intel_emit_post_sync_nonzero_flush(ring);
  197. if (ret)
  198. return ret;
  199. /* Just flush everything. Experiments have shown that reducing the
  200. * number of bits based on the write domains has little performance
  201. * impact.
  202. */
  203. if (flush_domains) {
  204. flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
  205. flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
  206. /*
  207. * Ensure that any following seqno writes only happen
  208. * when the render cache is indeed flushed.
  209. */
  210. flags |= PIPE_CONTROL_CS_STALL;
  211. }
  212. if (invalidate_domains) {
  213. flags |= PIPE_CONTROL_TLB_INVALIDATE;
  214. flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
  215. flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
  216. flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
  217. flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
  218. flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
  219. /*
  220. * TLB invalidate requires a post-sync write.
  221. */
  222. flags |= PIPE_CONTROL_QW_WRITE | PIPE_CONTROL_CS_STALL;
  223. }
  224. ret = intel_ring_begin(ring, 4);
  225. if (ret)
  226. return ret;
  227. intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
  228. intel_ring_emit(ring, flags);
  229. intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT);
  230. intel_ring_emit(ring, 0);
  231. intel_ring_advance(ring);
  232. return 0;
  233. }
  234. static int
  235. gen7_render_ring_cs_stall_wa(struct intel_ring_buffer *ring)
  236. {
  237. int ret;
  238. ret = intel_ring_begin(ring, 4);
  239. if (ret)
  240. return ret;
  241. intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
  242. intel_ring_emit(ring, PIPE_CONTROL_CS_STALL |
  243. PIPE_CONTROL_STALL_AT_SCOREBOARD);
  244. intel_ring_emit(ring, 0);
  245. intel_ring_emit(ring, 0);
  246. intel_ring_advance(ring);
  247. return 0;
  248. }
  249. static int gen7_ring_fbc_flush(struct intel_ring_buffer *ring, u32 value)
  250. {
  251. int ret;
  252. if (!ring->fbc_dirty)
  253. return 0;
  254. ret = intel_ring_begin(ring, 4);
  255. if (ret)
  256. return ret;
  257. intel_ring_emit(ring, MI_NOOP);
  258. /* WaFbcNukeOn3DBlt:ivb/hsw */
  259. intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
  260. intel_ring_emit(ring, MSG_FBC_REND_STATE);
  261. intel_ring_emit(ring, value);
  262. intel_ring_advance(ring);
  263. ring->fbc_dirty = false;
  264. return 0;
  265. }
  266. static int
  267. gen7_render_ring_flush(struct intel_ring_buffer *ring,
  268. u32 invalidate_domains, u32 flush_domains)
  269. {
  270. u32 flags = 0;
  271. struct pipe_control *pc = ring->private;
  272. u32 scratch_addr = pc->gtt_offset + 128;
  273. int ret;
  274. /*
  275. * Ensure that any following seqno writes only happen when the render
  276. * cache is indeed flushed.
  277. *
  278. * Workaround: 4th PIPE_CONTROL command (except the ones with only
  279. * read-cache invalidate bits set) must have the CS_STALL bit set. We
  280. * don't try to be clever and just set it unconditionally.
  281. */
  282. flags |= PIPE_CONTROL_CS_STALL;
  283. /* Just flush everything. Experiments have shown that reducing the
  284. * number of bits based on the write domains has little performance
  285. * impact.
  286. */
  287. if (flush_domains) {
  288. flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
  289. flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
  290. }
  291. if (invalidate_domains) {
  292. flags |= PIPE_CONTROL_TLB_INVALIDATE;
  293. flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
  294. flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
  295. flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
  296. flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
  297. flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
  298. /*
  299. * TLB invalidate requires a post-sync write.
  300. */
  301. flags |= PIPE_CONTROL_QW_WRITE;
  302. flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;
  303. /* Workaround: we must issue a pipe_control with CS-stall bit
  304. * set before a pipe_control command that has the state cache
  305. * invalidate bit set. */
  306. gen7_render_ring_cs_stall_wa(ring);
  307. }
  308. ret = intel_ring_begin(ring, 4);
  309. if (ret)
  310. return ret;
  311. intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
  312. intel_ring_emit(ring, flags);
  313. intel_ring_emit(ring, scratch_addr);
  314. intel_ring_emit(ring, 0);
  315. intel_ring_advance(ring);
  316. if (flush_domains)
  317. return gen7_ring_fbc_flush(ring, FBC_REND_NUKE);
  318. return 0;
  319. }
  320. static void ring_write_tail(struct intel_ring_buffer *ring,
  321. u32 value)
  322. {
  323. drm_i915_private_t *dev_priv = ring->dev->dev_private;
  324. I915_WRITE_TAIL(ring, value);
  325. }
  326. u32 intel_ring_get_active_head(struct intel_ring_buffer *ring)
  327. {
  328. drm_i915_private_t *dev_priv = ring->dev->dev_private;
  329. u32 acthd_reg = INTEL_INFO(ring->dev)->gen >= 4 ?
  330. RING_ACTHD(ring->mmio_base) : ACTHD;
  331. return I915_READ(acthd_reg);
  332. }
  333. static void ring_setup_phys_status_page(struct intel_ring_buffer *ring)
  334. {
  335. struct drm_i915_private *dev_priv = ring->dev->dev_private;
  336. u32 addr;
  337. addr = dev_priv->status_page_dmah->busaddr;
  338. if (INTEL_INFO(ring->dev)->gen >= 4)
  339. addr |= (dev_priv->status_page_dmah->busaddr >> 28) & 0xf0;
  340. I915_WRITE(HWS_PGA, addr);
  341. }
  342. static int init_ring_common(struct intel_ring_buffer *ring)
  343. {
  344. struct drm_device *dev = ring->dev;
  345. drm_i915_private_t *dev_priv = dev->dev_private;
  346. struct drm_i915_gem_object *obj = ring->obj;
  347. int ret = 0;
  348. u32 head;
  349. if (HAS_FORCE_WAKE(dev))
  350. gen6_gt_force_wake_get(dev_priv);
  351. if (I915_NEED_GFX_HWS(dev))
  352. intel_ring_setup_status_page(ring);
  353. else
  354. ring_setup_phys_status_page(ring);
  355. /* Stop the ring if it's running. */
  356. I915_WRITE_CTL(ring, 0);
  357. I915_WRITE_HEAD(ring, 0);
  358. ring->write_tail(ring, 0);
  359. head = I915_READ_HEAD(ring) & HEAD_ADDR;
  360. /* G45 ring initialization fails to reset head to zero */
  361. if (head != 0) {
  362. DRM_DEBUG_KMS("%s head not reset to zero "
  363. "ctl %08x head %08x tail %08x start %08x\n",
  364. ring->name,
  365. I915_READ_CTL(ring),
  366. I915_READ_HEAD(ring),
  367. I915_READ_TAIL(ring),
  368. I915_READ_START(ring));
  369. I915_WRITE_HEAD(ring, 0);
  370. if (I915_READ_HEAD(ring) & HEAD_ADDR) {
  371. DRM_ERROR("failed to set %s head to zero "
  372. "ctl %08x head %08x tail %08x start %08x\n",
  373. ring->name,
  374. I915_READ_CTL(ring),
  375. I915_READ_HEAD(ring),
  376. I915_READ_TAIL(ring),
  377. I915_READ_START(ring));
  378. }
  379. }
  380. /* Initialize the ring. This must happen _after_ we've cleared the ring
  381. * registers with the above sequence (the readback of the HEAD registers
  382. * also enforces ordering), otherwise the hw might lose the new ring
  383. * register values. */
  384. I915_WRITE_START(ring, i915_gem_obj_ggtt_offset(obj));
  385. I915_WRITE_CTL(ring,
  386. ((ring->size - PAGE_SIZE) & RING_NR_PAGES)
  387. | RING_VALID);
  388. /* If the head is still not zero, the ring is dead */
  389. if (wait_for((I915_READ_CTL(ring) & RING_VALID) != 0 &&
  390. I915_READ_START(ring) == i915_gem_obj_ggtt_offset(obj) &&
  391. (I915_READ_HEAD(ring) & HEAD_ADDR) == 0, 50)) {
  392. DRM_ERROR("%s initialization failed "
  393. "ctl %08x head %08x tail %08x start %08x\n",
  394. ring->name,
  395. I915_READ_CTL(ring),
  396. I915_READ_HEAD(ring),
  397. I915_READ_TAIL(ring),
  398. I915_READ_START(ring));
  399. ret = -EIO;
  400. goto out;
  401. }
  402. if (!drm_core_check_feature(ring->dev, DRIVER_MODESET))
  403. i915_kernel_lost_context(ring->dev);
  404. else {
  405. ring->head = I915_READ_HEAD(ring);
  406. ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
  407. ring->space = ring_space(ring);
  408. ring->last_retired_head = -1;
  409. }
  410. memset(&ring->hangcheck, 0, sizeof(ring->hangcheck));
  411. out:
  412. if (HAS_FORCE_WAKE(dev))
  413. gen6_gt_force_wake_put(dev_priv);
  414. return ret;
  415. }
  416. static int
  417. init_pipe_control(struct intel_ring_buffer *ring)
  418. {
  419. struct pipe_control *pc;
  420. struct drm_i915_gem_object *obj;
  421. int ret;
  422. if (ring->private)
  423. return 0;
  424. pc = kmalloc(sizeof(*pc), GFP_KERNEL);
  425. if (!pc)
  426. return -ENOMEM;
  427. obj = i915_gem_alloc_object(ring->dev, 4096);
  428. if (obj == NULL) {
  429. DRM_ERROR("Failed to allocate seqno page\n");
  430. ret = -ENOMEM;
  431. goto err;
  432. }
  433. i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
  434. ret = i915_gem_obj_ggtt_pin(obj, 4096, true, false);
  435. if (ret)
  436. goto err_unref;
  437. pc->gtt_offset = i915_gem_obj_ggtt_offset(obj);
  438. pc->cpu_page = kmap(sg_page(obj->pages->sgl));
  439. if (pc->cpu_page == NULL) {
  440. ret = -ENOMEM;
  441. goto err_unpin;
  442. }
  443. DRM_DEBUG_DRIVER("%s pipe control offset: 0x%08x\n",
  444. ring->name, pc->gtt_offset);
  445. pc->obj = obj;
  446. ring->private = pc;
  447. return 0;
  448. err_unpin:
  449. i915_gem_object_unpin(obj);
  450. err_unref:
  451. drm_gem_object_unreference(&obj->base);
  452. err:
  453. kfree(pc);
  454. return ret;
  455. }
  456. static void
  457. cleanup_pipe_control(struct intel_ring_buffer *ring)
  458. {
  459. struct pipe_control *pc = ring->private;
  460. struct drm_i915_gem_object *obj;
  461. obj = pc->obj;
  462. kunmap(sg_page(obj->pages->sgl));
  463. i915_gem_object_unpin(obj);
  464. drm_gem_object_unreference(&obj->base);
  465. kfree(pc);
  466. }
  467. static int init_render_ring(struct intel_ring_buffer *ring)
  468. {
  469. struct drm_device *dev = ring->dev;
  470. struct drm_i915_private *dev_priv = dev->dev_private;
  471. int ret = init_ring_common(ring);
  472. if (INTEL_INFO(dev)->gen > 3)
  473. I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH));
  474. /* We need to disable the AsyncFlip performance optimisations in order
  475. * to use MI_WAIT_FOR_EVENT within the CS. It should already be
  476. * programmed to '1' on all products.
  477. *
  478. * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv
  479. */
  480. if (INTEL_INFO(dev)->gen >= 6)
  481. I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE));
  482. /* Required for the hardware to program scanline values for waiting */
  483. if (INTEL_INFO(dev)->gen == 6)
  484. I915_WRITE(GFX_MODE,
  485. _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_ALWAYS));
  486. if (IS_GEN7(dev))
  487. I915_WRITE(GFX_MODE_GEN7,
  488. _MASKED_BIT_DISABLE(GFX_TLB_INVALIDATE_ALWAYS) |
  489. _MASKED_BIT_ENABLE(GFX_REPLAY_MODE));
  490. if (INTEL_INFO(dev)->gen >= 5) {
  491. ret = init_pipe_control(ring);
  492. if (ret)
  493. return ret;
  494. }
  495. if (IS_GEN6(dev)) {
  496. /* From the Sandybridge PRM, volume 1 part 3, page 24:
  497. * "If this bit is set, STCunit will have LRA as replacement
  498. * policy. [...] This bit must be reset. LRA replacement
  499. * policy is not supported."
  500. */
  501. I915_WRITE(CACHE_MODE_0,
  502. _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
  503. /* This is not explicitly set for GEN6, so read the register.
  504. * see intel_ring_mi_set_context() for why we care.
  505. * TODO: consider explicitly setting the bit for GEN5
  506. */
  507. ring->itlb_before_ctx_switch =
  508. !!(I915_READ(GFX_MODE) & GFX_TLB_INVALIDATE_ALWAYS);
  509. }
  510. if (INTEL_INFO(dev)->gen >= 6)
  511. I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));
  512. if (HAS_L3_GPU_CACHE(dev))
  513. I915_WRITE_IMR(ring, ~GT_RENDER_L3_PARITY_ERROR_INTERRUPT);
  514. return ret;
  515. }
  516. static void render_ring_cleanup(struct intel_ring_buffer *ring)
  517. {
  518. struct drm_device *dev = ring->dev;
  519. if (!ring->private)
  520. return;
  521. if (HAS_BROKEN_CS_TLB(dev))
  522. drm_gem_object_unreference(to_gem_object(ring->private));
  523. if (INTEL_INFO(dev)->gen >= 5)
  524. cleanup_pipe_control(ring);
  525. ring->private = NULL;
  526. }
  527. static void
  528. update_mboxes(struct intel_ring_buffer *ring,
  529. u32 mmio_offset)
  530. {
  531. /* NB: In order to be able to do semaphore MBOX updates for varying number
  532. * of rings, it's easiest if we round up each individual update to a
  533. * multiple of 2 (since ring updates must always be a multiple of 2)
  534. * even though the actual update only requires 3 dwords.
  535. */
  536. #define MBOX_UPDATE_DWORDS 4
  537. intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
  538. intel_ring_emit(ring, mmio_offset);
  539. intel_ring_emit(ring, ring->outstanding_lazy_request);
  540. intel_ring_emit(ring, MI_NOOP);
  541. }
  542. /**
  543. * gen6_add_request - Update the semaphore mailbox registers
  544. *
  545. * @ring - ring that is adding a request
  546. * @seqno - return seqno stuck into the ring
  547. *
  548. * Update the mailbox registers in the *other* rings with the current seqno.
  549. * This acts like a signal in the canonical semaphore.
  550. */
  551. static int
  552. gen6_add_request(struct intel_ring_buffer *ring)
  553. {
  554. struct drm_device *dev = ring->dev;
  555. struct drm_i915_private *dev_priv = dev->dev_private;
  556. struct intel_ring_buffer *useless;
  557. int i, ret;
  558. ret = intel_ring_begin(ring, ((I915_NUM_RINGS-1) *
  559. MBOX_UPDATE_DWORDS) +
  560. 4);
  561. if (ret)
  562. return ret;
  563. #undef MBOX_UPDATE_DWORDS
  564. for_each_ring(useless, dev_priv, i) {
  565. u32 mbox_reg = ring->signal_mbox[i];
  566. if (mbox_reg != GEN6_NOSYNC)
  567. update_mboxes(ring, mbox_reg);
  568. }
  569. intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
  570. intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
  571. intel_ring_emit(ring, ring->outstanding_lazy_request);
  572. intel_ring_emit(ring, MI_USER_INTERRUPT);
  573. intel_ring_advance(ring);
  574. return 0;
  575. }
  576. static inline bool i915_gem_has_seqno_wrapped(struct drm_device *dev,
  577. u32 seqno)
  578. {
  579. struct drm_i915_private *dev_priv = dev->dev_private;
  580. return dev_priv->last_seqno < seqno;
  581. }
  582. /**
  583. * intel_ring_sync - sync the waiter to the signaller on seqno
  584. *
  585. * @waiter - ring that is waiting
  586. * @signaller - ring which has, or will signal
  587. * @seqno - seqno which the waiter will block on
  588. */
  589. static int
  590. gen6_ring_sync(struct intel_ring_buffer *waiter,
  591. struct intel_ring_buffer *signaller,
  592. u32 seqno)
  593. {
  594. int ret;
  595. u32 dw1 = MI_SEMAPHORE_MBOX |
  596. MI_SEMAPHORE_COMPARE |
  597. MI_SEMAPHORE_REGISTER;
  598. /* Throughout all of the GEM code, seqno passed implies our current
  599. * seqno is >= the last seqno executed. However for hardware the
  600. * comparison is strictly greater than.
  601. */
  602. seqno -= 1;
  603. WARN_ON(signaller->semaphore_register[waiter->id] ==
  604. MI_SEMAPHORE_SYNC_INVALID);
  605. ret = intel_ring_begin(waiter, 4);
  606. if (ret)
  607. return ret;
  608. /* If seqno wrap happened, omit the wait with no-ops */
  609. if (likely(!i915_gem_has_seqno_wrapped(waiter->dev, seqno))) {
  610. intel_ring_emit(waiter,
  611. dw1 |
  612. signaller->semaphore_register[waiter->id]);
  613. intel_ring_emit(waiter, seqno);
  614. intel_ring_emit(waiter, 0);
  615. intel_ring_emit(waiter, MI_NOOP);
  616. } else {
  617. intel_ring_emit(waiter, MI_NOOP);
  618. intel_ring_emit(waiter, MI_NOOP);
  619. intel_ring_emit(waiter, MI_NOOP);
  620. intel_ring_emit(waiter, MI_NOOP);
  621. }
  622. intel_ring_advance(waiter);
  623. return 0;
  624. }
  625. #define PIPE_CONTROL_FLUSH(ring__, addr__) \
  626. do { \
  627. intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE | \
  628. PIPE_CONTROL_DEPTH_STALL); \
  629. intel_ring_emit(ring__, (addr__) | PIPE_CONTROL_GLOBAL_GTT); \
  630. intel_ring_emit(ring__, 0); \
  631. intel_ring_emit(ring__, 0); \
  632. } while (0)
  633. static int
  634. pc_render_add_request(struct intel_ring_buffer *ring)
  635. {
  636. struct pipe_control *pc = ring->private;
  637. u32 scratch_addr = pc->gtt_offset + 128;
  638. int ret;
  639. /* For Ironlake, MI_USER_INTERRUPT was deprecated and apparently
  640. * incoherent with writes to memory, i.e. completely fubar,
  641. * so we need to use PIPE_NOTIFY instead.
  642. *
  643. * However, we also need to workaround the qword write
  644. * incoherence by flushing the 6 PIPE_NOTIFY buffers out to
  645. * memory before requesting an interrupt.
  646. */
  647. ret = intel_ring_begin(ring, 32);
  648. if (ret)
  649. return ret;
  650. intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
  651. PIPE_CONTROL_WRITE_FLUSH |
  652. PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
  653. intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
  654. intel_ring_emit(ring, ring->outstanding_lazy_request);
  655. intel_ring_emit(ring, 0);
  656. PIPE_CONTROL_FLUSH(ring, scratch_addr);
  657. scratch_addr += 128; /* write to separate cachelines */
  658. PIPE_CONTROL_FLUSH(ring, scratch_addr);
  659. scratch_addr += 128;
  660. PIPE_CONTROL_FLUSH(ring, scratch_addr);
  661. scratch_addr += 128;
  662. PIPE_CONTROL_FLUSH(ring, scratch_addr);
  663. scratch_addr += 128;
  664. PIPE_CONTROL_FLUSH(ring, scratch_addr);
  665. scratch_addr += 128;
  666. PIPE_CONTROL_FLUSH(ring, scratch_addr);
  667. intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
  668. PIPE_CONTROL_WRITE_FLUSH |
  669. PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
  670. PIPE_CONTROL_NOTIFY);
  671. intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
  672. intel_ring_emit(ring, ring->outstanding_lazy_request);
  673. intel_ring_emit(ring, 0);
  674. intel_ring_advance(ring);
  675. return 0;
  676. }
  677. static u32
  678. gen6_ring_get_seqno(struct intel_ring_buffer *ring, bool lazy_coherency)
  679. {
  680. /* Workaround to force correct ordering between irq and seqno writes on
  681. * ivb (and maybe also on snb) by reading from a CS register (like
  682. * ACTHD) before reading the status page. */
  683. if (!lazy_coherency)
  684. intel_ring_get_active_head(ring);
  685. return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
  686. }
  687. static u32
  688. ring_get_seqno(struct intel_ring_buffer *ring, bool lazy_coherency)
  689. {
  690. return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
  691. }
  692. static void
  693. ring_set_seqno(struct intel_ring_buffer *ring, u32 seqno)
  694. {
  695. intel_write_status_page(ring, I915_GEM_HWS_INDEX, seqno);
  696. }
  697. static u32
  698. pc_render_get_seqno(struct intel_ring_buffer *ring, bool lazy_coherency)
  699. {
  700. struct pipe_control *pc = ring->private;
  701. return pc->cpu_page[0];
  702. }
  703. static void
  704. pc_render_set_seqno(struct intel_ring_buffer *ring, u32 seqno)
  705. {
  706. struct pipe_control *pc = ring->private;
  707. pc->cpu_page[0] = seqno;
  708. }
  709. static bool
  710. gen5_ring_get_irq(struct intel_ring_buffer *ring)
  711. {
  712. struct drm_device *dev = ring->dev;
  713. drm_i915_private_t *dev_priv = dev->dev_private;
  714. unsigned long flags;
  715. if (!dev->irq_enabled)
  716. return false;
  717. spin_lock_irqsave(&dev_priv->irq_lock, flags);
  718. if (ring->irq_refcount++ == 0)
  719. ilk_enable_gt_irq(dev_priv, ring->irq_enable_mask);
  720. spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  721. return true;
  722. }
  723. static void
  724. gen5_ring_put_irq(struct intel_ring_buffer *ring)
  725. {
  726. struct drm_device *dev = ring->dev;
  727. drm_i915_private_t *dev_priv = dev->dev_private;
  728. unsigned long flags;
  729. spin_lock_irqsave(&dev_priv->irq_lock, flags);
  730. if (--ring->irq_refcount == 0)
  731. ilk_disable_gt_irq(dev_priv, ring->irq_enable_mask);
  732. spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  733. }
  734. static bool
  735. i9xx_ring_get_irq(struct intel_ring_buffer *ring)
  736. {
  737. struct drm_device *dev = ring->dev;
  738. drm_i915_private_t *dev_priv = dev->dev_private;
  739. unsigned long flags;
  740. if (!dev->irq_enabled)
  741. return false;
  742. spin_lock_irqsave(&dev_priv->irq_lock, flags);
  743. if (ring->irq_refcount++ == 0) {
  744. dev_priv->irq_mask &= ~ring->irq_enable_mask;
  745. I915_WRITE(IMR, dev_priv->irq_mask);
  746. POSTING_READ(IMR);
  747. }
  748. spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  749. return true;
  750. }
  751. static void
  752. i9xx_ring_put_irq(struct intel_ring_buffer *ring)
  753. {
  754. struct drm_device *dev = ring->dev;
  755. drm_i915_private_t *dev_priv = dev->dev_private;
  756. unsigned long flags;
  757. spin_lock_irqsave(&dev_priv->irq_lock, flags);
  758. if (--ring->irq_refcount == 0) {
  759. dev_priv->irq_mask |= ring->irq_enable_mask;
  760. I915_WRITE(IMR, dev_priv->irq_mask);
  761. POSTING_READ(IMR);
  762. }
  763. spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  764. }
  765. static bool
  766. i8xx_ring_get_irq(struct intel_ring_buffer *ring)
  767. {
  768. struct drm_device *dev = ring->dev;
  769. drm_i915_private_t *dev_priv = dev->dev_private;
  770. unsigned long flags;
  771. if (!dev->irq_enabled)
  772. return false;
  773. spin_lock_irqsave(&dev_priv->irq_lock, flags);
  774. if (ring->irq_refcount++ == 0) {
  775. dev_priv->irq_mask &= ~ring->irq_enable_mask;
  776. I915_WRITE16(IMR, dev_priv->irq_mask);
  777. POSTING_READ16(IMR);
  778. }
  779. spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  780. return true;
  781. }
  782. static void
  783. i8xx_ring_put_irq(struct intel_ring_buffer *ring)
  784. {
  785. struct drm_device *dev = ring->dev;
  786. drm_i915_private_t *dev_priv = dev->dev_private;
  787. unsigned long flags;
  788. spin_lock_irqsave(&dev_priv->irq_lock, flags);
  789. if (--ring->irq_refcount == 0) {
  790. dev_priv->irq_mask |= ring->irq_enable_mask;
  791. I915_WRITE16(IMR, dev_priv->irq_mask);
  792. POSTING_READ16(IMR);
  793. }
  794. spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  795. }
  796. void intel_ring_setup_status_page(struct intel_ring_buffer *ring)
  797. {
  798. struct drm_device *dev = ring->dev;
  799. drm_i915_private_t *dev_priv = ring->dev->dev_private;
  800. u32 mmio = 0;
  801. /* The ring status page addresses are no longer next to the rest of
  802. * the ring registers as of gen7.
  803. */
  804. if (IS_GEN7(dev)) {
  805. switch (ring->id) {
  806. case RCS:
  807. mmio = RENDER_HWS_PGA_GEN7;
  808. break;
  809. case BCS:
  810. mmio = BLT_HWS_PGA_GEN7;
  811. break;
  812. case VCS:
  813. mmio = BSD_HWS_PGA_GEN7;
  814. break;
  815. case VECS:
  816. mmio = VEBOX_HWS_PGA_GEN7;
  817. break;
  818. }
  819. } else if (IS_GEN6(ring->dev)) {
  820. mmio = RING_HWS_PGA_GEN6(ring->mmio_base);
  821. } else {
  822. mmio = RING_HWS_PGA(ring->mmio_base);
  823. }
  824. I915_WRITE(mmio, (u32)ring->status_page.gfx_addr);
  825. POSTING_READ(mmio);
  826. }
  827. static int
  828. bsd_ring_flush(struct intel_ring_buffer *ring,
  829. u32 invalidate_domains,
  830. u32 flush_domains)
  831. {
  832. int ret;
  833. ret = intel_ring_begin(ring, 2);
  834. if (ret)
  835. return ret;
  836. intel_ring_emit(ring, MI_FLUSH);
  837. intel_ring_emit(ring, MI_NOOP);
  838. intel_ring_advance(ring);
  839. return 0;
  840. }
  841. static int
  842. i9xx_add_request(struct intel_ring_buffer *ring)
  843. {
  844. int ret;
  845. ret = intel_ring_begin(ring, 4);
  846. if (ret)
  847. return ret;
  848. intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
  849. intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
  850. intel_ring_emit(ring, ring->outstanding_lazy_request);
  851. intel_ring_emit(ring, MI_USER_INTERRUPT);
  852. intel_ring_advance(ring);
  853. return 0;
  854. }
  855. static bool
  856. gen6_ring_get_irq(struct intel_ring_buffer *ring)
  857. {
  858. struct drm_device *dev = ring->dev;
  859. drm_i915_private_t *dev_priv = dev->dev_private;
  860. unsigned long flags;
  861. if (!dev->irq_enabled)
  862. return false;
  863. /* It looks like we need to prevent the gt from suspending while waiting
  864. * for an notifiy irq, otherwise irqs seem to get lost on at least the
  865. * blt/bsd rings on ivb. */
  866. gen6_gt_force_wake_get(dev_priv);
  867. spin_lock_irqsave(&dev_priv->irq_lock, flags);
  868. if (ring->irq_refcount++ == 0) {
  869. if (HAS_L3_GPU_CACHE(dev) && ring->id == RCS)
  870. I915_WRITE_IMR(ring,
  871. ~(ring->irq_enable_mask |
  872. GT_RENDER_L3_PARITY_ERROR_INTERRUPT));
  873. else
  874. I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
  875. ilk_enable_gt_irq(dev_priv, ring->irq_enable_mask);
  876. }
  877. spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  878. return true;
  879. }
  880. static void
  881. gen6_ring_put_irq(struct intel_ring_buffer *ring)
  882. {
  883. struct drm_device *dev = ring->dev;
  884. drm_i915_private_t *dev_priv = dev->dev_private;
  885. unsigned long flags;
  886. spin_lock_irqsave(&dev_priv->irq_lock, flags);
  887. if (--ring->irq_refcount == 0) {
  888. if (HAS_L3_GPU_CACHE(dev) && ring->id == RCS)
  889. I915_WRITE_IMR(ring,
  890. ~GT_RENDER_L3_PARITY_ERROR_INTERRUPT);
  891. else
  892. I915_WRITE_IMR(ring, ~0);
  893. ilk_disable_gt_irq(dev_priv, ring->irq_enable_mask);
  894. }
  895. spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  896. gen6_gt_force_wake_put(dev_priv);
  897. }
  898. static bool
  899. hsw_vebox_get_irq(struct intel_ring_buffer *ring)
  900. {
  901. struct drm_device *dev = ring->dev;
  902. struct drm_i915_private *dev_priv = dev->dev_private;
  903. unsigned long flags;
  904. if (!dev->irq_enabled)
  905. return false;
  906. spin_lock_irqsave(&dev_priv->irq_lock, flags);
  907. if (ring->irq_refcount++ == 0) {
  908. I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
  909. snb_enable_pm_irq(dev_priv, ring->irq_enable_mask);
  910. }
  911. spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  912. return true;
  913. }
  914. static void
  915. hsw_vebox_put_irq(struct intel_ring_buffer *ring)
  916. {
  917. struct drm_device *dev = ring->dev;
  918. struct drm_i915_private *dev_priv = dev->dev_private;
  919. unsigned long flags;
  920. if (!dev->irq_enabled)
  921. return;
  922. spin_lock_irqsave(&dev_priv->irq_lock, flags);
  923. if (--ring->irq_refcount == 0) {
  924. I915_WRITE_IMR(ring, ~0);
  925. snb_disable_pm_irq(dev_priv, ring->irq_enable_mask);
  926. }
  927. spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  928. }
  929. static int
  930. i965_dispatch_execbuffer(struct intel_ring_buffer *ring,
  931. u32 offset, u32 length,
  932. unsigned flags)
  933. {
  934. int ret;
  935. ret = intel_ring_begin(ring, 2);
  936. if (ret)
  937. return ret;
  938. intel_ring_emit(ring,
  939. MI_BATCH_BUFFER_START |
  940. MI_BATCH_GTT |
  941. (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_I965));
  942. intel_ring_emit(ring, offset);
  943. intel_ring_advance(ring);
  944. return 0;
  945. }
  946. /* Just userspace ABI convention to limit the wa batch bo to a resonable size */
  947. #define I830_BATCH_LIMIT (256*1024)
  948. static int
  949. i830_dispatch_execbuffer(struct intel_ring_buffer *ring,
  950. u32 offset, u32 len,
  951. unsigned flags)
  952. {
  953. int ret;
  954. if (flags & I915_DISPATCH_PINNED) {
  955. ret = intel_ring_begin(ring, 4);
  956. if (ret)
  957. return ret;
  958. intel_ring_emit(ring, MI_BATCH_BUFFER);
  959. intel_ring_emit(ring, offset | (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE));
  960. intel_ring_emit(ring, offset + len - 8);
  961. intel_ring_emit(ring, MI_NOOP);
  962. intel_ring_advance(ring);
  963. } else {
  964. struct drm_i915_gem_object *obj = ring->private;
  965. u32 cs_offset = i915_gem_obj_ggtt_offset(obj);
  966. if (len > I830_BATCH_LIMIT)
  967. return -ENOSPC;
  968. ret = intel_ring_begin(ring, 9+3);
  969. if (ret)
  970. return ret;
  971. /* Blit the batch (which has now all relocs applied) to the stable batch
  972. * scratch bo area (so that the CS never stumbles over its tlb
  973. * invalidation bug) ... */
  974. intel_ring_emit(ring, XY_SRC_COPY_BLT_CMD |
  975. XY_SRC_COPY_BLT_WRITE_ALPHA |
  976. XY_SRC_COPY_BLT_WRITE_RGB);
  977. intel_ring_emit(ring, BLT_DEPTH_32 | BLT_ROP_GXCOPY | 4096);
  978. intel_ring_emit(ring, 0);
  979. intel_ring_emit(ring, (DIV_ROUND_UP(len, 4096) << 16) | 1024);
  980. intel_ring_emit(ring, cs_offset);
  981. intel_ring_emit(ring, 0);
  982. intel_ring_emit(ring, 4096);
  983. intel_ring_emit(ring, offset);
  984. intel_ring_emit(ring, MI_FLUSH);
  985. /* ... and execute it. */
  986. intel_ring_emit(ring, MI_BATCH_BUFFER);
  987. intel_ring_emit(ring, cs_offset | (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE));
  988. intel_ring_emit(ring, cs_offset + len - 8);
  989. intel_ring_advance(ring);
  990. }
  991. return 0;
  992. }
  993. static int
  994. i915_dispatch_execbuffer(struct intel_ring_buffer *ring,
  995. u32 offset, u32 len,
  996. unsigned flags)
  997. {
  998. int ret;
  999. ret = intel_ring_begin(ring, 2);
  1000. if (ret)
  1001. return ret;
  1002. intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_GTT);
  1003. intel_ring_emit(ring, offset | (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE));
  1004. intel_ring_advance(ring);
  1005. return 0;
  1006. }
  1007. static void cleanup_status_page(struct intel_ring_buffer *ring)
  1008. {
  1009. struct drm_i915_gem_object *obj;
  1010. obj = ring->status_page.obj;
  1011. if (obj == NULL)
  1012. return;
  1013. kunmap(sg_page(obj->pages->sgl));
  1014. i915_gem_object_unpin(obj);
  1015. drm_gem_object_unreference(&obj->base);
  1016. ring->status_page.obj = NULL;
  1017. }
  1018. static int init_status_page(struct intel_ring_buffer *ring)
  1019. {
  1020. struct drm_device *dev = ring->dev;
  1021. struct drm_i915_gem_object *obj;
  1022. int ret;
  1023. obj = i915_gem_alloc_object(dev, 4096);
  1024. if (obj == NULL) {
  1025. DRM_ERROR("Failed to allocate status page\n");
  1026. ret = -ENOMEM;
  1027. goto err;
  1028. }
  1029. i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
  1030. ret = i915_gem_obj_ggtt_pin(obj, 4096, true, false);
  1031. if (ret != 0) {
  1032. goto err_unref;
  1033. }
  1034. ring->status_page.gfx_addr = i915_gem_obj_ggtt_offset(obj);
  1035. ring->status_page.page_addr = kmap(sg_page(obj->pages->sgl));
  1036. if (ring->status_page.page_addr == NULL) {
  1037. ret = -ENOMEM;
  1038. goto err_unpin;
  1039. }
  1040. ring->status_page.obj = obj;
  1041. memset(ring->status_page.page_addr, 0, PAGE_SIZE);
  1042. DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
  1043. ring->name, ring->status_page.gfx_addr);
  1044. return 0;
  1045. err_unpin:
  1046. i915_gem_object_unpin(obj);
  1047. err_unref:
  1048. drm_gem_object_unreference(&obj->base);
  1049. err:
  1050. return ret;
  1051. }
  1052. static int init_phys_status_page(struct intel_ring_buffer *ring)
  1053. {
  1054. struct drm_i915_private *dev_priv = ring->dev->dev_private;
  1055. if (!dev_priv->status_page_dmah) {
  1056. dev_priv->status_page_dmah =
  1057. drm_pci_alloc(ring->dev, PAGE_SIZE, PAGE_SIZE);
  1058. if (!dev_priv->status_page_dmah)
  1059. return -ENOMEM;
  1060. }
  1061. ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
  1062. memset(ring->status_page.page_addr, 0, PAGE_SIZE);
  1063. return 0;
  1064. }
  1065. static int intel_init_ring_buffer(struct drm_device *dev,
  1066. struct intel_ring_buffer *ring)
  1067. {
  1068. struct drm_i915_gem_object *obj;
  1069. struct drm_i915_private *dev_priv = dev->dev_private;
  1070. int ret;
  1071. ring->dev = dev;
  1072. INIT_LIST_HEAD(&ring->active_list);
  1073. INIT_LIST_HEAD(&ring->request_list);
  1074. ring->size = 32 * PAGE_SIZE;
  1075. memset(ring->sync_seqno, 0, sizeof(ring->sync_seqno));
  1076. init_waitqueue_head(&ring->irq_queue);
  1077. if (I915_NEED_GFX_HWS(dev)) {
  1078. ret = init_status_page(ring);
  1079. if (ret)
  1080. return ret;
  1081. } else {
  1082. BUG_ON(ring->id != RCS);
  1083. ret = init_phys_status_page(ring);
  1084. if (ret)
  1085. return ret;
  1086. }
  1087. obj = NULL;
  1088. if (!HAS_LLC(dev))
  1089. obj = i915_gem_object_create_stolen(dev, ring->size);
  1090. if (obj == NULL)
  1091. obj = i915_gem_alloc_object(dev, ring->size);
  1092. if (obj == NULL) {
  1093. DRM_ERROR("Failed to allocate ringbuffer\n");
  1094. ret = -ENOMEM;
  1095. goto err_hws;
  1096. }
  1097. ring->obj = obj;
  1098. ret = i915_gem_obj_ggtt_pin(obj, PAGE_SIZE, true, false);
  1099. if (ret)
  1100. goto err_unref;
  1101. ret = i915_gem_object_set_to_gtt_domain(obj, true);
  1102. if (ret)
  1103. goto err_unpin;
  1104. ring->virtual_start =
  1105. ioremap_wc(dev_priv->gtt.mappable_base + i915_gem_obj_ggtt_offset(obj),
  1106. ring->size);
  1107. if (ring->virtual_start == NULL) {
  1108. DRM_ERROR("Failed to map ringbuffer.\n");
  1109. ret = -EINVAL;
  1110. goto err_unpin;
  1111. }
  1112. ret = ring->init(ring);
  1113. if (ret)
  1114. goto err_unmap;
  1115. /* Workaround an erratum on the i830 which causes a hang if
  1116. * the TAIL pointer points to within the last 2 cachelines
  1117. * of the buffer.
  1118. */
  1119. ring->effective_size = ring->size;
  1120. if (IS_I830(ring->dev) || IS_845G(ring->dev))
  1121. ring->effective_size -= 128;
  1122. return 0;
  1123. err_unmap:
  1124. iounmap(ring->virtual_start);
  1125. err_unpin:
  1126. i915_gem_object_unpin(obj);
  1127. err_unref:
  1128. drm_gem_object_unreference(&obj->base);
  1129. ring->obj = NULL;
  1130. err_hws:
  1131. cleanup_status_page(ring);
  1132. return ret;
  1133. }
  1134. void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring)
  1135. {
  1136. struct drm_i915_private *dev_priv;
  1137. int ret;
  1138. if (ring->obj == NULL)
  1139. return;
  1140. /* Disable the ring buffer. The ring must be idle at this point */
  1141. dev_priv = ring->dev->dev_private;
  1142. ret = intel_ring_idle(ring);
  1143. if (ret)
  1144. DRM_ERROR("failed to quiesce %s whilst cleaning up: %d\n",
  1145. ring->name, ret);
  1146. I915_WRITE_CTL(ring, 0);
  1147. iounmap(ring->virtual_start);
  1148. i915_gem_object_unpin(ring->obj);
  1149. drm_gem_object_unreference(&ring->obj->base);
  1150. ring->obj = NULL;
  1151. if (ring->cleanup)
  1152. ring->cleanup(ring);
  1153. cleanup_status_page(ring);
  1154. }
  1155. static int intel_ring_wait_seqno(struct intel_ring_buffer *ring, u32 seqno)
  1156. {
  1157. int ret;
  1158. ret = i915_wait_seqno(ring, seqno);
  1159. if (!ret)
  1160. i915_gem_retire_requests_ring(ring);
  1161. return ret;
  1162. }
  1163. static int intel_ring_wait_request(struct intel_ring_buffer *ring, int n)
  1164. {
  1165. struct drm_i915_gem_request *request;
  1166. u32 seqno = 0;
  1167. int ret;
  1168. i915_gem_retire_requests_ring(ring);
  1169. if (ring->last_retired_head != -1) {
  1170. ring->head = ring->last_retired_head;
  1171. ring->last_retired_head = -1;
  1172. ring->space = ring_space(ring);
  1173. if (ring->space >= n)
  1174. return 0;
  1175. }
  1176. list_for_each_entry(request, &ring->request_list, list) {
  1177. int space;
  1178. if (request->tail == -1)
  1179. continue;
  1180. space = request->tail - (ring->tail + I915_RING_FREE_SPACE);
  1181. if (space < 0)
  1182. space += ring->size;
  1183. if (space >= n) {
  1184. seqno = request->seqno;
  1185. break;
  1186. }
  1187. /* Consume this request in case we need more space than
  1188. * is available and so need to prevent a race between
  1189. * updating last_retired_head and direct reads of
  1190. * I915_RING_HEAD. It also provides a nice sanity check.
  1191. */
  1192. request->tail = -1;
  1193. }
  1194. if (seqno == 0)
  1195. return -ENOSPC;
  1196. ret = intel_ring_wait_seqno(ring, seqno);
  1197. if (ret)
  1198. return ret;
  1199. if (WARN_ON(ring->last_retired_head == -1))
  1200. return -ENOSPC;
  1201. ring->head = ring->last_retired_head;
  1202. ring->last_retired_head = -1;
  1203. ring->space = ring_space(ring);
  1204. if (WARN_ON(ring->space < n))
  1205. return -ENOSPC;
  1206. return 0;
  1207. }
  1208. static int ring_wait_for_space(struct intel_ring_buffer *ring, int n)
  1209. {
  1210. struct drm_device *dev = ring->dev;
  1211. struct drm_i915_private *dev_priv = dev->dev_private;
  1212. unsigned long end;
  1213. int ret;
  1214. ret = intel_ring_wait_request(ring, n);
  1215. if (ret != -ENOSPC)
  1216. return ret;
  1217. trace_i915_ring_wait_begin(ring);
  1218. /* With GEM the hangcheck timer should kick us out of the loop,
  1219. * leaving it early runs the risk of corrupting GEM state (due
  1220. * to running on almost untested codepaths). But on resume
  1221. * timers don't work yet, so prevent a complete hang in that
  1222. * case by choosing an insanely large timeout. */
  1223. end = jiffies + 60 * HZ;
  1224. do {
  1225. ring->head = I915_READ_HEAD(ring);
  1226. ring->space = ring_space(ring);
  1227. if (ring->space >= n) {
  1228. trace_i915_ring_wait_end(ring);
  1229. return 0;
  1230. }
  1231. if (dev->primary->master) {
  1232. struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
  1233. if (master_priv->sarea_priv)
  1234. master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
  1235. }
  1236. msleep(1);
  1237. ret = i915_gem_check_wedge(&dev_priv->gpu_error,
  1238. dev_priv->mm.interruptible);
  1239. if (ret)
  1240. return ret;
  1241. } while (!time_after(jiffies, end));
  1242. trace_i915_ring_wait_end(ring);
  1243. return -EBUSY;
  1244. }
  1245. static int intel_wrap_ring_buffer(struct intel_ring_buffer *ring)
  1246. {
  1247. uint32_t __iomem *virt;
  1248. int rem = ring->size - ring->tail;
  1249. if (ring->space < rem) {
  1250. int ret = ring_wait_for_space(ring, rem);
  1251. if (ret)
  1252. return ret;
  1253. }
  1254. virt = ring->virtual_start + ring->tail;
  1255. rem /= 4;
  1256. while (rem--)
  1257. iowrite32(MI_NOOP, virt++);
  1258. ring->tail = 0;
  1259. ring->space = ring_space(ring);
  1260. return 0;
  1261. }
  1262. int intel_ring_idle(struct intel_ring_buffer *ring)
  1263. {
  1264. u32 seqno;
  1265. int ret;
  1266. /* We need to add any requests required to flush the objects and ring */
  1267. if (ring->outstanding_lazy_request) {
  1268. ret = i915_add_request(ring, NULL);
  1269. if (ret)
  1270. return ret;
  1271. }
  1272. /* Wait upon the last request to be completed */
  1273. if (list_empty(&ring->request_list))
  1274. return 0;
  1275. seqno = list_entry(ring->request_list.prev,
  1276. struct drm_i915_gem_request,
  1277. list)->seqno;
  1278. return i915_wait_seqno(ring, seqno);
  1279. }
  1280. static int
  1281. intel_ring_alloc_seqno(struct intel_ring_buffer *ring)
  1282. {
  1283. if (ring->outstanding_lazy_request)
  1284. return 0;
  1285. return i915_gem_get_seqno(ring->dev, &ring->outstanding_lazy_request);
  1286. }
  1287. static int __intel_ring_begin(struct intel_ring_buffer *ring,
  1288. int bytes)
  1289. {
  1290. int ret;
  1291. if (unlikely(ring->tail + bytes > ring->effective_size)) {
  1292. ret = intel_wrap_ring_buffer(ring);
  1293. if (unlikely(ret))
  1294. return ret;
  1295. }
  1296. if (unlikely(ring->space < bytes)) {
  1297. ret = ring_wait_for_space(ring, bytes);
  1298. if (unlikely(ret))
  1299. return ret;
  1300. }
  1301. ring->space -= bytes;
  1302. return 0;
  1303. }
  1304. int intel_ring_begin(struct intel_ring_buffer *ring,
  1305. int num_dwords)
  1306. {
  1307. drm_i915_private_t *dev_priv = ring->dev->dev_private;
  1308. int ret;
  1309. ret = i915_gem_check_wedge(&dev_priv->gpu_error,
  1310. dev_priv->mm.interruptible);
  1311. if (ret)
  1312. return ret;
  1313. /* Preallocate the olr before touching the ring */
  1314. ret = intel_ring_alloc_seqno(ring);
  1315. if (ret)
  1316. return ret;
  1317. return __intel_ring_begin(ring, num_dwords * sizeof(uint32_t));
  1318. }
  1319. void intel_ring_init_seqno(struct intel_ring_buffer *ring, u32 seqno)
  1320. {
  1321. struct drm_i915_private *dev_priv = ring->dev->dev_private;
  1322. BUG_ON(ring->outstanding_lazy_request);
  1323. if (INTEL_INFO(ring->dev)->gen >= 6) {
  1324. I915_WRITE(RING_SYNC_0(ring->mmio_base), 0);
  1325. I915_WRITE(RING_SYNC_1(ring->mmio_base), 0);
  1326. if (HAS_VEBOX(ring->dev))
  1327. I915_WRITE(RING_SYNC_2(ring->mmio_base), 0);
  1328. }
  1329. ring->set_seqno(ring, seqno);
  1330. ring->hangcheck.seqno = seqno;
  1331. }
  1332. void intel_ring_advance(struct intel_ring_buffer *ring)
  1333. {
  1334. struct drm_i915_private *dev_priv = ring->dev->dev_private;
  1335. ring->tail &= ring->size - 1;
  1336. if (dev_priv->gpu_error.stop_rings & intel_ring_flag(ring))
  1337. return;
  1338. ring->write_tail(ring, ring->tail);
  1339. }
  1340. static void gen6_bsd_ring_write_tail(struct intel_ring_buffer *ring,
  1341. u32 value)
  1342. {
  1343. drm_i915_private_t *dev_priv = ring->dev->dev_private;
  1344. /* Every tail move must follow the sequence below */
  1345. /* Disable notification that the ring is IDLE. The GT
  1346. * will then assume that it is busy and bring it out of rc6.
  1347. */
  1348. I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
  1349. _MASKED_BIT_ENABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
  1350. /* Clear the context id. Here be magic! */
  1351. I915_WRITE64(GEN6_BSD_RNCID, 0x0);
  1352. /* Wait for the ring not to be idle, i.e. for it to wake up. */
  1353. if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) &
  1354. GEN6_BSD_SLEEP_INDICATOR) == 0,
  1355. 50))
  1356. DRM_ERROR("timed out waiting for the BSD ring to wake up\n");
  1357. /* Now that the ring is fully powered up, update the tail */
  1358. I915_WRITE_TAIL(ring, value);
  1359. POSTING_READ(RING_TAIL(ring->mmio_base));
  1360. /* Let the ring send IDLE messages to the GT again,
  1361. * and so let it sleep to conserve power when idle.
  1362. */
  1363. I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
  1364. _MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
  1365. }
  1366. static int gen6_bsd_ring_flush(struct intel_ring_buffer *ring,
  1367. u32 invalidate, u32 flush)
  1368. {
  1369. uint32_t cmd;
  1370. int ret;
  1371. ret = intel_ring_begin(ring, 4);
  1372. if (ret)
  1373. return ret;
  1374. cmd = MI_FLUSH_DW;
  1375. /*
  1376. * Bspec vol 1c.5 - video engine command streamer:
  1377. * "If ENABLED, all TLBs will be invalidated once the flush
  1378. * operation is complete. This bit is only valid when the
  1379. * Post-Sync Operation field is a value of 1h or 3h."
  1380. */
  1381. if (invalidate & I915_GEM_GPU_DOMAINS)
  1382. cmd |= MI_INVALIDATE_TLB | MI_INVALIDATE_BSD |
  1383. MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW;
  1384. intel_ring_emit(ring, cmd);
  1385. intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT);
  1386. intel_ring_emit(ring, 0);
  1387. intel_ring_emit(ring, MI_NOOP);
  1388. intel_ring_advance(ring);
  1389. return 0;
  1390. }
  1391. static int
  1392. hsw_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
  1393. u32 offset, u32 len,
  1394. unsigned flags)
  1395. {
  1396. int ret;
  1397. ret = intel_ring_begin(ring, 2);
  1398. if (ret)
  1399. return ret;
  1400. intel_ring_emit(ring,
  1401. MI_BATCH_BUFFER_START | MI_BATCH_PPGTT_HSW |
  1402. (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_HSW));
  1403. /* bit0-7 is the length on GEN6+ */
  1404. intel_ring_emit(ring, offset);
  1405. intel_ring_advance(ring);
  1406. return 0;
  1407. }
  1408. static int
  1409. gen6_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
  1410. u32 offset, u32 len,
  1411. unsigned flags)
  1412. {
  1413. int ret;
  1414. ret = intel_ring_begin(ring, 2);
  1415. if (ret)
  1416. return ret;
  1417. intel_ring_emit(ring,
  1418. MI_BATCH_BUFFER_START |
  1419. (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_I965));
  1420. /* bit0-7 is the length on GEN6+ */
  1421. intel_ring_emit(ring, offset);
  1422. intel_ring_advance(ring);
  1423. return 0;
  1424. }
  1425. /* Blitter support (SandyBridge+) */
  1426. static int gen6_ring_flush(struct intel_ring_buffer *ring,
  1427. u32 invalidate, u32 flush)
  1428. {
  1429. struct drm_device *dev = ring->dev;
  1430. uint32_t cmd;
  1431. int ret;
  1432. ret = intel_ring_begin(ring, 4);
  1433. if (ret)
  1434. return ret;
  1435. cmd = MI_FLUSH_DW;
  1436. /*
  1437. * Bspec vol 1c.3 - blitter engine command streamer:
  1438. * "If ENABLED, all TLBs will be invalidated once the flush
  1439. * operation is complete. This bit is only valid when the
  1440. * Post-Sync Operation field is a value of 1h or 3h."
  1441. */
  1442. if (invalidate & I915_GEM_DOMAIN_RENDER)
  1443. cmd |= MI_INVALIDATE_TLB | MI_FLUSH_DW_STORE_INDEX |
  1444. MI_FLUSH_DW_OP_STOREDW;
  1445. intel_ring_emit(ring, cmd);
  1446. intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT);
  1447. intel_ring_emit(ring, 0);
  1448. intel_ring_emit(ring, MI_NOOP);
  1449. intel_ring_advance(ring);
  1450. if (IS_GEN7(dev) && flush)
  1451. return gen7_ring_fbc_flush(ring, FBC_REND_CACHE_CLEAN);
  1452. return 0;
  1453. }
  1454. int intel_init_render_ring_buffer(struct drm_device *dev)
  1455. {
  1456. drm_i915_private_t *dev_priv = dev->dev_private;
  1457. struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
  1458. ring->name = "render ring";
  1459. ring->id = RCS;
  1460. ring->mmio_base = RENDER_RING_BASE;
  1461. if (INTEL_INFO(dev)->gen >= 6) {
  1462. ring->add_request = gen6_add_request;
  1463. ring->flush = gen7_render_ring_flush;
  1464. if (INTEL_INFO(dev)->gen == 6)
  1465. ring->flush = gen6_render_ring_flush;
  1466. ring->irq_get = gen6_ring_get_irq;
  1467. ring->irq_put = gen6_ring_put_irq;
  1468. ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT;
  1469. ring->get_seqno = gen6_ring_get_seqno;
  1470. ring->set_seqno = ring_set_seqno;
  1471. ring->sync_to = gen6_ring_sync;
  1472. ring->semaphore_register[RCS] = MI_SEMAPHORE_SYNC_INVALID;
  1473. ring->semaphore_register[VCS] = MI_SEMAPHORE_SYNC_RV;
  1474. ring->semaphore_register[BCS] = MI_SEMAPHORE_SYNC_RB;
  1475. ring->semaphore_register[VECS] = MI_SEMAPHORE_SYNC_RVE;
  1476. ring->signal_mbox[RCS] = GEN6_NOSYNC;
  1477. ring->signal_mbox[VCS] = GEN6_VRSYNC;
  1478. ring->signal_mbox[BCS] = GEN6_BRSYNC;
  1479. ring->signal_mbox[VECS] = GEN6_VERSYNC;
  1480. } else if (IS_GEN5(dev)) {
  1481. ring->add_request = pc_render_add_request;
  1482. ring->flush = gen4_render_ring_flush;
  1483. ring->get_seqno = pc_render_get_seqno;
  1484. ring->set_seqno = pc_render_set_seqno;
  1485. ring->irq_get = gen5_ring_get_irq;
  1486. ring->irq_put = gen5_ring_put_irq;
  1487. ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT |
  1488. GT_RENDER_PIPECTL_NOTIFY_INTERRUPT;
  1489. } else {
  1490. ring->add_request = i9xx_add_request;
  1491. if (INTEL_INFO(dev)->gen < 4)
  1492. ring->flush = gen2_render_ring_flush;
  1493. else
  1494. ring->flush = gen4_render_ring_flush;
  1495. ring->get_seqno = ring_get_seqno;
  1496. ring->set_seqno = ring_set_seqno;
  1497. if (IS_GEN2(dev)) {
  1498. ring->irq_get = i8xx_ring_get_irq;
  1499. ring->irq_put = i8xx_ring_put_irq;
  1500. } else {
  1501. ring->irq_get = i9xx_ring_get_irq;
  1502. ring->irq_put = i9xx_ring_put_irq;
  1503. }
  1504. ring->irq_enable_mask = I915_USER_INTERRUPT;
  1505. }
  1506. ring->write_tail = ring_write_tail;
  1507. if (IS_HASWELL(dev))
  1508. ring->dispatch_execbuffer = hsw_ring_dispatch_execbuffer;
  1509. else if (INTEL_INFO(dev)->gen >= 6)
  1510. ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
  1511. else if (INTEL_INFO(dev)->gen >= 4)
  1512. ring->dispatch_execbuffer = i965_dispatch_execbuffer;
  1513. else if (IS_I830(dev) || IS_845G(dev))
  1514. ring->dispatch_execbuffer = i830_dispatch_execbuffer;
  1515. else
  1516. ring->dispatch_execbuffer = i915_dispatch_execbuffer;
  1517. ring->init = init_render_ring;
  1518. ring->cleanup = render_ring_cleanup;
  1519. /* Workaround batchbuffer to combat CS tlb bug. */
  1520. if (HAS_BROKEN_CS_TLB(dev)) {
  1521. struct drm_i915_gem_object *obj;
  1522. int ret;
  1523. obj = i915_gem_alloc_object(dev, I830_BATCH_LIMIT);
  1524. if (obj == NULL) {
  1525. DRM_ERROR("Failed to allocate batch bo\n");
  1526. return -ENOMEM;
  1527. }
  1528. ret = i915_gem_obj_ggtt_pin(obj, 0, true, false);
  1529. if (ret != 0) {
  1530. drm_gem_object_unreference(&obj->base);
  1531. DRM_ERROR("Failed to ping batch bo\n");
  1532. return ret;
  1533. }
  1534. ring->private = obj;
  1535. }
  1536. return intel_init_ring_buffer(dev, ring);
  1537. }
  1538. int intel_render_ring_init_dri(struct drm_device *dev, u64 start, u32 size)
  1539. {
  1540. drm_i915_private_t *dev_priv = dev->dev_private;
  1541. struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
  1542. int ret;
  1543. ring->name = "render ring";
  1544. ring->id = RCS;
  1545. ring->mmio_base = RENDER_RING_BASE;
  1546. if (INTEL_INFO(dev)->gen >= 6) {
  1547. /* non-kms not supported on gen6+ */
  1548. return -ENODEV;
  1549. }
  1550. /* Note: gem is not supported on gen5/ilk without kms (the corresponding
  1551. * gem_init ioctl returns with -ENODEV). Hence we do not need to set up
  1552. * the special gen5 functions. */
  1553. ring->add_request = i9xx_add_request;
  1554. if (INTEL_INFO(dev)->gen < 4)
  1555. ring->flush = gen2_render_ring_flush;
  1556. else
  1557. ring->flush = gen4_render_ring_flush;
  1558. ring->get_seqno = ring_get_seqno;
  1559. ring->set_seqno = ring_set_seqno;
  1560. if (IS_GEN2(dev)) {
  1561. ring->irq_get = i8xx_ring_get_irq;
  1562. ring->irq_put = i8xx_ring_put_irq;
  1563. } else {
  1564. ring->irq_get = i9xx_ring_get_irq;
  1565. ring->irq_put = i9xx_ring_put_irq;
  1566. }
  1567. ring->irq_enable_mask = I915_USER_INTERRUPT;
  1568. ring->write_tail = ring_write_tail;
  1569. if (INTEL_INFO(dev)->gen >= 4)
  1570. ring->dispatch_execbuffer = i965_dispatch_execbuffer;
  1571. else if (IS_I830(dev) || IS_845G(dev))
  1572. ring->dispatch_execbuffer = i830_dispatch_execbuffer;
  1573. else
  1574. ring->dispatch_execbuffer = i915_dispatch_execbuffer;
  1575. ring->init = init_render_ring;
  1576. ring->cleanup = render_ring_cleanup;
  1577. ring->dev = dev;
  1578. INIT_LIST_HEAD(&ring->active_list);
  1579. INIT_LIST_HEAD(&ring->request_list);
  1580. ring->size = size;
  1581. ring->effective_size = ring->size;
  1582. if (IS_I830(ring->dev) || IS_845G(ring->dev))
  1583. ring->effective_size -= 128;
  1584. ring->virtual_start = ioremap_wc(start, size);
  1585. if (ring->virtual_start == NULL) {
  1586. DRM_ERROR("can not ioremap virtual address for"
  1587. " ring buffer\n");
  1588. return -ENOMEM;
  1589. }
  1590. if (!I915_NEED_GFX_HWS(dev)) {
  1591. ret = init_phys_status_page(ring);
  1592. if (ret)
  1593. return ret;
  1594. }
  1595. return 0;
  1596. }
  1597. int intel_init_bsd_ring_buffer(struct drm_device *dev)
  1598. {
  1599. drm_i915_private_t *dev_priv = dev->dev_private;
  1600. struct intel_ring_buffer *ring = &dev_priv->ring[VCS];
  1601. ring->name = "bsd ring";
  1602. ring->id = VCS;
  1603. ring->write_tail = ring_write_tail;
  1604. if (IS_GEN6(dev) || IS_GEN7(dev)) {
  1605. ring->mmio_base = GEN6_BSD_RING_BASE;
  1606. /* gen6 bsd needs a special wa for tail updates */
  1607. if (IS_GEN6(dev))
  1608. ring->write_tail = gen6_bsd_ring_write_tail;
  1609. ring->flush = gen6_bsd_ring_flush;
  1610. ring->add_request = gen6_add_request;
  1611. ring->get_seqno = gen6_ring_get_seqno;
  1612. ring->set_seqno = ring_set_seqno;
  1613. ring->irq_enable_mask = GT_BSD_USER_INTERRUPT;
  1614. ring->irq_get = gen6_ring_get_irq;
  1615. ring->irq_put = gen6_ring_put_irq;
  1616. ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
  1617. ring->sync_to = gen6_ring_sync;
  1618. ring->semaphore_register[RCS] = MI_SEMAPHORE_SYNC_VR;
  1619. ring->semaphore_register[VCS] = MI_SEMAPHORE_SYNC_INVALID;
  1620. ring->semaphore_register[BCS] = MI_SEMAPHORE_SYNC_VB;
  1621. ring->semaphore_register[VECS] = MI_SEMAPHORE_SYNC_VVE;
  1622. ring->signal_mbox[RCS] = GEN6_RVSYNC;
  1623. ring->signal_mbox[VCS] = GEN6_NOSYNC;
  1624. ring->signal_mbox[BCS] = GEN6_BVSYNC;
  1625. ring->signal_mbox[VECS] = GEN6_VEVSYNC;
  1626. } else {
  1627. ring->mmio_base = BSD_RING_BASE;
  1628. ring->flush = bsd_ring_flush;
  1629. ring->add_request = i9xx_add_request;
  1630. ring->get_seqno = ring_get_seqno;
  1631. ring->set_seqno = ring_set_seqno;
  1632. if (IS_GEN5(dev)) {
  1633. ring->irq_enable_mask = ILK_BSD_USER_INTERRUPT;
  1634. ring->irq_get = gen5_ring_get_irq;
  1635. ring->irq_put = gen5_ring_put_irq;
  1636. } else {
  1637. ring->irq_enable_mask = I915_BSD_USER_INTERRUPT;
  1638. ring->irq_get = i9xx_ring_get_irq;
  1639. ring->irq_put = i9xx_ring_put_irq;
  1640. }
  1641. ring->dispatch_execbuffer = i965_dispatch_execbuffer;
  1642. }
  1643. ring->init = init_ring_common;
  1644. return intel_init_ring_buffer(dev, ring);
  1645. }
  1646. int intel_init_blt_ring_buffer(struct drm_device *dev)
  1647. {
  1648. drm_i915_private_t *dev_priv = dev->dev_private;
  1649. struct intel_ring_buffer *ring = &dev_priv->ring[BCS];
  1650. ring->name = "blitter ring";
  1651. ring->id = BCS;
  1652. ring->mmio_base = BLT_RING_BASE;
  1653. ring->write_tail = ring_write_tail;
  1654. ring->flush = gen6_ring_flush;
  1655. ring->add_request = gen6_add_request;
  1656. ring->get_seqno = gen6_ring_get_seqno;
  1657. ring->set_seqno = ring_set_seqno;
  1658. ring->irq_enable_mask = GT_BLT_USER_INTERRUPT;
  1659. ring->irq_get = gen6_ring_get_irq;
  1660. ring->irq_put = gen6_ring_put_irq;
  1661. ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
  1662. ring->sync_to = gen6_ring_sync;
  1663. ring->semaphore_register[RCS] = MI_SEMAPHORE_SYNC_BR;
  1664. ring->semaphore_register[VCS] = MI_SEMAPHORE_SYNC_BV;
  1665. ring->semaphore_register[BCS] = MI_SEMAPHORE_SYNC_INVALID;
  1666. ring->semaphore_register[VECS] = MI_SEMAPHORE_SYNC_BVE;
  1667. ring->signal_mbox[RCS] = GEN6_RBSYNC;
  1668. ring->signal_mbox[VCS] = GEN6_VBSYNC;
  1669. ring->signal_mbox[BCS] = GEN6_NOSYNC;
  1670. ring->signal_mbox[VECS] = GEN6_VEBSYNC;
  1671. ring->init = init_ring_common;
  1672. return intel_init_ring_buffer(dev, ring);
  1673. }
  1674. int intel_init_vebox_ring_buffer(struct drm_device *dev)
  1675. {
  1676. drm_i915_private_t *dev_priv = dev->dev_private;
  1677. struct intel_ring_buffer *ring = &dev_priv->ring[VECS];
  1678. ring->name = "video enhancement ring";
  1679. ring->id = VECS;
  1680. ring->mmio_base = VEBOX_RING_BASE;
  1681. ring->write_tail = ring_write_tail;
  1682. ring->flush = gen6_ring_flush;
  1683. ring->add_request = gen6_add_request;
  1684. ring->get_seqno = gen6_ring_get_seqno;
  1685. ring->set_seqno = ring_set_seqno;
  1686. ring->irq_enable_mask = PM_VEBOX_USER_INTERRUPT;
  1687. ring->irq_get = hsw_vebox_get_irq;
  1688. ring->irq_put = hsw_vebox_put_irq;
  1689. ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
  1690. ring->sync_to = gen6_ring_sync;
  1691. ring->semaphore_register[RCS] = MI_SEMAPHORE_SYNC_VER;
  1692. ring->semaphore_register[VCS] = MI_SEMAPHORE_SYNC_VEV;
  1693. ring->semaphore_register[BCS] = MI_SEMAPHORE_SYNC_VEB;
  1694. ring->semaphore_register[VECS] = MI_SEMAPHORE_SYNC_INVALID;
  1695. ring->signal_mbox[RCS] = GEN6_RVESYNC;
  1696. ring->signal_mbox[VCS] = GEN6_VVESYNC;
  1697. ring->signal_mbox[BCS] = GEN6_BVESYNC;
  1698. ring->signal_mbox[VECS] = GEN6_NOSYNC;
  1699. ring->init = init_ring_common;
  1700. return intel_init_ring_buffer(dev, ring);
  1701. }
  1702. int
  1703. intel_ring_flush_all_caches(struct intel_ring_buffer *ring)
  1704. {
  1705. int ret;
  1706. if (!ring->gpu_caches_dirty)
  1707. return 0;
  1708. ret = ring->flush(ring, 0, I915_GEM_GPU_DOMAINS);
  1709. if (ret)
  1710. return ret;
  1711. trace_i915_gem_ring_flush(ring, 0, I915_GEM_GPU_DOMAINS);
  1712. ring->gpu_caches_dirty = false;
  1713. return 0;
  1714. }
  1715. int
  1716. intel_ring_invalidate_all_caches(struct intel_ring_buffer *ring)
  1717. {
  1718. uint32_t flush_domains;
  1719. int ret;
  1720. flush_domains = 0;
  1721. if (ring->gpu_caches_dirty)
  1722. flush_domains = I915_GEM_GPU_DOMAINS;
  1723. ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, flush_domains);
  1724. if (ret)
  1725. return ret;
  1726. trace_i915_gem_ring_flush(ring, I915_GEM_GPU_DOMAINS, flush_domains);
  1727. ring->gpu_caches_dirty = false;
  1728. return 0;
  1729. }