intel_ringbuffer.c 33 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399
  1. /*
  2. * Copyright © 2008-2010 Intel Corporation
  3. *
  4. * Permission is hereby granted, free of charge, to any person obtaining a
  5. * copy of this software and associated documentation files (the "Software"),
  6. * to deal in the Software without restriction, including without limitation
  7. * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8. * and/or sell copies of the Software, and to permit persons to whom the
  9. * Software is furnished to do so, subject to the following conditions:
  10. *
  11. * The above copyright notice and this permission notice (including the next
  12. * paragraph) shall be included in all copies or substantial portions of the
  13. * Software.
  14. *
  15. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17. * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  18. * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19. * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20. * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21. * IN THE SOFTWARE.
  22. *
  23. * Authors:
  24. * Eric Anholt <eric@anholt.net>
  25. * Zou Nan hai <nanhai.zou@intel.com>
  26. * Xiang Hai hao<haihao.xiang@intel.com>
  27. *
  28. */
  29. #include "drmP.h"
  30. #include "drm.h"
  31. #include "i915_drv.h"
  32. #include "i915_drm.h"
  33. #include "i915_trace.h"
  34. #include "intel_drv.h"
  35. static inline int ring_space(struct intel_ring_buffer *ring)
  36. {
  37. int space = (ring->head & HEAD_ADDR) - (ring->tail + 8);
  38. if (space < 0)
  39. space += ring->size;
  40. return space;
  41. }
  42. static u32 i915_gem_get_seqno(struct drm_device *dev)
  43. {
  44. drm_i915_private_t *dev_priv = dev->dev_private;
  45. u32 seqno;
  46. seqno = dev_priv->next_seqno;
  47. /* reserve 0 for non-seqno */
  48. if (++dev_priv->next_seqno == 0)
  49. dev_priv->next_seqno = 1;
  50. return seqno;
  51. }
  52. static int
  53. render_ring_flush(struct intel_ring_buffer *ring,
  54. u32 invalidate_domains,
  55. u32 flush_domains)
  56. {
  57. struct drm_device *dev = ring->dev;
  58. u32 cmd;
  59. int ret;
  60. /*
  61. * read/write caches:
  62. *
  63. * I915_GEM_DOMAIN_RENDER is always invalidated, but is
  64. * only flushed if MI_NO_WRITE_FLUSH is unset. On 965, it is
  65. * also flushed at 2d versus 3d pipeline switches.
  66. *
  67. * read-only caches:
  68. *
  69. * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
  70. * MI_READ_FLUSH is set, and is always flushed on 965.
  71. *
  72. * I915_GEM_DOMAIN_COMMAND may not exist?
  73. *
  74. * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
  75. * invalidated when MI_EXE_FLUSH is set.
  76. *
  77. * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
  78. * invalidated with every MI_FLUSH.
  79. *
  80. * TLBs:
  81. *
  82. * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
  83. * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
  84. * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
  85. * are flushed at any MI_FLUSH.
  86. */
  87. cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
  88. if ((invalidate_domains|flush_domains) &
  89. I915_GEM_DOMAIN_RENDER)
  90. cmd &= ~MI_NO_WRITE_FLUSH;
  91. if (INTEL_INFO(dev)->gen < 4) {
  92. /*
  93. * On the 965, the sampler cache always gets flushed
  94. * and this bit is reserved.
  95. */
  96. if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
  97. cmd |= MI_READ_FLUSH;
  98. }
  99. if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
  100. cmd |= MI_EXE_FLUSH;
  101. if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&
  102. (IS_G4X(dev) || IS_GEN5(dev)))
  103. cmd |= MI_INVALIDATE_ISP;
  104. ret = intel_ring_begin(ring, 2);
  105. if (ret)
  106. return ret;
  107. intel_ring_emit(ring, cmd);
  108. intel_ring_emit(ring, MI_NOOP);
  109. intel_ring_advance(ring);
  110. return 0;
  111. }
  112. static void ring_write_tail(struct intel_ring_buffer *ring,
  113. u32 value)
  114. {
  115. drm_i915_private_t *dev_priv = ring->dev->dev_private;
  116. I915_WRITE_TAIL(ring, value);
  117. }
  118. u32 intel_ring_get_active_head(struct intel_ring_buffer *ring)
  119. {
  120. drm_i915_private_t *dev_priv = ring->dev->dev_private;
  121. u32 acthd_reg = INTEL_INFO(ring->dev)->gen >= 4 ?
  122. RING_ACTHD(ring->mmio_base) : ACTHD;
  123. return I915_READ(acthd_reg);
  124. }
  125. static int init_ring_common(struct intel_ring_buffer *ring)
  126. {
  127. drm_i915_private_t *dev_priv = ring->dev->dev_private;
  128. struct drm_i915_gem_object *obj = ring->obj;
  129. u32 head;
  130. /* Stop the ring if it's running. */
  131. I915_WRITE_CTL(ring, 0);
  132. I915_WRITE_HEAD(ring, 0);
  133. ring->write_tail(ring, 0);
  134. /* Initialize the ring. */
  135. I915_WRITE_START(ring, obj->gtt_offset);
  136. head = I915_READ_HEAD(ring) & HEAD_ADDR;
  137. /* G45 ring initialization fails to reset head to zero */
  138. if (head != 0) {
  139. DRM_DEBUG_KMS("%s head not reset to zero "
  140. "ctl %08x head %08x tail %08x start %08x\n",
  141. ring->name,
  142. I915_READ_CTL(ring),
  143. I915_READ_HEAD(ring),
  144. I915_READ_TAIL(ring),
  145. I915_READ_START(ring));
  146. I915_WRITE_HEAD(ring, 0);
  147. if (I915_READ_HEAD(ring) & HEAD_ADDR) {
  148. DRM_ERROR("failed to set %s head to zero "
  149. "ctl %08x head %08x tail %08x start %08x\n",
  150. ring->name,
  151. I915_READ_CTL(ring),
  152. I915_READ_HEAD(ring),
  153. I915_READ_TAIL(ring),
  154. I915_READ_START(ring));
  155. }
  156. }
  157. I915_WRITE_CTL(ring,
  158. ((ring->size - PAGE_SIZE) & RING_NR_PAGES)
  159. | RING_REPORT_64K | RING_VALID);
  160. /* If the head is still not zero, the ring is dead */
  161. if ((I915_READ_CTL(ring) & RING_VALID) == 0 ||
  162. I915_READ_START(ring) != obj->gtt_offset ||
  163. (I915_READ_HEAD(ring) & HEAD_ADDR) != 0) {
  164. DRM_ERROR("%s initialization failed "
  165. "ctl %08x head %08x tail %08x start %08x\n",
  166. ring->name,
  167. I915_READ_CTL(ring),
  168. I915_READ_HEAD(ring),
  169. I915_READ_TAIL(ring),
  170. I915_READ_START(ring));
  171. return -EIO;
  172. }
  173. if (!drm_core_check_feature(ring->dev, DRIVER_MODESET))
  174. i915_kernel_lost_context(ring->dev);
  175. else {
  176. ring->head = I915_READ_HEAD(ring);
  177. ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
  178. ring->space = ring_space(ring);
  179. }
  180. return 0;
  181. }
  182. /*
  183. * 965+ support PIPE_CONTROL commands, which provide finer grained control
  184. * over cache flushing.
  185. */
  186. struct pipe_control {
  187. struct drm_i915_gem_object *obj;
  188. volatile u32 *cpu_page;
  189. u32 gtt_offset;
  190. };
  191. static int
  192. init_pipe_control(struct intel_ring_buffer *ring)
  193. {
  194. struct pipe_control *pc;
  195. struct drm_i915_gem_object *obj;
  196. int ret;
  197. if (ring->private)
  198. return 0;
  199. pc = kmalloc(sizeof(*pc), GFP_KERNEL);
  200. if (!pc)
  201. return -ENOMEM;
  202. obj = i915_gem_alloc_object(ring->dev, 4096);
  203. if (obj == NULL) {
  204. DRM_ERROR("Failed to allocate seqno page\n");
  205. ret = -ENOMEM;
  206. goto err;
  207. }
  208. obj->cache_level = I915_CACHE_LLC;
  209. ret = i915_gem_object_pin(obj, 4096, true);
  210. if (ret)
  211. goto err_unref;
  212. pc->gtt_offset = obj->gtt_offset;
  213. pc->cpu_page = kmap(obj->pages[0]);
  214. if (pc->cpu_page == NULL)
  215. goto err_unpin;
  216. pc->obj = obj;
  217. ring->private = pc;
  218. return 0;
  219. err_unpin:
  220. i915_gem_object_unpin(obj);
  221. err_unref:
  222. drm_gem_object_unreference(&obj->base);
  223. err:
  224. kfree(pc);
  225. return ret;
  226. }
  227. static void
  228. cleanup_pipe_control(struct intel_ring_buffer *ring)
  229. {
  230. struct pipe_control *pc = ring->private;
  231. struct drm_i915_gem_object *obj;
  232. if (!ring->private)
  233. return;
  234. obj = pc->obj;
  235. kunmap(obj->pages[0]);
  236. i915_gem_object_unpin(obj);
  237. drm_gem_object_unreference(&obj->base);
  238. kfree(pc);
  239. ring->private = NULL;
  240. }
  241. static int init_render_ring(struct intel_ring_buffer *ring)
  242. {
  243. struct drm_device *dev = ring->dev;
  244. struct drm_i915_private *dev_priv = dev->dev_private;
  245. int ret = init_ring_common(ring);
  246. if (INTEL_INFO(dev)->gen > 3) {
  247. int mode = VS_TIMER_DISPATCH << 16 | VS_TIMER_DISPATCH;
  248. if (IS_GEN6(dev) || IS_GEN7(dev))
  249. mode |= MI_FLUSH_ENABLE << 16 | MI_FLUSH_ENABLE;
  250. I915_WRITE(MI_MODE, mode);
  251. }
  252. if (INTEL_INFO(dev)->gen >= 6) {
  253. } else if (IS_GEN5(dev)) {
  254. ret = init_pipe_control(ring);
  255. if (ret)
  256. return ret;
  257. }
  258. return ret;
  259. }
  260. static void render_ring_cleanup(struct intel_ring_buffer *ring)
  261. {
  262. if (!ring->private)
  263. return;
  264. cleanup_pipe_control(ring);
  265. }
  266. static void
  267. update_semaphore(struct intel_ring_buffer *ring, int i, u32 seqno)
  268. {
  269. struct drm_device *dev = ring->dev;
  270. struct drm_i915_private *dev_priv = dev->dev_private;
  271. int id;
  272. /*
  273. * cs -> 1 = vcs, 0 = bcs
  274. * vcs -> 1 = bcs, 0 = cs,
  275. * bcs -> 1 = cs, 0 = vcs.
  276. */
  277. id = ring - dev_priv->ring;
  278. id += 2 - i;
  279. id %= 3;
  280. intel_ring_emit(ring,
  281. MI_SEMAPHORE_MBOX |
  282. MI_SEMAPHORE_REGISTER |
  283. MI_SEMAPHORE_UPDATE);
  284. intel_ring_emit(ring, seqno);
  285. intel_ring_emit(ring,
  286. RING_SYNC_0(dev_priv->ring[id].mmio_base) + 4*i);
  287. }
  288. static int
  289. gen6_add_request(struct intel_ring_buffer *ring,
  290. u32 *result)
  291. {
  292. u32 seqno;
  293. int ret;
  294. ret = intel_ring_begin(ring, 10);
  295. if (ret)
  296. return ret;
  297. seqno = i915_gem_get_seqno(ring->dev);
  298. update_semaphore(ring, 0, seqno);
  299. update_semaphore(ring, 1, seqno);
  300. intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
  301. intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
  302. intel_ring_emit(ring, seqno);
  303. intel_ring_emit(ring, MI_USER_INTERRUPT);
  304. intel_ring_advance(ring);
  305. *result = seqno;
  306. return 0;
  307. }
  308. int
  309. intel_ring_sync(struct intel_ring_buffer *ring,
  310. struct intel_ring_buffer *to,
  311. u32 seqno)
  312. {
  313. int ret;
  314. ret = intel_ring_begin(ring, 4);
  315. if (ret)
  316. return ret;
  317. intel_ring_emit(ring,
  318. MI_SEMAPHORE_MBOX |
  319. MI_SEMAPHORE_REGISTER |
  320. intel_ring_sync_index(ring, to) << 17 |
  321. MI_SEMAPHORE_COMPARE);
  322. intel_ring_emit(ring, seqno);
  323. intel_ring_emit(ring, 0);
  324. intel_ring_emit(ring, MI_NOOP);
  325. intel_ring_advance(ring);
  326. return 0;
  327. }
  328. #define PIPE_CONTROL_FLUSH(ring__, addr__) \
  329. do { \
  330. intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE | \
  331. PIPE_CONTROL_DEPTH_STALL | 2); \
  332. intel_ring_emit(ring__, (addr__) | PIPE_CONTROL_GLOBAL_GTT); \
  333. intel_ring_emit(ring__, 0); \
  334. intel_ring_emit(ring__, 0); \
  335. } while (0)
  336. static int
  337. pc_render_add_request(struct intel_ring_buffer *ring,
  338. u32 *result)
  339. {
  340. struct drm_device *dev = ring->dev;
  341. u32 seqno = i915_gem_get_seqno(dev);
  342. struct pipe_control *pc = ring->private;
  343. u32 scratch_addr = pc->gtt_offset + 128;
  344. int ret;
  345. /* For Ironlake, MI_USER_INTERRUPT was deprecated and apparently
  346. * incoherent with writes to memory, i.e. completely fubar,
  347. * so we need to use PIPE_NOTIFY instead.
  348. *
  349. * However, we also need to workaround the qword write
  350. * incoherence by flushing the 6 PIPE_NOTIFY buffers out to
  351. * memory before requesting an interrupt.
  352. */
  353. ret = intel_ring_begin(ring, 32);
  354. if (ret)
  355. return ret;
  356. intel_ring_emit(ring, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
  357. PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH);
  358. intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
  359. intel_ring_emit(ring, seqno);
  360. intel_ring_emit(ring, 0);
  361. PIPE_CONTROL_FLUSH(ring, scratch_addr);
  362. scratch_addr += 128; /* write to separate cachelines */
  363. PIPE_CONTROL_FLUSH(ring, scratch_addr);
  364. scratch_addr += 128;
  365. PIPE_CONTROL_FLUSH(ring, scratch_addr);
  366. scratch_addr += 128;
  367. PIPE_CONTROL_FLUSH(ring, scratch_addr);
  368. scratch_addr += 128;
  369. PIPE_CONTROL_FLUSH(ring, scratch_addr);
  370. scratch_addr += 128;
  371. PIPE_CONTROL_FLUSH(ring, scratch_addr);
  372. intel_ring_emit(ring, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
  373. PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH |
  374. PIPE_CONTROL_NOTIFY);
  375. intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
  376. intel_ring_emit(ring, seqno);
  377. intel_ring_emit(ring, 0);
  378. intel_ring_advance(ring);
  379. *result = seqno;
  380. return 0;
  381. }
  382. static int
  383. render_ring_add_request(struct intel_ring_buffer *ring,
  384. u32 *result)
  385. {
  386. struct drm_device *dev = ring->dev;
  387. u32 seqno = i915_gem_get_seqno(dev);
  388. int ret;
  389. ret = intel_ring_begin(ring, 4);
  390. if (ret)
  391. return ret;
  392. intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
  393. intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
  394. intel_ring_emit(ring, seqno);
  395. intel_ring_emit(ring, MI_USER_INTERRUPT);
  396. intel_ring_advance(ring);
  397. *result = seqno;
  398. return 0;
  399. }
  400. static u32
  401. ring_get_seqno(struct intel_ring_buffer *ring)
  402. {
  403. return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
  404. }
  405. static u32
  406. pc_render_get_seqno(struct intel_ring_buffer *ring)
  407. {
  408. struct pipe_control *pc = ring->private;
  409. return pc->cpu_page[0];
  410. }
  411. static void
  412. ironlake_enable_irq(drm_i915_private_t *dev_priv, u32 mask)
  413. {
  414. dev_priv->gt_irq_mask &= ~mask;
  415. I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
  416. POSTING_READ(GTIMR);
  417. }
  418. static void
  419. ironlake_disable_irq(drm_i915_private_t *dev_priv, u32 mask)
  420. {
  421. dev_priv->gt_irq_mask |= mask;
  422. I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
  423. POSTING_READ(GTIMR);
  424. }
  425. static void
  426. i915_enable_irq(drm_i915_private_t *dev_priv, u32 mask)
  427. {
  428. dev_priv->irq_mask &= ~mask;
  429. I915_WRITE(IMR, dev_priv->irq_mask);
  430. POSTING_READ(IMR);
  431. }
  432. static void
  433. i915_disable_irq(drm_i915_private_t *dev_priv, u32 mask)
  434. {
  435. dev_priv->irq_mask |= mask;
  436. I915_WRITE(IMR, dev_priv->irq_mask);
  437. POSTING_READ(IMR);
  438. }
  439. static bool
  440. render_ring_get_irq(struct intel_ring_buffer *ring)
  441. {
  442. struct drm_device *dev = ring->dev;
  443. drm_i915_private_t *dev_priv = dev->dev_private;
  444. if (!dev->irq_enabled)
  445. return false;
  446. spin_lock(&ring->irq_lock);
  447. if (ring->irq_refcount++ == 0) {
  448. if (HAS_PCH_SPLIT(dev))
  449. ironlake_enable_irq(dev_priv,
  450. GT_PIPE_NOTIFY | GT_USER_INTERRUPT);
  451. else
  452. i915_enable_irq(dev_priv, I915_USER_INTERRUPT);
  453. }
  454. spin_unlock(&ring->irq_lock);
  455. return true;
  456. }
  457. static void
  458. render_ring_put_irq(struct intel_ring_buffer *ring)
  459. {
  460. struct drm_device *dev = ring->dev;
  461. drm_i915_private_t *dev_priv = dev->dev_private;
  462. spin_lock(&ring->irq_lock);
  463. if (--ring->irq_refcount == 0) {
  464. if (HAS_PCH_SPLIT(dev))
  465. ironlake_disable_irq(dev_priv,
  466. GT_USER_INTERRUPT |
  467. GT_PIPE_NOTIFY);
  468. else
  469. i915_disable_irq(dev_priv, I915_USER_INTERRUPT);
  470. }
  471. spin_unlock(&ring->irq_lock);
  472. }
  473. void intel_ring_setup_status_page(struct intel_ring_buffer *ring)
  474. {
  475. struct drm_device *dev = ring->dev;
  476. drm_i915_private_t *dev_priv = ring->dev->dev_private;
  477. u32 mmio = 0;
  478. /* The ring status page addresses are no longer next to the rest of
  479. * the ring registers as of gen7.
  480. */
  481. if (IS_GEN7(dev)) {
  482. switch (ring->id) {
  483. case RING_RENDER:
  484. mmio = RENDER_HWS_PGA_GEN7;
  485. break;
  486. case RING_BLT:
  487. mmio = BLT_HWS_PGA_GEN7;
  488. break;
  489. case RING_BSD:
  490. mmio = BSD_HWS_PGA_GEN7;
  491. break;
  492. }
  493. } else if (IS_GEN6(ring->dev)) {
  494. mmio = RING_HWS_PGA_GEN6(ring->mmio_base);
  495. } else {
  496. mmio = RING_HWS_PGA(ring->mmio_base);
  497. }
  498. I915_WRITE(mmio, (u32)ring->status_page.gfx_addr);
  499. POSTING_READ(mmio);
  500. }
  501. static int
  502. bsd_ring_flush(struct intel_ring_buffer *ring,
  503. u32 invalidate_domains,
  504. u32 flush_domains)
  505. {
  506. int ret;
  507. ret = intel_ring_begin(ring, 2);
  508. if (ret)
  509. return ret;
  510. intel_ring_emit(ring, MI_FLUSH);
  511. intel_ring_emit(ring, MI_NOOP);
  512. intel_ring_advance(ring);
  513. return 0;
  514. }
  515. static int
  516. ring_add_request(struct intel_ring_buffer *ring,
  517. u32 *result)
  518. {
  519. u32 seqno;
  520. int ret;
  521. ret = intel_ring_begin(ring, 4);
  522. if (ret)
  523. return ret;
  524. seqno = i915_gem_get_seqno(ring->dev);
  525. intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
  526. intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
  527. intel_ring_emit(ring, seqno);
  528. intel_ring_emit(ring, MI_USER_INTERRUPT);
  529. intel_ring_advance(ring);
  530. *result = seqno;
  531. return 0;
  532. }
  533. static bool
  534. ring_get_irq(struct intel_ring_buffer *ring, u32 flag)
  535. {
  536. struct drm_device *dev = ring->dev;
  537. drm_i915_private_t *dev_priv = dev->dev_private;
  538. if (!dev->irq_enabled)
  539. return false;
  540. spin_lock(&ring->irq_lock);
  541. if (ring->irq_refcount++ == 0)
  542. ironlake_enable_irq(dev_priv, flag);
  543. spin_unlock(&ring->irq_lock);
  544. return true;
  545. }
  546. static void
  547. ring_put_irq(struct intel_ring_buffer *ring, u32 flag)
  548. {
  549. struct drm_device *dev = ring->dev;
  550. drm_i915_private_t *dev_priv = dev->dev_private;
  551. spin_lock(&ring->irq_lock);
  552. if (--ring->irq_refcount == 0)
  553. ironlake_disable_irq(dev_priv, flag);
  554. spin_unlock(&ring->irq_lock);
  555. }
  556. static bool
  557. gen6_ring_get_irq(struct intel_ring_buffer *ring, u32 gflag, u32 rflag)
  558. {
  559. struct drm_device *dev = ring->dev;
  560. drm_i915_private_t *dev_priv = dev->dev_private;
  561. if (!dev->irq_enabled)
  562. return false;
  563. spin_lock(&ring->irq_lock);
  564. if (ring->irq_refcount++ == 0) {
  565. ring->irq_mask &= ~rflag;
  566. I915_WRITE_IMR(ring, ring->irq_mask);
  567. ironlake_enable_irq(dev_priv, gflag);
  568. }
  569. spin_unlock(&ring->irq_lock);
  570. return true;
  571. }
  572. static void
  573. gen6_ring_put_irq(struct intel_ring_buffer *ring, u32 gflag, u32 rflag)
  574. {
  575. struct drm_device *dev = ring->dev;
  576. drm_i915_private_t *dev_priv = dev->dev_private;
  577. spin_lock(&ring->irq_lock);
  578. if (--ring->irq_refcount == 0) {
  579. ring->irq_mask |= rflag;
  580. I915_WRITE_IMR(ring, ring->irq_mask);
  581. ironlake_disable_irq(dev_priv, gflag);
  582. }
  583. spin_unlock(&ring->irq_lock);
  584. }
  585. static bool
  586. bsd_ring_get_irq(struct intel_ring_buffer *ring)
  587. {
  588. struct drm_device *dev = ring->dev;
  589. drm_i915_private_t *dev_priv = dev->dev_private;
  590. if (!dev->irq_enabled)
  591. return false;
  592. spin_lock(&ring->irq_lock);
  593. if (ring->irq_refcount++ == 0) {
  594. if (IS_G4X(dev))
  595. i915_enable_irq(dev_priv, I915_BSD_USER_INTERRUPT);
  596. else
  597. ironlake_enable_irq(dev_priv, GT_BSD_USER_INTERRUPT);
  598. }
  599. spin_unlock(&ring->irq_lock);
  600. return true;
  601. }
  602. static void
  603. bsd_ring_put_irq(struct intel_ring_buffer *ring)
  604. {
  605. struct drm_device *dev = ring->dev;
  606. drm_i915_private_t *dev_priv = dev->dev_private;
  607. spin_lock(&ring->irq_lock);
  608. if (--ring->irq_refcount == 0) {
  609. if (IS_G4X(dev))
  610. i915_disable_irq(dev_priv, I915_BSD_USER_INTERRUPT);
  611. else
  612. ironlake_disable_irq(dev_priv, GT_BSD_USER_INTERRUPT);
  613. }
  614. spin_unlock(&ring->irq_lock);
  615. }
  616. static int
  617. ring_dispatch_execbuffer(struct intel_ring_buffer *ring, u32 offset, u32 length)
  618. {
  619. int ret;
  620. ret = intel_ring_begin(ring, 2);
  621. if (ret)
  622. return ret;
  623. intel_ring_emit(ring,
  624. MI_BATCH_BUFFER_START | (2 << 6) |
  625. MI_BATCH_NON_SECURE_I965);
  626. intel_ring_emit(ring, offset);
  627. intel_ring_advance(ring);
  628. return 0;
  629. }
  630. static int
  631. render_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
  632. u32 offset, u32 len)
  633. {
  634. struct drm_device *dev = ring->dev;
  635. int ret;
  636. if (IS_I830(dev) || IS_845G(dev)) {
  637. ret = intel_ring_begin(ring, 4);
  638. if (ret)
  639. return ret;
  640. intel_ring_emit(ring, MI_BATCH_BUFFER);
  641. intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE);
  642. intel_ring_emit(ring, offset + len - 8);
  643. intel_ring_emit(ring, 0);
  644. } else {
  645. ret = intel_ring_begin(ring, 2);
  646. if (ret)
  647. return ret;
  648. if (INTEL_INFO(dev)->gen >= 4) {
  649. intel_ring_emit(ring,
  650. MI_BATCH_BUFFER_START | (2 << 6) |
  651. MI_BATCH_NON_SECURE_I965);
  652. intel_ring_emit(ring, offset);
  653. } else {
  654. intel_ring_emit(ring,
  655. MI_BATCH_BUFFER_START | (2 << 6));
  656. intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE);
  657. }
  658. }
  659. intel_ring_advance(ring);
  660. return 0;
  661. }
  662. static void cleanup_status_page(struct intel_ring_buffer *ring)
  663. {
  664. drm_i915_private_t *dev_priv = ring->dev->dev_private;
  665. struct drm_i915_gem_object *obj;
  666. obj = ring->status_page.obj;
  667. if (obj == NULL)
  668. return;
  669. kunmap(obj->pages[0]);
  670. i915_gem_object_unpin(obj);
  671. drm_gem_object_unreference(&obj->base);
  672. ring->status_page.obj = NULL;
  673. memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
  674. }
  675. static int init_status_page(struct intel_ring_buffer *ring)
  676. {
  677. struct drm_device *dev = ring->dev;
  678. drm_i915_private_t *dev_priv = dev->dev_private;
  679. struct drm_i915_gem_object *obj;
  680. int ret;
  681. obj = i915_gem_alloc_object(dev, 4096);
  682. if (obj == NULL) {
  683. DRM_ERROR("Failed to allocate status page\n");
  684. ret = -ENOMEM;
  685. goto err;
  686. }
  687. obj->cache_level = I915_CACHE_LLC;
  688. ret = i915_gem_object_pin(obj, 4096, true);
  689. if (ret != 0) {
  690. goto err_unref;
  691. }
  692. ring->status_page.gfx_addr = obj->gtt_offset;
  693. ring->status_page.page_addr = kmap(obj->pages[0]);
  694. if (ring->status_page.page_addr == NULL) {
  695. memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
  696. goto err_unpin;
  697. }
  698. ring->status_page.obj = obj;
  699. memset(ring->status_page.page_addr, 0, PAGE_SIZE);
  700. intel_ring_setup_status_page(ring);
  701. DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
  702. ring->name, ring->status_page.gfx_addr);
  703. return 0;
  704. err_unpin:
  705. i915_gem_object_unpin(obj);
  706. err_unref:
  707. drm_gem_object_unreference(&obj->base);
  708. err:
  709. return ret;
  710. }
  711. int intel_init_ring_buffer(struct drm_device *dev,
  712. struct intel_ring_buffer *ring)
  713. {
  714. struct drm_i915_gem_object *obj;
  715. int ret;
  716. ring->dev = dev;
  717. INIT_LIST_HEAD(&ring->active_list);
  718. INIT_LIST_HEAD(&ring->request_list);
  719. INIT_LIST_HEAD(&ring->gpu_write_list);
  720. init_waitqueue_head(&ring->irq_queue);
  721. spin_lock_init(&ring->irq_lock);
  722. ring->irq_mask = ~0;
  723. if (I915_NEED_GFX_HWS(dev)) {
  724. ret = init_status_page(ring);
  725. if (ret)
  726. return ret;
  727. }
  728. obj = i915_gem_alloc_object(dev, ring->size);
  729. if (obj == NULL) {
  730. DRM_ERROR("Failed to allocate ringbuffer\n");
  731. ret = -ENOMEM;
  732. goto err_hws;
  733. }
  734. ring->obj = obj;
  735. ret = i915_gem_object_pin(obj, PAGE_SIZE, true);
  736. if (ret)
  737. goto err_unref;
  738. ring->map.size = ring->size;
  739. ring->map.offset = dev->agp->base + obj->gtt_offset;
  740. ring->map.type = 0;
  741. ring->map.flags = 0;
  742. ring->map.mtrr = 0;
  743. drm_core_ioremap_wc(&ring->map, dev);
  744. if (ring->map.handle == NULL) {
  745. DRM_ERROR("Failed to map ringbuffer.\n");
  746. ret = -EINVAL;
  747. goto err_unpin;
  748. }
  749. ring->virtual_start = ring->map.handle;
  750. ret = ring->init(ring);
  751. if (ret)
  752. goto err_unmap;
  753. /* Workaround an erratum on the i830 which causes a hang if
  754. * the TAIL pointer points to within the last 2 cachelines
  755. * of the buffer.
  756. */
  757. ring->effective_size = ring->size;
  758. if (IS_I830(ring->dev))
  759. ring->effective_size -= 128;
  760. return 0;
  761. err_unmap:
  762. drm_core_ioremapfree(&ring->map, dev);
  763. err_unpin:
  764. i915_gem_object_unpin(obj);
  765. err_unref:
  766. drm_gem_object_unreference(&obj->base);
  767. ring->obj = NULL;
  768. err_hws:
  769. cleanup_status_page(ring);
  770. return ret;
  771. }
  772. void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring)
  773. {
  774. struct drm_i915_private *dev_priv;
  775. int ret;
  776. if (ring->obj == NULL)
  777. return;
  778. /* Disable the ring buffer. The ring must be idle at this point */
  779. dev_priv = ring->dev->dev_private;
  780. ret = intel_wait_ring_idle(ring);
  781. if (ret)
  782. DRM_ERROR("failed to quiesce %s whilst cleaning up: %d\n",
  783. ring->name, ret);
  784. I915_WRITE_CTL(ring, 0);
  785. drm_core_ioremapfree(&ring->map, ring->dev);
  786. i915_gem_object_unpin(ring->obj);
  787. drm_gem_object_unreference(&ring->obj->base);
  788. ring->obj = NULL;
  789. if (ring->cleanup)
  790. ring->cleanup(ring);
  791. cleanup_status_page(ring);
  792. }
  793. static int intel_wrap_ring_buffer(struct intel_ring_buffer *ring)
  794. {
  795. unsigned int *virt;
  796. int rem = ring->size - ring->tail;
  797. if (ring->space < rem) {
  798. int ret = intel_wait_ring_buffer(ring, rem);
  799. if (ret)
  800. return ret;
  801. }
  802. virt = (unsigned int *)(ring->virtual_start + ring->tail);
  803. rem /= 8;
  804. while (rem--) {
  805. *virt++ = MI_NOOP;
  806. *virt++ = MI_NOOP;
  807. }
  808. ring->tail = 0;
  809. ring->space = ring_space(ring);
  810. return 0;
  811. }
  812. int intel_wait_ring_buffer(struct intel_ring_buffer *ring, int n)
  813. {
  814. struct drm_device *dev = ring->dev;
  815. struct drm_i915_private *dev_priv = dev->dev_private;
  816. unsigned long end;
  817. u32 head;
  818. /* If the reported head position has wrapped or hasn't advanced,
  819. * fallback to the slow and accurate path.
  820. */
  821. head = intel_read_status_page(ring, 4);
  822. if (head > ring->head) {
  823. ring->head = head;
  824. ring->space = ring_space(ring);
  825. if (ring->space >= n)
  826. return 0;
  827. }
  828. trace_i915_ring_wait_begin(ring);
  829. end = jiffies + 3 * HZ;
  830. do {
  831. ring->head = I915_READ_HEAD(ring);
  832. ring->space = ring_space(ring);
  833. if (ring->space >= n) {
  834. trace_i915_ring_wait_end(ring);
  835. return 0;
  836. }
  837. if (dev->primary->master) {
  838. struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
  839. if (master_priv->sarea_priv)
  840. master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
  841. }
  842. msleep(1);
  843. if (atomic_read(&dev_priv->mm.wedged))
  844. return -EAGAIN;
  845. } while (!time_after(jiffies, end));
  846. trace_i915_ring_wait_end(ring);
  847. return -EBUSY;
  848. }
  849. int intel_ring_begin(struct intel_ring_buffer *ring,
  850. int num_dwords)
  851. {
  852. struct drm_i915_private *dev_priv = ring->dev->dev_private;
  853. int n = 4*num_dwords;
  854. int ret;
  855. if (unlikely(atomic_read(&dev_priv->mm.wedged)))
  856. return -EIO;
  857. if (unlikely(ring->tail + n > ring->effective_size)) {
  858. ret = intel_wrap_ring_buffer(ring);
  859. if (unlikely(ret))
  860. return ret;
  861. }
  862. if (unlikely(ring->space < n)) {
  863. ret = intel_wait_ring_buffer(ring, n);
  864. if (unlikely(ret))
  865. return ret;
  866. }
  867. ring->space -= n;
  868. return 0;
  869. }
  870. void intel_ring_advance(struct intel_ring_buffer *ring)
  871. {
  872. ring->tail &= ring->size - 1;
  873. ring->write_tail(ring, ring->tail);
  874. }
  875. static const struct intel_ring_buffer render_ring = {
  876. .name = "render ring",
  877. .id = RING_RENDER,
  878. .mmio_base = RENDER_RING_BASE,
  879. .size = 32 * PAGE_SIZE,
  880. .init = init_render_ring,
  881. .write_tail = ring_write_tail,
  882. .flush = render_ring_flush,
  883. .add_request = render_ring_add_request,
  884. .get_seqno = ring_get_seqno,
  885. .irq_get = render_ring_get_irq,
  886. .irq_put = render_ring_put_irq,
  887. .dispatch_execbuffer = render_ring_dispatch_execbuffer,
  888. .cleanup = render_ring_cleanup,
  889. };
  890. /* ring buffer for bit-stream decoder */
  891. static const struct intel_ring_buffer bsd_ring = {
  892. .name = "bsd ring",
  893. .id = RING_BSD,
  894. .mmio_base = BSD_RING_BASE,
  895. .size = 32 * PAGE_SIZE,
  896. .init = init_ring_common,
  897. .write_tail = ring_write_tail,
  898. .flush = bsd_ring_flush,
  899. .add_request = ring_add_request,
  900. .get_seqno = ring_get_seqno,
  901. .irq_get = bsd_ring_get_irq,
  902. .irq_put = bsd_ring_put_irq,
  903. .dispatch_execbuffer = ring_dispatch_execbuffer,
  904. };
  905. static void gen6_bsd_ring_write_tail(struct intel_ring_buffer *ring,
  906. u32 value)
  907. {
  908. drm_i915_private_t *dev_priv = ring->dev->dev_private;
  909. /* Every tail move must follow the sequence below */
  910. I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
  911. GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK |
  912. GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_DISABLE);
  913. I915_WRITE(GEN6_BSD_RNCID, 0x0);
  914. if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) &
  915. GEN6_BSD_SLEEP_PSMI_CONTROL_IDLE_INDICATOR) == 0,
  916. 50))
  917. DRM_ERROR("timed out waiting for IDLE Indicator\n");
  918. I915_WRITE_TAIL(ring, value);
  919. I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
  920. GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK |
  921. GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_ENABLE);
  922. }
  923. static int gen6_ring_flush(struct intel_ring_buffer *ring,
  924. u32 invalidate, u32 flush)
  925. {
  926. uint32_t cmd;
  927. int ret;
  928. ret = intel_ring_begin(ring, 4);
  929. if (ret)
  930. return ret;
  931. cmd = MI_FLUSH_DW;
  932. if (invalidate & I915_GEM_GPU_DOMAINS)
  933. cmd |= MI_INVALIDATE_TLB | MI_INVALIDATE_BSD;
  934. intel_ring_emit(ring, cmd);
  935. intel_ring_emit(ring, 0);
  936. intel_ring_emit(ring, 0);
  937. intel_ring_emit(ring, MI_NOOP);
  938. intel_ring_advance(ring);
  939. return 0;
  940. }
  941. static int
  942. gen6_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
  943. u32 offset, u32 len)
  944. {
  945. int ret;
  946. ret = intel_ring_begin(ring, 2);
  947. if (ret)
  948. return ret;
  949. intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_NON_SECURE_I965);
  950. /* bit0-7 is the length on GEN6+ */
  951. intel_ring_emit(ring, offset);
  952. intel_ring_advance(ring);
  953. return 0;
  954. }
  955. static bool
  956. gen6_render_ring_get_irq(struct intel_ring_buffer *ring)
  957. {
  958. return gen6_ring_get_irq(ring,
  959. GT_USER_INTERRUPT,
  960. GEN6_RENDER_USER_INTERRUPT);
  961. }
  962. static void
  963. gen6_render_ring_put_irq(struct intel_ring_buffer *ring)
  964. {
  965. return gen6_ring_put_irq(ring,
  966. GT_USER_INTERRUPT,
  967. GEN6_RENDER_USER_INTERRUPT);
  968. }
  969. static bool
  970. gen6_bsd_ring_get_irq(struct intel_ring_buffer *ring)
  971. {
  972. return gen6_ring_get_irq(ring,
  973. GT_GEN6_BSD_USER_INTERRUPT,
  974. GEN6_BSD_USER_INTERRUPT);
  975. }
  976. static void
  977. gen6_bsd_ring_put_irq(struct intel_ring_buffer *ring)
  978. {
  979. return gen6_ring_put_irq(ring,
  980. GT_GEN6_BSD_USER_INTERRUPT,
  981. GEN6_BSD_USER_INTERRUPT);
  982. }
  983. /* ring buffer for Video Codec for Gen6+ */
  984. static const struct intel_ring_buffer gen6_bsd_ring = {
  985. .name = "gen6 bsd ring",
  986. .id = RING_BSD,
  987. .mmio_base = GEN6_BSD_RING_BASE,
  988. .size = 32 * PAGE_SIZE,
  989. .init = init_ring_common,
  990. .write_tail = gen6_bsd_ring_write_tail,
  991. .flush = gen6_ring_flush,
  992. .add_request = gen6_add_request,
  993. .get_seqno = ring_get_seqno,
  994. .irq_get = gen6_bsd_ring_get_irq,
  995. .irq_put = gen6_bsd_ring_put_irq,
  996. .dispatch_execbuffer = gen6_ring_dispatch_execbuffer,
  997. };
  998. /* Blitter support (SandyBridge+) */
  999. static bool
  1000. blt_ring_get_irq(struct intel_ring_buffer *ring)
  1001. {
  1002. return gen6_ring_get_irq(ring,
  1003. GT_BLT_USER_INTERRUPT,
  1004. GEN6_BLITTER_USER_INTERRUPT);
  1005. }
  1006. static void
  1007. blt_ring_put_irq(struct intel_ring_buffer *ring)
  1008. {
  1009. gen6_ring_put_irq(ring,
  1010. GT_BLT_USER_INTERRUPT,
  1011. GEN6_BLITTER_USER_INTERRUPT);
  1012. }
  1013. /* Workaround for some stepping of SNB,
  1014. * each time when BLT engine ring tail moved,
  1015. * the first command in the ring to be parsed
  1016. * should be MI_BATCH_BUFFER_START
  1017. */
  1018. #define NEED_BLT_WORKAROUND(dev) \
  1019. (IS_GEN6(dev) && (dev->pdev->revision < 8))
  1020. static inline struct drm_i915_gem_object *
  1021. to_blt_workaround(struct intel_ring_buffer *ring)
  1022. {
  1023. return ring->private;
  1024. }
  1025. static int blt_ring_init(struct intel_ring_buffer *ring)
  1026. {
  1027. if (NEED_BLT_WORKAROUND(ring->dev)) {
  1028. struct drm_i915_gem_object *obj;
  1029. u32 *ptr;
  1030. int ret;
  1031. obj = i915_gem_alloc_object(ring->dev, 4096);
  1032. if (obj == NULL)
  1033. return -ENOMEM;
  1034. ret = i915_gem_object_pin(obj, 4096, true);
  1035. if (ret) {
  1036. drm_gem_object_unreference(&obj->base);
  1037. return ret;
  1038. }
  1039. ptr = kmap(obj->pages[0]);
  1040. *ptr++ = MI_BATCH_BUFFER_END;
  1041. *ptr++ = MI_NOOP;
  1042. kunmap(obj->pages[0]);
  1043. ret = i915_gem_object_set_to_gtt_domain(obj, false);
  1044. if (ret) {
  1045. i915_gem_object_unpin(obj);
  1046. drm_gem_object_unreference(&obj->base);
  1047. return ret;
  1048. }
  1049. ring->private = obj;
  1050. }
  1051. return init_ring_common(ring);
  1052. }
  1053. static int blt_ring_begin(struct intel_ring_buffer *ring,
  1054. int num_dwords)
  1055. {
  1056. if (ring->private) {
  1057. int ret = intel_ring_begin(ring, num_dwords+2);
  1058. if (ret)
  1059. return ret;
  1060. intel_ring_emit(ring, MI_BATCH_BUFFER_START);
  1061. intel_ring_emit(ring, to_blt_workaround(ring)->gtt_offset);
  1062. return 0;
  1063. } else
  1064. return intel_ring_begin(ring, 4);
  1065. }
  1066. static int blt_ring_flush(struct intel_ring_buffer *ring,
  1067. u32 invalidate, u32 flush)
  1068. {
  1069. uint32_t cmd;
  1070. int ret;
  1071. ret = blt_ring_begin(ring, 4);
  1072. if (ret)
  1073. return ret;
  1074. cmd = MI_FLUSH_DW;
  1075. if (invalidate & I915_GEM_DOMAIN_RENDER)
  1076. cmd |= MI_INVALIDATE_TLB;
  1077. intel_ring_emit(ring, cmd);
  1078. intel_ring_emit(ring, 0);
  1079. intel_ring_emit(ring, 0);
  1080. intel_ring_emit(ring, MI_NOOP);
  1081. intel_ring_advance(ring);
  1082. return 0;
  1083. }
  1084. static void blt_ring_cleanup(struct intel_ring_buffer *ring)
  1085. {
  1086. if (!ring->private)
  1087. return;
  1088. i915_gem_object_unpin(ring->private);
  1089. drm_gem_object_unreference(ring->private);
  1090. ring->private = NULL;
  1091. }
  1092. static const struct intel_ring_buffer gen6_blt_ring = {
  1093. .name = "blt ring",
  1094. .id = RING_BLT,
  1095. .mmio_base = BLT_RING_BASE,
  1096. .size = 32 * PAGE_SIZE,
  1097. .init = blt_ring_init,
  1098. .write_tail = ring_write_tail,
  1099. .flush = blt_ring_flush,
  1100. .add_request = gen6_add_request,
  1101. .get_seqno = ring_get_seqno,
  1102. .irq_get = blt_ring_get_irq,
  1103. .irq_put = blt_ring_put_irq,
  1104. .dispatch_execbuffer = gen6_ring_dispatch_execbuffer,
  1105. .cleanup = blt_ring_cleanup,
  1106. };
  1107. int intel_init_render_ring_buffer(struct drm_device *dev)
  1108. {
  1109. drm_i915_private_t *dev_priv = dev->dev_private;
  1110. struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
  1111. *ring = render_ring;
  1112. if (INTEL_INFO(dev)->gen >= 6) {
  1113. ring->add_request = gen6_add_request;
  1114. ring->irq_get = gen6_render_ring_get_irq;
  1115. ring->irq_put = gen6_render_ring_put_irq;
  1116. } else if (IS_GEN5(dev)) {
  1117. ring->add_request = pc_render_add_request;
  1118. ring->get_seqno = pc_render_get_seqno;
  1119. }
  1120. if (!I915_NEED_GFX_HWS(dev)) {
  1121. ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
  1122. memset(ring->status_page.page_addr, 0, PAGE_SIZE);
  1123. }
  1124. return intel_init_ring_buffer(dev, ring);
  1125. }
  1126. int intel_render_ring_init_dri(struct drm_device *dev, u64 start, u32 size)
  1127. {
  1128. drm_i915_private_t *dev_priv = dev->dev_private;
  1129. struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
  1130. *ring = render_ring;
  1131. if (INTEL_INFO(dev)->gen >= 6) {
  1132. ring->add_request = gen6_add_request;
  1133. ring->irq_get = gen6_render_ring_get_irq;
  1134. ring->irq_put = gen6_render_ring_put_irq;
  1135. } else if (IS_GEN5(dev)) {
  1136. ring->add_request = pc_render_add_request;
  1137. ring->get_seqno = pc_render_get_seqno;
  1138. }
  1139. ring->dev = dev;
  1140. INIT_LIST_HEAD(&ring->active_list);
  1141. INIT_LIST_HEAD(&ring->request_list);
  1142. INIT_LIST_HEAD(&ring->gpu_write_list);
  1143. ring->size = size;
  1144. ring->effective_size = ring->size;
  1145. if (IS_I830(ring->dev))
  1146. ring->effective_size -= 128;
  1147. ring->map.offset = start;
  1148. ring->map.size = size;
  1149. ring->map.type = 0;
  1150. ring->map.flags = 0;
  1151. ring->map.mtrr = 0;
  1152. drm_core_ioremap_wc(&ring->map, dev);
  1153. if (ring->map.handle == NULL) {
  1154. DRM_ERROR("can not ioremap virtual address for"
  1155. " ring buffer\n");
  1156. return -ENOMEM;
  1157. }
  1158. ring->virtual_start = (void __force __iomem *)ring->map.handle;
  1159. return 0;
  1160. }
  1161. int intel_init_bsd_ring_buffer(struct drm_device *dev)
  1162. {
  1163. drm_i915_private_t *dev_priv = dev->dev_private;
  1164. struct intel_ring_buffer *ring = &dev_priv->ring[VCS];
  1165. if (IS_GEN6(dev) || IS_GEN7(dev))
  1166. *ring = gen6_bsd_ring;
  1167. else
  1168. *ring = bsd_ring;
  1169. return intel_init_ring_buffer(dev, ring);
  1170. }
  1171. int intel_init_blt_ring_buffer(struct drm_device *dev)
  1172. {
  1173. drm_i915_private_t *dev_priv = dev->dev_private;
  1174. struct intel_ring_buffer *ring = &dev_priv->ring[BCS];
  1175. *ring = gen6_blt_ring;
  1176. return intel_init_ring_buffer(dev, ring);
  1177. }