i915_gem_gtt.c 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913
  1. /*
  2. * Copyright © 2010 Daniel Vetter
  3. *
  4. * Permission is hereby granted, free of charge, to any person obtaining a
  5. * copy of this software and associated documentation files (the "Software"),
  6. * to deal in the Software without restriction, including without limitation
  7. * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8. * and/or sell copies of the Software, and to permit persons to whom the
  9. * Software is furnished to do so, subject to the following conditions:
  10. *
  11. * The above copyright notice and this permission notice (including the next
  12. * paragraph) shall be included in all copies or substantial portions of the
  13. * Software.
  14. *
  15. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17. * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  18. * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19. * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20. * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21. * IN THE SOFTWARE.
  22. *
  23. */
  24. #include <drm/drmP.h>
  25. #include <drm/i915_drm.h>
  26. #include "i915_drv.h"
  27. #include "i915_trace.h"
  28. #include "intel_drv.h"
  29. #define GEN6_PPGTT_PD_ENTRIES 512
  30. #define I915_PPGTT_PT_ENTRIES (PAGE_SIZE / sizeof(gen6_gtt_pte_t))
  31. /* PPGTT stuff */
  32. #define GEN6_GTT_ADDR_ENCODE(addr) ((addr) | (((addr) >> 28) & 0xff0))
  33. #define HSW_GTT_ADDR_ENCODE(addr) ((addr) | (((addr) >> 28) & 0x7f0))
  34. #define GEN6_PDE_VALID (1 << 0)
  35. /* gen6+ has bit 11-4 for physical addr bit 39-32 */
  36. #define GEN6_PDE_ADDR_ENCODE(addr) GEN6_GTT_ADDR_ENCODE(addr)
  37. #define GEN6_PTE_VALID (1 << 0)
  38. #define GEN6_PTE_UNCACHED (1 << 1)
  39. #define HSW_PTE_UNCACHED (0)
  40. #define GEN6_PTE_CACHE_LLC (2 << 1)
  41. #define GEN6_PTE_CACHE_LLC_MLC (3 << 1)
  42. #define GEN6_PTE_ADDR_ENCODE(addr) GEN6_GTT_ADDR_ENCODE(addr)
  43. #define HSW_PTE_ADDR_ENCODE(addr) HSW_GTT_ADDR_ENCODE(addr)
  44. /* Cacheability Control is a 4-bit value. The low three bits are stored in *
  45. * bits 3:1 of the PTE, while the fourth bit is stored in bit 11 of the PTE.
  46. */
  47. #define HSW_CACHEABILITY_CONTROL(bits) ((((bits) & 0x7) << 1) | \
  48. (((bits) & 0x8) << (11 - 3)))
  49. #define HSW_WB_LLC_AGE0 HSW_CACHEABILITY_CONTROL(0x3)
  50. #define HSW_WB_ELLC_LLC_AGE0 HSW_CACHEABILITY_CONTROL(0xb)
  51. static gen6_gtt_pte_t gen6_pte_encode(dma_addr_t addr,
  52. enum i915_cache_level level)
  53. {
  54. gen6_gtt_pte_t pte = GEN6_PTE_VALID;
  55. pte |= GEN6_PTE_ADDR_ENCODE(addr);
  56. switch (level) {
  57. case I915_CACHE_LLC_MLC:
  58. pte |= GEN6_PTE_CACHE_LLC_MLC;
  59. break;
  60. case I915_CACHE_LLC:
  61. pte |= GEN6_PTE_CACHE_LLC;
  62. break;
  63. case I915_CACHE_NONE:
  64. pte |= GEN6_PTE_UNCACHED;
  65. break;
  66. default:
  67. BUG();
  68. }
  69. return pte;
  70. }
  71. #define BYT_PTE_WRITEABLE (1 << 1)
  72. #define BYT_PTE_SNOOPED_BY_CPU_CACHES (1 << 2)
  73. static gen6_gtt_pte_t byt_pte_encode(dma_addr_t addr,
  74. enum i915_cache_level level)
  75. {
  76. gen6_gtt_pte_t pte = GEN6_PTE_VALID;
  77. pte |= GEN6_PTE_ADDR_ENCODE(addr);
  78. /* Mark the page as writeable. Other platforms don't have a
  79. * setting for read-only/writable, so this matches that behavior.
  80. */
  81. pte |= BYT_PTE_WRITEABLE;
  82. if (level != I915_CACHE_NONE)
  83. pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES;
  84. return pte;
  85. }
  86. static gen6_gtt_pte_t hsw_pte_encode(dma_addr_t addr,
  87. enum i915_cache_level level)
  88. {
  89. gen6_gtt_pte_t pte = GEN6_PTE_VALID;
  90. pte |= HSW_PTE_ADDR_ENCODE(addr);
  91. if (level != I915_CACHE_NONE)
  92. pte |= HSW_WB_LLC_AGE0;
  93. return pte;
  94. }
  95. static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr,
  96. enum i915_cache_level level)
  97. {
  98. gen6_gtt_pte_t pte = GEN6_PTE_VALID;
  99. pte |= HSW_PTE_ADDR_ENCODE(addr);
  100. if (level != I915_CACHE_NONE)
  101. pte |= HSW_WB_ELLC_LLC_AGE0;
  102. return pte;
  103. }
  104. static void gen6_write_pdes(struct i915_hw_ppgtt *ppgtt)
  105. {
  106. struct drm_i915_private *dev_priv = ppgtt->base.dev->dev_private;
  107. gen6_gtt_pte_t __iomem *pd_addr;
  108. uint32_t pd_entry;
  109. int i;
  110. WARN_ON(ppgtt->pd_offset & 0x3f);
  111. pd_addr = (gen6_gtt_pte_t __iomem*)dev_priv->gtt.gsm +
  112. ppgtt->pd_offset / sizeof(gen6_gtt_pte_t);
  113. for (i = 0; i < ppgtt->num_pd_entries; i++) {
  114. dma_addr_t pt_addr;
  115. pt_addr = ppgtt->pt_dma_addr[i];
  116. pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr);
  117. pd_entry |= GEN6_PDE_VALID;
  118. writel(pd_entry, pd_addr + i);
  119. }
  120. readl(pd_addr);
  121. }
  122. static int gen6_ppgtt_enable(struct drm_device *dev)
  123. {
  124. drm_i915_private_t *dev_priv = dev->dev_private;
  125. uint32_t pd_offset;
  126. struct intel_ring_buffer *ring;
  127. struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
  128. int i;
  129. BUG_ON(ppgtt->pd_offset & 0x3f);
  130. gen6_write_pdes(ppgtt);
  131. pd_offset = ppgtt->pd_offset;
  132. pd_offset /= 64; /* in cachelines, */
  133. pd_offset <<= 16;
  134. if (INTEL_INFO(dev)->gen == 6) {
  135. uint32_t ecochk, gab_ctl, ecobits;
  136. ecobits = I915_READ(GAC_ECO_BITS);
  137. I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT |
  138. ECOBITS_PPGTT_CACHE64B);
  139. gab_ctl = I915_READ(GAB_CTL);
  140. I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT);
  141. ecochk = I915_READ(GAM_ECOCHK);
  142. I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT |
  143. ECOCHK_PPGTT_CACHE64B);
  144. I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
  145. } else if (INTEL_INFO(dev)->gen >= 7) {
  146. uint32_t ecochk, ecobits;
  147. ecobits = I915_READ(GAC_ECO_BITS);
  148. I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B);
  149. ecochk = I915_READ(GAM_ECOCHK);
  150. if (IS_HASWELL(dev)) {
  151. ecochk |= ECOCHK_PPGTT_WB_HSW;
  152. } else {
  153. ecochk |= ECOCHK_PPGTT_LLC_IVB;
  154. ecochk &= ~ECOCHK_PPGTT_GFDT_IVB;
  155. }
  156. I915_WRITE(GAM_ECOCHK, ecochk);
  157. /* GFX_MODE is per-ring on gen7+ */
  158. }
  159. for_each_ring(ring, dev_priv, i) {
  160. if (INTEL_INFO(dev)->gen >= 7)
  161. I915_WRITE(RING_MODE_GEN7(ring),
  162. _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
  163. I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
  164. I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset);
  165. }
  166. return 0;
  167. }
  168. /* PPGTT support for Sandybdrige/Gen6 and later */
  169. static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
  170. unsigned first_entry,
  171. unsigned num_entries)
  172. {
  173. struct i915_hw_ppgtt *ppgtt =
  174. container_of(vm, struct i915_hw_ppgtt, base);
  175. gen6_gtt_pte_t *pt_vaddr, scratch_pte;
  176. unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES;
  177. unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
  178. unsigned last_pte, i;
  179. scratch_pte = vm->pte_encode(vm->scratch.addr, I915_CACHE_LLC);
  180. while (num_entries) {
  181. last_pte = first_pte + num_entries;
  182. if (last_pte > I915_PPGTT_PT_ENTRIES)
  183. last_pte = I915_PPGTT_PT_ENTRIES;
  184. pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pt]);
  185. for (i = first_pte; i < last_pte; i++)
  186. pt_vaddr[i] = scratch_pte;
  187. kunmap_atomic(pt_vaddr);
  188. num_entries -= last_pte - first_pte;
  189. first_pte = 0;
  190. act_pt++;
  191. }
  192. }
  193. static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
  194. struct sg_table *pages,
  195. unsigned first_entry,
  196. enum i915_cache_level cache_level)
  197. {
  198. struct i915_hw_ppgtt *ppgtt =
  199. container_of(vm, struct i915_hw_ppgtt, base);
  200. gen6_gtt_pte_t *pt_vaddr;
  201. unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES;
  202. unsigned act_pte = first_entry % I915_PPGTT_PT_ENTRIES;
  203. struct sg_page_iter sg_iter;
  204. pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pt]);
  205. for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) {
  206. dma_addr_t page_addr;
  207. page_addr = sg_page_iter_dma_address(&sg_iter);
  208. pt_vaddr[act_pte] = vm->pte_encode(page_addr, cache_level);
  209. if (++act_pte == I915_PPGTT_PT_ENTRIES) {
  210. kunmap_atomic(pt_vaddr);
  211. act_pt++;
  212. pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pt]);
  213. act_pte = 0;
  214. }
  215. }
  216. kunmap_atomic(pt_vaddr);
  217. }
  218. static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
  219. {
  220. struct i915_hw_ppgtt *ppgtt =
  221. container_of(vm, struct i915_hw_ppgtt, base);
  222. int i;
  223. drm_mm_takedown(&ppgtt->base.mm);
  224. if (ppgtt->pt_dma_addr) {
  225. for (i = 0; i < ppgtt->num_pd_entries; i++)
  226. pci_unmap_page(ppgtt->base.dev->pdev,
  227. ppgtt->pt_dma_addr[i],
  228. 4096, PCI_DMA_BIDIRECTIONAL);
  229. }
  230. kfree(ppgtt->pt_dma_addr);
  231. for (i = 0; i < ppgtt->num_pd_entries; i++)
  232. __free_page(ppgtt->pt_pages[i]);
  233. kfree(ppgtt->pt_pages);
  234. kfree(ppgtt);
  235. }
  236. static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
  237. {
  238. struct drm_device *dev = ppgtt->base.dev;
  239. struct drm_i915_private *dev_priv = dev->dev_private;
  240. unsigned first_pd_entry_in_global_pt;
  241. int i;
  242. int ret = -ENOMEM;
  243. /* ppgtt PDEs reside in the global gtt pagetable, which has 512*1024
  244. * entries. For aliasing ppgtt support we just steal them at the end for
  245. * now. */
  246. first_pd_entry_in_global_pt = gtt_total_entries(dev_priv->gtt);
  247. ppgtt->base.pte_encode = dev_priv->gtt.base.pte_encode;
  248. ppgtt->num_pd_entries = GEN6_PPGTT_PD_ENTRIES;
  249. ppgtt->enable = gen6_ppgtt_enable;
  250. ppgtt->base.clear_range = gen6_ppgtt_clear_range;
  251. ppgtt->base.insert_entries = gen6_ppgtt_insert_entries;
  252. ppgtt->base.cleanup = gen6_ppgtt_cleanup;
  253. ppgtt->base.scratch = dev_priv->gtt.base.scratch;
  254. ppgtt->pt_pages = kzalloc(sizeof(struct page *)*ppgtt->num_pd_entries,
  255. GFP_KERNEL);
  256. if (!ppgtt->pt_pages)
  257. return -ENOMEM;
  258. for (i = 0; i < ppgtt->num_pd_entries; i++) {
  259. ppgtt->pt_pages[i] = alloc_page(GFP_KERNEL);
  260. if (!ppgtt->pt_pages[i])
  261. goto err_pt_alloc;
  262. }
  263. ppgtt->pt_dma_addr = kzalloc(sizeof(dma_addr_t) *ppgtt->num_pd_entries,
  264. GFP_KERNEL);
  265. if (!ppgtt->pt_dma_addr)
  266. goto err_pt_alloc;
  267. for (i = 0; i < ppgtt->num_pd_entries; i++) {
  268. dma_addr_t pt_addr;
  269. pt_addr = pci_map_page(dev->pdev, ppgtt->pt_pages[i], 0, 4096,
  270. PCI_DMA_BIDIRECTIONAL);
  271. if (pci_dma_mapping_error(dev->pdev, pt_addr)) {
  272. ret = -EIO;
  273. goto err_pd_pin;
  274. }
  275. ppgtt->pt_dma_addr[i] = pt_addr;
  276. }
  277. ppgtt->base.clear_range(&ppgtt->base, 0,
  278. ppgtt->num_pd_entries * I915_PPGTT_PT_ENTRIES);
  279. ppgtt->pd_offset = first_pd_entry_in_global_pt * sizeof(gen6_gtt_pte_t);
  280. return 0;
  281. err_pd_pin:
  282. if (ppgtt->pt_dma_addr) {
  283. for (i--; i >= 0; i--)
  284. pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i],
  285. 4096, PCI_DMA_BIDIRECTIONAL);
  286. }
  287. err_pt_alloc:
  288. kfree(ppgtt->pt_dma_addr);
  289. for (i = 0; i < ppgtt->num_pd_entries; i++) {
  290. if (ppgtt->pt_pages[i])
  291. __free_page(ppgtt->pt_pages[i]);
  292. }
  293. kfree(ppgtt->pt_pages);
  294. return ret;
  295. }
  296. static int i915_gem_init_aliasing_ppgtt(struct drm_device *dev)
  297. {
  298. struct drm_i915_private *dev_priv = dev->dev_private;
  299. struct i915_hw_ppgtt *ppgtt;
  300. int ret;
  301. ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
  302. if (!ppgtt)
  303. return -ENOMEM;
  304. ppgtt->base.dev = dev;
  305. if (INTEL_INFO(dev)->gen < 8)
  306. ret = gen6_ppgtt_init(ppgtt);
  307. else
  308. BUG();
  309. if (ret)
  310. kfree(ppgtt);
  311. else {
  312. dev_priv->mm.aliasing_ppgtt = ppgtt;
  313. drm_mm_init(&ppgtt->base.mm, ppgtt->base.start,
  314. ppgtt->base.total);
  315. }
  316. return ret;
  317. }
  318. void i915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev)
  319. {
  320. struct drm_i915_private *dev_priv = dev->dev_private;
  321. struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
  322. if (!ppgtt)
  323. return;
  324. ppgtt->base.cleanup(&ppgtt->base);
  325. dev_priv->mm.aliasing_ppgtt = NULL;
  326. }
  327. void i915_ppgtt_bind_object(struct i915_hw_ppgtt *ppgtt,
  328. struct drm_i915_gem_object *obj,
  329. enum i915_cache_level cache_level)
  330. {
  331. ppgtt->base.insert_entries(&ppgtt->base, obj->pages,
  332. i915_gem_obj_ggtt_offset(obj) >> PAGE_SHIFT,
  333. cache_level);
  334. }
  335. void i915_ppgtt_unbind_object(struct i915_hw_ppgtt *ppgtt,
  336. struct drm_i915_gem_object *obj)
  337. {
  338. ppgtt->base.clear_range(&ppgtt->base,
  339. i915_gem_obj_ggtt_offset(obj) >> PAGE_SHIFT,
  340. obj->base.size >> PAGE_SHIFT);
  341. }
  342. extern int intel_iommu_gfx_mapped;
  343. /* Certain Gen5 chipsets require require idling the GPU before
  344. * unmapping anything from the GTT when VT-d is enabled.
  345. */
  346. static inline bool needs_idle_maps(struct drm_device *dev)
  347. {
  348. #ifdef CONFIG_INTEL_IOMMU
  349. /* Query intel_iommu to see if we need the workaround. Presumably that
  350. * was loaded first.
  351. */
  352. if (IS_GEN5(dev) && IS_MOBILE(dev) && intel_iommu_gfx_mapped)
  353. return true;
  354. #endif
  355. return false;
  356. }
  357. static bool do_idling(struct drm_i915_private *dev_priv)
  358. {
  359. bool ret = dev_priv->mm.interruptible;
  360. if (unlikely(dev_priv->gtt.do_idle_maps)) {
  361. dev_priv->mm.interruptible = false;
  362. if (i915_gpu_idle(dev_priv->dev)) {
  363. DRM_ERROR("Couldn't idle GPU\n");
  364. /* Wait a bit, in hopes it avoids the hang */
  365. udelay(10);
  366. }
  367. }
  368. return ret;
  369. }
  370. static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible)
  371. {
  372. if (unlikely(dev_priv->gtt.do_idle_maps))
  373. dev_priv->mm.interruptible = interruptible;
  374. }
  375. void i915_gem_restore_gtt_mappings(struct drm_device *dev)
  376. {
  377. struct drm_i915_private *dev_priv = dev->dev_private;
  378. struct drm_i915_gem_object *obj;
  379. /* First fill our portion of the GTT with scratch pages */
  380. dev_priv->gtt.base.clear_range(&dev_priv->gtt.base,
  381. dev_priv->gtt.base.start / PAGE_SIZE,
  382. dev_priv->gtt.base.total / PAGE_SIZE);
  383. list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
  384. i915_gem_clflush_object(obj);
  385. i915_gem_gtt_bind_object(obj, obj->cache_level);
  386. }
  387. i915_gem_chipset_flush(dev);
  388. }
  389. int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj)
  390. {
  391. if (obj->has_dma_mapping)
  392. return 0;
  393. if (!dma_map_sg(&obj->base.dev->pdev->dev,
  394. obj->pages->sgl, obj->pages->nents,
  395. PCI_DMA_BIDIRECTIONAL))
  396. return -ENOSPC;
  397. return 0;
  398. }
  399. /*
  400. * Binds an object into the global gtt with the specified cache level. The object
  401. * will be accessible to the GPU via commands whose operands reference offsets
  402. * within the global GTT as well as accessible by the GPU through the GMADR
  403. * mapped BAR (dev_priv->mm.gtt->gtt).
  404. */
  405. static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
  406. struct sg_table *st,
  407. unsigned int first_entry,
  408. enum i915_cache_level level)
  409. {
  410. struct drm_i915_private *dev_priv = vm->dev->dev_private;
  411. gen6_gtt_pte_t __iomem *gtt_entries =
  412. (gen6_gtt_pte_t __iomem *)dev_priv->gtt.gsm + first_entry;
  413. int i = 0;
  414. struct sg_page_iter sg_iter;
  415. dma_addr_t addr;
  416. for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) {
  417. addr = sg_page_iter_dma_address(&sg_iter);
  418. iowrite32(vm->pte_encode(addr, level), &gtt_entries[i]);
  419. i++;
  420. }
  421. /* XXX: This serves as a posting read to make sure that the PTE has
  422. * actually been updated. There is some concern that even though
  423. * registers and PTEs are within the same BAR that they are potentially
  424. * of NUMA access patterns. Therefore, even with the way we assume
  425. * hardware should work, we must keep this posting read for paranoia.
  426. */
  427. if (i != 0)
  428. WARN_ON(readl(&gtt_entries[i-1]) !=
  429. vm->pte_encode(addr, level));
  430. /* This next bit makes the above posting read even more important. We
  431. * want to flush the TLBs only after we're certain all the PTE updates
  432. * have finished.
  433. */
  434. I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
  435. POSTING_READ(GFX_FLSH_CNTL_GEN6);
  436. }
  437. static void gen6_ggtt_clear_range(struct i915_address_space *vm,
  438. unsigned int first_entry,
  439. unsigned int num_entries)
  440. {
  441. struct drm_i915_private *dev_priv = vm->dev->dev_private;
  442. gen6_gtt_pte_t scratch_pte, __iomem *gtt_base =
  443. (gen6_gtt_pte_t __iomem *) dev_priv->gtt.gsm + first_entry;
  444. const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry;
  445. int i;
  446. if (WARN(num_entries > max_entries,
  447. "First entry = %d; Num entries = %d (max=%d)\n",
  448. first_entry, num_entries, max_entries))
  449. num_entries = max_entries;
  450. scratch_pte = vm->pte_encode(vm->scratch.addr, I915_CACHE_LLC);
  451. for (i = 0; i < num_entries; i++)
  452. iowrite32(scratch_pte, &gtt_base[i]);
  453. readl(gtt_base);
  454. }
  455. static void i915_ggtt_insert_entries(struct i915_address_space *vm,
  456. struct sg_table *st,
  457. unsigned int pg_start,
  458. enum i915_cache_level cache_level)
  459. {
  460. unsigned int flags = (cache_level == I915_CACHE_NONE) ?
  461. AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
  462. intel_gtt_insert_sg_entries(st, pg_start, flags);
  463. }
  464. static void i915_ggtt_clear_range(struct i915_address_space *vm,
  465. unsigned int first_entry,
  466. unsigned int num_entries)
  467. {
  468. intel_gtt_clear_range(first_entry, num_entries);
  469. }
  470. void i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj,
  471. enum i915_cache_level cache_level)
  472. {
  473. struct drm_device *dev = obj->base.dev;
  474. struct drm_i915_private *dev_priv = dev->dev_private;
  475. const unsigned long entry = i915_gem_obj_ggtt_offset(obj) >> PAGE_SHIFT;
  476. dev_priv->gtt.base.insert_entries(&dev_priv->gtt.base, obj->pages,
  477. entry,
  478. cache_level);
  479. obj->has_global_gtt_mapping = 1;
  480. }
  481. void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj)
  482. {
  483. struct drm_device *dev = obj->base.dev;
  484. struct drm_i915_private *dev_priv = dev->dev_private;
  485. const unsigned long entry = i915_gem_obj_ggtt_offset(obj) >> PAGE_SHIFT;
  486. dev_priv->gtt.base.clear_range(&dev_priv->gtt.base,
  487. entry,
  488. obj->base.size >> PAGE_SHIFT);
  489. obj->has_global_gtt_mapping = 0;
  490. }
  491. void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj)
  492. {
  493. struct drm_device *dev = obj->base.dev;
  494. struct drm_i915_private *dev_priv = dev->dev_private;
  495. bool interruptible;
  496. interruptible = do_idling(dev_priv);
  497. if (!obj->has_dma_mapping)
  498. dma_unmap_sg(&dev->pdev->dev,
  499. obj->pages->sgl, obj->pages->nents,
  500. PCI_DMA_BIDIRECTIONAL);
  501. undo_idling(dev_priv, interruptible);
  502. }
  503. static void i915_gtt_color_adjust(struct drm_mm_node *node,
  504. unsigned long color,
  505. unsigned long *start,
  506. unsigned long *end)
  507. {
  508. if (node->color != color)
  509. *start += 4096;
  510. if (!list_empty(&node->node_list)) {
  511. node = list_entry(node->node_list.next,
  512. struct drm_mm_node,
  513. node_list);
  514. if (node->allocated && node->color != color)
  515. *end -= 4096;
  516. }
  517. }
  518. void i915_gem_setup_global_gtt(struct drm_device *dev,
  519. unsigned long start,
  520. unsigned long mappable_end,
  521. unsigned long end)
  522. {
  523. /* Let GEM Manage all of the aperture.
  524. *
  525. * However, leave one page at the end still bound to the scratch page.
  526. * There are a number of places where the hardware apparently prefetches
  527. * past the end of the object, and we've seen multiple hangs with the
  528. * GPU head pointer stuck in a batchbuffer bound at the last page of the
  529. * aperture. One page should be enough to keep any prefetching inside
  530. * of the aperture.
  531. */
  532. drm_i915_private_t *dev_priv = dev->dev_private;
  533. struct drm_mm_node *entry;
  534. struct drm_i915_gem_object *obj;
  535. unsigned long hole_start, hole_end;
  536. BUG_ON(mappable_end > end);
  537. /* Subtract the guard page ... */
  538. drm_mm_init(&dev_priv->gtt.base.mm, start, end - start - PAGE_SIZE);
  539. if (!HAS_LLC(dev))
  540. dev_priv->gtt.base.mm.color_adjust = i915_gtt_color_adjust;
  541. /* Mark any preallocated objects as occupied */
  542. list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
  543. struct i915_vma *vma = __i915_gem_obj_to_vma(obj);
  544. int ret;
  545. DRM_DEBUG_KMS("reserving preallocated space: %lx + %zx\n",
  546. i915_gem_obj_ggtt_offset(obj), obj->base.size);
  547. WARN_ON(i915_gem_obj_ggtt_bound(obj));
  548. ret = drm_mm_reserve_node(&dev_priv->gtt.base.mm, &vma->node);
  549. if (ret)
  550. DRM_DEBUG_KMS("Reservation failed\n");
  551. obj->has_global_gtt_mapping = 1;
  552. list_add(&vma->vma_link, &obj->vma_list);
  553. }
  554. dev_priv->gtt.base.start = start;
  555. dev_priv->gtt.base.total = end - start;
  556. /* Clear any non-preallocated blocks */
  557. drm_mm_for_each_hole(entry, &dev_priv->gtt.base.mm,
  558. hole_start, hole_end) {
  559. const unsigned long count = (hole_end - hole_start) / PAGE_SIZE;
  560. DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n",
  561. hole_start, hole_end);
  562. dev_priv->gtt.base.clear_range(&dev_priv->gtt.base,
  563. hole_start / PAGE_SIZE,
  564. count);
  565. }
  566. /* And finally clear the reserved guard page */
  567. dev_priv->gtt.base.clear_range(&dev_priv->gtt.base,
  568. end / PAGE_SIZE - 1, 1);
  569. }
  570. static bool
  571. intel_enable_ppgtt(struct drm_device *dev)
  572. {
  573. if (i915_enable_ppgtt >= 0)
  574. return i915_enable_ppgtt;
  575. #ifdef CONFIG_INTEL_IOMMU
  576. /* Disable ppgtt on SNB if VT-d is on. */
  577. if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped)
  578. return false;
  579. #endif
  580. return true;
  581. }
  582. void i915_gem_init_global_gtt(struct drm_device *dev)
  583. {
  584. struct drm_i915_private *dev_priv = dev->dev_private;
  585. unsigned long gtt_size, mappable_size;
  586. gtt_size = dev_priv->gtt.base.total;
  587. mappable_size = dev_priv->gtt.mappable_end;
  588. if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) {
  589. int ret;
  590. if (INTEL_INFO(dev)->gen <= 7) {
  591. /* PPGTT pdes are stolen from global gtt ptes, so shrink the
  592. * aperture accordingly when using aliasing ppgtt. */
  593. gtt_size -= GEN6_PPGTT_PD_ENTRIES * PAGE_SIZE;
  594. }
  595. i915_gem_setup_global_gtt(dev, 0, mappable_size, gtt_size);
  596. ret = i915_gem_init_aliasing_ppgtt(dev);
  597. if (!ret)
  598. return;
  599. DRM_ERROR("Aliased PPGTT setup failed %d\n", ret);
  600. drm_mm_takedown(&dev_priv->gtt.base.mm);
  601. gtt_size += GEN6_PPGTT_PD_ENTRIES * PAGE_SIZE;
  602. }
  603. i915_gem_setup_global_gtt(dev, 0, mappable_size, gtt_size);
  604. }
  605. static int setup_scratch_page(struct drm_device *dev)
  606. {
  607. struct drm_i915_private *dev_priv = dev->dev_private;
  608. struct page *page;
  609. dma_addr_t dma_addr;
  610. page = alloc_page(GFP_KERNEL | GFP_DMA32 | __GFP_ZERO);
  611. if (page == NULL)
  612. return -ENOMEM;
  613. get_page(page);
  614. set_pages_uc(page, 1);
  615. #ifdef CONFIG_INTEL_IOMMU
  616. dma_addr = pci_map_page(dev->pdev, page, 0, PAGE_SIZE,
  617. PCI_DMA_BIDIRECTIONAL);
  618. if (pci_dma_mapping_error(dev->pdev, dma_addr))
  619. return -EINVAL;
  620. #else
  621. dma_addr = page_to_phys(page);
  622. #endif
  623. dev_priv->gtt.base.scratch.page = page;
  624. dev_priv->gtt.base.scratch.addr = dma_addr;
  625. return 0;
  626. }
  627. static void teardown_scratch_page(struct drm_device *dev)
  628. {
  629. struct drm_i915_private *dev_priv = dev->dev_private;
  630. struct page *page = dev_priv->gtt.base.scratch.page;
  631. set_pages_wb(page, 1);
  632. pci_unmap_page(dev->pdev, dev_priv->gtt.base.scratch.addr,
  633. PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
  634. put_page(page);
  635. __free_page(page);
  636. }
  637. static inline unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl)
  638. {
  639. snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT;
  640. snb_gmch_ctl &= SNB_GMCH_GGMS_MASK;
  641. return snb_gmch_ctl << 20;
  642. }
  643. static inline size_t gen6_get_stolen_size(u16 snb_gmch_ctl)
  644. {
  645. snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT;
  646. snb_gmch_ctl &= SNB_GMCH_GMS_MASK;
  647. return snb_gmch_ctl << 25; /* 32 MB units */
  648. }
  649. static int gen6_gmch_probe(struct drm_device *dev,
  650. size_t *gtt_total,
  651. size_t *stolen,
  652. phys_addr_t *mappable_base,
  653. unsigned long *mappable_end)
  654. {
  655. struct drm_i915_private *dev_priv = dev->dev_private;
  656. phys_addr_t gtt_bus_addr;
  657. unsigned int gtt_size;
  658. u16 snb_gmch_ctl;
  659. int ret;
  660. *mappable_base = pci_resource_start(dev->pdev, 2);
  661. *mappable_end = pci_resource_len(dev->pdev, 2);
  662. /* 64/512MB is the current min/max we actually know of, but this is just
  663. * a coarse sanity check.
  664. */
  665. if ((*mappable_end < (64<<20) || (*mappable_end > (512<<20)))) {
  666. DRM_ERROR("Unknown GMADR size (%lx)\n",
  667. dev_priv->gtt.mappable_end);
  668. return -ENXIO;
  669. }
  670. if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(40)))
  671. pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(40));
  672. pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
  673. gtt_size = gen6_get_total_gtt_size(snb_gmch_ctl);
  674. *stolen = gen6_get_stolen_size(snb_gmch_ctl);
  675. *gtt_total = (gtt_size / sizeof(gen6_gtt_pte_t)) << PAGE_SHIFT;
  676. /* For Modern GENs the PTEs and register space are split in the BAR */
  677. gtt_bus_addr = pci_resource_start(dev->pdev, 0) +
  678. (pci_resource_len(dev->pdev, 0) / 2);
  679. dev_priv->gtt.gsm = ioremap_wc(gtt_bus_addr, gtt_size);
  680. if (!dev_priv->gtt.gsm) {
  681. DRM_ERROR("Failed to map the gtt page table\n");
  682. return -ENOMEM;
  683. }
  684. ret = setup_scratch_page(dev);
  685. if (ret)
  686. DRM_ERROR("Scratch setup failed\n");
  687. dev_priv->gtt.base.clear_range = gen6_ggtt_clear_range;
  688. dev_priv->gtt.base.insert_entries = gen6_ggtt_insert_entries;
  689. return ret;
  690. }
  691. static void gen6_gmch_remove(struct i915_address_space *vm)
  692. {
  693. struct i915_gtt *gtt = container_of(vm, struct i915_gtt, base);
  694. iounmap(gtt->gsm);
  695. teardown_scratch_page(vm->dev);
  696. }
  697. static int i915_gmch_probe(struct drm_device *dev,
  698. size_t *gtt_total,
  699. size_t *stolen,
  700. phys_addr_t *mappable_base,
  701. unsigned long *mappable_end)
  702. {
  703. struct drm_i915_private *dev_priv = dev->dev_private;
  704. int ret;
  705. ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->dev->pdev, NULL);
  706. if (!ret) {
  707. DRM_ERROR("failed to set up gmch\n");
  708. return -EIO;
  709. }
  710. intel_gtt_get(gtt_total, stolen, mappable_base, mappable_end);
  711. dev_priv->gtt.do_idle_maps = needs_idle_maps(dev_priv->dev);
  712. dev_priv->gtt.base.clear_range = i915_ggtt_clear_range;
  713. dev_priv->gtt.base.insert_entries = i915_ggtt_insert_entries;
  714. return 0;
  715. }
  716. static void i915_gmch_remove(struct i915_address_space *vm)
  717. {
  718. intel_gmch_remove();
  719. }
  720. int i915_gem_gtt_init(struct drm_device *dev)
  721. {
  722. struct drm_i915_private *dev_priv = dev->dev_private;
  723. struct i915_gtt *gtt = &dev_priv->gtt;
  724. int ret;
  725. if (INTEL_INFO(dev)->gen <= 5) {
  726. gtt->gtt_probe = i915_gmch_probe;
  727. gtt->base.cleanup = i915_gmch_remove;
  728. } else {
  729. gtt->gtt_probe = gen6_gmch_probe;
  730. gtt->base.cleanup = gen6_gmch_remove;
  731. if (IS_HASWELL(dev) && dev_priv->ellc_size)
  732. gtt->base.pte_encode = iris_pte_encode;
  733. else if (IS_HASWELL(dev))
  734. gtt->base.pte_encode = hsw_pte_encode;
  735. else if (IS_VALLEYVIEW(dev))
  736. gtt->base.pte_encode = byt_pte_encode;
  737. else
  738. gtt->base.pte_encode = gen6_pte_encode;
  739. }
  740. ret = gtt->gtt_probe(dev, &gtt->base.total, &gtt->stolen_size,
  741. &gtt->mappable_base, &gtt->mappable_end);
  742. if (ret)
  743. return ret;
  744. gtt->base.dev = dev;
  745. /* GMADR is the PCI mmio aperture into the global GTT. */
  746. DRM_INFO("Memory usable by graphics device = %zdM\n",
  747. gtt->base.total >> 20);
  748. DRM_DEBUG_DRIVER("GMADR size = %ldM\n", gtt->mappable_end >> 20);
  749. DRM_DEBUG_DRIVER("GTT stolen size = %zdM\n", gtt->stolen_size >> 20);
  750. return 0;
  751. }