i915_gem_execbuffer.c 30 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154
  1. /*
  2. * Copyright © 2008,2010 Intel Corporation
  3. *
  4. * Permission is hereby granted, free of charge, to any person obtaining a
  5. * copy of this software and associated documentation files (the "Software"),
  6. * to deal in the Software without restriction, including without limitation
  7. * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8. * and/or sell copies of the Software, and to permit persons to whom the
  9. * Software is furnished to do so, subject to the following conditions:
  10. *
  11. * The above copyright notice and this permission notice (including the next
  12. * paragraph) shall be included in all copies or substantial portions of the
  13. * Software.
  14. *
  15. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17. * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  18. * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19. * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20. * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21. * IN THE SOFTWARE.
  22. *
  23. * Authors:
  24. * Eric Anholt <eric@anholt.net>
  25. * Chris Wilson <chris@chris-wilson.co.uk>
  26. *
  27. */
  28. #include <drm/drmP.h>
  29. #include <drm/i915_drm.h>
  30. #include "i915_drv.h"
  31. #include "i915_trace.h"
  32. #include "intel_drv.h"
  33. #include <linux/dma_remapping.h>
  34. struct eb_objects {
  35. struct list_head objects;
  36. int and;
  37. struct hlist_head buckets[0];
  38. };
  39. static struct eb_objects *
  40. eb_create(int size)
  41. {
  42. struct eb_objects *eb;
  43. int count = PAGE_SIZE / sizeof(struct hlist_head) / 2;
  44. BUILD_BUG_ON(!is_power_of_2(PAGE_SIZE / sizeof(struct hlist_head)));
  45. while (count > size)
  46. count >>= 1;
  47. eb = kzalloc(count*sizeof(struct hlist_head) +
  48. sizeof(struct eb_objects),
  49. GFP_KERNEL);
  50. if (eb == NULL)
  51. return eb;
  52. eb->and = count - 1;
  53. INIT_LIST_HEAD(&eb->objects);
  54. return eb;
  55. }
  56. static void
  57. eb_reset(struct eb_objects *eb)
  58. {
  59. memset(eb->buckets, 0, (eb->and+1)*sizeof(struct hlist_head));
  60. }
  61. static void
  62. eb_add_object(struct eb_objects *eb, struct drm_i915_gem_object *obj)
  63. {
  64. hlist_add_head(&obj->exec_node,
  65. &eb->buckets[obj->exec_handle & eb->and]);
  66. }
  67. static int
  68. eb_lookup_objects(struct eb_objects *eb,
  69. struct drm_i915_gem_exec_object2 *exec,
  70. int count,
  71. struct drm_file *file)
  72. {
  73. int i;
  74. spin_lock(&file->table_lock);
  75. for (i = 0; i < count; i++) {
  76. struct drm_i915_gem_object *obj;
  77. obj = to_intel_bo(idr_find(&file->object_idr, exec[i].handle));
  78. if (obj == NULL) {
  79. spin_unlock(&file->table_lock);
  80. DRM_DEBUG("Invalid object handle %d at index %d\n",
  81. exec[i].handle, i);
  82. return -ENOENT;
  83. }
  84. if (!list_empty(&obj->exec_list)) {
  85. spin_unlock(&file->table_lock);
  86. DRM_DEBUG("Object %p [handle %d, index %d] appears more than once in object list\n",
  87. obj, exec[i].handle, i);
  88. return -EINVAL;
  89. }
  90. drm_gem_object_reference(&obj->base);
  91. list_add_tail(&obj->exec_list, &eb->objects);
  92. obj->exec_handle = exec[i].handle;
  93. obj->exec_entry = &exec[i];
  94. eb_add_object(eb, obj);
  95. }
  96. spin_unlock(&file->table_lock);
  97. return 0;
  98. }
  99. static struct drm_i915_gem_object *
  100. eb_get_object(struct eb_objects *eb, unsigned long handle)
  101. {
  102. struct hlist_head *head;
  103. struct hlist_node *node;
  104. struct drm_i915_gem_object *obj;
  105. head = &eb->buckets[handle & eb->and];
  106. hlist_for_each(node, head) {
  107. obj = hlist_entry(node, struct drm_i915_gem_object, exec_node);
  108. if (obj->exec_handle == handle)
  109. return obj;
  110. }
  111. return NULL;
  112. }
  113. static void
  114. eb_destroy(struct eb_objects *eb)
  115. {
  116. while (!list_empty(&eb->objects)) {
  117. struct drm_i915_gem_object *obj;
  118. obj = list_first_entry(&eb->objects,
  119. struct drm_i915_gem_object,
  120. exec_list);
  121. list_del_init(&obj->exec_list);
  122. drm_gem_object_unreference(&obj->base);
  123. }
  124. kfree(eb);
  125. }
  126. static inline int use_cpu_reloc(struct drm_i915_gem_object *obj)
  127. {
  128. return (obj->base.write_domain == I915_GEM_DOMAIN_CPU ||
  129. !obj->map_and_fenceable ||
  130. obj->cache_level != I915_CACHE_NONE);
  131. }
  132. static int
  133. i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
  134. struct eb_objects *eb,
  135. struct drm_i915_gem_relocation_entry *reloc)
  136. {
  137. struct drm_device *dev = obj->base.dev;
  138. struct drm_gem_object *target_obj;
  139. struct drm_i915_gem_object *target_i915_obj;
  140. uint32_t target_offset;
  141. int ret = -EINVAL;
  142. /* we've already hold a reference to all valid objects */
  143. target_obj = &eb_get_object(eb, reloc->target_handle)->base;
  144. if (unlikely(target_obj == NULL))
  145. return -ENOENT;
  146. target_i915_obj = to_intel_bo(target_obj);
  147. target_offset = target_i915_obj->gtt_offset;
  148. /* Sandybridge PPGTT errata: We need a global gtt mapping for MI and
  149. * pipe_control writes because the gpu doesn't properly redirect them
  150. * through the ppgtt for non_secure batchbuffers. */
  151. if (unlikely(IS_GEN6(dev) &&
  152. reloc->write_domain == I915_GEM_DOMAIN_INSTRUCTION &&
  153. !target_i915_obj->has_global_gtt_mapping)) {
  154. i915_gem_gtt_bind_object(target_i915_obj,
  155. target_i915_obj->cache_level);
  156. }
  157. /* Validate that the target is in a valid r/w GPU domain */
  158. if (unlikely(reloc->write_domain & (reloc->write_domain - 1))) {
  159. DRM_DEBUG("reloc with multiple write domains: "
  160. "obj %p target %d offset %d "
  161. "read %08x write %08x",
  162. obj, reloc->target_handle,
  163. (int) reloc->offset,
  164. reloc->read_domains,
  165. reloc->write_domain);
  166. return ret;
  167. }
  168. if (unlikely((reloc->write_domain | reloc->read_domains)
  169. & ~I915_GEM_GPU_DOMAINS)) {
  170. DRM_DEBUG("reloc with read/write non-GPU domains: "
  171. "obj %p target %d offset %d "
  172. "read %08x write %08x",
  173. obj, reloc->target_handle,
  174. (int) reloc->offset,
  175. reloc->read_domains,
  176. reloc->write_domain);
  177. return ret;
  178. }
  179. target_obj->pending_read_domains |= reloc->read_domains;
  180. target_obj->pending_write_domain |= reloc->write_domain;
  181. /* If the relocation already has the right value in it, no
  182. * more work needs to be done.
  183. */
  184. if (target_offset == reloc->presumed_offset)
  185. return 0;
  186. /* Check that the relocation address is valid... */
  187. if (unlikely(reloc->offset > obj->base.size - 4)) {
  188. DRM_DEBUG("Relocation beyond object bounds: "
  189. "obj %p target %d offset %d size %d.\n",
  190. obj, reloc->target_handle,
  191. (int) reloc->offset,
  192. (int) obj->base.size);
  193. return ret;
  194. }
  195. if (unlikely(reloc->offset & 3)) {
  196. DRM_DEBUG("Relocation not 4-byte aligned: "
  197. "obj %p target %d offset %d.\n",
  198. obj, reloc->target_handle,
  199. (int) reloc->offset);
  200. return ret;
  201. }
  202. /* We can't wait for rendering with pagefaults disabled */
  203. if (obj->active && in_atomic())
  204. return -EFAULT;
  205. reloc->delta += target_offset;
  206. if (use_cpu_reloc(obj)) {
  207. uint32_t page_offset = reloc->offset & ~PAGE_MASK;
  208. char *vaddr;
  209. ret = i915_gem_object_set_to_cpu_domain(obj, 1);
  210. if (ret)
  211. return ret;
  212. vaddr = kmap_atomic(i915_gem_object_get_page(obj,
  213. reloc->offset >> PAGE_SHIFT));
  214. *(uint32_t *)(vaddr + page_offset) = reloc->delta;
  215. kunmap_atomic(vaddr);
  216. } else {
  217. struct drm_i915_private *dev_priv = dev->dev_private;
  218. uint32_t __iomem *reloc_entry;
  219. void __iomem *reloc_page;
  220. ret = i915_gem_object_set_to_gtt_domain(obj, true);
  221. if (ret)
  222. return ret;
  223. ret = i915_gem_object_put_fence(obj);
  224. if (ret)
  225. return ret;
  226. /* Map the page containing the relocation we're going to perform. */
  227. reloc->offset += obj->gtt_offset;
  228. reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
  229. reloc->offset & PAGE_MASK);
  230. reloc_entry = (uint32_t __iomem *)
  231. (reloc_page + (reloc->offset & ~PAGE_MASK));
  232. iowrite32(reloc->delta, reloc_entry);
  233. io_mapping_unmap_atomic(reloc_page);
  234. }
  235. /* and update the user's relocation entry */
  236. reloc->presumed_offset = target_offset;
  237. return 0;
  238. }
  239. static int
  240. i915_gem_execbuffer_relocate_object(struct drm_i915_gem_object *obj,
  241. struct eb_objects *eb)
  242. {
  243. #define N_RELOC(x) ((x) / sizeof(struct drm_i915_gem_relocation_entry))
  244. struct drm_i915_gem_relocation_entry stack_reloc[N_RELOC(512)];
  245. struct drm_i915_gem_relocation_entry __user *user_relocs;
  246. struct drm_i915_gem_exec_object2 *entry = obj->exec_entry;
  247. int remain, ret;
  248. user_relocs = (void __user *)(uintptr_t)entry->relocs_ptr;
  249. remain = entry->relocation_count;
  250. while (remain) {
  251. struct drm_i915_gem_relocation_entry *r = stack_reloc;
  252. int count = remain;
  253. if (count > ARRAY_SIZE(stack_reloc))
  254. count = ARRAY_SIZE(stack_reloc);
  255. remain -= count;
  256. if (__copy_from_user_inatomic(r, user_relocs, count*sizeof(r[0])))
  257. return -EFAULT;
  258. do {
  259. u64 offset = r->presumed_offset;
  260. ret = i915_gem_execbuffer_relocate_entry(obj, eb, r);
  261. if (ret)
  262. return ret;
  263. if (r->presumed_offset != offset &&
  264. __copy_to_user_inatomic(&user_relocs->presumed_offset,
  265. &r->presumed_offset,
  266. sizeof(r->presumed_offset))) {
  267. return -EFAULT;
  268. }
  269. user_relocs++;
  270. r++;
  271. } while (--count);
  272. }
  273. return 0;
  274. #undef N_RELOC
  275. }
  276. static int
  277. i915_gem_execbuffer_relocate_object_slow(struct drm_i915_gem_object *obj,
  278. struct eb_objects *eb,
  279. struct drm_i915_gem_relocation_entry *relocs)
  280. {
  281. const struct drm_i915_gem_exec_object2 *entry = obj->exec_entry;
  282. int i, ret;
  283. for (i = 0; i < entry->relocation_count; i++) {
  284. ret = i915_gem_execbuffer_relocate_entry(obj, eb, &relocs[i]);
  285. if (ret)
  286. return ret;
  287. }
  288. return 0;
  289. }
  290. static int
  291. i915_gem_execbuffer_relocate(struct drm_device *dev,
  292. struct eb_objects *eb)
  293. {
  294. struct drm_i915_gem_object *obj;
  295. int ret = 0;
  296. /* This is the fast path and we cannot handle a pagefault whilst
  297. * holding the struct mutex lest the user pass in the relocations
  298. * contained within a mmaped bo. For in such a case we, the page
  299. * fault handler would call i915_gem_fault() and we would try to
  300. * acquire the struct mutex again. Obviously this is bad and so
  301. * lockdep complains vehemently.
  302. */
  303. pagefault_disable();
  304. list_for_each_entry(obj, &eb->objects, exec_list) {
  305. ret = i915_gem_execbuffer_relocate_object(obj, eb);
  306. if (ret)
  307. break;
  308. }
  309. pagefault_enable();
  310. return ret;
  311. }
  312. #define __EXEC_OBJECT_HAS_PIN (1<<31)
  313. #define __EXEC_OBJECT_HAS_FENCE (1<<30)
  314. static int
  315. need_reloc_mappable(struct drm_i915_gem_object *obj)
  316. {
  317. struct drm_i915_gem_exec_object2 *entry = obj->exec_entry;
  318. return entry->relocation_count && !use_cpu_reloc(obj);
  319. }
  320. static int
  321. i915_gem_execbuffer_reserve_object(struct drm_i915_gem_object *obj,
  322. struct intel_ring_buffer *ring)
  323. {
  324. struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  325. struct drm_i915_gem_exec_object2 *entry = obj->exec_entry;
  326. bool has_fenced_gpu_access = INTEL_INFO(ring->dev)->gen < 4;
  327. bool need_fence, need_mappable;
  328. int ret;
  329. need_fence =
  330. has_fenced_gpu_access &&
  331. entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
  332. obj->tiling_mode != I915_TILING_NONE;
  333. need_mappable = need_fence || need_reloc_mappable(obj);
  334. ret = i915_gem_object_pin(obj, entry->alignment, need_mappable, false);
  335. if (ret)
  336. return ret;
  337. entry->flags |= __EXEC_OBJECT_HAS_PIN;
  338. if (has_fenced_gpu_access) {
  339. if (entry->flags & EXEC_OBJECT_NEEDS_FENCE) {
  340. ret = i915_gem_object_get_fence(obj);
  341. if (ret)
  342. return ret;
  343. if (i915_gem_object_pin_fence(obj))
  344. entry->flags |= __EXEC_OBJECT_HAS_FENCE;
  345. obj->pending_fenced_gpu_access = true;
  346. }
  347. }
  348. /* Ensure ppgtt mapping exists if needed */
  349. if (dev_priv->mm.aliasing_ppgtt && !obj->has_aliasing_ppgtt_mapping) {
  350. i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
  351. obj, obj->cache_level);
  352. obj->has_aliasing_ppgtt_mapping = 1;
  353. }
  354. entry->offset = obj->gtt_offset;
  355. return 0;
  356. }
  357. static void
  358. i915_gem_execbuffer_unreserve_object(struct drm_i915_gem_object *obj)
  359. {
  360. struct drm_i915_gem_exec_object2 *entry;
  361. if (!obj->gtt_space)
  362. return;
  363. entry = obj->exec_entry;
  364. if (entry->flags & __EXEC_OBJECT_HAS_FENCE)
  365. i915_gem_object_unpin_fence(obj);
  366. if (entry->flags & __EXEC_OBJECT_HAS_PIN)
  367. i915_gem_object_unpin(obj);
  368. entry->flags &= ~(__EXEC_OBJECT_HAS_FENCE | __EXEC_OBJECT_HAS_PIN);
  369. }
  370. static int
  371. i915_gem_execbuffer_reserve(struct intel_ring_buffer *ring,
  372. struct drm_file *file,
  373. struct list_head *objects)
  374. {
  375. struct drm_i915_gem_object *obj;
  376. struct list_head ordered_objects;
  377. bool has_fenced_gpu_access = INTEL_INFO(ring->dev)->gen < 4;
  378. int retry;
  379. INIT_LIST_HEAD(&ordered_objects);
  380. while (!list_empty(objects)) {
  381. struct drm_i915_gem_exec_object2 *entry;
  382. bool need_fence, need_mappable;
  383. obj = list_first_entry(objects,
  384. struct drm_i915_gem_object,
  385. exec_list);
  386. entry = obj->exec_entry;
  387. need_fence =
  388. has_fenced_gpu_access &&
  389. entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
  390. obj->tiling_mode != I915_TILING_NONE;
  391. need_mappable = need_fence || need_reloc_mappable(obj);
  392. if (need_mappable)
  393. list_move(&obj->exec_list, &ordered_objects);
  394. else
  395. list_move_tail(&obj->exec_list, &ordered_objects);
  396. obj->base.pending_read_domains = 0;
  397. obj->base.pending_write_domain = 0;
  398. obj->pending_fenced_gpu_access = false;
  399. }
  400. list_splice(&ordered_objects, objects);
  401. /* Attempt to pin all of the buffers into the GTT.
  402. * This is done in 3 phases:
  403. *
  404. * 1a. Unbind all objects that do not match the GTT constraints for
  405. * the execbuffer (fenceable, mappable, alignment etc).
  406. * 1b. Increment pin count for already bound objects.
  407. * 2. Bind new objects.
  408. * 3. Decrement pin count.
  409. *
  410. * This avoid unnecessary unbinding of later objects in order to make
  411. * room for the earlier objects *unless* we need to defragment.
  412. */
  413. retry = 0;
  414. do {
  415. int ret = 0;
  416. /* Unbind any ill-fitting objects or pin. */
  417. list_for_each_entry(obj, objects, exec_list) {
  418. struct drm_i915_gem_exec_object2 *entry = obj->exec_entry;
  419. bool need_fence, need_mappable;
  420. if (!obj->gtt_space)
  421. continue;
  422. need_fence =
  423. has_fenced_gpu_access &&
  424. entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
  425. obj->tiling_mode != I915_TILING_NONE;
  426. need_mappable = need_fence || need_reloc_mappable(obj);
  427. if ((entry->alignment && obj->gtt_offset & (entry->alignment - 1)) ||
  428. (need_mappable && !obj->map_and_fenceable))
  429. ret = i915_gem_object_unbind(obj);
  430. else
  431. ret = i915_gem_execbuffer_reserve_object(obj, ring);
  432. if (ret)
  433. goto err;
  434. }
  435. /* Bind fresh objects */
  436. list_for_each_entry(obj, objects, exec_list) {
  437. if (obj->gtt_space)
  438. continue;
  439. ret = i915_gem_execbuffer_reserve_object(obj, ring);
  440. if (ret)
  441. goto err;
  442. }
  443. err: /* Decrement pin count for bound objects */
  444. list_for_each_entry(obj, objects, exec_list)
  445. i915_gem_execbuffer_unreserve_object(obj);
  446. if (ret != -ENOSPC || retry++)
  447. return ret;
  448. ret = i915_gem_evict_everything(ring->dev);
  449. if (ret)
  450. return ret;
  451. } while (1);
  452. }
  453. static int
  454. i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
  455. struct drm_file *file,
  456. struct intel_ring_buffer *ring,
  457. struct eb_objects *eb,
  458. struct drm_i915_gem_exec_object2 *exec,
  459. int count)
  460. {
  461. struct drm_i915_gem_relocation_entry *reloc;
  462. struct drm_i915_gem_object *obj;
  463. int *reloc_offset;
  464. int i, total, ret;
  465. /* We may process another execbuffer during the unlock... */
  466. while (!list_empty(&eb->objects)) {
  467. obj = list_first_entry(&eb->objects,
  468. struct drm_i915_gem_object,
  469. exec_list);
  470. list_del_init(&obj->exec_list);
  471. drm_gem_object_unreference(&obj->base);
  472. }
  473. mutex_unlock(&dev->struct_mutex);
  474. total = 0;
  475. for (i = 0; i < count; i++)
  476. total += exec[i].relocation_count;
  477. reloc_offset = drm_malloc_ab(count, sizeof(*reloc_offset));
  478. reloc = drm_malloc_ab(total, sizeof(*reloc));
  479. if (reloc == NULL || reloc_offset == NULL) {
  480. drm_free_large(reloc);
  481. drm_free_large(reloc_offset);
  482. mutex_lock(&dev->struct_mutex);
  483. return -ENOMEM;
  484. }
  485. total = 0;
  486. for (i = 0; i < count; i++) {
  487. struct drm_i915_gem_relocation_entry __user *user_relocs;
  488. user_relocs = (void __user *)(uintptr_t)exec[i].relocs_ptr;
  489. if (copy_from_user(reloc+total, user_relocs,
  490. exec[i].relocation_count * sizeof(*reloc))) {
  491. ret = -EFAULT;
  492. mutex_lock(&dev->struct_mutex);
  493. goto err;
  494. }
  495. reloc_offset[i] = total;
  496. total += exec[i].relocation_count;
  497. }
  498. ret = i915_mutex_lock_interruptible(dev);
  499. if (ret) {
  500. mutex_lock(&dev->struct_mutex);
  501. goto err;
  502. }
  503. /* reacquire the objects */
  504. eb_reset(eb);
  505. ret = eb_lookup_objects(eb, exec, count, file);
  506. if (ret)
  507. goto err;
  508. ret = i915_gem_execbuffer_reserve(ring, file, &eb->objects);
  509. if (ret)
  510. goto err;
  511. list_for_each_entry(obj, &eb->objects, exec_list) {
  512. int offset = obj->exec_entry - exec;
  513. ret = i915_gem_execbuffer_relocate_object_slow(obj, eb,
  514. reloc + reloc_offset[offset]);
  515. if (ret)
  516. goto err;
  517. }
  518. /* Leave the user relocations as are, this is the painfully slow path,
  519. * and we want to avoid the complication of dropping the lock whilst
  520. * having buffers reserved in the aperture and so causing spurious
  521. * ENOSPC for random operations.
  522. */
  523. err:
  524. drm_free_large(reloc);
  525. drm_free_large(reloc_offset);
  526. return ret;
  527. }
  528. static int
  529. i915_gem_execbuffer_move_to_gpu(struct intel_ring_buffer *ring,
  530. struct list_head *objects)
  531. {
  532. struct drm_i915_gem_object *obj;
  533. uint32_t flush_domains = 0;
  534. int ret;
  535. list_for_each_entry(obj, objects, exec_list) {
  536. ret = i915_gem_object_sync(obj, ring);
  537. if (ret)
  538. return ret;
  539. if (obj->base.write_domain & I915_GEM_DOMAIN_CPU)
  540. i915_gem_clflush_object(obj);
  541. flush_domains |= obj->base.write_domain;
  542. }
  543. if (flush_domains & I915_GEM_DOMAIN_CPU)
  544. i915_gem_chipset_flush(ring->dev);
  545. if (flush_domains & I915_GEM_DOMAIN_GTT)
  546. wmb();
  547. /* Unconditionally invalidate gpu caches and ensure that we do flush
  548. * any residual writes from the previous batch.
  549. */
  550. return intel_ring_invalidate_all_caches(ring);
  551. }
  552. static bool
  553. i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec)
  554. {
  555. return ((exec->batch_start_offset | exec->batch_len) & 0x7) == 0;
  556. }
  557. static int
  558. validate_exec_list(struct drm_i915_gem_exec_object2 *exec,
  559. int count)
  560. {
  561. int i;
  562. for (i = 0; i < count; i++) {
  563. char __user *ptr = (char __user *)(uintptr_t)exec[i].relocs_ptr;
  564. int length; /* limited by fault_in_pages_readable() */
  565. /* First check for malicious input causing overflow */
  566. if (exec[i].relocation_count >
  567. INT_MAX / sizeof(struct drm_i915_gem_relocation_entry))
  568. return -EINVAL;
  569. length = exec[i].relocation_count *
  570. sizeof(struct drm_i915_gem_relocation_entry);
  571. if (!access_ok(VERIFY_READ, ptr, length))
  572. return -EFAULT;
  573. /* we may also need to update the presumed offsets */
  574. if (!access_ok(VERIFY_WRITE, ptr, length))
  575. return -EFAULT;
  576. if (fault_in_multipages_readable(ptr, length))
  577. return -EFAULT;
  578. }
  579. return 0;
  580. }
  581. static void
  582. i915_gem_execbuffer_move_to_active(struct list_head *objects,
  583. struct intel_ring_buffer *ring)
  584. {
  585. struct drm_i915_gem_object *obj;
  586. list_for_each_entry(obj, objects, exec_list) {
  587. u32 old_read = obj->base.read_domains;
  588. u32 old_write = obj->base.write_domain;
  589. obj->base.read_domains = obj->base.pending_read_domains;
  590. obj->base.write_domain = obj->base.pending_write_domain;
  591. obj->fenced_gpu_access = obj->pending_fenced_gpu_access;
  592. i915_gem_object_move_to_active(obj, ring);
  593. if (obj->base.write_domain) {
  594. obj->dirty = 1;
  595. obj->last_write_seqno = intel_ring_get_seqno(ring);
  596. if (obj->pin_count) /* check for potential scanout */
  597. intel_mark_fb_busy(obj);
  598. }
  599. trace_i915_gem_object_change_domain(obj, old_read, old_write);
  600. }
  601. }
  602. static void
  603. i915_gem_execbuffer_retire_commands(struct drm_device *dev,
  604. struct drm_file *file,
  605. struct intel_ring_buffer *ring)
  606. {
  607. /* Unconditionally force add_request to emit a full flush. */
  608. ring->gpu_caches_dirty = true;
  609. /* Add a breadcrumb for the completion of the batch buffer */
  610. (void)i915_add_request(ring, file, NULL);
  611. }
  612. static int
  613. i915_reset_gen7_sol_offsets(struct drm_device *dev,
  614. struct intel_ring_buffer *ring)
  615. {
  616. drm_i915_private_t *dev_priv = dev->dev_private;
  617. int ret, i;
  618. if (!IS_GEN7(dev) || ring != &dev_priv->ring[RCS])
  619. return 0;
  620. ret = intel_ring_begin(ring, 4 * 3);
  621. if (ret)
  622. return ret;
  623. for (i = 0; i < 4; i++) {
  624. intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
  625. intel_ring_emit(ring, GEN7_SO_WRITE_OFFSET(i));
  626. intel_ring_emit(ring, 0);
  627. }
  628. intel_ring_advance(ring);
  629. return 0;
  630. }
  631. static int
  632. i915_gem_do_execbuffer(struct drm_device *dev, void *data,
  633. struct drm_file *file,
  634. struct drm_i915_gem_execbuffer2 *args,
  635. struct drm_i915_gem_exec_object2 *exec)
  636. {
  637. drm_i915_private_t *dev_priv = dev->dev_private;
  638. struct eb_objects *eb;
  639. struct drm_i915_gem_object *batch_obj;
  640. struct drm_clip_rect *cliprects = NULL;
  641. struct intel_ring_buffer *ring;
  642. u32 ctx_id = i915_execbuffer2_get_context_id(*args);
  643. u32 exec_start, exec_len;
  644. u32 mask;
  645. u32 flags;
  646. int ret, mode, i;
  647. if (!i915_gem_check_execbuffer(args)) {
  648. DRM_DEBUG("execbuf with invalid offset/length\n");
  649. return -EINVAL;
  650. }
  651. ret = validate_exec_list(exec, args->buffer_count);
  652. if (ret)
  653. return ret;
  654. flags = 0;
  655. if (args->flags & I915_EXEC_SECURE) {
  656. if (!file->is_master || !capable(CAP_SYS_ADMIN))
  657. return -EPERM;
  658. flags |= I915_DISPATCH_SECURE;
  659. }
  660. if (args->flags & I915_EXEC_IS_PINNED)
  661. flags |= I915_DISPATCH_PINNED;
  662. switch (args->flags & I915_EXEC_RING_MASK) {
  663. case I915_EXEC_DEFAULT:
  664. case I915_EXEC_RENDER:
  665. ring = &dev_priv->ring[RCS];
  666. break;
  667. case I915_EXEC_BSD:
  668. ring = &dev_priv->ring[VCS];
  669. if (ctx_id != 0) {
  670. DRM_DEBUG("Ring %s doesn't support contexts\n",
  671. ring->name);
  672. return -EPERM;
  673. }
  674. break;
  675. case I915_EXEC_BLT:
  676. ring = &dev_priv->ring[BCS];
  677. if (ctx_id != 0) {
  678. DRM_DEBUG("Ring %s doesn't support contexts\n",
  679. ring->name);
  680. return -EPERM;
  681. }
  682. break;
  683. default:
  684. DRM_DEBUG("execbuf with unknown ring: %d\n",
  685. (int)(args->flags & I915_EXEC_RING_MASK));
  686. return -EINVAL;
  687. }
  688. if (!intel_ring_initialized(ring)) {
  689. DRM_DEBUG("execbuf with invalid ring: %d\n",
  690. (int)(args->flags & I915_EXEC_RING_MASK));
  691. return -EINVAL;
  692. }
  693. mode = args->flags & I915_EXEC_CONSTANTS_MASK;
  694. mask = I915_EXEC_CONSTANTS_MASK;
  695. switch (mode) {
  696. case I915_EXEC_CONSTANTS_REL_GENERAL:
  697. case I915_EXEC_CONSTANTS_ABSOLUTE:
  698. case I915_EXEC_CONSTANTS_REL_SURFACE:
  699. if (ring == &dev_priv->ring[RCS] &&
  700. mode != dev_priv->relative_constants_mode) {
  701. if (INTEL_INFO(dev)->gen < 4)
  702. return -EINVAL;
  703. if (INTEL_INFO(dev)->gen > 5 &&
  704. mode == I915_EXEC_CONSTANTS_REL_SURFACE)
  705. return -EINVAL;
  706. /* The HW changed the meaning on this bit on gen6 */
  707. if (INTEL_INFO(dev)->gen >= 6)
  708. mask &= ~I915_EXEC_CONSTANTS_REL_SURFACE;
  709. }
  710. break;
  711. default:
  712. DRM_DEBUG("execbuf with unknown constants: %d\n", mode);
  713. return -EINVAL;
  714. }
  715. if (args->buffer_count < 1) {
  716. DRM_DEBUG("execbuf with %d buffers\n", args->buffer_count);
  717. return -EINVAL;
  718. }
  719. if (args->num_cliprects != 0) {
  720. if (ring != &dev_priv->ring[RCS]) {
  721. DRM_DEBUG("clip rectangles are only valid with the render ring\n");
  722. return -EINVAL;
  723. }
  724. if (INTEL_INFO(dev)->gen >= 5) {
  725. DRM_DEBUG("clip rectangles are only valid on pre-gen5\n");
  726. return -EINVAL;
  727. }
  728. if (args->num_cliprects > UINT_MAX / sizeof(*cliprects)) {
  729. DRM_DEBUG("execbuf with %u cliprects\n",
  730. args->num_cliprects);
  731. return -EINVAL;
  732. }
  733. cliprects = kmalloc(args->num_cliprects * sizeof(*cliprects),
  734. GFP_KERNEL);
  735. if (cliprects == NULL) {
  736. ret = -ENOMEM;
  737. goto pre_mutex_err;
  738. }
  739. if (copy_from_user(cliprects,
  740. (struct drm_clip_rect __user *)(uintptr_t)
  741. args->cliprects_ptr,
  742. sizeof(*cliprects)*args->num_cliprects)) {
  743. ret = -EFAULT;
  744. goto pre_mutex_err;
  745. }
  746. }
  747. ret = i915_mutex_lock_interruptible(dev);
  748. if (ret)
  749. goto pre_mutex_err;
  750. if (dev_priv->mm.suspended) {
  751. mutex_unlock(&dev->struct_mutex);
  752. ret = -EBUSY;
  753. goto pre_mutex_err;
  754. }
  755. eb = eb_create(args->buffer_count);
  756. if (eb == NULL) {
  757. mutex_unlock(&dev->struct_mutex);
  758. ret = -ENOMEM;
  759. goto pre_mutex_err;
  760. }
  761. /* Look up object handles */
  762. ret = eb_lookup_objects(eb, exec, args->buffer_count, file);
  763. if (ret)
  764. goto err;
  765. /* take note of the batch buffer before we might reorder the lists */
  766. batch_obj = list_entry(eb->objects.prev,
  767. struct drm_i915_gem_object,
  768. exec_list);
  769. /* Move the objects en-masse into the GTT, evicting if necessary. */
  770. ret = i915_gem_execbuffer_reserve(ring, file, &eb->objects);
  771. if (ret)
  772. goto err;
  773. /* The objects are in their final locations, apply the relocations. */
  774. ret = i915_gem_execbuffer_relocate(dev, eb);
  775. if (ret) {
  776. if (ret == -EFAULT) {
  777. ret = i915_gem_execbuffer_relocate_slow(dev, file, ring,
  778. eb, exec,
  779. args->buffer_count);
  780. BUG_ON(!mutex_is_locked(&dev->struct_mutex));
  781. }
  782. if (ret)
  783. goto err;
  784. }
  785. /* Set the pending read domains for the batch buffer to COMMAND */
  786. if (batch_obj->base.pending_write_domain) {
  787. DRM_DEBUG("Attempting to use self-modifying batch buffer\n");
  788. ret = -EINVAL;
  789. goto err;
  790. }
  791. batch_obj->base.pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
  792. /* snb/ivb/vlv conflate the "batch in ppgtt" bit with the "non-secure
  793. * batch" bit. Hence we need to pin secure batches into the global gtt.
  794. * hsw should have this fixed, but let's be paranoid and do it
  795. * unconditionally for now. */
  796. if (flags & I915_DISPATCH_SECURE && !batch_obj->has_global_gtt_mapping)
  797. i915_gem_gtt_bind_object(batch_obj, batch_obj->cache_level);
  798. ret = i915_gem_execbuffer_move_to_gpu(ring, &eb->objects);
  799. if (ret)
  800. goto err;
  801. ret = i915_switch_context(ring, file, ctx_id);
  802. if (ret)
  803. goto err;
  804. if (ring == &dev_priv->ring[RCS] &&
  805. mode != dev_priv->relative_constants_mode) {
  806. ret = intel_ring_begin(ring, 4);
  807. if (ret)
  808. goto err;
  809. intel_ring_emit(ring, MI_NOOP);
  810. intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
  811. intel_ring_emit(ring, INSTPM);
  812. intel_ring_emit(ring, mask << 16 | mode);
  813. intel_ring_advance(ring);
  814. dev_priv->relative_constants_mode = mode;
  815. }
  816. if (args->flags & I915_EXEC_GEN7_SOL_RESET) {
  817. ret = i915_reset_gen7_sol_offsets(dev, ring);
  818. if (ret)
  819. goto err;
  820. }
  821. exec_start = batch_obj->gtt_offset + args->batch_start_offset;
  822. exec_len = args->batch_len;
  823. if (cliprects) {
  824. for (i = 0; i < args->num_cliprects; i++) {
  825. ret = i915_emit_box(dev, &cliprects[i],
  826. args->DR1, args->DR4);
  827. if (ret)
  828. goto err;
  829. ret = ring->dispatch_execbuffer(ring,
  830. exec_start, exec_len,
  831. flags);
  832. if (ret)
  833. goto err;
  834. }
  835. } else {
  836. ret = ring->dispatch_execbuffer(ring,
  837. exec_start, exec_len,
  838. flags);
  839. if (ret)
  840. goto err;
  841. }
  842. trace_i915_gem_ring_dispatch(ring, intel_ring_get_seqno(ring), flags);
  843. i915_gem_execbuffer_move_to_active(&eb->objects, ring);
  844. i915_gem_execbuffer_retire_commands(dev, file, ring);
  845. err:
  846. eb_destroy(eb);
  847. mutex_unlock(&dev->struct_mutex);
  848. pre_mutex_err:
  849. kfree(cliprects);
  850. return ret;
  851. }
  852. /*
  853. * Legacy execbuffer just creates an exec2 list from the original exec object
  854. * list array and passes it to the real function.
  855. */
  856. int
  857. i915_gem_execbuffer(struct drm_device *dev, void *data,
  858. struct drm_file *file)
  859. {
  860. struct drm_i915_gem_execbuffer *args = data;
  861. struct drm_i915_gem_execbuffer2 exec2;
  862. struct drm_i915_gem_exec_object *exec_list = NULL;
  863. struct drm_i915_gem_exec_object2 *exec2_list = NULL;
  864. int ret, i;
  865. if (args->buffer_count < 1) {
  866. DRM_DEBUG("execbuf with %d buffers\n", args->buffer_count);
  867. return -EINVAL;
  868. }
  869. /* Copy in the exec list from userland */
  870. exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
  871. exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
  872. if (exec_list == NULL || exec2_list == NULL) {
  873. DRM_DEBUG("Failed to allocate exec list for %d buffers\n",
  874. args->buffer_count);
  875. drm_free_large(exec_list);
  876. drm_free_large(exec2_list);
  877. return -ENOMEM;
  878. }
  879. ret = copy_from_user(exec_list,
  880. (void __user *)(uintptr_t)args->buffers_ptr,
  881. sizeof(*exec_list) * args->buffer_count);
  882. if (ret != 0) {
  883. DRM_DEBUG("copy %d exec entries failed %d\n",
  884. args->buffer_count, ret);
  885. drm_free_large(exec_list);
  886. drm_free_large(exec2_list);
  887. return -EFAULT;
  888. }
  889. for (i = 0; i < args->buffer_count; i++) {
  890. exec2_list[i].handle = exec_list[i].handle;
  891. exec2_list[i].relocation_count = exec_list[i].relocation_count;
  892. exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
  893. exec2_list[i].alignment = exec_list[i].alignment;
  894. exec2_list[i].offset = exec_list[i].offset;
  895. if (INTEL_INFO(dev)->gen < 4)
  896. exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
  897. else
  898. exec2_list[i].flags = 0;
  899. }
  900. exec2.buffers_ptr = args->buffers_ptr;
  901. exec2.buffer_count = args->buffer_count;
  902. exec2.batch_start_offset = args->batch_start_offset;
  903. exec2.batch_len = args->batch_len;
  904. exec2.DR1 = args->DR1;
  905. exec2.DR4 = args->DR4;
  906. exec2.num_cliprects = args->num_cliprects;
  907. exec2.cliprects_ptr = args->cliprects_ptr;
  908. exec2.flags = I915_EXEC_RENDER;
  909. i915_execbuffer2_set_context_id(exec2, 0);
  910. ret = i915_gem_do_execbuffer(dev, data, file, &exec2, exec2_list);
  911. if (!ret) {
  912. /* Copy the new buffer offsets back to the user's exec list. */
  913. for (i = 0; i < args->buffer_count; i++)
  914. exec_list[i].offset = exec2_list[i].offset;
  915. /* ... and back out to userspace */
  916. ret = copy_to_user((void __user *)(uintptr_t)args->buffers_ptr,
  917. exec_list,
  918. sizeof(*exec_list) * args->buffer_count);
  919. if (ret) {
  920. ret = -EFAULT;
  921. DRM_DEBUG("failed to copy %d exec entries "
  922. "back to user (%d)\n",
  923. args->buffer_count, ret);
  924. }
  925. }
  926. drm_free_large(exec_list);
  927. drm_free_large(exec2_list);
  928. return ret;
  929. }
  930. int
  931. i915_gem_execbuffer2(struct drm_device *dev, void *data,
  932. struct drm_file *file)
  933. {
  934. struct drm_i915_gem_execbuffer2 *args = data;
  935. struct drm_i915_gem_exec_object2 *exec2_list = NULL;
  936. int ret;
  937. if (args->buffer_count < 1 ||
  938. args->buffer_count > UINT_MAX / sizeof(*exec2_list)) {
  939. DRM_DEBUG("execbuf2 with %d buffers\n", args->buffer_count);
  940. return -EINVAL;
  941. }
  942. exec2_list = kmalloc(sizeof(*exec2_list)*args->buffer_count,
  943. GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY);
  944. if (exec2_list == NULL)
  945. exec2_list = drm_malloc_ab(sizeof(*exec2_list),
  946. args->buffer_count);
  947. if (exec2_list == NULL) {
  948. DRM_DEBUG("Failed to allocate exec list for %d buffers\n",
  949. args->buffer_count);
  950. return -ENOMEM;
  951. }
  952. ret = copy_from_user(exec2_list,
  953. (struct drm_i915_relocation_entry __user *)
  954. (uintptr_t) args->buffers_ptr,
  955. sizeof(*exec2_list) * args->buffer_count);
  956. if (ret != 0) {
  957. DRM_DEBUG("copy %d exec entries failed %d\n",
  958. args->buffer_count, ret);
  959. drm_free_large(exec2_list);
  960. return -EFAULT;
  961. }
  962. ret = i915_gem_do_execbuffer(dev, data, file, args, exec2_list);
  963. if (!ret) {
  964. /* Copy the new buffer offsets back to the user's exec list. */
  965. ret = copy_to_user((void __user *)(uintptr_t)args->buffers_ptr,
  966. exec2_list,
  967. sizeof(*exec2_list) * args->buffer_count);
  968. if (ret) {
  969. ret = -EFAULT;
  970. DRM_DEBUG("failed to copy %d exec entries "
  971. "back to user (%d)\n",
  972. args->buffer_count, ret);
  973. }
  974. }
  975. drm_free_large(exec2_list);
  976. return ret;
  977. }