i915_gem_execbuffer.c 33 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237
  1. /*
  2. * Copyright © 2008,2010 Intel Corporation
  3. *
  4. * Permission is hereby granted, free of charge, to any person obtaining a
  5. * copy of this software and associated documentation files (the "Software"),
  6. * to deal in the Software without restriction, including without limitation
  7. * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8. * and/or sell copies of the Software, and to permit persons to whom the
  9. * Software is furnished to do so, subject to the following conditions:
  10. *
  11. * The above copyright notice and this permission notice (including the next
  12. * paragraph) shall be included in all copies or substantial portions of the
  13. * Software.
  14. *
  15. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17. * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  18. * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19. * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20. * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21. * IN THE SOFTWARE.
  22. *
  23. * Authors:
  24. * Eric Anholt <eric@anholt.net>
  25. * Chris Wilson <chris@chris-wilson.co.uk>
  26. *
  27. */
  28. #include <drm/drmP.h>
  29. #include <drm/i915_drm.h>
  30. #include "i915_drv.h"
  31. #include "i915_trace.h"
  32. #include "intel_drv.h"
  33. #include <linux/dma_remapping.h>
  34. struct eb_objects {
  35. struct list_head objects;
  36. int and;
  37. union {
  38. struct drm_i915_gem_object *lut[0];
  39. struct hlist_head buckets[0];
  40. };
  41. };
  42. static struct eb_objects *
  43. eb_create(struct drm_i915_gem_execbuffer2 *args)
  44. {
  45. struct eb_objects *eb = NULL;
  46. if (args->flags & I915_EXEC_HANDLE_LUT) {
  47. int size = args->buffer_count;
  48. size *= sizeof(struct drm_i915_gem_object *);
  49. size += sizeof(struct eb_objects);
  50. eb = kmalloc(size, GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY);
  51. }
  52. if (eb == NULL) {
  53. int size = args->buffer_count;
  54. int count = PAGE_SIZE / sizeof(struct hlist_head) / 2;
  55. BUILD_BUG_ON_NOT_POWER_OF_2(PAGE_SIZE / sizeof(struct hlist_head));
  56. while (count > 2*size)
  57. count >>= 1;
  58. eb = kzalloc(count*sizeof(struct hlist_head) +
  59. sizeof(struct eb_objects),
  60. GFP_TEMPORARY);
  61. if (eb == NULL)
  62. return eb;
  63. eb->and = count - 1;
  64. } else
  65. eb->and = -args->buffer_count;
  66. INIT_LIST_HEAD(&eb->objects);
  67. return eb;
  68. }
  69. static void
  70. eb_reset(struct eb_objects *eb)
  71. {
  72. if (eb->and >= 0)
  73. memset(eb->buckets, 0, (eb->and+1)*sizeof(struct hlist_head));
  74. }
  75. static int
  76. eb_lookup_objects(struct eb_objects *eb,
  77. struct drm_i915_gem_exec_object2 *exec,
  78. const struct drm_i915_gem_execbuffer2 *args,
  79. struct drm_file *file)
  80. {
  81. int i;
  82. spin_lock(&file->table_lock);
  83. for (i = 0; i < args->buffer_count; i++) {
  84. struct drm_i915_gem_object *obj;
  85. obj = to_intel_bo(idr_find(&file->object_idr, exec[i].handle));
  86. if (obj == NULL) {
  87. spin_unlock(&file->table_lock);
  88. DRM_DEBUG("Invalid object handle %d at index %d\n",
  89. exec[i].handle, i);
  90. return -ENOENT;
  91. }
  92. if (!list_empty(&obj->exec_list)) {
  93. spin_unlock(&file->table_lock);
  94. DRM_DEBUG("Object %p [handle %d, index %d] appears more than once in object list\n",
  95. obj, exec[i].handle, i);
  96. return -EINVAL;
  97. }
  98. drm_gem_object_reference(&obj->base);
  99. list_add_tail(&obj->exec_list, &eb->objects);
  100. obj->exec_entry = &exec[i];
  101. if (eb->and < 0) {
  102. eb->lut[i] = obj;
  103. } else {
  104. uint32_t handle = args->flags & I915_EXEC_HANDLE_LUT ? i : exec[i].handle;
  105. obj->exec_handle = handle;
  106. hlist_add_head(&obj->exec_node,
  107. &eb->buckets[handle & eb->and]);
  108. }
  109. }
  110. spin_unlock(&file->table_lock);
  111. return 0;
  112. }
  113. static struct drm_i915_gem_object *
  114. eb_get_object(struct eb_objects *eb, unsigned long handle)
  115. {
  116. if (eb->and < 0) {
  117. if (handle >= -eb->and)
  118. return NULL;
  119. return eb->lut[handle];
  120. } else {
  121. struct hlist_head *head;
  122. struct hlist_node *node;
  123. head = &eb->buckets[handle & eb->and];
  124. hlist_for_each(node, head) {
  125. struct drm_i915_gem_object *obj;
  126. obj = hlist_entry(node, struct drm_i915_gem_object, exec_node);
  127. if (obj->exec_handle == handle)
  128. return obj;
  129. }
  130. return NULL;
  131. }
  132. }
  133. static void
  134. eb_destroy(struct eb_objects *eb)
  135. {
  136. while (!list_empty(&eb->objects)) {
  137. struct drm_i915_gem_object *obj;
  138. obj = list_first_entry(&eb->objects,
  139. struct drm_i915_gem_object,
  140. exec_list);
  141. list_del_init(&obj->exec_list);
  142. drm_gem_object_unreference(&obj->base);
  143. }
  144. kfree(eb);
  145. }
  146. static inline int use_cpu_reloc(struct drm_i915_gem_object *obj)
  147. {
  148. return (obj->base.write_domain == I915_GEM_DOMAIN_CPU ||
  149. !obj->map_and_fenceable ||
  150. obj->cache_level != I915_CACHE_NONE);
  151. }
  152. static int
  153. i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
  154. struct eb_objects *eb,
  155. struct drm_i915_gem_relocation_entry *reloc)
  156. {
  157. struct drm_device *dev = obj->base.dev;
  158. struct drm_gem_object *target_obj;
  159. struct drm_i915_gem_object *target_i915_obj;
  160. uint32_t target_offset;
  161. int ret = -EINVAL;
  162. /* we've already hold a reference to all valid objects */
  163. target_obj = &eb_get_object(eb, reloc->target_handle)->base;
  164. if (unlikely(target_obj == NULL))
  165. return -ENOENT;
  166. target_i915_obj = to_intel_bo(target_obj);
  167. target_offset = i915_gem_obj_ggtt_offset(target_i915_obj);
  168. /* Sandybridge PPGTT errata: We need a global gtt mapping for MI and
  169. * pipe_control writes because the gpu doesn't properly redirect them
  170. * through the ppgtt for non_secure batchbuffers. */
  171. if (unlikely(IS_GEN6(dev) &&
  172. reloc->write_domain == I915_GEM_DOMAIN_INSTRUCTION &&
  173. !target_i915_obj->has_global_gtt_mapping)) {
  174. i915_gem_gtt_bind_object(target_i915_obj,
  175. target_i915_obj->cache_level);
  176. }
  177. /* Validate that the target is in a valid r/w GPU domain */
  178. if (unlikely(reloc->write_domain & (reloc->write_domain - 1))) {
  179. DRM_DEBUG("reloc with multiple write domains: "
  180. "obj %p target %d offset %d "
  181. "read %08x write %08x",
  182. obj, reloc->target_handle,
  183. (int) reloc->offset,
  184. reloc->read_domains,
  185. reloc->write_domain);
  186. return ret;
  187. }
  188. if (unlikely((reloc->write_domain | reloc->read_domains)
  189. & ~I915_GEM_GPU_DOMAINS)) {
  190. DRM_DEBUG("reloc with read/write non-GPU domains: "
  191. "obj %p target %d offset %d "
  192. "read %08x write %08x",
  193. obj, reloc->target_handle,
  194. (int) reloc->offset,
  195. reloc->read_domains,
  196. reloc->write_domain);
  197. return ret;
  198. }
  199. target_obj->pending_read_domains |= reloc->read_domains;
  200. target_obj->pending_write_domain |= reloc->write_domain;
  201. /* If the relocation already has the right value in it, no
  202. * more work needs to be done.
  203. */
  204. if (target_offset == reloc->presumed_offset)
  205. return 0;
  206. /* Check that the relocation address is valid... */
  207. if (unlikely(reloc->offset > obj->base.size - 4)) {
  208. DRM_DEBUG("Relocation beyond object bounds: "
  209. "obj %p target %d offset %d size %d.\n",
  210. obj, reloc->target_handle,
  211. (int) reloc->offset,
  212. (int) obj->base.size);
  213. return ret;
  214. }
  215. if (unlikely(reloc->offset & 3)) {
  216. DRM_DEBUG("Relocation not 4-byte aligned: "
  217. "obj %p target %d offset %d.\n",
  218. obj, reloc->target_handle,
  219. (int) reloc->offset);
  220. return ret;
  221. }
  222. /* We can't wait for rendering with pagefaults disabled */
  223. if (obj->active && in_atomic())
  224. return -EFAULT;
  225. reloc->delta += target_offset;
  226. if (use_cpu_reloc(obj)) {
  227. uint32_t page_offset = offset_in_page(reloc->offset);
  228. char *vaddr;
  229. ret = i915_gem_object_set_to_cpu_domain(obj, 1);
  230. if (ret)
  231. return ret;
  232. vaddr = kmap_atomic(i915_gem_object_get_page(obj,
  233. reloc->offset >> PAGE_SHIFT));
  234. *(uint32_t *)(vaddr + page_offset) = reloc->delta;
  235. kunmap_atomic(vaddr);
  236. } else {
  237. struct drm_i915_private *dev_priv = dev->dev_private;
  238. uint32_t __iomem *reloc_entry;
  239. void __iomem *reloc_page;
  240. ret = i915_gem_object_set_to_gtt_domain(obj, true);
  241. if (ret)
  242. return ret;
  243. ret = i915_gem_object_put_fence(obj);
  244. if (ret)
  245. return ret;
  246. /* Map the page containing the relocation we're going to perform. */
  247. reloc->offset += i915_gem_obj_ggtt_offset(obj);
  248. reloc_page = io_mapping_map_atomic_wc(dev_priv->gtt.mappable,
  249. reloc->offset & PAGE_MASK);
  250. reloc_entry = (uint32_t __iomem *)
  251. (reloc_page + offset_in_page(reloc->offset));
  252. iowrite32(reloc->delta, reloc_entry);
  253. io_mapping_unmap_atomic(reloc_page);
  254. }
  255. /* and update the user's relocation entry */
  256. reloc->presumed_offset = target_offset;
  257. return 0;
  258. }
  259. static int
  260. i915_gem_execbuffer_relocate_object(struct drm_i915_gem_object *obj,
  261. struct eb_objects *eb)
  262. {
  263. #define N_RELOC(x) ((x) / sizeof(struct drm_i915_gem_relocation_entry))
  264. struct drm_i915_gem_relocation_entry stack_reloc[N_RELOC(512)];
  265. struct drm_i915_gem_relocation_entry __user *user_relocs;
  266. struct drm_i915_gem_exec_object2 *entry = obj->exec_entry;
  267. int remain, ret;
  268. user_relocs = to_user_ptr(entry->relocs_ptr);
  269. remain = entry->relocation_count;
  270. while (remain) {
  271. struct drm_i915_gem_relocation_entry *r = stack_reloc;
  272. int count = remain;
  273. if (count > ARRAY_SIZE(stack_reloc))
  274. count = ARRAY_SIZE(stack_reloc);
  275. remain -= count;
  276. if (__copy_from_user_inatomic(r, user_relocs, count*sizeof(r[0])))
  277. return -EFAULT;
  278. do {
  279. u64 offset = r->presumed_offset;
  280. ret = i915_gem_execbuffer_relocate_entry(obj, eb, r);
  281. if (ret)
  282. return ret;
  283. if (r->presumed_offset != offset &&
  284. __copy_to_user_inatomic(&user_relocs->presumed_offset,
  285. &r->presumed_offset,
  286. sizeof(r->presumed_offset))) {
  287. return -EFAULT;
  288. }
  289. user_relocs++;
  290. r++;
  291. } while (--count);
  292. }
  293. return 0;
  294. #undef N_RELOC
  295. }
  296. static int
  297. i915_gem_execbuffer_relocate_object_slow(struct drm_i915_gem_object *obj,
  298. struct eb_objects *eb,
  299. struct drm_i915_gem_relocation_entry *relocs)
  300. {
  301. const struct drm_i915_gem_exec_object2 *entry = obj->exec_entry;
  302. int i, ret;
  303. for (i = 0; i < entry->relocation_count; i++) {
  304. ret = i915_gem_execbuffer_relocate_entry(obj, eb, &relocs[i]);
  305. if (ret)
  306. return ret;
  307. }
  308. return 0;
  309. }
  310. static int
  311. i915_gem_execbuffer_relocate(struct eb_objects *eb)
  312. {
  313. struct drm_i915_gem_object *obj;
  314. int ret = 0;
  315. /* This is the fast path and we cannot handle a pagefault whilst
  316. * holding the struct mutex lest the user pass in the relocations
  317. * contained within a mmaped bo. For in such a case we, the page
  318. * fault handler would call i915_gem_fault() and we would try to
  319. * acquire the struct mutex again. Obviously this is bad and so
  320. * lockdep complains vehemently.
  321. */
  322. pagefault_disable();
  323. list_for_each_entry(obj, &eb->objects, exec_list) {
  324. ret = i915_gem_execbuffer_relocate_object(obj, eb);
  325. if (ret)
  326. break;
  327. }
  328. pagefault_enable();
  329. return ret;
  330. }
  331. #define __EXEC_OBJECT_HAS_PIN (1<<31)
  332. #define __EXEC_OBJECT_HAS_FENCE (1<<30)
  333. static int
  334. need_reloc_mappable(struct drm_i915_gem_object *obj)
  335. {
  336. struct drm_i915_gem_exec_object2 *entry = obj->exec_entry;
  337. return entry->relocation_count && !use_cpu_reloc(obj);
  338. }
  339. static int
  340. i915_gem_execbuffer_reserve_object(struct drm_i915_gem_object *obj,
  341. struct intel_ring_buffer *ring,
  342. bool *need_reloc)
  343. {
  344. struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  345. struct drm_i915_gem_exec_object2 *entry = obj->exec_entry;
  346. bool has_fenced_gpu_access = INTEL_INFO(ring->dev)->gen < 4;
  347. bool need_fence, need_mappable;
  348. int ret;
  349. need_fence =
  350. has_fenced_gpu_access &&
  351. entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
  352. obj->tiling_mode != I915_TILING_NONE;
  353. need_mappable = need_fence || need_reloc_mappable(obj);
  354. ret = i915_gem_object_pin(obj, entry->alignment, need_mappable, false);
  355. if (ret)
  356. return ret;
  357. entry->flags |= __EXEC_OBJECT_HAS_PIN;
  358. if (has_fenced_gpu_access) {
  359. if (entry->flags & EXEC_OBJECT_NEEDS_FENCE) {
  360. ret = i915_gem_object_get_fence(obj);
  361. if (ret)
  362. return ret;
  363. if (i915_gem_object_pin_fence(obj))
  364. entry->flags |= __EXEC_OBJECT_HAS_FENCE;
  365. obj->pending_fenced_gpu_access = true;
  366. }
  367. }
  368. /* Ensure ppgtt mapping exists if needed */
  369. if (dev_priv->mm.aliasing_ppgtt && !obj->has_aliasing_ppgtt_mapping) {
  370. i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
  371. obj, obj->cache_level);
  372. obj->has_aliasing_ppgtt_mapping = 1;
  373. }
  374. if (entry->offset != i915_gem_obj_ggtt_offset(obj)) {
  375. entry->offset = i915_gem_obj_ggtt_offset(obj);
  376. *need_reloc = true;
  377. }
  378. if (entry->flags & EXEC_OBJECT_WRITE) {
  379. obj->base.pending_read_domains = I915_GEM_DOMAIN_RENDER;
  380. obj->base.pending_write_domain = I915_GEM_DOMAIN_RENDER;
  381. }
  382. if (entry->flags & EXEC_OBJECT_NEEDS_GTT &&
  383. !obj->has_global_gtt_mapping)
  384. i915_gem_gtt_bind_object(obj, obj->cache_level);
  385. return 0;
  386. }
  387. static void
  388. i915_gem_execbuffer_unreserve_object(struct drm_i915_gem_object *obj)
  389. {
  390. struct drm_i915_gem_exec_object2 *entry;
  391. if (!i915_gem_obj_ggtt_bound(obj))
  392. return;
  393. entry = obj->exec_entry;
  394. if (entry->flags & __EXEC_OBJECT_HAS_FENCE)
  395. i915_gem_object_unpin_fence(obj);
  396. if (entry->flags & __EXEC_OBJECT_HAS_PIN)
  397. i915_gem_object_unpin(obj);
  398. entry->flags &= ~(__EXEC_OBJECT_HAS_FENCE | __EXEC_OBJECT_HAS_PIN);
  399. }
  400. static int
  401. i915_gem_execbuffer_reserve(struct intel_ring_buffer *ring,
  402. struct list_head *objects,
  403. bool *need_relocs)
  404. {
  405. struct drm_i915_gem_object *obj;
  406. struct list_head ordered_objects;
  407. bool has_fenced_gpu_access = INTEL_INFO(ring->dev)->gen < 4;
  408. int retry;
  409. INIT_LIST_HEAD(&ordered_objects);
  410. while (!list_empty(objects)) {
  411. struct drm_i915_gem_exec_object2 *entry;
  412. bool need_fence, need_mappable;
  413. obj = list_first_entry(objects,
  414. struct drm_i915_gem_object,
  415. exec_list);
  416. entry = obj->exec_entry;
  417. need_fence =
  418. has_fenced_gpu_access &&
  419. entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
  420. obj->tiling_mode != I915_TILING_NONE;
  421. need_mappable = need_fence || need_reloc_mappable(obj);
  422. if (need_mappable)
  423. list_move(&obj->exec_list, &ordered_objects);
  424. else
  425. list_move_tail(&obj->exec_list, &ordered_objects);
  426. obj->base.pending_read_domains = I915_GEM_GPU_DOMAINS & ~I915_GEM_DOMAIN_COMMAND;
  427. obj->base.pending_write_domain = 0;
  428. obj->pending_fenced_gpu_access = false;
  429. }
  430. list_splice(&ordered_objects, objects);
  431. /* Attempt to pin all of the buffers into the GTT.
  432. * This is done in 3 phases:
  433. *
  434. * 1a. Unbind all objects that do not match the GTT constraints for
  435. * the execbuffer (fenceable, mappable, alignment etc).
  436. * 1b. Increment pin count for already bound objects.
  437. * 2. Bind new objects.
  438. * 3. Decrement pin count.
  439. *
  440. * This avoid unnecessary unbinding of later objects in order to make
  441. * room for the earlier objects *unless* we need to defragment.
  442. */
  443. retry = 0;
  444. do {
  445. int ret = 0;
  446. /* Unbind any ill-fitting objects or pin. */
  447. list_for_each_entry(obj, objects, exec_list) {
  448. struct drm_i915_gem_exec_object2 *entry = obj->exec_entry;
  449. bool need_fence, need_mappable;
  450. if (!i915_gem_obj_ggtt_bound(obj))
  451. continue;
  452. need_fence =
  453. has_fenced_gpu_access &&
  454. entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
  455. obj->tiling_mode != I915_TILING_NONE;
  456. need_mappable = need_fence || need_reloc_mappable(obj);
  457. if ((entry->alignment &&
  458. i915_gem_obj_ggtt_offset(obj) & (entry->alignment - 1)) ||
  459. (need_mappable && !obj->map_and_fenceable))
  460. ret = i915_gem_object_unbind(obj);
  461. else
  462. ret = i915_gem_execbuffer_reserve_object(obj, ring, need_relocs);
  463. if (ret)
  464. goto err;
  465. }
  466. /* Bind fresh objects */
  467. list_for_each_entry(obj, objects, exec_list) {
  468. if (i915_gem_obj_ggtt_bound(obj))
  469. continue;
  470. ret = i915_gem_execbuffer_reserve_object(obj, ring, need_relocs);
  471. if (ret)
  472. goto err;
  473. }
  474. err: /* Decrement pin count for bound objects */
  475. list_for_each_entry(obj, objects, exec_list)
  476. i915_gem_execbuffer_unreserve_object(obj);
  477. if (ret != -ENOSPC || retry++)
  478. return ret;
  479. ret = i915_gem_evict_everything(ring->dev);
  480. if (ret)
  481. return ret;
  482. } while (1);
  483. }
  484. static int
  485. i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
  486. struct drm_i915_gem_execbuffer2 *args,
  487. struct drm_file *file,
  488. struct intel_ring_buffer *ring,
  489. struct eb_objects *eb,
  490. struct drm_i915_gem_exec_object2 *exec)
  491. {
  492. struct drm_i915_gem_relocation_entry *reloc;
  493. struct drm_i915_gem_object *obj;
  494. bool need_relocs;
  495. int *reloc_offset;
  496. int i, total, ret;
  497. int count = args->buffer_count;
  498. /* We may process another execbuffer during the unlock... */
  499. while (!list_empty(&eb->objects)) {
  500. obj = list_first_entry(&eb->objects,
  501. struct drm_i915_gem_object,
  502. exec_list);
  503. list_del_init(&obj->exec_list);
  504. drm_gem_object_unreference(&obj->base);
  505. }
  506. mutex_unlock(&dev->struct_mutex);
  507. total = 0;
  508. for (i = 0; i < count; i++)
  509. total += exec[i].relocation_count;
  510. reloc_offset = drm_malloc_ab(count, sizeof(*reloc_offset));
  511. reloc = drm_malloc_ab(total, sizeof(*reloc));
  512. if (reloc == NULL || reloc_offset == NULL) {
  513. drm_free_large(reloc);
  514. drm_free_large(reloc_offset);
  515. mutex_lock(&dev->struct_mutex);
  516. return -ENOMEM;
  517. }
  518. total = 0;
  519. for (i = 0; i < count; i++) {
  520. struct drm_i915_gem_relocation_entry __user *user_relocs;
  521. u64 invalid_offset = (u64)-1;
  522. int j;
  523. user_relocs = to_user_ptr(exec[i].relocs_ptr);
  524. if (copy_from_user(reloc+total, user_relocs,
  525. exec[i].relocation_count * sizeof(*reloc))) {
  526. ret = -EFAULT;
  527. mutex_lock(&dev->struct_mutex);
  528. goto err;
  529. }
  530. /* As we do not update the known relocation offsets after
  531. * relocating (due to the complexities in lock handling),
  532. * we need to mark them as invalid now so that we force the
  533. * relocation processing next time. Just in case the target
  534. * object is evicted and then rebound into its old
  535. * presumed_offset before the next execbuffer - if that
  536. * happened we would make the mistake of assuming that the
  537. * relocations were valid.
  538. */
  539. for (j = 0; j < exec[i].relocation_count; j++) {
  540. if (copy_to_user(&user_relocs[j].presumed_offset,
  541. &invalid_offset,
  542. sizeof(invalid_offset))) {
  543. ret = -EFAULT;
  544. mutex_lock(&dev->struct_mutex);
  545. goto err;
  546. }
  547. }
  548. reloc_offset[i] = total;
  549. total += exec[i].relocation_count;
  550. }
  551. ret = i915_mutex_lock_interruptible(dev);
  552. if (ret) {
  553. mutex_lock(&dev->struct_mutex);
  554. goto err;
  555. }
  556. /* reacquire the objects */
  557. eb_reset(eb);
  558. ret = eb_lookup_objects(eb, exec, args, file);
  559. if (ret)
  560. goto err;
  561. need_relocs = (args->flags & I915_EXEC_NO_RELOC) == 0;
  562. ret = i915_gem_execbuffer_reserve(ring, &eb->objects, &need_relocs);
  563. if (ret)
  564. goto err;
  565. list_for_each_entry(obj, &eb->objects, exec_list) {
  566. int offset = obj->exec_entry - exec;
  567. ret = i915_gem_execbuffer_relocate_object_slow(obj, eb,
  568. reloc + reloc_offset[offset]);
  569. if (ret)
  570. goto err;
  571. }
  572. /* Leave the user relocations as are, this is the painfully slow path,
  573. * and we want to avoid the complication of dropping the lock whilst
  574. * having buffers reserved in the aperture and so causing spurious
  575. * ENOSPC for random operations.
  576. */
  577. err:
  578. drm_free_large(reloc);
  579. drm_free_large(reloc_offset);
  580. return ret;
  581. }
  582. static int
  583. i915_gem_execbuffer_move_to_gpu(struct intel_ring_buffer *ring,
  584. struct list_head *objects)
  585. {
  586. struct drm_i915_gem_object *obj;
  587. uint32_t flush_domains = 0;
  588. int ret;
  589. list_for_each_entry(obj, objects, exec_list) {
  590. ret = i915_gem_object_sync(obj, ring);
  591. if (ret)
  592. return ret;
  593. if (obj->base.write_domain & I915_GEM_DOMAIN_CPU)
  594. i915_gem_clflush_object(obj);
  595. flush_domains |= obj->base.write_domain;
  596. }
  597. if (flush_domains & I915_GEM_DOMAIN_CPU)
  598. i915_gem_chipset_flush(ring->dev);
  599. if (flush_domains & I915_GEM_DOMAIN_GTT)
  600. wmb();
  601. /* Unconditionally invalidate gpu caches and ensure that we do flush
  602. * any residual writes from the previous batch.
  603. */
  604. return intel_ring_invalidate_all_caches(ring);
  605. }
  606. static bool
  607. i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec)
  608. {
  609. if (exec->flags & __I915_EXEC_UNKNOWN_FLAGS)
  610. return false;
  611. return ((exec->batch_start_offset | exec->batch_len) & 0x7) == 0;
  612. }
  613. static int
  614. validate_exec_list(struct drm_i915_gem_exec_object2 *exec,
  615. int count)
  616. {
  617. int i;
  618. int relocs_total = 0;
  619. int relocs_max = INT_MAX / sizeof(struct drm_i915_gem_relocation_entry);
  620. for (i = 0; i < count; i++) {
  621. char __user *ptr = to_user_ptr(exec[i].relocs_ptr);
  622. int length; /* limited by fault_in_pages_readable() */
  623. if (exec[i].flags & __EXEC_OBJECT_UNKNOWN_FLAGS)
  624. return -EINVAL;
  625. /* First check for malicious input causing overflow in
  626. * the worst case where we need to allocate the entire
  627. * relocation tree as a single array.
  628. */
  629. if (exec[i].relocation_count > relocs_max - relocs_total)
  630. return -EINVAL;
  631. relocs_total += exec[i].relocation_count;
  632. length = exec[i].relocation_count *
  633. sizeof(struct drm_i915_gem_relocation_entry);
  634. /*
  635. * We must check that the entire relocation array is safe
  636. * to read, but since we may need to update the presumed
  637. * offsets during execution, check for full write access.
  638. */
  639. if (!access_ok(VERIFY_WRITE, ptr, length))
  640. return -EFAULT;
  641. if (likely(!i915_prefault_disable)) {
  642. if (fault_in_multipages_readable(ptr, length))
  643. return -EFAULT;
  644. }
  645. }
  646. return 0;
  647. }
  648. static void
  649. i915_gem_execbuffer_move_to_active(struct list_head *objects,
  650. struct intel_ring_buffer *ring)
  651. {
  652. struct drm_i915_gem_object *obj;
  653. list_for_each_entry(obj, objects, exec_list) {
  654. u32 old_read = obj->base.read_domains;
  655. u32 old_write = obj->base.write_domain;
  656. obj->base.write_domain = obj->base.pending_write_domain;
  657. if (obj->base.write_domain == 0)
  658. obj->base.pending_read_domains |= obj->base.read_domains;
  659. obj->base.read_domains = obj->base.pending_read_domains;
  660. obj->fenced_gpu_access = obj->pending_fenced_gpu_access;
  661. i915_gem_object_move_to_active(obj, ring);
  662. if (obj->base.write_domain) {
  663. obj->dirty = 1;
  664. obj->last_write_seqno = intel_ring_get_seqno(ring);
  665. if (obj->pin_count) /* check for potential scanout */
  666. intel_mark_fb_busy(obj, ring);
  667. }
  668. trace_i915_gem_object_change_domain(obj, old_read, old_write);
  669. }
  670. }
  671. static void
  672. i915_gem_execbuffer_retire_commands(struct drm_device *dev,
  673. struct drm_file *file,
  674. struct intel_ring_buffer *ring,
  675. struct drm_i915_gem_object *obj)
  676. {
  677. /* Unconditionally force add_request to emit a full flush. */
  678. ring->gpu_caches_dirty = true;
  679. /* Add a breadcrumb for the completion of the batch buffer */
  680. (void)__i915_add_request(ring, file, obj, NULL);
  681. }
  682. static int
  683. i915_reset_gen7_sol_offsets(struct drm_device *dev,
  684. struct intel_ring_buffer *ring)
  685. {
  686. drm_i915_private_t *dev_priv = dev->dev_private;
  687. int ret, i;
  688. if (!IS_GEN7(dev) || ring != &dev_priv->ring[RCS])
  689. return 0;
  690. ret = intel_ring_begin(ring, 4 * 3);
  691. if (ret)
  692. return ret;
  693. for (i = 0; i < 4; i++) {
  694. intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
  695. intel_ring_emit(ring, GEN7_SO_WRITE_OFFSET(i));
  696. intel_ring_emit(ring, 0);
  697. }
  698. intel_ring_advance(ring);
  699. return 0;
  700. }
  701. static int
  702. i915_gem_do_execbuffer(struct drm_device *dev, void *data,
  703. struct drm_file *file,
  704. struct drm_i915_gem_execbuffer2 *args,
  705. struct drm_i915_gem_exec_object2 *exec)
  706. {
  707. drm_i915_private_t *dev_priv = dev->dev_private;
  708. struct eb_objects *eb;
  709. struct drm_i915_gem_object *batch_obj;
  710. struct drm_clip_rect *cliprects = NULL;
  711. struct intel_ring_buffer *ring;
  712. u32 ctx_id = i915_execbuffer2_get_context_id(*args);
  713. u32 exec_start, exec_len;
  714. u32 mask, flags;
  715. int ret, mode, i;
  716. bool need_relocs;
  717. if (!i915_gem_check_execbuffer(args))
  718. return -EINVAL;
  719. ret = validate_exec_list(exec, args->buffer_count);
  720. if (ret)
  721. return ret;
  722. flags = 0;
  723. if (args->flags & I915_EXEC_SECURE) {
  724. if (!file->is_master || !capable(CAP_SYS_ADMIN))
  725. return -EPERM;
  726. flags |= I915_DISPATCH_SECURE;
  727. }
  728. if (args->flags & I915_EXEC_IS_PINNED)
  729. flags |= I915_DISPATCH_PINNED;
  730. switch (args->flags & I915_EXEC_RING_MASK) {
  731. case I915_EXEC_DEFAULT:
  732. case I915_EXEC_RENDER:
  733. ring = &dev_priv->ring[RCS];
  734. break;
  735. case I915_EXEC_BSD:
  736. ring = &dev_priv->ring[VCS];
  737. if (ctx_id != DEFAULT_CONTEXT_ID) {
  738. DRM_DEBUG("Ring %s doesn't support contexts\n",
  739. ring->name);
  740. return -EPERM;
  741. }
  742. break;
  743. case I915_EXEC_BLT:
  744. ring = &dev_priv->ring[BCS];
  745. if (ctx_id != DEFAULT_CONTEXT_ID) {
  746. DRM_DEBUG("Ring %s doesn't support contexts\n",
  747. ring->name);
  748. return -EPERM;
  749. }
  750. break;
  751. case I915_EXEC_VEBOX:
  752. ring = &dev_priv->ring[VECS];
  753. if (ctx_id != DEFAULT_CONTEXT_ID) {
  754. DRM_DEBUG("Ring %s doesn't support contexts\n",
  755. ring->name);
  756. return -EPERM;
  757. }
  758. break;
  759. default:
  760. DRM_DEBUG("execbuf with unknown ring: %d\n",
  761. (int)(args->flags & I915_EXEC_RING_MASK));
  762. return -EINVAL;
  763. }
  764. if (!intel_ring_initialized(ring)) {
  765. DRM_DEBUG("execbuf with invalid ring: %d\n",
  766. (int)(args->flags & I915_EXEC_RING_MASK));
  767. return -EINVAL;
  768. }
  769. mode = args->flags & I915_EXEC_CONSTANTS_MASK;
  770. mask = I915_EXEC_CONSTANTS_MASK;
  771. switch (mode) {
  772. case I915_EXEC_CONSTANTS_REL_GENERAL:
  773. case I915_EXEC_CONSTANTS_ABSOLUTE:
  774. case I915_EXEC_CONSTANTS_REL_SURFACE:
  775. if (ring == &dev_priv->ring[RCS] &&
  776. mode != dev_priv->relative_constants_mode) {
  777. if (INTEL_INFO(dev)->gen < 4)
  778. return -EINVAL;
  779. if (INTEL_INFO(dev)->gen > 5 &&
  780. mode == I915_EXEC_CONSTANTS_REL_SURFACE)
  781. return -EINVAL;
  782. /* The HW changed the meaning on this bit on gen6 */
  783. if (INTEL_INFO(dev)->gen >= 6)
  784. mask &= ~I915_EXEC_CONSTANTS_REL_SURFACE;
  785. }
  786. break;
  787. default:
  788. DRM_DEBUG("execbuf with unknown constants: %d\n", mode);
  789. return -EINVAL;
  790. }
  791. if (args->buffer_count < 1) {
  792. DRM_DEBUG("execbuf with %d buffers\n", args->buffer_count);
  793. return -EINVAL;
  794. }
  795. if (args->num_cliprects != 0) {
  796. if (ring != &dev_priv->ring[RCS]) {
  797. DRM_DEBUG("clip rectangles are only valid with the render ring\n");
  798. return -EINVAL;
  799. }
  800. if (INTEL_INFO(dev)->gen >= 5) {
  801. DRM_DEBUG("clip rectangles are only valid on pre-gen5\n");
  802. return -EINVAL;
  803. }
  804. if (args->num_cliprects > UINT_MAX / sizeof(*cliprects)) {
  805. DRM_DEBUG("execbuf with %u cliprects\n",
  806. args->num_cliprects);
  807. return -EINVAL;
  808. }
  809. cliprects = kmalloc(args->num_cliprects * sizeof(*cliprects),
  810. GFP_KERNEL);
  811. if (cliprects == NULL) {
  812. ret = -ENOMEM;
  813. goto pre_mutex_err;
  814. }
  815. if (copy_from_user(cliprects,
  816. to_user_ptr(args->cliprects_ptr),
  817. sizeof(*cliprects)*args->num_cliprects)) {
  818. ret = -EFAULT;
  819. goto pre_mutex_err;
  820. }
  821. }
  822. ret = i915_mutex_lock_interruptible(dev);
  823. if (ret)
  824. goto pre_mutex_err;
  825. if (dev_priv->ums.mm_suspended) {
  826. mutex_unlock(&dev->struct_mutex);
  827. ret = -EBUSY;
  828. goto pre_mutex_err;
  829. }
  830. eb = eb_create(args);
  831. if (eb == NULL) {
  832. mutex_unlock(&dev->struct_mutex);
  833. ret = -ENOMEM;
  834. goto pre_mutex_err;
  835. }
  836. /* Look up object handles */
  837. ret = eb_lookup_objects(eb, exec, args, file);
  838. if (ret)
  839. goto err;
  840. /* take note of the batch buffer before we might reorder the lists */
  841. batch_obj = list_entry(eb->objects.prev,
  842. struct drm_i915_gem_object,
  843. exec_list);
  844. /* Move the objects en-masse into the GTT, evicting if necessary. */
  845. need_relocs = (args->flags & I915_EXEC_NO_RELOC) == 0;
  846. ret = i915_gem_execbuffer_reserve(ring, &eb->objects, &need_relocs);
  847. if (ret)
  848. goto err;
  849. /* The objects are in their final locations, apply the relocations. */
  850. if (need_relocs)
  851. ret = i915_gem_execbuffer_relocate(eb);
  852. if (ret) {
  853. if (ret == -EFAULT) {
  854. ret = i915_gem_execbuffer_relocate_slow(dev, args, file, ring,
  855. eb, exec);
  856. BUG_ON(!mutex_is_locked(&dev->struct_mutex));
  857. }
  858. if (ret)
  859. goto err;
  860. }
  861. /* Set the pending read domains for the batch buffer to COMMAND */
  862. if (batch_obj->base.pending_write_domain) {
  863. DRM_DEBUG("Attempting to use self-modifying batch buffer\n");
  864. ret = -EINVAL;
  865. goto err;
  866. }
  867. batch_obj->base.pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
  868. /* snb/ivb/vlv conflate the "batch in ppgtt" bit with the "non-secure
  869. * batch" bit. Hence we need to pin secure batches into the global gtt.
  870. * hsw should have this fixed, but let's be paranoid and do it
  871. * unconditionally for now. */
  872. if (flags & I915_DISPATCH_SECURE && !batch_obj->has_global_gtt_mapping)
  873. i915_gem_gtt_bind_object(batch_obj, batch_obj->cache_level);
  874. ret = i915_gem_execbuffer_move_to_gpu(ring, &eb->objects);
  875. if (ret)
  876. goto err;
  877. ret = i915_switch_context(ring, file, ctx_id);
  878. if (ret)
  879. goto err;
  880. if (ring == &dev_priv->ring[RCS] &&
  881. mode != dev_priv->relative_constants_mode) {
  882. ret = intel_ring_begin(ring, 4);
  883. if (ret)
  884. goto err;
  885. intel_ring_emit(ring, MI_NOOP);
  886. intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
  887. intel_ring_emit(ring, INSTPM);
  888. intel_ring_emit(ring, mask << 16 | mode);
  889. intel_ring_advance(ring);
  890. dev_priv->relative_constants_mode = mode;
  891. }
  892. if (args->flags & I915_EXEC_GEN7_SOL_RESET) {
  893. ret = i915_reset_gen7_sol_offsets(dev, ring);
  894. if (ret)
  895. goto err;
  896. }
  897. exec_start = i915_gem_obj_ggtt_offset(batch_obj) + args->batch_start_offset;
  898. exec_len = args->batch_len;
  899. if (cliprects) {
  900. for (i = 0; i < args->num_cliprects; i++) {
  901. ret = i915_emit_box(dev, &cliprects[i],
  902. args->DR1, args->DR4);
  903. if (ret)
  904. goto err;
  905. ret = ring->dispatch_execbuffer(ring,
  906. exec_start, exec_len,
  907. flags);
  908. if (ret)
  909. goto err;
  910. }
  911. } else {
  912. ret = ring->dispatch_execbuffer(ring,
  913. exec_start, exec_len,
  914. flags);
  915. if (ret)
  916. goto err;
  917. }
  918. trace_i915_gem_ring_dispatch(ring, intel_ring_get_seqno(ring), flags);
  919. i915_gem_execbuffer_move_to_active(&eb->objects, ring);
  920. i915_gem_execbuffer_retire_commands(dev, file, ring, batch_obj);
  921. err:
  922. eb_destroy(eb);
  923. mutex_unlock(&dev->struct_mutex);
  924. pre_mutex_err:
  925. kfree(cliprects);
  926. return ret;
  927. }
  928. /*
  929. * Legacy execbuffer just creates an exec2 list from the original exec object
  930. * list array and passes it to the real function.
  931. */
  932. int
  933. i915_gem_execbuffer(struct drm_device *dev, void *data,
  934. struct drm_file *file)
  935. {
  936. struct drm_i915_gem_execbuffer *args = data;
  937. struct drm_i915_gem_execbuffer2 exec2;
  938. struct drm_i915_gem_exec_object *exec_list = NULL;
  939. struct drm_i915_gem_exec_object2 *exec2_list = NULL;
  940. int ret, i;
  941. if (args->buffer_count < 1) {
  942. DRM_DEBUG("execbuf with %d buffers\n", args->buffer_count);
  943. return -EINVAL;
  944. }
  945. /* Copy in the exec list from userland */
  946. exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
  947. exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
  948. if (exec_list == NULL || exec2_list == NULL) {
  949. DRM_DEBUG("Failed to allocate exec list for %d buffers\n",
  950. args->buffer_count);
  951. drm_free_large(exec_list);
  952. drm_free_large(exec2_list);
  953. return -ENOMEM;
  954. }
  955. ret = copy_from_user(exec_list,
  956. to_user_ptr(args->buffers_ptr),
  957. sizeof(*exec_list) * args->buffer_count);
  958. if (ret != 0) {
  959. DRM_DEBUG("copy %d exec entries failed %d\n",
  960. args->buffer_count, ret);
  961. drm_free_large(exec_list);
  962. drm_free_large(exec2_list);
  963. return -EFAULT;
  964. }
  965. for (i = 0; i < args->buffer_count; i++) {
  966. exec2_list[i].handle = exec_list[i].handle;
  967. exec2_list[i].relocation_count = exec_list[i].relocation_count;
  968. exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
  969. exec2_list[i].alignment = exec_list[i].alignment;
  970. exec2_list[i].offset = exec_list[i].offset;
  971. if (INTEL_INFO(dev)->gen < 4)
  972. exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
  973. else
  974. exec2_list[i].flags = 0;
  975. }
  976. exec2.buffers_ptr = args->buffers_ptr;
  977. exec2.buffer_count = args->buffer_count;
  978. exec2.batch_start_offset = args->batch_start_offset;
  979. exec2.batch_len = args->batch_len;
  980. exec2.DR1 = args->DR1;
  981. exec2.DR4 = args->DR4;
  982. exec2.num_cliprects = args->num_cliprects;
  983. exec2.cliprects_ptr = args->cliprects_ptr;
  984. exec2.flags = I915_EXEC_RENDER;
  985. i915_execbuffer2_set_context_id(exec2, 0);
  986. ret = i915_gem_do_execbuffer(dev, data, file, &exec2, exec2_list);
  987. if (!ret) {
  988. /* Copy the new buffer offsets back to the user's exec list. */
  989. for (i = 0; i < args->buffer_count; i++)
  990. exec_list[i].offset = exec2_list[i].offset;
  991. /* ... and back out to userspace */
  992. ret = copy_to_user(to_user_ptr(args->buffers_ptr),
  993. exec_list,
  994. sizeof(*exec_list) * args->buffer_count);
  995. if (ret) {
  996. ret = -EFAULT;
  997. DRM_DEBUG("failed to copy %d exec entries "
  998. "back to user (%d)\n",
  999. args->buffer_count, ret);
  1000. }
  1001. }
  1002. drm_free_large(exec_list);
  1003. drm_free_large(exec2_list);
  1004. return ret;
  1005. }
  1006. int
  1007. i915_gem_execbuffer2(struct drm_device *dev, void *data,
  1008. struct drm_file *file)
  1009. {
  1010. struct drm_i915_gem_execbuffer2 *args = data;
  1011. struct drm_i915_gem_exec_object2 *exec2_list = NULL;
  1012. int ret;
  1013. if (args->buffer_count < 1 ||
  1014. args->buffer_count > UINT_MAX / sizeof(*exec2_list)) {
  1015. DRM_DEBUG("execbuf2 with %d buffers\n", args->buffer_count);
  1016. return -EINVAL;
  1017. }
  1018. exec2_list = kmalloc(sizeof(*exec2_list)*args->buffer_count,
  1019. GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY);
  1020. if (exec2_list == NULL)
  1021. exec2_list = drm_malloc_ab(sizeof(*exec2_list),
  1022. args->buffer_count);
  1023. if (exec2_list == NULL) {
  1024. DRM_DEBUG("Failed to allocate exec list for %d buffers\n",
  1025. args->buffer_count);
  1026. return -ENOMEM;
  1027. }
  1028. ret = copy_from_user(exec2_list,
  1029. to_user_ptr(args->buffers_ptr),
  1030. sizeof(*exec2_list) * args->buffer_count);
  1031. if (ret != 0) {
  1032. DRM_DEBUG("copy %d exec entries failed %d\n",
  1033. args->buffer_count, ret);
  1034. drm_free_large(exec2_list);
  1035. return -EFAULT;
  1036. }
  1037. ret = i915_gem_do_execbuffer(dev, data, file, args, exec2_list);
  1038. if (!ret) {
  1039. /* Copy the new buffer offsets back to the user's exec list. */
  1040. ret = copy_to_user(to_user_ptr(args->buffers_ptr),
  1041. exec2_list,
  1042. sizeof(*exec2_list) * args->buffer_count);
  1043. if (ret) {
  1044. ret = -EFAULT;
  1045. DRM_DEBUG("failed to copy %d exec entries "
  1046. "back to user (%d)\n",
  1047. args->buffer_count, ret);
  1048. }
  1049. }
  1050. drm_free_large(exec2_list);
  1051. return ret;
  1052. }