i915_gem_execbuffer.c 33 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235
  1. /*
  2. * Copyright © 2008,2010 Intel Corporation
  3. *
  4. * Permission is hereby granted, free of charge, to any person obtaining a
  5. * copy of this software and associated documentation files (the "Software"),
  6. * to deal in the Software without restriction, including without limitation
  7. * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8. * and/or sell copies of the Software, and to permit persons to whom the
  9. * Software is furnished to do so, subject to the following conditions:
  10. *
  11. * The above copyright notice and this permission notice (including the next
  12. * paragraph) shall be included in all copies or substantial portions of the
  13. * Software.
  14. *
  15. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17. * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  18. * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19. * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20. * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21. * IN THE SOFTWARE.
  22. *
  23. * Authors:
  24. * Eric Anholt <eric@anholt.net>
  25. * Chris Wilson <chris@chris-wilson.co.uk>
  26. *
  27. */
  28. #include <drm/drmP.h>
  29. #include <drm/i915_drm.h>
  30. #include "i915_drv.h"
  31. #include "i915_trace.h"
  32. #include "intel_drv.h"
  33. #include <linux/dma_remapping.h>
  34. struct eb_objects {
  35. struct list_head objects;
  36. int and;
  37. union {
  38. struct drm_i915_gem_object *lut[0];
  39. struct hlist_head buckets[0];
  40. };
  41. };
  42. static struct eb_objects *
  43. eb_create(struct drm_i915_gem_execbuffer2 *args)
  44. {
  45. struct eb_objects *eb = NULL;
  46. if (args->flags & I915_EXEC_HANDLE_LUT) {
  47. int size = args->buffer_count;
  48. size *= sizeof(struct drm_i915_gem_object *);
  49. size += sizeof(struct eb_objects);
  50. eb = kmalloc(size, GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY);
  51. }
  52. if (eb == NULL) {
  53. int size = args->buffer_count;
  54. int count = PAGE_SIZE / sizeof(struct hlist_head) / 2;
  55. BUILD_BUG_ON_NOT_POWER_OF_2(PAGE_SIZE / sizeof(struct hlist_head));
  56. while (count > 2*size)
  57. count >>= 1;
  58. eb = kzalloc(count*sizeof(struct hlist_head) +
  59. sizeof(struct eb_objects),
  60. GFP_TEMPORARY);
  61. if (eb == NULL)
  62. return eb;
  63. eb->and = count - 1;
  64. } else
  65. eb->and = -args->buffer_count;
  66. INIT_LIST_HEAD(&eb->objects);
  67. return eb;
  68. }
  69. static void
  70. eb_reset(struct eb_objects *eb)
  71. {
  72. if (eb->and >= 0)
  73. memset(eb->buckets, 0, (eb->and+1)*sizeof(struct hlist_head));
  74. }
  75. static int
  76. eb_lookup_objects(struct eb_objects *eb,
  77. struct drm_i915_gem_exec_object2 *exec,
  78. const struct drm_i915_gem_execbuffer2 *args,
  79. struct drm_file *file)
  80. {
  81. int i;
  82. spin_lock(&file->table_lock);
  83. for (i = 0; i < args->buffer_count; i++) {
  84. struct drm_i915_gem_object *obj;
  85. obj = to_intel_bo(idr_find(&file->object_idr, exec[i].handle));
  86. if (obj == NULL) {
  87. spin_unlock(&file->table_lock);
  88. DRM_DEBUG("Invalid object handle %d at index %d\n",
  89. exec[i].handle, i);
  90. return -ENOENT;
  91. }
  92. if (!list_empty(&obj->exec_list)) {
  93. spin_unlock(&file->table_lock);
  94. DRM_DEBUG("Object %p [handle %d, index %d] appears more than once in object list\n",
  95. obj, exec[i].handle, i);
  96. return -EINVAL;
  97. }
  98. drm_gem_object_reference(&obj->base);
  99. list_add_tail(&obj->exec_list, &eb->objects);
  100. obj->exec_entry = &exec[i];
  101. if (eb->and < 0) {
  102. eb->lut[i] = obj;
  103. } else {
  104. uint32_t handle = args->flags & I915_EXEC_HANDLE_LUT ? i : exec[i].handle;
  105. obj->exec_handle = handle;
  106. hlist_add_head(&obj->exec_node,
  107. &eb->buckets[handle & eb->and]);
  108. }
  109. }
  110. spin_unlock(&file->table_lock);
  111. return 0;
  112. }
  113. static struct drm_i915_gem_object *
  114. eb_get_object(struct eb_objects *eb, unsigned long handle)
  115. {
  116. if (eb->and < 0) {
  117. if (handle >= -eb->and)
  118. return NULL;
  119. return eb->lut[handle];
  120. } else {
  121. struct hlist_head *head;
  122. struct hlist_node *node;
  123. head = &eb->buckets[handle & eb->and];
  124. hlist_for_each(node, head) {
  125. struct drm_i915_gem_object *obj;
  126. obj = hlist_entry(node, struct drm_i915_gem_object, exec_node);
  127. if (obj->exec_handle == handle)
  128. return obj;
  129. }
  130. return NULL;
  131. }
  132. }
  133. static void
  134. eb_destroy(struct eb_objects *eb)
  135. {
  136. while (!list_empty(&eb->objects)) {
  137. struct drm_i915_gem_object *obj;
  138. obj = list_first_entry(&eb->objects,
  139. struct drm_i915_gem_object,
  140. exec_list);
  141. list_del_init(&obj->exec_list);
  142. drm_gem_object_unreference(&obj->base);
  143. }
  144. kfree(eb);
  145. }
  146. static inline int use_cpu_reloc(struct drm_i915_gem_object *obj)
  147. {
  148. return (obj->base.write_domain == I915_GEM_DOMAIN_CPU ||
  149. !obj->map_and_fenceable ||
  150. obj->cache_level != I915_CACHE_NONE);
  151. }
  152. static int
  153. i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
  154. struct eb_objects *eb,
  155. struct drm_i915_gem_relocation_entry *reloc)
  156. {
  157. struct drm_device *dev = obj->base.dev;
  158. struct drm_gem_object *target_obj;
  159. struct drm_i915_gem_object *target_i915_obj;
  160. uint32_t target_offset;
  161. int ret = -EINVAL;
  162. /* we've already hold a reference to all valid objects */
  163. target_obj = &eb_get_object(eb, reloc->target_handle)->base;
  164. if (unlikely(target_obj == NULL))
  165. return -ENOENT;
  166. target_i915_obj = to_intel_bo(target_obj);
  167. target_offset = i915_gem_obj_ggtt_offset(target_i915_obj);
  168. /* Sandybridge PPGTT errata: We need a global gtt mapping for MI and
  169. * pipe_control writes because the gpu doesn't properly redirect them
  170. * through the ppgtt for non_secure batchbuffers. */
  171. if (unlikely(IS_GEN6(dev) &&
  172. reloc->write_domain == I915_GEM_DOMAIN_INSTRUCTION &&
  173. !target_i915_obj->has_global_gtt_mapping)) {
  174. i915_gem_gtt_bind_object(target_i915_obj,
  175. target_i915_obj->cache_level);
  176. }
  177. /* Validate that the target is in a valid r/w GPU domain */
  178. if (unlikely(reloc->write_domain & (reloc->write_domain - 1))) {
  179. DRM_DEBUG("reloc with multiple write domains: "
  180. "obj %p target %d offset %d "
  181. "read %08x write %08x",
  182. obj, reloc->target_handle,
  183. (int) reloc->offset,
  184. reloc->read_domains,
  185. reloc->write_domain);
  186. return ret;
  187. }
  188. if (unlikely((reloc->write_domain | reloc->read_domains)
  189. & ~I915_GEM_GPU_DOMAINS)) {
  190. DRM_DEBUG("reloc with read/write non-GPU domains: "
  191. "obj %p target %d offset %d "
  192. "read %08x write %08x",
  193. obj, reloc->target_handle,
  194. (int) reloc->offset,
  195. reloc->read_domains,
  196. reloc->write_domain);
  197. return ret;
  198. }
  199. target_obj->pending_read_domains |= reloc->read_domains;
  200. target_obj->pending_write_domain |= reloc->write_domain;
  201. /* If the relocation already has the right value in it, no
  202. * more work needs to be done.
  203. */
  204. if (target_offset == reloc->presumed_offset)
  205. return 0;
  206. /* Check that the relocation address is valid... */
  207. if (unlikely(reloc->offset > obj->base.size - 4)) {
  208. DRM_DEBUG("Relocation beyond object bounds: "
  209. "obj %p target %d offset %d size %d.\n",
  210. obj, reloc->target_handle,
  211. (int) reloc->offset,
  212. (int) obj->base.size);
  213. return ret;
  214. }
  215. if (unlikely(reloc->offset & 3)) {
  216. DRM_DEBUG("Relocation not 4-byte aligned: "
  217. "obj %p target %d offset %d.\n",
  218. obj, reloc->target_handle,
  219. (int) reloc->offset);
  220. return ret;
  221. }
  222. /* We can't wait for rendering with pagefaults disabled */
  223. if (obj->active && in_atomic())
  224. return -EFAULT;
  225. reloc->delta += target_offset;
  226. if (use_cpu_reloc(obj)) {
  227. uint32_t page_offset = reloc->offset & ~PAGE_MASK;
  228. char *vaddr;
  229. ret = i915_gem_object_set_to_cpu_domain(obj, 1);
  230. if (ret)
  231. return ret;
  232. vaddr = kmap_atomic(i915_gem_object_get_page(obj,
  233. reloc->offset >> PAGE_SHIFT));
  234. *(uint32_t *)(vaddr + page_offset) = reloc->delta;
  235. kunmap_atomic(vaddr);
  236. } else {
  237. struct drm_i915_private *dev_priv = dev->dev_private;
  238. uint32_t __iomem *reloc_entry;
  239. void __iomem *reloc_page;
  240. ret = i915_gem_object_set_to_gtt_domain(obj, true);
  241. if (ret)
  242. return ret;
  243. ret = i915_gem_object_put_fence(obj);
  244. if (ret)
  245. return ret;
  246. /* Map the page containing the relocation we're going to perform. */
  247. reloc->offset += i915_gem_obj_ggtt_offset(obj);
  248. reloc_page = io_mapping_map_atomic_wc(dev_priv->gtt.mappable,
  249. reloc->offset & PAGE_MASK);
  250. reloc_entry = (uint32_t __iomem *)
  251. (reloc_page + (reloc->offset & ~PAGE_MASK));
  252. iowrite32(reloc->delta, reloc_entry);
  253. io_mapping_unmap_atomic(reloc_page);
  254. }
  255. /* and update the user's relocation entry */
  256. reloc->presumed_offset = target_offset;
  257. return 0;
  258. }
  259. static int
  260. i915_gem_execbuffer_relocate_object(struct drm_i915_gem_object *obj,
  261. struct eb_objects *eb)
  262. {
  263. #define N_RELOC(x) ((x) / sizeof(struct drm_i915_gem_relocation_entry))
  264. struct drm_i915_gem_relocation_entry stack_reloc[N_RELOC(512)];
  265. struct drm_i915_gem_relocation_entry __user *user_relocs;
  266. struct drm_i915_gem_exec_object2 *entry = obj->exec_entry;
  267. int remain, ret;
  268. user_relocs = to_user_ptr(entry->relocs_ptr);
  269. remain = entry->relocation_count;
  270. while (remain) {
  271. struct drm_i915_gem_relocation_entry *r = stack_reloc;
  272. int count = remain;
  273. if (count > ARRAY_SIZE(stack_reloc))
  274. count = ARRAY_SIZE(stack_reloc);
  275. remain -= count;
  276. if (__copy_from_user_inatomic(r, user_relocs, count*sizeof(r[0])))
  277. return -EFAULT;
  278. do {
  279. u64 offset = r->presumed_offset;
  280. ret = i915_gem_execbuffer_relocate_entry(obj, eb, r);
  281. if (ret)
  282. return ret;
  283. if (r->presumed_offset != offset &&
  284. __copy_to_user_inatomic(&user_relocs->presumed_offset,
  285. &r->presumed_offset,
  286. sizeof(r->presumed_offset))) {
  287. return -EFAULT;
  288. }
  289. user_relocs++;
  290. r++;
  291. } while (--count);
  292. }
  293. return 0;
  294. #undef N_RELOC
  295. }
  296. static int
  297. i915_gem_execbuffer_relocate_object_slow(struct drm_i915_gem_object *obj,
  298. struct eb_objects *eb,
  299. struct drm_i915_gem_relocation_entry *relocs)
  300. {
  301. const struct drm_i915_gem_exec_object2 *entry = obj->exec_entry;
  302. int i, ret;
  303. for (i = 0; i < entry->relocation_count; i++) {
  304. ret = i915_gem_execbuffer_relocate_entry(obj, eb, &relocs[i]);
  305. if (ret)
  306. return ret;
  307. }
  308. return 0;
  309. }
  310. static int
  311. i915_gem_execbuffer_relocate(struct eb_objects *eb)
  312. {
  313. struct drm_i915_gem_object *obj;
  314. int ret = 0;
  315. /* This is the fast path and we cannot handle a pagefault whilst
  316. * holding the struct mutex lest the user pass in the relocations
  317. * contained within a mmaped bo. For in such a case we, the page
  318. * fault handler would call i915_gem_fault() and we would try to
  319. * acquire the struct mutex again. Obviously this is bad and so
  320. * lockdep complains vehemently.
  321. */
  322. pagefault_disable();
  323. list_for_each_entry(obj, &eb->objects, exec_list) {
  324. ret = i915_gem_execbuffer_relocate_object(obj, eb);
  325. if (ret)
  326. break;
  327. }
  328. pagefault_enable();
  329. return ret;
  330. }
  331. #define __EXEC_OBJECT_HAS_PIN (1<<31)
  332. #define __EXEC_OBJECT_HAS_FENCE (1<<30)
  333. static int
  334. need_reloc_mappable(struct drm_i915_gem_object *obj)
  335. {
  336. struct drm_i915_gem_exec_object2 *entry = obj->exec_entry;
  337. return entry->relocation_count && !use_cpu_reloc(obj);
  338. }
  339. static int
  340. i915_gem_execbuffer_reserve_object(struct drm_i915_gem_object *obj,
  341. struct intel_ring_buffer *ring,
  342. bool *need_reloc)
  343. {
  344. struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  345. struct drm_i915_gem_exec_object2 *entry = obj->exec_entry;
  346. bool has_fenced_gpu_access = INTEL_INFO(ring->dev)->gen < 4;
  347. bool need_fence, need_mappable;
  348. int ret;
  349. need_fence =
  350. has_fenced_gpu_access &&
  351. entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
  352. obj->tiling_mode != I915_TILING_NONE;
  353. need_mappable = need_fence || need_reloc_mappable(obj);
  354. ret = i915_gem_object_pin(obj, entry->alignment, need_mappable, false);
  355. if (ret)
  356. return ret;
  357. entry->flags |= __EXEC_OBJECT_HAS_PIN;
  358. if (has_fenced_gpu_access) {
  359. if (entry->flags & EXEC_OBJECT_NEEDS_FENCE) {
  360. ret = i915_gem_object_get_fence(obj);
  361. if (ret)
  362. return ret;
  363. if (i915_gem_object_pin_fence(obj))
  364. entry->flags |= __EXEC_OBJECT_HAS_FENCE;
  365. obj->pending_fenced_gpu_access = true;
  366. }
  367. }
  368. /* Ensure ppgtt mapping exists if needed */
  369. if (dev_priv->mm.aliasing_ppgtt && !obj->has_aliasing_ppgtt_mapping) {
  370. i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
  371. obj, obj->cache_level);
  372. obj->has_aliasing_ppgtt_mapping = 1;
  373. }
  374. if (entry->offset != i915_gem_obj_ggtt_offset(obj)) {
  375. entry->offset = i915_gem_obj_ggtt_offset(obj);
  376. *need_reloc = true;
  377. }
  378. if (entry->flags & EXEC_OBJECT_WRITE) {
  379. obj->base.pending_read_domains = I915_GEM_DOMAIN_RENDER;
  380. obj->base.pending_write_domain = I915_GEM_DOMAIN_RENDER;
  381. }
  382. if (entry->flags & EXEC_OBJECT_NEEDS_GTT &&
  383. !obj->has_global_gtt_mapping)
  384. i915_gem_gtt_bind_object(obj, obj->cache_level);
  385. return 0;
  386. }
  387. static void
  388. i915_gem_execbuffer_unreserve_object(struct drm_i915_gem_object *obj)
  389. {
  390. struct drm_i915_gem_exec_object2 *entry;
  391. if (!i915_gem_obj_ggtt_bound(obj))
  392. return;
  393. entry = obj->exec_entry;
  394. if (entry->flags & __EXEC_OBJECT_HAS_FENCE)
  395. i915_gem_object_unpin_fence(obj);
  396. if (entry->flags & __EXEC_OBJECT_HAS_PIN)
  397. i915_gem_object_unpin(obj);
  398. entry->flags &= ~(__EXEC_OBJECT_HAS_FENCE | __EXEC_OBJECT_HAS_PIN);
  399. }
  400. static int
  401. i915_gem_execbuffer_reserve(struct intel_ring_buffer *ring,
  402. struct list_head *objects,
  403. bool *need_relocs)
  404. {
  405. struct drm_i915_gem_object *obj;
  406. struct list_head ordered_objects;
  407. bool has_fenced_gpu_access = INTEL_INFO(ring->dev)->gen < 4;
  408. int retry;
  409. INIT_LIST_HEAD(&ordered_objects);
  410. while (!list_empty(objects)) {
  411. struct drm_i915_gem_exec_object2 *entry;
  412. bool need_fence, need_mappable;
  413. obj = list_first_entry(objects,
  414. struct drm_i915_gem_object,
  415. exec_list);
  416. entry = obj->exec_entry;
  417. need_fence =
  418. has_fenced_gpu_access &&
  419. entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
  420. obj->tiling_mode != I915_TILING_NONE;
  421. need_mappable = need_fence || need_reloc_mappable(obj);
  422. if (need_mappable)
  423. list_move(&obj->exec_list, &ordered_objects);
  424. else
  425. list_move_tail(&obj->exec_list, &ordered_objects);
  426. obj->base.pending_read_domains = I915_GEM_GPU_DOMAINS & ~I915_GEM_DOMAIN_COMMAND;
  427. obj->base.pending_write_domain = 0;
  428. obj->pending_fenced_gpu_access = false;
  429. }
  430. list_splice(&ordered_objects, objects);
  431. /* Attempt to pin all of the buffers into the GTT.
  432. * This is done in 3 phases:
  433. *
  434. * 1a. Unbind all objects that do not match the GTT constraints for
  435. * the execbuffer (fenceable, mappable, alignment etc).
  436. * 1b. Increment pin count for already bound objects.
  437. * 2. Bind new objects.
  438. * 3. Decrement pin count.
  439. *
  440. * This avoid unnecessary unbinding of later objects in order to make
  441. * room for the earlier objects *unless* we need to defragment.
  442. */
  443. retry = 0;
  444. do {
  445. int ret = 0;
  446. /* Unbind any ill-fitting objects or pin. */
  447. list_for_each_entry(obj, objects, exec_list) {
  448. struct drm_i915_gem_exec_object2 *entry = obj->exec_entry;
  449. bool need_fence, need_mappable;
  450. if (!i915_gem_obj_ggtt_bound(obj))
  451. continue;
  452. need_fence =
  453. has_fenced_gpu_access &&
  454. entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
  455. obj->tiling_mode != I915_TILING_NONE;
  456. need_mappable = need_fence || need_reloc_mappable(obj);
  457. if ((entry->alignment &&
  458. i915_gem_obj_ggtt_offset(obj) & (entry->alignment - 1)) ||
  459. (need_mappable && !obj->map_and_fenceable))
  460. ret = i915_gem_object_unbind(obj);
  461. else
  462. ret = i915_gem_execbuffer_reserve_object(obj, ring, need_relocs);
  463. if (ret)
  464. goto err;
  465. }
  466. /* Bind fresh objects */
  467. list_for_each_entry(obj, objects, exec_list) {
  468. if (i915_gem_obj_ggtt_bound(obj))
  469. continue;
  470. ret = i915_gem_execbuffer_reserve_object(obj, ring, need_relocs);
  471. if (ret)
  472. goto err;
  473. }
  474. err: /* Decrement pin count for bound objects */
  475. list_for_each_entry(obj, objects, exec_list)
  476. i915_gem_execbuffer_unreserve_object(obj);
  477. if (ret != -ENOSPC || retry++)
  478. return ret;
  479. ret = i915_gem_evict_everything(ring->dev);
  480. if (ret)
  481. return ret;
  482. } while (1);
  483. }
  484. static int
  485. i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
  486. struct drm_i915_gem_execbuffer2 *args,
  487. struct drm_file *file,
  488. struct intel_ring_buffer *ring,
  489. struct eb_objects *eb,
  490. struct drm_i915_gem_exec_object2 *exec)
  491. {
  492. struct drm_i915_gem_relocation_entry *reloc;
  493. struct drm_i915_gem_object *obj;
  494. bool need_relocs;
  495. int *reloc_offset;
  496. int i, total, ret;
  497. int count = args->buffer_count;
  498. /* We may process another execbuffer during the unlock... */
  499. while (!list_empty(&eb->objects)) {
  500. obj = list_first_entry(&eb->objects,
  501. struct drm_i915_gem_object,
  502. exec_list);
  503. list_del_init(&obj->exec_list);
  504. drm_gem_object_unreference(&obj->base);
  505. }
  506. mutex_unlock(&dev->struct_mutex);
  507. total = 0;
  508. for (i = 0; i < count; i++)
  509. total += exec[i].relocation_count;
  510. reloc_offset = drm_malloc_ab(count, sizeof(*reloc_offset));
  511. reloc = drm_malloc_ab(total, sizeof(*reloc));
  512. if (reloc == NULL || reloc_offset == NULL) {
  513. drm_free_large(reloc);
  514. drm_free_large(reloc_offset);
  515. mutex_lock(&dev->struct_mutex);
  516. return -ENOMEM;
  517. }
  518. total = 0;
  519. for (i = 0; i < count; i++) {
  520. struct drm_i915_gem_relocation_entry __user *user_relocs;
  521. u64 invalid_offset = (u64)-1;
  522. int j;
  523. user_relocs = to_user_ptr(exec[i].relocs_ptr);
  524. if (copy_from_user(reloc+total, user_relocs,
  525. exec[i].relocation_count * sizeof(*reloc))) {
  526. ret = -EFAULT;
  527. mutex_lock(&dev->struct_mutex);
  528. goto err;
  529. }
  530. /* As we do not update the known relocation offsets after
  531. * relocating (due to the complexities in lock handling),
  532. * we need to mark them as invalid now so that we force the
  533. * relocation processing next time. Just in case the target
  534. * object is evicted and then rebound into its old
  535. * presumed_offset before the next execbuffer - if that
  536. * happened we would make the mistake of assuming that the
  537. * relocations were valid.
  538. */
  539. for (j = 0; j < exec[i].relocation_count; j++) {
  540. if (copy_to_user(&user_relocs[j].presumed_offset,
  541. &invalid_offset,
  542. sizeof(invalid_offset))) {
  543. ret = -EFAULT;
  544. mutex_lock(&dev->struct_mutex);
  545. goto err;
  546. }
  547. }
  548. reloc_offset[i] = total;
  549. total += exec[i].relocation_count;
  550. }
  551. ret = i915_mutex_lock_interruptible(dev);
  552. if (ret) {
  553. mutex_lock(&dev->struct_mutex);
  554. goto err;
  555. }
  556. /* reacquire the objects */
  557. eb_reset(eb);
  558. ret = eb_lookup_objects(eb, exec, args, file);
  559. if (ret)
  560. goto err;
  561. need_relocs = (args->flags & I915_EXEC_NO_RELOC) == 0;
  562. ret = i915_gem_execbuffer_reserve(ring, &eb->objects, &need_relocs);
  563. if (ret)
  564. goto err;
  565. list_for_each_entry(obj, &eb->objects, exec_list) {
  566. int offset = obj->exec_entry - exec;
  567. ret = i915_gem_execbuffer_relocate_object_slow(obj, eb,
  568. reloc + reloc_offset[offset]);
  569. if (ret)
  570. goto err;
  571. }
  572. /* Leave the user relocations as are, this is the painfully slow path,
  573. * and we want to avoid the complication of dropping the lock whilst
  574. * having buffers reserved in the aperture and so causing spurious
  575. * ENOSPC for random operations.
  576. */
  577. err:
  578. drm_free_large(reloc);
  579. drm_free_large(reloc_offset);
  580. return ret;
  581. }
  582. static int
  583. i915_gem_execbuffer_move_to_gpu(struct intel_ring_buffer *ring,
  584. struct list_head *objects)
  585. {
  586. struct drm_i915_gem_object *obj;
  587. uint32_t flush_domains = 0;
  588. int ret;
  589. list_for_each_entry(obj, objects, exec_list) {
  590. ret = i915_gem_object_sync(obj, ring);
  591. if (ret)
  592. return ret;
  593. if (obj->base.write_domain & I915_GEM_DOMAIN_CPU)
  594. i915_gem_clflush_object(obj);
  595. flush_domains |= obj->base.write_domain;
  596. }
  597. if (flush_domains & I915_GEM_DOMAIN_CPU)
  598. i915_gem_chipset_flush(ring->dev);
  599. if (flush_domains & I915_GEM_DOMAIN_GTT)
  600. wmb();
  601. /* Unconditionally invalidate gpu caches and ensure that we do flush
  602. * any residual writes from the previous batch.
  603. */
  604. return intel_ring_invalidate_all_caches(ring);
  605. }
  606. static bool
  607. i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec)
  608. {
  609. if (exec->flags & __I915_EXEC_UNKNOWN_FLAGS)
  610. return false;
  611. return ((exec->batch_start_offset | exec->batch_len) & 0x7) == 0;
  612. }
  613. static int
  614. validate_exec_list(struct drm_i915_gem_exec_object2 *exec,
  615. int count)
  616. {
  617. int i;
  618. int relocs_total = 0;
  619. int relocs_max = INT_MAX / sizeof(struct drm_i915_gem_relocation_entry);
  620. for (i = 0; i < count; i++) {
  621. char __user *ptr = to_user_ptr(exec[i].relocs_ptr);
  622. int length; /* limited by fault_in_pages_readable() */
  623. if (exec[i].flags & __EXEC_OBJECT_UNKNOWN_FLAGS)
  624. return -EINVAL;
  625. /* First check for malicious input causing overflow in
  626. * the worst case where we need to allocate the entire
  627. * relocation tree as a single array.
  628. */
  629. if (exec[i].relocation_count > relocs_max - relocs_total)
  630. return -EINVAL;
  631. relocs_total += exec[i].relocation_count;
  632. length = exec[i].relocation_count *
  633. sizeof(struct drm_i915_gem_relocation_entry);
  634. /*
  635. * We must check that the entire relocation array is safe
  636. * to read, but since we may need to update the presumed
  637. * offsets during execution, check for full write access.
  638. */
  639. if (!access_ok(VERIFY_WRITE, ptr, length))
  640. return -EFAULT;
  641. if (fault_in_multipages_readable(ptr, length))
  642. return -EFAULT;
  643. }
  644. return 0;
  645. }
  646. static void
  647. i915_gem_execbuffer_move_to_active(struct list_head *objects,
  648. struct intel_ring_buffer *ring)
  649. {
  650. struct drm_i915_gem_object *obj;
  651. list_for_each_entry(obj, objects, exec_list) {
  652. u32 old_read = obj->base.read_domains;
  653. u32 old_write = obj->base.write_domain;
  654. obj->base.write_domain = obj->base.pending_write_domain;
  655. if (obj->base.write_domain == 0)
  656. obj->base.pending_read_domains |= obj->base.read_domains;
  657. obj->base.read_domains = obj->base.pending_read_domains;
  658. obj->fenced_gpu_access = obj->pending_fenced_gpu_access;
  659. i915_gem_object_move_to_active(obj, ring);
  660. if (obj->base.write_domain) {
  661. obj->dirty = 1;
  662. obj->last_write_seqno = intel_ring_get_seqno(ring);
  663. if (obj->pin_count) /* check for potential scanout */
  664. intel_mark_fb_busy(obj, ring);
  665. }
  666. trace_i915_gem_object_change_domain(obj, old_read, old_write);
  667. }
  668. }
  669. static void
  670. i915_gem_execbuffer_retire_commands(struct drm_device *dev,
  671. struct drm_file *file,
  672. struct intel_ring_buffer *ring,
  673. struct drm_i915_gem_object *obj)
  674. {
  675. /* Unconditionally force add_request to emit a full flush. */
  676. ring->gpu_caches_dirty = true;
  677. /* Add a breadcrumb for the completion of the batch buffer */
  678. (void)__i915_add_request(ring, file, obj, NULL);
  679. }
  680. static int
  681. i915_reset_gen7_sol_offsets(struct drm_device *dev,
  682. struct intel_ring_buffer *ring)
  683. {
  684. drm_i915_private_t *dev_priv = dev->dev_private;
  685. int ret, i;
  686. if (!IS_GEN7(dev) || ring != &dev_priv->ring[RCS])
  687. return 0;
  688. ret = intel_ring_begin(ring, 4 * 3);
  689. if (ret)
  690. return ret;
  691. for (i = 0; i < 4; i++) {
  692. intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
  693. intel_ring_emit(ring, GEN7_SO_WRITE_OFFSET(i));
  694. intel_ring_emit(ring, 0);
  695. }
  696. intel_ring_advance(ring);
  697. return 0;
  698. }
  699. static int
  700. i915_gem_do_execbuffer(struct drm_device *dev, void *data,
  701. struct drm_file *file,
  702. struct drm_i915_gem_execbuffer2 *args,
  703. struct drm_i915_gem_exec_object2 *exec)
  704. {
  705. drm_i915_private_t *dev_priv = dev->dev_private;
  706. struct eb_objects *eb;
  707. struct drm_i915_gem_object *batch_obj;
  708. struct drm_clip_rect *cliprects = NULL;
  709. struct intel_ring_buffer *ring;
  710. u32 ctx_id = i915_execbuffer2_get_context_id(*args);
  711. u32 exec_start, exec_len;
  712. u32 mask, flags;
  713. int ret, mode, i;
  714. bool need_relocs;
  715. if (!i915_gem_check_execbuffer(args))
  716. return -EINVAL;
  717. ret = validate_exec_list(exec, args->buffer_count);
  718. if (ret)
  719. return ret;
  720. flags = 0;
  721. if (args->flags & I915_EXEC_SECURE) {
  722. if (!file->is_master || !capable(CAP_SYS_ADMIN))
  723. return -EPERM;
  724. flags |= I915_DISPATCH_SECURE;
  725. }
  726. if (args->flags & I915_EXEC_IS_PINNED)
  727. flags |= I915_DISPATCH_PINNED;
  728. switch (args->flags & I915_EXEC_RING_MASK) {
  729. case I915_EXEC_DEFAULT:
  730. case I915_EXEC_RENDER:
  731. ring = &dev_priv->ring[RCS];
  732. break;
  733. case I915_EXEC_BSD:
  734. ring = &dev_priv->ring[VCS];
  735. if (ctx_id != 0) {
  736. DRM_DEBUG("Ring %s doesn't support contexts\n",
  737. ring->name);
  738. return -EPERM;
  739. }
  740. break;
  741. case I915_EXEC_BLT:
  742. ring = &dev_priv->ring[BCS];
  743. if (ctx_id != 0) {
  744. DRM_DEBUG("Ring %s doesn't support contexts\n",
  745. ring->name);
  746. return -EPERM;
  747. }
  748. break;
  749. case I915_EXEC_VEBOX:
  750. ring = &dev_priv->ring[VECS];
  751. if (ctx_id != 0) {
  752. DRM_DEBUG("Ring %s doesn't support contexts\n",
  753. ring->name);
  754. return -EPERM;
  755. }
  756. break;
  757. default:
  758. DRM_DEBUG("execbuf with unknown ring: %d\n",
  759. (int)(args->flags & I915_EXEC_RING_MASK));
  760. return -EINVAL;
  761. }
  762. if (!intel_ring_initialized(ring)) {
  763. DRM_DEBUG("execbuf with invalid ring: %d\n",
  764. (int)(args->flags & I915_EXEC_RING_MASK));
  765. return -EINVAL;
  766. }
  767. mode = args->flags & I915_EXEC_CONSTANTS_MASK;
  768. mask = I915_EXEC_CONSTANTS_MASK;
  769. switch (mode) {
  770. case I915_EXEC_CONSTANTS_REL_GENERAL:
  771. case I915_EXEC_CONSTANTS_ABSOLUTE:
  772. case I915_EXEC_CONSTANTS_REL_SURFACE:
  773. if (ring == &dev_priv->ring[RCS] &&
  774. mode != dev_priv->relative_constants_mode) {
  775. if (INTEL_INFO(dev)->gen < 4)
  776. return -EINVAL;
  777. if (INTEL_INFO(dev)->gen > 5 &&
  778. mode == I915_EXEC_CONSTANTS_REL_SURFACE)
  779. return -EINVAL;
  780. /* The HW changed the meaning on this bit on gen6 */
  781. if (INTEL_INFO(dev)->gen >= 6)
  782. mask &= ~I915_EXEC_CONSTANTS_REL_SURFACE;
  783. }
  784. break;
  785. default:
  786. DRM_DEBUG("execbuf with unknown constants: %d\n", mode);
  787. return -EINVAL;
  788. }
  789. if (args->buffer_count < 1) {
  790. DRM_DEBUG("execbuf with %d buffers\n", args->buffer_count);
  791. return -EINVAL;
  792. }
  793. if (args->num_cliprects != 0) {
  794. if (ring != &dev_priv->ring[RCS]) {
  795. DRM_DEBUG("clip rectangles are only valid with the render ring\n");
  796. return -EINVAL;
  797. }
  798. if (INTEL_INFO(dev)->gen >= 5) {
  799. DRM_DEBUG("clip rectangles are only valid on pre-gen5\n");
  800. return -EINVAL;
  801. }
  802. if (args->num_cliprects > UINT_MAX / sizeof(*cliprects)) {
  803. DRM_DEBUG("execbuf with %u cliprects\n",
  804. args->num_cliprects);
  805. return -EINVAL;
  806. }
  807. cliprects = kmalloc(args->num_cliprects * sizeof(*cliprects),
  808. GFP_KERNEL);
  809. if (cliprects == NULL) {
  810. ret = -ENOMEM;
  811. goto pre_mutex_err;
  812. }
  813. if (copy_from_user(cliprects,
  814. to_user_ptr(args->cliprects_ptr),
  815. sizeof(*cliprects)*args->num_cliprects)) {
  816. ret = -EFAULT;
  817. goto pre_mutex_err;
  818. }
  819. }
  820. ret = i915_mutex_lock_interruptible(dev);
  821. if (ret)
  822. goto pre_mutex_err;
  823. if (dev_priv->ums.mm_suspended) {
  824. mutex_unlock(&dev->struct_mutex);
  825. ret = -EBUSY;
  826. goto pre_mutex_err;
  827. }
  828. eb = eb_create(args);
  829. if (eb == NULL) {
  830. mutex_unlock(&dev->struct_mutex);
  831. ret = -ENOMEM;
  832. goto pre_mutex_err;
  833. }
  834. /* Look up object handles */
  835. ret = eb_lookup_objects(eb, exec, args, file);
  836. if (ret)
  837. goto err;
  838. /* take note of the batch buffer before we might reorder the lists */
  839. batch_obj = list_entry(eb->objects.prev,
  840. struct drm_i915_gem_object,
  841. exec_list);
  842. /* Move the objects en-masse into the GTT, evicting if necessary. */
  843. need_relocs = (args->flags & I915_EXEC_NO_RELOC) == 0;
  844. ret = i915_gem_execbuffer_reserve(ring, &eb->objects, &need_relocs);
  845. if (ret)
  846. goto err;
  847. /* The objects are in their final locations, apply the relocations. */
  848. if (need_relocs)
  849. ret = i915_gem_execbuffer_relocate(eb);
  850. if (ret) {
  851. if (ret == -EFAULT) {
  852. ret = i915_gem_execbuffer_relocate_slow(dev, args, file, ring,
  853. eb, exec);
  854. BUG_ON(!mutex_is_locked(&dev->struct_mutex));
  855. }
  856. if (ret)
  857. goto err;
  858. }
  859. /* Set the pending read domains for the batch buffer to COMMAND */
  860. if (batch_obj->base.pending_write_domain) {
  861. DRM_DEBUG("Attempting to use self-modifying batch buffer\n");
  862. ret = -EINVAL;
  863. goto err;
  864. }
  865. batch_obj->base.pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
  866. /* snb/ivb/vlv conflate the "batch in ppgtt" bit with the "non-secure
  867. * batch" bit. Hence we need to pin secure batches into the global gtt.
  868. * hsw should have this fixed, but let's be paranoid and do it
  869. * unconditionally for now. */
  870. if (flags & I915_DISPATCH_SECURE && !batch_obj->has_global_gtt_mapping)
  871. i915_gem_gtt_bind_object(batch_obj, batch_obj->cache_level);
  872. ret = i915_gem_execbuffer_move_to_gpu(ring, &eb->objects);
  873. if (ret)
  874. goto err;
  875. ret = i915_switch_context(ring, file, ctx_id);
  876. if (ret)
  877. goto err;
  878. if (ring == &dev_priv->ring[RCS] &&
  879. mode != dev_priv->relative_constants_mode) {
  880. ret = intel_ring_begin(ring, 4);
  881. if (ret)
  882. goto err;
  883. intel_ring_emit(ring, MI_NOOP);
  884. intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
  885. intel_ring_emit(ring, INSTPM);
  886. intel_ring_emit(ring, mask << 16 | mode);
  887. intel_ring_advance(ring);
  888. dev_priv->relative_constants_mode = mode;
  889. }
  890. if (args->flags & I915_EXEC_GEN7_SOL_RESET) {
  891. ret = i915_reset_gen7_sol_offsets(dev, ring);
  892. if (ret)
  893. goto err;
  894. }
  895. exec_start = i915_gem_obj_ggtt_offset(batch_obj) + args->batch_start_offset;
  896. exec_len = args->batch_len;
  897. if (cliprects) {
  898. for (i = 0; i < args->num_cliprects; i++) {
  899. ret = i915_emit_box(dev, &cliprects[i],
  900. args->DR1, args->DR4);
  901. if (ret)
  902. goto err;
  903. ret = ring->dispatch_execbuffer(ring,
  904. exec_start, exec_len,
  905. flags);
  906. if (ret)
  907. goto err;
  908. }
  909. } else {
  910. ret = ring->dispatch_execbuffer(ring,
  911. exec_start, exec_len,
  912. flags);
  913. if (ret)
  914. goto err;
  915. }
  916. trace_i915_gem_ring_dispatch(ring, intel_ring_get_seqno(ring), flags);
  917. i915_gem_execbuffer_move_to_active(&eb->objects, ring);
  918. i915_gem_execbuffer_retire_commands(dev, file, ring, batch_obj);
  919. err:
  920. eb_destroy(eb);
  921. mutex_unlock(&dev->struct_mutex);
  922. pre_mutex_err:
  923. kfree(cliprects);
  924. return ret;
  925. }
  926. /*
  927. * Legacy execbuffer just creates an exec2 list from the original exec object
  928. * list array and passes it to the real function.
  929. */
  930. int
  931. i915_gem_execbuffer(struct drm_device *dev, void *data,
  932. struct drm_file *file)
  933. {
  934. struct drm_i915_gem_execbuffer *args = data;
  935. struct drm_i915_gem_execbuffer2 exec2;
  936. struct drm_i915_gem_exec_object *exec_list = NULL;
  937. struct drm_i915_gem_exec_object2 *exec2_list = NULL;
  938. int ret, i;
  939. if (args->buffer_count < 1) {
  940. DRM_DEBUG("execbuf with %d buffers\n", args->buffer_count);
  941. return -EINVAL;
  942. }
  943. /* Copy in the exec list from userland */
  944. exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
  945. exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
  946. if (exec_list == NULL || exec2_list == NULL) {
  947. DRM_DEBUG("Failed to allocate exec list for %d buffers\n",
  948. args->buffer_count);
  949. drm_free_large(exec_list);
  950. drm_free_large(exec2_list);
  951. return -ENOMEM;
  952. }
  953. ret = copy_from_user(exec_list,
  954. to_user_ptr(args->buffers_ptr),
  955. sizeof(*exec_list) * args->buffer_count);
  956. if (ret != 0) {
  957. DRM_DEBUG("copy %d exec entries failed %d\n",
  958. args->buffer_count, ret);
  959. drm_free_large(exec_list);
  960. drm_free_large(exec2_list);
  961. return -EFAULT;
  962. }
  963. for (i = 0; i < args->buffer_count; i++) {
  964. exec2_list[i].handle = exec_list[i].handle;
  965. exec2_list[i].relocation_count = exec_list[i].relocation_count;
  966. exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
  967. exec2_list[i].alignment = exec_list[i].alignment;
  968. exec2_list[i].offset = exec_list[i].offset;
  969. if (INTEL_INFO(dev)->gen < 4)
  970. exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
  971. else
  972. exec2_list[i].flags = 0;
  973. }
  974. exec2.buffers_ptr = args->buffers_ptr;
  975. exec2.buffer_count = args->buffer_count;
  976. exec2.batch_start_offset = args->batch_start_offset;
  977. exec2.batch_len = args->batch_len;
  978. exec2.DR1 = args->DR1;
  979. exec2.DR4 = args->DR4;
  980. exec2.num_cliprects = args->num_cliprects;
  981. exec2.cliprects_ptr = args->cliprects_ptr;
  982. exec2.flags = I915_EXEC_RENDER;
  983. i915_execbuffer2_set_context_id(exec2, 0);
  984. ret = i915_gem_do_execbuffer(dev, data, file, &exec2, exec2_list);
  985. if (!ret) {
  986. /* Copy the new buffer offsets back to the user's exec list. */
  987. for (i = 0; i < args->buffer_count; i++)
  988. exec_list[i].offset = exec2_list[i].offset;
  989. /* ... and back out to userspace */
  990. ret = copy_to_user(to_user_ptr(args->buffers_ptr),
  991. exec_list,
  992. sizeof(*exec_list) * args->buffer_count);
  993. if (ret) {
  994. ret = -EFAULT;
  995. DRM_DEBUG("failed to copy %d exec entries "
  996. "back to user (%d)\n",
  997. args->buffer_count, ret);
  998. }
  999. }
  1000. drm_free_large(exec_list);
  1001. drm_free_large(exec2_list);
  1002. return ret;
  1003. }
  1004. int
  1005. i915_gem_execbuffer2(struct drm_device *dev, void *data,
  1006. struct drm_file *file)
  1007. {
  1008. struct drm_i915_gem_execbuffer2 *args = data;
  1009. struct drm_i915_gem_exec_object2 *exec2_list = NULL;
  1010. int ret;
  1011. if (args->buffer_count < 1 ||
  1012. args->buffer_count > UINT_MAX / sizeof(*exec2_list)) {
  1013. DRM_DEBUG("execbuf2 with %d buffers\n", args->buffer_count);
  1014. return -EINVAL;
  1015. }
  1016. exec2_list = kmalloc(sizeof(*exec2_list)*args->buffer_count,
  1017. GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY);
  1018. if (exec2_list == NULL)
  1019. exec2_list = drm_malloc_ab(sizeof(*exec2_list),
  1020. args->buffer_count);
  1021. if (exec2_list == NULL) {
  1022. DRM_DEBUG("Failed to allocate exec list for %d buffers\n",
  1023. args->buffer_count);
  1024. return -ENOMEM;
  1025. }
  1026. ret = copy_from_user(exec2_list,
  1027. to_user_ptr(args->buffers_ptr),
  1028. sizeof(*exec2_list) * args->buffer_count);
  1029. if (ret != 0) {
  1030. DRM_DEBUG("copy %d exec entries failed %d\n",
  1031. args->buffer_count, ret);
  1032. drm_free_large(exec2_list);
  1033. return -EFAULT;
  1034. }
  1035. ret = i915_gem_do_execbuffer(dev, data, file, args, exec2_list);
  1036. if (!ret) {
  1037. /* Copy the new buffer offsets back to the user's exec list. */
  1038. ret = copy_to_user(to_user_ptr(args->buffers_ptr),
  1039. exec2_list,
  1040. sizeof(*exec2_list) * args->buffer_count);
  1041. if (ret) {
  1042. ret = -EFAULT;
  1043. DRM_DEBUG("failed to copy %d exec entries "
  1044. "back to user (%d)\n",
  1045. args->buffer_count, ret);
  1046. }
  1047. }
  1048. drm_free_large(exec2_list);
  1049. return ret;
  1050. }