i915_gem_execbuffer.c 35 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346
  1. /*
  2. * Copyright © 2008,2010 Intel Corporation
  3. *
  4. * Permission is hereby granted, free of charge, to any person obtaining a
  5. * copy of this software and associated documentation files (the "Software"),
  6. * to deal in the Software without restriction, including without limitation
  7. * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8. * and/or sell copies of the Software, and to permit persons to whom the
  9. * Software is furnished to do so, subject to the following conditions:
  10. *
  11. * The above copyright notice and this permission notice (including the next
  12. * paragraph) shall be included in all copies or substantial portions of the
  13. * Software.
  14. *
  15. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17. * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  18. * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19. * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20. * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21. * IN THE SOFTWARE.
  22. *
  23. * Authors:
  24. * Eric Anholt <eric@anholt.net>
  25. * Chris Wilson <chris@chris-wilson.co.uk>
  26. *
  27. */
  28. #include "drmP.h"
  29. #include "drm.h"
  30. #include "i915_drm.h"
  31. #include "i915_drv.h"
  32. #include "i915_trace.h"
  33. #include "intel_drv.h"
  34. struct change_domains {
  35. uint32_t invalidate_domains;
  36. uint32_t flush_domains;
  37. uint32_t flush_rings;
  38. };
  39. /*
  40. * Set the next domain for the specified object. This
  41. * may not actually perform the necessary flushing/invaliding though,
  42. * as that may want to be batched with other set_domain operations
  43. *
  44. * This is (we hope) the only really tricky part of gem. The goal
  45. * is fairly simple -- track which caches hold bits of the object
  46. * and make sure they remain coherent. A few concrete examples may
  47. * help to explain how it works. For shorthand, we use the notation
  48. * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
  49. * a pair of read and write domain masks.
  50. *
  51. * Case 1: the batch buffer
  52. *
  53. * 1. Allocated
  54. * 2. Written by CPU
  55. * 3. Mapped to GTT
  56. * 4. Read by GPU
  57. * 5. Unmapped from GTT
  58. * 6. Freed
  59. *
  60. * Let's take these a step at a time
  61. *
  62. * 1. Allocated
  63. * Pages allocated from the kernel may still have
  64. * cache contents, so we set them to (CPU, CPU) always.
  65. * 2. Written by CPU (using pwrite)
  66. * The pwrite function calls set_domain (CPU, CPU) and
  67. * this function does nothing (as nothing changes)
  68. * 3. Mapped by GTT
  69. * This function asserts that the object is not
  70. * currently in any GPU-based read or write domains
  71. * 4. Read by GPU
  72. * i915_gem_execbuffer calls set_domain (COMMAND, 0).
  73. * As write_domain is zero, this function adds in the
  74. * current read domains (CPU+COMMAND, 0).
  75. * flush_domains is set to CPU.
  76. * invalidate_domains is set to COMMAND
  77. * clflush is run to get data out of the CPU caches
  78. * then i915_dev_set_domain calls i915_gem_flush to
  79. * emit an MI_FLUSH and drm_agp_chipset_flush
  80. * 5. Unmapped from GTT
  81. * i915_gem_object_unbind calls set_domain (CPU, CPU)
  82. * flush_domains and invalidate_domains end up both zero
  83. * so no flushing/invalidating happens
  84. * 6. Freed
  85. * yay, done
  86. *
  87. * Case 2: The shared render buffer
  88. *
  89. * 1. Allocated
  90. * 2. Mapped to GTT
  91. * 3. Read/written by GPU
  92. * 4. set_domain to (CPU,CPU)
  93. * 5. Read/written by CPU
  94. * 6. Read/written by GPU
  95. *
  96. * 1. Allocated
  97. * Same as last example, (CPU, CPU)
  98. * 2. Mapped to GTT
  99. * Nothing changes (assertions find that it is not in the GPU)
  100. * 3. Read/written by GPU
  101. * execbuffer calls set_domain (RENDER, RENDER)
  102. * flush_domains gets CPU
  103. * invalidate_domains gets GPU
  104. * clflush (obj)
  105. * MI_FLUSH and drm_agp_chipset_flush
  106. * 4. set_domain (CPU, CPU)
  107. * flush_domains gets GPU
  108. * invalidate_domains gets CPU
  109. * wait_rendering (obj) to make sure all drawing is complete.
  110. * This will include an MI_FLUSH to get the data from GPU
  111. * to memory
  112. * clflush (obj) to invalidate the CPU cache
  113. * Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
  114. * 5. Read/written by CPU
  115. * cache lines are loaded and dirtied
  116. * 6. Read written by GPU
  117. * Same as last GPU access
  118. *
  119. * Case 3: The constant buffer
  120. *
  121. * 1. Allocated
  122. * 2. Written by CPU
  123. * 3. Read by GPU
  124. * 4. Updated (written) by CPU again
  125. * 5. Read by GPU
  126. *
  127. * 1. Allocated
  128. * (CPU, CPU)
  129. * 2. Written by CPU
  130. * (CPU, CPU)
  131. * 3. Read by GPU
  132. * (CPU+RENDER, 0)
  133. * flush_domains = CPU
  134. * invalidate_domains = RENDER
  135. * clflush (obj)
  136. * MI_FLUSH
  137. * drm_agp_chipset_flush
  138. * 4. Updated (written) by CPU again
  139. * (CPU, CPU)
  140. * flush_domains = 0 (no previous write domain)
  141. * invalidate_domains = 0 (no new read domains)
  142. * 5. Read by GPU
  143. * (CPU+RENDER, 0)
  144. * flush_domains = CPU
  145. * invalidate_domains = RENDER
  146. * clflush (obj)
  147. * MI_FLUSH
  148. * drm_agp_chipset_flush
  149. */
  150. static void
  151. i915_gem_object_set_to_gpu_domain(struct drm_i915_gem_object *obj,
  152. struct intel_ring_buffer *ring,
  153. struct change_domains *cd)
  154. {
  155. uint32_t invalidate_domains = 0, flush_domains = 0;
  156. /*
  157. * If the object isn't moving to a new write domain,
  158. * let the object stay in multiple read domains
  159. */
  160. if (obj->base.pending_write_domain == 0)
  161. obj->base.pending_read_domains |= obj->base.read_domains;
  162. /*
  163. * Flush the current write domain if
  164. * the new read domains don't match. Invalidate
  165. * any read domains which differ from the old
  166. * write domain
  167. */
  168. if (obj->base.write_domain &&
  169. (((obj->base.write_domain != obj->base.pending_read_domains ||
  170. obj->ring != ring)) ||
  171. (obj->fenced_gpu_access && !obj->pending_fenced_gpu_access))) {
  172. flush_domains |= obj->base.write_domain;
  173. invalidate_domains |=
  174. obj->base.pending_read_domains & ~obj->base.write_domain;
  175. }
  176. /*
  177. * Invalidate any read caches which may have
  178. * stale data. That is, any new read domains.
  179. */
  180. invalidate_domains |= obj->base.pending_read_domains & ~obj->base.read_domains;
  181. if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU)
  182. i915_gem_clflush_object(obj);
  183. /* blow away mappings if mapped through GTT */
  184. if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_GTT)
  185. i915_gem_release_mmap(obj);
  186. /* The actual obj->write_domain will be updated with
  187. * pending_write_domain after we emit the accumulated flush for all
  188. * of our domain changes in execbuffers (which clears objects'
  189. * write_domains). So if we have a current write domain that we
  190. * aren't changing, set pending_write_domain to that.
  191. */
  192. if (flush_domains == 0 && obj->base.pending_write_domain == 0)
  193. obj->base.pending_write_domain = obj->base.write_domain;
  194. cd->invalidate_domains |= invalidate_domains;
  195. cd->flush_domains |= flush_domains;
  196. if (flush_domains & I915_GEM_GPU_DOMAINS)
  197. cd->flush_rings |= obj->ring->id;
  198. if (invalidate_domains & I915_GEM_GPU_DOMAINS)
  199. cd->flush_rings |= ring->id;
  200. }
  201. struct eb_objects {
  202. int and;
  203. struct hlist_head buckets[0];
  204. };
  205. static struct eb_objects *
  206. eb_create(int size)
  207. {
  208. struct eb_objects *eb;
  209. int count = PAGE_SIZE / sizeof(struct hlist_head) / 2;
  210. while (count > size)
  211. count >>= 1;
  212. eb = kzalloc(count*sizeof(struct hlist_head) +
  213. sizeof(struct eb_objects),
  214. GFP_KERNEL);
  215. if (eb == NULL)
  216. return eb;
  217. eb->and = count - 1;
  218. return eb;
  219. }
  220. static void
  221. eb_reset(struct eb_objects *eb)
  222. {
  223. memset(eb->buckets, 0, (eb->and+1)*sizeof(struct hlist_head));
  224. }
  225. static void
  226. eb_add_object(struct eb_objects *eb, struct drm_i915_gem_object *obj)
  227. {
  228. hlist_add_head(&obj->exec_node,
  229. &eb->buckets[obj->exec_handle & eb->and]);
  230. }
  231. static struct drm_i915_gem_object *
  232. eb_get_object(struct eb_objects *eb, unsigned long handle)
  233. {
  234. struct hlist_head *head;
  235. struct hlist_node *node;
  236. struct drm_i915_gem_object *obj;
  237. head = &eb->buckets[handle & eb->and];
  238. hlist_for_each(node, head) {
  239. obj = hlist_entry(node, struct drm_i915_gem_object, exec_node);
  240. if (obj->exec_handle == handle)
  241. return obj;
  242. }
  243. return NULL;
  244. }
  245. static void
  246. eb_destroy(struct eb_objects *eb)
  247. {
  248. kfree(eb);
  249. }
  250. static int
  251. i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
  252. struct eb_objects *eb,
  253. struct drm_i915_gem_exec_object2 *entry,
  254. struct drm_i915_gem_relocation_entry *reloc)
  255. {
  256. struct drm_device *dev = obj->base.dev;
  257. struct drm_gem_object *target_obj;
  258. uint32_t target_offset;
  259. int ret = -EINVAL;
  260. /* we've already hold a reference to all valid objects */
  261. target_obj = &eb_get_object(eb, reloc->target_handle)->base;
  262. if (unlikely(target_obj == NULL))
  263. return -ENOENT;
  264. target_offset = to_intel_bo(target_obj)->gtt_offset;
  265. #if WATCH_RELOC
  266. DRM_INFO("%s: obj %p offset %08x target %d "
  267. "read %08x write %08x gtt %08x "
  268. "presumed %08x delta %08x\n",
  269. __func__,
  270. obj,
  271. (int) reloc->offset,
  272. (int) reloc->target_handle,
  273. (int) reloc->read_domains,
  274. (int) reloc->write_domain,
  275. (int) target_offset,
  276. (int) reloc->presumed_offset,
  277. reloc->delta);
  278. #endif
  279. /* The target buffer should have appeared before us in the
  280. * exec_object list, so it should have a GTT space bound by now.
  281. */
  282. if (unlikely(target_offset == 0)) {
  283. DRM_ERROR("No GTT space found for object %d\n",
  284. reloc->target_handle);
  285. return ret;
  286. }
  287. /* Validate that the target is in a valid r/w GPU domain */
  288. if (unlikely(reloc->write_domain & (reloc->write_domain - 1))) {
  289. DRM_ERROR("reloc with multiple write domains: "
  290. "obj %p target %d offset %d "
  291. "read %08x write %08x",
  292. obj, reloc->target_handle,
  293. (int) reloc->offset,
  294. reloc->read_domains,
  295. reloc->write_domain);
  296. return ret;
  297. }
  298. if (unlikely((reloc->write_domain | reloc->read_domains) & I915_GEM_DOMAIN_CPU)) {
  299. DRM_ERROR("reloc with read/write CPU domains: "
  300. "obj %p target %d offset %d "
  301. "read %08x write %08x",
  302. obj, reloc->target_handle,
  303. (int) reloc->offset,
  304. reloc->read_domains,
  305. reloc->write_domain);
  306. return ret;
  307. }
  308. if (unlikely(reloc->write_domain && target_obj->pending_write_domain &&
  309. reloc->write_domain != target_obj->pending_write_domain)) {
  310. DRM_ERROR("Write domain conflict: "
  311. "obj %p target %d offset %d "
  312. "new %08x old %08x\n",
  313. obj, reloc->target_handle,
  314. (int) reloc->offset,
  315. reloc->write_domain,
  316. target_obj->pending_write_domain);
  317. return ret;
  318. }
  319. target_obj->pending_read_domains |= reloc->read_domains;
  320. target_obj->pending_write_domain |= reloc->write_domain;
  321. /* If the relocation already has the right value in it, no
  322. * more work needs to be done.
  323. */
  324. if (target_offset == reloc->presumed_offset)
  325. return 0;
  326. /* Check that the relocation address is valid... */
  327. if (unlikely(reloc->offset > obj->base.size - 4)) {
  328. DRM_ERROR("Relocation beyond object bounds: "
  329. "obj %p target %d offset %d size %d.\n",
  330. obj, reloc->target_handle,
  331. (int) reloc->offset,
  332. (int) obj->base.size);
  333. return ret;
  334. }
  335. if (unlikely(reloc->offset & 3)) {
  336. DRM_ERROR("Relocation not 4-byte aligned: "
  337. "obj %p target %d offset %d.\n",
  338. obj, reloc->target_handle,
  339. (int) reloc->offset);
  340. return ret;
  341. }
  342. /* and points to somewhere within the target object. */
  343. if (unlikely(reloc->delta >= target_obj->size)) {
  344. DRM_ERROR("Relocation beyond target object bounds: "
  345. "obj %p target %d delta %d size %d.\n",
  346. obj, reloc->target_handle,
  347. (int) reloc->delta,
  348. (int) target_obj->size);
  349. return ret;
  350. }
  351. reloc->delta += target_offset;
  352. if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) {
  353. uint32_t page_offset = reloc->offset & ~PAGE_MASK;
  354. char *vaddr;
  355. vaddr = kmap_atomic(obj->pages[reloc->offset >> PAGE_SHIFT]);
  356. *(uint32_t *)(vaddr + page_offset) = reloc->delta;
  357. kunmap_atomic(vaddr);
  358. } else {
  359. struct drm_i915_private *dev_priv = dev->dev_private;
  360. uint32_t __iomem *reloc_entry;
  361. void __iomem *reloc_page;
  362. ret = i915_gem_object_set_to_gtt_domain(obj, 1);
  363. if (ret)
  364. return ret;
  365. /* Map the page containing the relocation we're going to perform. */
  366. reloc->offset += obj->gtt_offset;
  367. reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
  368. reloc->offset & PAGE_MASK);
  369. reloc_entry = (uint32_t __iomem *)
  370. (reloc_page + (reloc->offset & ~PAGE_MASK));
  371. iowrite32(reloc->delta, reloc_entry);
  372. io_mapping_unmap_atomic(reloc_page);
  373. }
  374. /* and update the user's relocation entry */
  375. reloc->presumed_offset = target_offset;
  376. return 0;
  377. }
  378. static int
  379. i915_gem_execbuffer_relocate_object(struct drm_i915_gem_object *obj,
  380. struct eb_objects *eb,
  381. struct drm_i915_gem_exec_object2 *entry)
  382. {
  383. struct drm_i915_gem_relocation_entry __user *user_relocs;
  384. int i, ret;
  385. user_relocs = (void __user *)(uintptr_t)entry->relocs_ptr;
  386. for (i = 0; i < entry->relocation_count; i++) {
  387. struct drm_i915_gem_relocation_entry reloc;
  388. if (__copy_from_user_inatomic(&reloc,
  389. user_relocs+i,
  390. sizeof(reloc)))
  391. return -EFAULT;
  392. ret = i915_gem_execbuffer_relocate_entry(obj, eb, entry, &reloc);
  393. if (ret)
  394. return ret;
  395. if (__copy_to_user_inatomic(&user_relocs[i].presumed_offset,
  396. &reloc.presumed_offset,
  397. sizeof(reloc.presumed_offset)))
  398. return -EFAULT;
  399. }
  400. return 0;
  401. }
  402. static int
  403. i915_gem_execbuffer_relocate_object_slow(struct drm_i915_gem_object *obj,
  404. struct eb_objects *eb,
  405. struct drm_i915_gem_exec_object2 *entry,
  406. struct drm_i915_gem_relocation_entry *relocs)
  407. {
  408. int i, ret;
  409. for (i = 0; i < entry->relocation_count; i++) {
  410. ret = i915_gem_execbuffer_relocate_entry(obj, eb, entry, &relocs[i]);
  411. if (ret)
  412. return ret;
  413. }
  414. return 0;
  415. }
  416. static int
  417. i915_gem_execbuffer_relocate(struct drm_device *dev,
  418. struct eb_objects *eb,
  419. struct list_head *objects,
  420. struct drm_i915_gem_exec_object2 *exec)
  421. {
  422. struct drm_i915_gem_object *obj;
  423. int ret;
  424. list_for_each_entry(obj, objects, exec_list) {
  425. obj->base.pending_read_domains = 0;
  426. obj->base.pending_write_domain = 0;
  427. ret = i915_gem_execbuffer_relocate_object(obj, eb, exec++);
  428. if (ret)
  429. return ret;
  430. }
  431. return 0;
  432. }
  433. static int
  434. i915_gem_execbuffer_reserve(struct intel_ring_buffer *ring,
  435. struct drm_file *file,
  436. struct list_head *objects,
  437. struct drm_i915_gem_exec_object2 *exec)
  438. {
  439. struct drm_i915_gem_object *obj;
  440. struct drm_i915_gem_exec_object2 *entry;
  441. int ret, retry;
  442. bool has_fenced_gpu_access = INTEL_INFO(ring->dev)->gen < 4;
  443. /* Attempt to pin all of the buffers into the GTT.
  444. * This is done in 3 phases:
  445. *
  446. * 1a. Unbind all objects that do not match the GTT constraints for
  447. * the execbuffer (fenceable, mappable, alignment etc).
  448. * 1b. Increment pin count for already bound objects.
  449. * 2. Bind new objects.
  450. * 3. Decrement pin count.
  451. *
  452. * This avoid unnecessary unbinding of later objects in order to makr
  453. * room for the earlier objects *unless* we need to defragment.
  454. */
  455. retry = 0;
  456. do {
  457. ret = 0;
  458. /* Unbind any ill-fitting objects or pin. */
  459. entry = exec;
  460. list_for_each_entry(obj, objects, exec_list) {
  461. bool need_fence, need_mappable;
  462. if (!obj->gtt_space) {
  463. entry++;
  464. continue;
  465. }
  466. need_fence =
  467. has_fenced_gpu_access &&
  468. entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
  469. obj->tiling_mode != I915_TILING_NONE;
  470. need_mappable =
  471. entry->relocation_count ? true : need_fence;
  472. if ((entry->alignment && obj->gtt_offset & (entry->alignment - 1)) ||
  473. (need_mappable && !obj->map_and_fenceable))
  474. ret = i915_gem_object_unbind(obj);
  475. else
  476. ret = i915_gem_object_pin(obj,
  477. entry->alignment,
  478. need_mappable);
  479. if (ret)
  480. goto err;
  481. entry++;
  482. }
  483. /* Bind fresh objects */
  484. entry = exec;
  485. list_for_each_entry(obj, objects, exec_list) {
  486. bool need_fence;
  487. need_fence =
  488. has_fenced_gpu_access &&
  489. entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
  490. obj->tiling_mode != I915_TILING_NONE;
  491. if (!obj->gtt_space) {
  492. bool need_mappable =
  493. entry->relocation_count ? true : need_fence;
  494. ret = i915_gem_object_pin(obj,
  495. entry->alignment,
  496. need_mappable);
  497. if (ret)
  498. break;
  499. }
  500. if (has_fenced_gpu_access) {
  501. if (need_fence) {
  502. ret = i915_gem_object_get_fence(obj, ring, 1);
  503. if (ret)
  504. break;
  505. } else if (entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
  506. obj->tiling_mode == I915_TILING_NONE) {
  507. /* XXX pipelined! */
  508. ret = i915_gem_object_put_fence(obj);
  509. if (ret)
  510. break;
  511. }
  512. obj->pending_fenced_gpu_access = need_fence;
  513. }
  514. entry->offset = obj->gtt_offset;
  515. entry++;
  516. }
  517. /* Decrement pin count for bound objects */
  518. list_for_each_entry(obj, objects, exec_list) {
  519. if (obj->gtt_space)
  520. i915_gem_object_unpin(obj);
  521. }
  522. if (ret != -ENOSPC || retry > 1)
  523. return ret;
  524. /* First attempt, just clear anything that is purgeable.
  525. * Second attempt, clear the entire GTT.
  526. */
  527. ret = i915_gem_evict_everything(ring->dev, retry == 0);
  528. if (ret)
  529. return ret;
  530. retry++;
  531. } while (1);
  532. err:
  533. obj = list_entry(obj->exec_list.prev,
  534. struct drm_i915_gem_object,
  535. exec_list);
  536. while (objects != &obj->exec_list) {
  537. if (obj->gtt_space)
  538. i915_gem_object_unpin(obj);
  539. obj = list_entry(obj->exec_list.prev,
  540. struct drm_i915_gem_object,
  541. exec_list);
  542. }
  543. return ret;
  544. }
  545. static int
  546. i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
  547. struct drm_file *file,
  548. struct intel_ring_buffer *ring,
  549. struct list_head *objects,
  550. struct eb_objects *eb,
  551. struct drm_i915_gem_exec_object2 *exec,
  552. int count)
  553. {
  554. struct drm_i915_gem_relocation_entry *reloc;
  555. struct drm_i915_gem_object *obj;
  556. int i, total, ret;
  557. /* We may process another execbuffer during the unlock... */
  558. while (list_empty(objects)) {
  559. obj = list_first_entry(objects,
  560. struct drm_i915_gem_object,
  561. exec_list);
  562. list_del_init(&obj->exec_list);
  563. drm_gem_object_unreference(&obj->base);
  564. }
  565. mutex_unlock(&dev->struct_mutex);
  566. total = 0;
  567. for (i = 0; i < count; i++)
  568. total += exec[i].relocation_count;
  569. reloc = drm_malloc_ab(total, sizeof(*reloc));
  570. if (reloc == NULL) {
  571. mutex_lock(&dev->struct_mutex);
  572. return -ENOMEM;
  573. }
  574. total = 0;
  575. for (i = 0; i < count; i++) {
  576. struct drm_i915_gem_relocation_entry __user *user_relocs;
  577. user_relocs = (void __user *)(uintptr_t)exec[i].relocs_ptr;
  578. if (copy_from_user(reloc+total, user_relocs,
  579. exec[i].relocation_count * sizeof(*reloc))) {
  580. ret = -EFAULT;
  581. mutex_lock(&dev->struct_mutex);
  582. goto err;
  583. }
  584. total += exec[i].relocation_count;
  585. }
  586. ret = i915_mutex_lock_interruptible(dev);
  587. if (ret) {
  588. mutex_lock(&dev->struct_mutex);
  589. goto err;
  590. }
  591. /* reacquire the objects */
  592. INIT_LIST_HEAD(objects);
  593. eb_reset(eb);
  594. for (i = 0; i < count; i++) {
  595. struct drm_i915_gem_object *obj;
  596. obj = to_intel_bo(drm_gem_object_lookup(dev, file,
  597. exec[i].handle));
  598. if (obj == NULL) {
  599. DRM_ERROR("Invalid object handle %d at index %d\n",
  600. exec[i].handle, i);
  601. ret = -ENOENT;
  602. goto err;
  603. }
  604. list_add_tail(&obj->exec_list, objects);
  605. obj->exec_handle = exec[i].handle;
  606. eb_add_object(eb, obj);
  607. }
  608. ret = i915_gem_execbuffer_reserve(ring, file, objects, exec);
  609. if (ret)
  610. goto err;
  611. total = 0;
  612. list_for_each_entry(obj, objects, exec_list) {
  613. obj->base.pending_read_domains = 0;
  614. obj->base.pending_write_domain = 0;
  615. ret = i915_gem_execbuffer_relocate_object_slow(obj, eb,
  616. exec,
  617. reloc + total);
  618. if (ret)
  619. goto err;
  620. total += exec->relocation_count;
  621. exec++;
  622. }
  623. /* Leave the user relocations as are, this is the painfully slow path,
  624. * and we want to avoid the complication of dropping the lock whilst
  625. * having buffers reserved in the aperture and so causing spurious
  626. * ENOSPC for random operations.
  627. */
  628. err:
  629. drm_free_large(reloc);
  630. return ret;
  631. }
  632. static void
  633. i915_gem_execbuffer_flush(struct drm_device *dev,
  634. uint32_t invalidate_domains,
  635. uint32_t flush_domains,
  636. uint32_t flush_rings)
  637. {
  638. drm_i915_private_t *dev_priv = dev->dev_private;
  639. int i;
  640. if (flush_domains & I915_GEM_DOMAIN_CPU)
  641. intel_gtt_chipset_flush();
  642. if (flush_domains & I915_GEM_DOMAIN_GTT)
  643. wmb();
  644. if ((flush_domains | invalidate_domains) & I915_GEM_GPU_DOMAINS) {
  645. for (i = 0; i < I915_NUM_RINGS; i++)
  646. if (flush_rings & (1 << i))
  647. i915_gem_flush_ring(dev, &dev_priv->ring[i],
  648. invalidate_domains,
  649. flush_domains);
  650. }
  651. }
  652. static int
  653. i915_gem_execbuffer_sync_rings(struct drm_i915_gem_object *obj,
  654. struct intel_ring_buffer *to)
  655. {
  656. struct intel_ring_buffer *from = obj->ring;
  657. u32 seqno;
  658. int ret, idx;
  659. if (from == NULL || to == from)
  660. return 0;
  661. if (INTEL_INFO(obj->base.dev)->gen < 6)
  662. return i915_gem_object_wait_rendering(obj, true);
  663. idx = intel_ring_sync_index(from, to);
  664. seqno = obj->last_rendering_seqno;
  665. if (seqno <= from->sync_seqno[idx])
  666. return 0;
  667. if (seqno == from->outstanding_lazy_request) {
  668. struct drm_i915_gem_request *request;
  669. request = kzalloc(sizeof(*request), GFP_KERNEL);
  670. if (request == NULL)
  671. return -ENOMEM;
  672. ret = i915_add_request(obj->base.dev, NULL, request, from);
  673. if (ret) {
  674. kfree(request);
  675. return ret;
  676. }
  677. seqno = request->seqno;
  678. }
  679. from->sync_seqno[idx] = seqno;
  680. return intel_ring_sync(to, from, seqno - 1);
  681. }
  682. static int
  683. i915_gem_execbuffer_move_to_gpu(struct intel_ring_buffer *ring,
  684. struct list_head *objects)
  685. {
  686. struct drm_i915_gem_object *obj;
  687. struct change_domains cd;
  688. int ret;
  689. cd.invalidate_domains = 0;
  690. cd.flush_domains = 0;
  691. cd.flush_rings = 0;
  692. list_for_each_entry(obj, objects, exec_list)
  693. i915_gem_object_set_to_gpu_domain(obj, ring, &cd);
  694. if (cd.invalidate_domains | cd.flush_domains) {
  695. #if WATCH_EXEC
  696. DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
  697. __func__,
  698. cd.invalidate_domains,
  699. cd.flush_domains);
  700. #endif
  701. i915_gem_execbuffer_flush(ring->dev,
  702. cd.invalidate_domains,
  703. cd.flush_domains,
  704. cd.flush_rings);
  705. }
  706. list_for_each_entry(obj, objects, exec_list) {
  707. ret = i915_gem_execbuffer_sync_rings(obj, ring);
  708. if (ret)
  709. return ret;
  710. }
  711. return 0;
  712. }
  713. static bool
  714. i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec)
  715. {
  716. return ((exec->batch_start_offset | exec->batch_len) & 0x7) == 0;
  717. }
  718. static int
  719. validate_exec_list(struct drm_i915_gem_exec_object2 *exec,
  720. int count)
  721. {
  722. int i;
  723. for (i = 0; i < count; i++) {
  724. char __user *ptr = (char __user *)(uintptr_t)exec[i].relocs_ptr;
  725. int length; /* limited by fault_in_pages_readable() */
  726. /* First check for malicious input causing overflow */
  727. if (exec[i].relocation_count >
  728. INT_MAX / sizeof(struct drm_i915_gem_relocation_entry))
  729. return -EINVAL;
  730. length = exec[i].relocation_count *
  731. sizeof(struct drm_i915_gem_relocation_entry);
  732. if (!access_ok(VERIFY_READ, ptr, length))
  733. return -EFAULT;
  734. /* we may also need to update the presumed offsets */
  735. if (!access_ok(VERIFY_WRITE, ptr, length))
  736. return -EFAULT;
  737. if (fault_in_pages_readable(ptr, length))
  738. return -EFAULT;
  739. }
  740. return 0;
  741. }
  742. static int
  743. i915_gem_execbuffer_wait_for_flips(struct intel_ring_buffer *ring,
  744. struct list_head *objects)
  745. {
  746. struct drm_i915_gem_object *obj;
  747. int flips;
  748. /* Check for any pending flips. As we only maintain a flip queue depth
  749. * of 1, we can simply insert a WAIT for the next display flip prior
  750. * to executing the batch and avoid stalling the CPU.
  751. */
  752. flips = 0;
  753. list_for_each_entry(obj, objects, exec_list) {
  754. if (obj->base.write_domain)
  755. flips |= atomic_read(&obj->pending_flip);
  756. }
  757. if (flips) {
  758. int plane, flip_mask, ret;
  759. for (plane = 0; flips >> plane; plane++) {
  760. if (((flips >> plane) & 1) == 0)
  761. continue;
  762. if (plane)
  763. flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
  764. else
  765. flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
  766. ret = intel_ring_begin(ring, 2);
  767. if (ret)
  768. return ret;
  769. intel_ring_emit(ring, MI_WAIT_FOR_EVENT | flip_mask);
  770. intel_ring_emit(ring, MI_NOOP);
  771. intel_ring_advance(ring);
  772. }
  773. }
  774. return 0;
  775. }
  776. static void
  777. i915_gem_execbuffer_move_to_active(struct list_head *objects,
  778. struct intel_ring_buffer *ring,
  779. u32 seqno)
  780. {
  781. struct drm_i915_gem_object *obj;
  782. list_for_each_entry(obj, objects, exec_list) {
  783. obj->base.read_domains = obj->base.pending_read_domains;
  784. obj->base.write_domain = obj->base.pending_write_domain;
  785. obj->fenced_gpu_access = obj->pending_fenced_gpu_access;
  786. i915_gem_object_move_to_active(obj, ring, seqno);
  787. if (obj->base.write_domain) {
  788. obj->dirty = 1;
  789. obj->pending_gpu_write = true;
  790. list_move_tail(&obj->gpu_write_list,
  791. &ring->gpu_write_list);
  792. intel_mark_busy(ring->dev, obj);
  793. }
  794. trace_i915_gem_object_change_domain(obj,
  795. obj->base.read_domains,
  796. obj->base.write_domain);
  797. }
  798. }
  799. static void
  800. i915_gem_execbuffer_retire_commands(struct drm_device *dev,
  801. struct drm_file *file,
  802. struct intel_ring_buffer *ring)
  803. {
  804. struct drm_i915_gem_request *request;
  805. u32 flush_domains;
  806. /*
  807. * Ensure that the commands in the batch buffer are
  808. * finished before the interrupt fires.
  809. *
  810. * The sampler always gets flushed on i965 (sigh).
  811. */
  812. flush_domains = 0;
  813. if (INTEL_INFO(dev)->gen >= 4)
  814. flush_domains |= I915_GEM_DOMAIN_SAMPLER;
  815. ring->flush(ring, I915_GEM_DOMAIN_COMMAND, flush_domains);
  816. /* Add a breadcrumb for the completion of the batch buffer */
  817. request = kzalloc(sizeof(*request), GFP_KERNEL);
  818. if (request == NULL || i915_add_request(dev, file, request, ring)) {
  819. i915_gem_next_request_seqno(dev, ring);
  820. kfree(request);
  821. }
  822. }
  823. static int
  824. i915_gem_do_execbuffer(struct drm_device *dev, void *data,
  825. struct drm_file *file,
  826. struct drm_i915_gem_execbuffer2 *args,
  827. struct drm_i915_gem_exec_object2 *exec)
  828. {
  829. drm_i915_private_t *dev_priv = dev->dev_private;
  830. struct list_head objects;
  831. struct eb_objects *eb;
  832. struct drm_i915_gem_object *batch_obj;
  833. struct drm_clip_rect *cliprects = NULL;
  834. struct intel_ring_buffer *ring;
  835. u32 exec_start, exec_len;
  836. u32 seqno;
  837. int ret, mode, i;
  838. if (!i915_gem_check_execbuffer(args)) {
  839. DRM_ERROR("execbuf with invalid offset/length\n");
  840. return -EINVAL;
  841. }
  842. ret = validate_exec_list(exec, args->buffer_count);
  843. if (ret)
  844. return ret;
  845. #if WATCH_EXEC
  846. DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
  847. (int) args->buffers_ptr, args->buffer_count, args->batch_len);
  848. #endif
  849. switch (args->flags & I915_EXEC_RING_MASK) {
  850. case I915_EXEC_DEFAULT:
  851. case I915_EXEC_RENDER:
  852. ring = &dev_priv->ring[RCS];
  853. break;
  854. case I915_EXEC_BSD:
  855. if (!HAS_BSD(dev)) {
  856. DRM_ERROR("execbuf with invalid ring (BSD)\n");
  857. return -EINVAL;
  858. }
  859. ring = &dev_priv->ring[VCS];
  860. break;
  861. case I915_EXEC_BLT:
  862. if (!HAS_BLT(dev)) {
  863. DRM_ERROR("execbuf with invalid ring (BLT)\n");
  864. return -EINVAL;
  865. }
  866. ring = &dev_priv->ring[BCS];
  867. break;
  868. default:
  869. DRM_ERROR("execbuf with unknown ring: %d\n",
  870. (int)(args->flags & I915_EXEC_RING_MASK));
  871. return -EINVAL;
  872. }
  873. mode = args->flags & I915_EXEC_CONSTANTS_MASK;
  874. switch (mode) {
  875. case I915_EXEC_CONSTANTS_REL_GENERAL:
  876. case I915_EXEC_CONSTANTS_ABSOLUTE:
  877. case I915_EXEC_CONSTANTS_REL_SURFACE:
  878. if (ring == &dev_priv->ring[RCS] &&
  879. mode != dev_priv->relative_constants_mode) {
  880. if (INTEL_INFO(dev)->gen < 4)
  881. return -EINVAL;
  882. if (INTEL_INFO(dev)->gen > 5 &&
  883. mode == I915_EXEC_CONSTANTS_REL_SURFACE)
  884. return -EINVAL;
  885. ret = intel_ring_begin(ring, 4);
  886. if (ret)
  887. return ret;
  888. intel_ring_emit(ring, MI_NOOP);
  889. intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
  890. intel_ring_emit(ring, INSTPM);
  891. intel_ring_emit(ring,
  892. I915_EXEC_CONSTANTS_MASK << 16 | mode);
  893. intel_ring_advance(ring);
  894. dev_priv->relative_constants_mode = mode;
  895. }
  896. break;
  897. default:
  898. DRM_ERROR("execbuf with unknown constants: %d\n", mode);
  899. return -EINVAL;
  900. }
  901. if (args->buffer_count < 1) {
  902. DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
  903. return -EINVAL;
  904. }
  905. if (args->num_cliprects != 0) {
  906. if (ring != &dev_priv->ring[RCS]) {
  907. DRM_ERROR("clip rectangles are only valid with the render ring\n");
  908. return -EINVAL;
  909. }
  910. cliprects = kmalloc(args->num_cliprects * sizeof(*cliprects),
  911. GFP_KERNEL);
  912. if (cliprects == NULL) {
  913. ret = -ENOMEM;
  914. goto pre_mutex_err;
  915. }
  916. if (copy_from_user(cliprects,
  917. (struct drm_clip_rect __user *)(uintptr_t)
  918. args->cliprects_ptr,
  919. sizeof(*cliprects)*args->num_cliprects)) {
  920. ret = -EFAULT;
  921. goto pre_mutex_err;
  922. }
  923. }
  924. ret = i915_mutex_lock_interruptible(dev);
  925. if (ret)
  926. goto pre_mutex_err;
  927. if (dev_priv->mm.suspended) {
  928. mutex_unlock(&dev->struct_mutex);
  929. ret = -EBUSY;
  930. goto pre_mutex_err;
  931. }
  932. eb = eb_create(args->buffer_count);
  933. if (eb == NULL) {
  934. mutex_unlock(&dev->struct_mutex);
  935. ret = -ENOMEM;
  936. goto pre_mutex_err;
  937. }
  938. /* Look up object handles */
  939. INIT_LIST_HEAD(&objects);
  940. for (i = 0; i < args->buffer_count; i++) {
  941. struct drm_i915_gem_object *obj;
  942. obj = to_intel_bo(drm_gem_object_lookup(dev, file,
  943. exec[i].handle));
  944. if (obj == NULL) {
  945. DRM_ERROR("Invalid object handle %d at index %d\n",
  946. exec[i].handle, i);
  947. /* prevent error path from reading uninitialized data */
  948. ret = -ENOENT;
  949. goto err;
  950. }
  951. if (!list_empty(&obj->exec_list)) {
  952. DRM_ERROR("Object %p [handle %d, index %d] appears more than once in object list\n",
  953. obj, exec[i].handle, i);
  954. ret = -EINVAL;
  955. goto err;
  956. }
  957. list_add_tail(&obj->exec_list, &objects);
  958. obj->exec_handle = exec[i].handle;
  959. eb_add_object(eb, obj);
  960. }
  961. /* Move the objects en-masse into the GTT, evicting if necessary. */
  962. ret = i915_gem_execbuffer_reserve(ring, file, &objects, exec);
  963. if (ret)
  964. goto err;
  965. /* The objects are in their final locations, apply the relocations. */
  966. ret = i915_gem_execbuffer_relocate(dev, eb, &objects, exec);
  967. if (ret) {
  968. if (ret == -EFAULT) {
  969. ret = i915_gem_execbuffer_relocate_slow(dev, file, ring,
  970. &objects, eb,
  971. exec,
  972. args->buffer_count);
  973. BUG_ON(!mutex_is_locked(&dev->struct_mutex));
  974. }
  975. if (ret)
  976. goto err;
  977. }
  978. /* Set the pending read domains for the batch buffer to COMMAND */
  979. batch_obj = list_entry(objects.prev,
  980. struct drm_i915_gem_object,
  981. exec_list);
  982. if (batch_obj->base.pending_write_domain) {
  983. DRM_ERROR("Attempting to use self-modifying batch buffer\n");
  984. ret = -EINVAL;
  985. goto err;
  986. }
  987. batch_obj->base.pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
  988. ret = i915_gem_execbuffer_move_to_gpu(ring, &objects);
  989. if (ret)
  990. goto err;
  991. ret = i915_gem_execbuffer_wait_for_flips(ring, &objects);
  992. if (ret)
  993. goto err;
  994. seqno = i915_gem_next_request_seqno(dev, ring);
  995. for (i = 0; i < I915_NUM_RINGS-1; i++) {
  996. if (seqno < ring->sync_seqno[i]) {
  997. /* The GPU can not handle its semaphore value wrapping,
  998. * so every billion or so execbuffers, we need to stall
  999. * the GPU in order to reset the counters.
  1000. */
  1001. ret = i915_gpu_idle(dev);
  1002. if (ret)
  1003. goto err;
  1004. BUG_ON(ring->sync_seqno[i]);
  1005. }
  1006. }
  1007. exec_start = batch_obj->gtt_offset + args->batch_start_offset;
  1008. exec_len = args->batch_len;
  1009. if (cliprects) {
  1010. for (i = 0; i < args->num_cliprects; i++) {
  1011. ret = i915_emit_box(dev, &cliprects[i],
  1012. args->DR1, args->DR4);
  1013. if (ret)
  1014. goto err;
  1015. ret = ring->dispatch_execbuffer(ring,
  1016. exec_start, exec_len);
  1017. if (ret)
  1018. goto err;
  1019. }
  1020. } else {
  1021. ret = ring->dispatch_execbuffer(ring, exec_start, exec_len);
  1022. if (ret)
  1023. goto err;
  1024. }
  1025. i915_gem_execbuffer_move_to_active(&objects, ring, seqno);
  1026. i915_gem_execbuffer_retire_commands(dev, file, ring);
  1027. err:
  1028. eb_destroy(eb);
  1029. while (!list_empty(&objects)) {
  1030. struct drm_i915_gem_object *obj;
  1031. obj = list_first_entry(&objects,
  1032. struct drm_i915_gem_object,
  1033. exec_list);
  1034. list_del_init(&obj->exec_list);
  1035. drm_gem_object_unreference(&obj->base);
  1036. }
  1037. mutex_unlock(&dev->struct_mutex);
  1038. pre_mutex_err:
  1039. kfree(cliprects);
  1040. return ret;
  1041. }
  1042. /*
  1043. * Legacy execbuffer just creates an exec2 list from the original exec object
  1044. * list array and passes it to the real function.
  1045. */
  1046. int
  1047. i915_gem_execbuffer(struct drm_device *dev, void *data,
  1048. struct drm_file *file)
  1049. {
  1050. struct drm_i915_gem_execbuffer *args = data;
  1051. struct drm_i915_gem_execbuffer2 exec2;
  1052. struct drm_i915_gem_exec_object *exec_list = NULL;
  1053. struct drm_i915_gem_exec_object2 *exec2_list = NULL;
  1054. int ret, i;
  1055. #if WATCH_EXEC
  1056. DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
  1057. (int) args->buffers_ptr, args->buffer_count, args->batch_len);
  1058. #endif
  1059. if (args->buffer_count < 1) {
  1060. DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
  1061. return -EINVAL;
  1062. }
  1063. /* Copy in the exec list from userland */
  1064. exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
  1065. exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
  1066. if (exec_list == NULL || exec2_list == NULL) {
  1067. DRM_ERROR("Failed to allocate exec list for %d buffers\n",
  1068. args->buffer_count);
  1069. drm_free_large(exec_list);
  1070. drm_free_large(exec2_list);
  1071. return -ENOMEM;
  1072. }
  1073. ret = copy_from_user(exec_list,
  1074. (struct drm_i915_relocation_entry __user *)
  1075. (uintptr_t) args->buffers_ptr,
  1076. sizeof(*exec_list) * args->buffer_count);
  1077. if (ret != 0) {
  1078. DRM_ERROR("copy %d exec entries failed %d\n",
  1079. args->buffer_count, ret);
  1080. drm_free_large(exec_list);
  1081. drm_free_large(exec2_list);
  1082. return -EFAULT;
  1083. }
  1084. for (i = 0; i < args->buffer_count; i++) {
  1085. exec2_list[i].handle = exec_list[i].handle;
  1086. exec2_list[i].relocation_count = exec_list[i].relocation_count;
  1087. exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
  1088. exec2_list[i].alignment = exec_list[i].alignment;
  1089. exec2_list[i].offset = exec_list[i].offset;
  1090. if (INTEL_INFO(dev)->gen < 4)
  1091. exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
  1092. else
  1093. exec2_list[i].flags = 0;
  1094. }
  1095. exec2.buffers_ptr = args->buffers_ptr;
  1096. exec2.buffer_count = args->buffer_count;
  1097. exec2.batch_start_offset = args->batch_start_offset;
  1098. exec2.batch_len = args->batch_len;
  1099. exec2.DR1 = args->DR1;
  1100. exec2.DR4 = args->DR4;
  1101. exec2.num_cliprects = args->num_cliprects;
  1102. exec2.cliprects_ptr = args->cliprects_ptr;
  1103. exec2.flags = I915_EXEC_RENDER;
  1104. ret = i915_gem_do_execbuffer(dev, data, file, &exec2, exec2_list);
  1105. if (!ret) {
  1106. /* Copy the new buffer offsets back to the user's exec list. */
  1107. for (i = 0; i < args->buffer_count; i++)
  1108. exec_list[i].offset = exec2_list[i].offset;
  1109. /* ... and back out to userspace */
  1110. ret = copy_to_user((struct drm_i915_relocation_entry __user *)
  1111. (uintptr_t) args->buffers_ptr,
  1112. exec_list,
  1113. sizeof(*exec_list) * args->buffer_count);
  1114. if (ret) {
  1115. ret = -EFAULT;
  1116. DRM_ERROR("failed to copy %d exec entries "
  1117. "back to user (%d)\n",
  1118. args->buffer_count, ret);
  1119. }
  1120. }
  1121. drm_free_large(exec_list);
  1122. drm_free_large(exec2_list);
  1123. return ret;
  1124. }
  1125. int
  1126. i915_gem_execbuffer2(struct drm_device *dev, void *data,
  1127. struct drm_file *file)
  1128. {
  1129. struct drm_i915_gem_execbuffer2 *args = data;
  1130. struct drm_i915_gem_exec_object2 *exec2_list = NULL;
  1131. int ret;
  1132. #if WATCH_EXEC
  1133. DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
  1134. (int) args->buffers_ptr, args->buffer_count, args->batch_len);
  1135. #endif
  1136. if (args->buffer_count < 1) {
  1137. DRM_ERROR("execbuf2 with %d buffers\n", args->buffer_count);
  1138. return -EINVAL;
  1139. }
  1140. exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
  1141. if (exec2_list == NULL) {
  1142. DRM_ERROR("Failed to allocate exec list for %d buffers\n",
  1143. args->buffer_count);
  1144. return -ENOMEM;
  1145. }
  1146. ret = copy_from_user(exec2_list,
  1147. (struct drm_i915_relocation_entry __user *)
  1148. (uintptr_t) args->buffers_ptr,
  1149. sizeof(*exec2_list) * args->buffer_count);
  1150. if (ret != 0) {
  1151. DRM_ERROR("copy %d exec entries failed %d\n",
  1152. args->buffer_count, ret);
  1153. drm_free_large(exec2_list);
  1154. return -EFAULT;
  1155. }
  1156. ret = i915_gem_do_execbuffer(dev, data, file, args, exec2_list);
  1157. if (!ret) {
  1158. /* Copy the new buffer offsets back to the user's exec list. */
  1159. ret = copy_to_user((struct drm_i915_relocation_entry __user *)
  1160. (uintptr_t) args->buffers_ptr,
  1161. exec2_list,
  1162. sizeof(*exec2_list) * args->buffer_count);
  1163. if (ret) {
  1164. ret = -EFAULT;
  1165. DRM_ERROR("failed to copy %d exec entries "
  1166. "back to user (%d)\n",
  1167. args->buffer_count, ret);
  1168. }
  1169. }
  1170. drm_free_large(exec2_list);
  1171. return ret;
  1172. }