ttm_execbuf_util.c 5.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219
  1. /**************************************************************************
  2. *
  3. * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
  4. * All Rights Reserved.
  5. *
  6. * Permission is hereby granted, free of charge, to any person obtaining a
  7. * copy of this software and associated documentation files (the
  8. * "Software"), to deal in the Software without restriction, including
  9. * without limitation the rights to use, copy, modify, merge, publish,
  10. * distribute, sub license, and/or sell copies of the Software, and to
  11. * permit persons to whom the Software is furnished to do so, subject to
  12. * the following conditions:
  13. *
  14. * The above copyright notice and this permission notice (including the
  15. * next paragraph) shall be included in all copies or substantial portions
  16. * of the Software.
  17. *
  18. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20. * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  21. * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  22. * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  23. * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  24. * USE OR OTHER DEALINGS IN THE SOFTWARE.
  25. *
  26. **************************************************************************/
  27. #include "ttm/ttm_execbuf_util.h"
  28. #include "ttm/ttm_bo_driver.h"
  29. #include "ttm/ttm_placement.h"
  30. #include <linux/wait.h>
  31. #include <linux/sched.h>
  32. #include <linux/module.h>
  33. static void ttm_eu_backoff_reservation_locked(struct list_head *list)
  34. {
  35. struct ttm_validate_buffer *entry;
  36. list_for_each_entry(entry, list, head) {
  37. struct ttm_buffer_object *bo = entry->bo;
  38. if (!entry->reserved)
  39. continue;
  40. if (entry->removed) {
  41. ttm_bo_add_to_lru(bo);
  42. entry->removed = false;
  43. }
  44. entry->reserved = false;
  45. atomic_set(&bo->reserved, 0);
  46. wake_up_all(&bo->event_queue);
  47. }
  48. }
  49. static void ttm_eu_del_from_lru_locked(struct list_head *list)
  50. {
  51. struct ttm_validate_buffer *entry;
  52. list_for_each_entry(entry, list, head) {
  53. struct ttm_buffer_object *bo = entry->bo;
  54. if (!entry->reserved)
  55. continue;
  56. if (!entry->removed) {
  57. entry->put_count = ttm_bo_del_from_lru(bo);
  58. entry->removed = true;
  59. }
  60. }
  61. }
  62. static void ttm_eu_list_ref_sub(struct list_head *list)
  63. {
  64. struct ttm_validate_buffer *entry;
  65. list_for_each_entry(entry, list, head) {
  66. struct ttm_buffer_object *bo = entry->bo;
  67. if (entry->put_count) {
  68. ttm_bo_list_ref_sub(bo, entry->put_count, true);
  69. entry->put_count = 0;
  70. }
  71. }
  72. }
  73. static int ttm_eu_wait_unreserved_locked(struct list_head *list,
  74. struct ttm_buffer_object *bo)
  75. {
  76. struct ttm_bo_global *glob = bo->glob;
  77. int ret;
  78. ttm_eu_del_from_lru_locked(list);
  79. spin_unlock(&glob->lru_lock);
  80. ret = ttm_bo_wait_unreserved(bo, true);
  81. spin_lock(&glob->lru_lock);
  82. if (unlikely(ret != 0))
  83. ttm_eu_backoff_reservation_locked(list);
  84. return ret;
  85. }
  86. void ttm_eu_backoff_reservation(struct list_head *list)
  87. {
  88. struct ttm_validate_buffer *entry;
  89. list_for_each_entry(entry, list, head) {
  90. struct ttm_buffer_object *bo = entry->bo;
  91. if (!entry->reserved)
  92. continue;
  93. entry->reserved = false;
  94. ttm_bo_unreserve(bo);
  95. }
  96. }
  97. EXPORT_SYMBOL(ttm_eu_backoff_reservation);
  98. /*
  99. * Reserve buffers for validation.
  100. *
  101. * If a buffer in the list is marked for CPU access, we back off and
  102. * wait for that buffer to become free for GPU access.
  103. *
  104. * If a buffer is reserved for another validation, the validator with
  105. * the highest validation sequence backs off and waits for that buffer
  106. * to become unreserved. This prevents deadlocks when validating multiple
  107. * buffers in different orders.
  108. */
  109. int ttm_eu_reserve_buffers(struct list_head *list, uint32_t val_seq)
  110. {
  111. struct ttm_bo_global *glob;
  112. struct ttm_validate_buffer *entry;
  113. int ret;
  114. if (list_empty(list))
  115. return 0;
  116. list_for_each_entry(entry, list, head) {
  117. entry->reserved = false;
  118. entry->put_count = 0;
  119. entry->removed = false;
  120. }
  121. entry = list_first_entry(list, struct ttm_validate_buffer, head);
  122. glob = entry->bo->glob;
  123. retry:
  124. spin_lock(&glob->lru_lock);
  125. list_for_each_entry(entry, list, head) {
  126. struct ttm_buffer_object *bo = entry->bo;
  127. retry_this_bo:
  128. ret = ttm_bo_reserve_locked(bo, true, true, true, val_seq);
  129. switch (ret) {
  130. case 0:
  131. break;
  132. case -EBUSY:
  133. ret = ttm_eu_wait_unreserved_locked(list, bo);
  134. if (unlikely(ret != 0)) {
  135. spin_unlock(&glob->lru_lock);
  136. ttm_eu_list_ref_sub(list);
  137. return ret;
  138. }
  139. goto retry_this_bo;
  140. case -EAGAIN:
  141. ttm_eu_backoff_reservation_locked(list);
  142. spin_unlock(&glob->lru_lock);
  143. ttm_eu_list_ref_sub(list);
  144. ret = ttm_bo_wait_unreserved(bo, true);
  145. if (unlikely(ret != 0))
  146. return ret;
  147. goto retry;
  148. default:
  149. ttm_eu_backoff_reservation_locked(list);
  150. spin_unlock(&glob->lru_lock);
  151. ttm_eu_list_ref_sub(list);
  152. return ret;
  153. }
  154. entry->reserved = true;
  155. if (unlikely(atomic_read(&bo->cpu_writers) > 0)) {
  156. ttm_eu_backoff_reservation_locked(list);
  157. spin_unlock(&glob->lru_lock);
  158. ttm_eu_list_ref_sub(list);
  159. ret = ttm_bo_wait_cpu(bo, false);
  160. if (ret)
  161. return ret;
  162. goto retry;
  163. }
  164. }
  165. ttm_eu_del_from_lru_locked(list);
  166. spin_unlock(&glob->lru_lock);
  167. ttm_eu_list_ref_sub(list);
  168. return 0;
  169. }
  170. EXPORT_SYMBOL(ttm_eu_reserve_buffers);
  171. void ttm_eu_fence_buffer_objects(struct list_head *list, void *sync_obj)
  172. {
  173. struct ttm_validate_buffer *entry;
  174. list_for_each_entry(entry, list, head) {
  175. struct ttm_buffer_object *bo = entry->bo;
  176. struct ttm_bo_driver *driver = bo->bdev->driver;
  177. void *old_sync_obj;
  178. spin_lock(&bo->lock);
  179. old_sync_obj = bo->sync_obj;
  180. bo->sync_obj = driver->sync_obj_ref(sync_obj);
  181. bo->sync_obj_arg = entry->new_sync_obj_arg;
  182. spin_unlock(&bo->lock);
  183. ttm_bo_unreserve(bo);
  184. entry->reserved = false;
  185. if (old_sync_obj)
  186. driver->sync_obj_unref(&old_sync_obj);
  187. }
  188. }
  189. EXPORT_SYMBOL(ttm_eu_fence_buffer_objects);