radeon_fence.c 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401
  1. /*
  2. * Copyright 2009 Jerome Glisse.
  3. * All Rights Reserved.
  4. *
  5. * Permission is hereby granted, free of charge, to any person obtaining a
  6. * copy of this software and associated documentation files (the
  7. * "Software"), to deal in the Software without restriction, including
  8. * without limitation the rights to use, copy, modify, merge, publish,
  9. * distribute, sub license, and/or sell copies of the Software, and to
  10. * permit persons to whom the Software is furnished to do so, subject to
  11. * the following conditions:
  12. *
  13. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  14. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  15. * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  16. * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  17. * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  18. * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  19. * USE OR OTHER DEALINGS IN THE SOFTWARE.
  20. *
  21. * The above copyright notice and this permission notice (including the
  22. * next paragraph) shall be included in all copies or substantial portions
  23. * of the Software.
  24. *
  25. */
  26. /*
  27. * Authors:
  28. * Jerome Glisse <glisse@freedesktop.org>
  29. * Dave Airlie
  30. */
  31. #include <linux/seq_file.h>
  32. #include <asm/atomic.h>
  33. #include <linux/wait.h>
  34. #include <linux/list.h>
  35. #include <linux/kref.h>
  36. #include "drmP.h"
  37. #include "drm.h"
  38. #include "radeon_reg.h"
  39. #include "radeon.h"
  40. int radeon_fence_emit(struct radeon_device *rdev, struct radeon_fence *fence)
  41. {
  42. unsigned long irq_flags;
  43. write_lock_irqsave(&rdev->fence_drv.lock, irq_flags);
  44. if (fence->emited) {
  45. write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags);
  46. return 0;
  47. }
  48. fence->seq = atomic_add_return(1, &rdev->fence_drv.seq);
  49. if (!rdev->cp.ready) {
  50. /* FIXME: cp is not running assume everythings is done right
  51. * away
  52. */
  53. WREG32(rdev->fence_drv.scratch_reg, fence->seq);
  54. } else
  55. radeon_fence_ring_emit(rdev, fence);
  56. fence->emited = true;
  57. list_del(&fence->list);
  58. list_add_tail(&fence->list, &rdev->fence_drv.emited);
  59. write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags);
  60. return 0;
  61. }
  62. static bool radeon_fence_poll_locked(struct radeon_device *rdev)
  63. {
  64. struct radeon_fence *fence;
  65. struct list_head *i, *n;
  66. uint32_t seq;
  67. bool wake = false;
  68. unsigned long cjiffies;
  69. seq = RREG32(rdev->fence_drv.scratch_reg);
  70. if (seq != rdev->fence_drv.last_seq) {
  71. rdev->fence_drv.last_seq = seq;
  72. rdev->fence_drv.last_jiffies = jiffies;
  73. rdev->fence_drv.last_timeout = RADEON_FENCE_JIFFIES_TIMEOUT;
  74. } else {
  75. cjiffies = jiffies;
  76. if (time_after(cjiffies, rdev->fence_drv.last_jiffies)) {
  77. cjiffies -= rdev->fence_drv.last_jiffies;
  78. if (time_after(rdev->fence_drv.last_timeout, cjiffies)) {
  79. /* update the timeout */
  80. rdev->fence_drv.last_timeout -= cjiffies;
  81. } else {
  82. /* the 500ms timeout is elapsed we should test
  83. * for GPU lockup
  84. */
  85. rdev->fence_drv.last_timeout = 1;
  86. }
  87. } else {
  88. /* wrap around update last jiffies, we will just wait
  89. * a little longer
  90. */
  91. rdev->fence_drv.last_jiffies = cjiffies;
  92. }
  93. return false;
  94. }
  95. n = NULL;
  96. list_for_each(i, &rdev->fence_drv.emited) {
  97. fence = list_entry(i, struct radeon_fence, list);
  98. if (fence->seq == seq) {
  99. n = i;
  100. break;
  101. }
  102. }
  103. /* all fence previous to this one are considered as signaled */
  104. if (n) {
  105. i = n;
  106. do {
  107. n = i->prev;
  108. list_del(i);
  109. list_add_tail(i, &rdev->fence_drv.signaled);
  110. fence = list_entry(i, struct radeon_fence, list);
  111. fence->signaled = true;
  112. i = n;
  113. } while (i != &rdev->fence_drv.emited);
  114. wake = true;
  115. }
  116. return wake;
  117. }
  118. static void radeon_fence_destroy(struct kref *kref)
  119. {
  120. unsigned long irq_flags;
  121. struct radeon_fence *fence;
  122. fence = container_of(kref, struct radeon_fence, kref);
  123. write_lock_irqsave(&fence->rdev->fence_drv.lock, irq_flags);
  124. list_del(&fence->list);
  125. fence->emited = false;
  126. write_unlock_irqrestore(&fence->rdev->fence_drv.lock, irq_flags);
  127. kfree(fence);
  128. }
  129. int radeon_fence_create(struct radeon_device *rdev, struct radeon_fence **fence)
  130. {
  131. unsigned long irq_flags;
  132. *fence = kmalloc(sizeof(struct radeon_fence), GFP_KERNEL);
  133. if ((*fence) == NULL) {
  134. return -ENOMEM;
  135. }
  136. kref_init(&((*fence)->kref));
  137. (*fence)->rdev = rdev;
  138. (*fence)->emited = false;
  139. (*fence)->signaled = false;
  140. (*fence)->seq = 0;
  141. INIT_LIST_HEAD(&(*fence)->list);
  142. write_lock_irqsave(&rdev->fence_drv.lock, irq_flags);
  143. list_add_tail(&(*fence)->list, &rdev->fence_drv.created);
  144. write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags);
  145. return 0;
  146. }
  147. bool radeon_fence_signaled(struct radeon_fence *fence)
  148. {
  149. unsigned long irq_flags;
  150. bool signaled = false;
  151. if (!fence)
  152. return true;
  153. if (fence->rdev->gpu_lockup)
  154. return true;
  155. write_lock_irqsave(&fence->rdev->fence_drv.lock, irq_flags);
  156. signaled = fence->signaled;
  157. /* if we are shuting down report all fence as signaled */
  158. if (fence->rdev->shutdown) {
  159. signaled = true;
  160. }
  161. if (!fence->emited) {
  162. WARN(1, "Querying an unemited fence : %p !\n", fence);
  163. signaled = true;
  164. }
  165. if (!signaled) {
  166. radeon_fence_poll_locked(fence->rdev);
  167. signaled = fence->signaled;
  168. }
  169. write_unlock_irqrestore(&fence->rdev->fence_drv.lock, irq_flags);
  170. return signaled;
  171. }
  172. int radeon_fence_wait(struct radeon_fence *fence, bool intr)
  173. {
  174. struct radeon_device *rdev;
  175. unsigned long irq_flags, timeout;
  176. u32 seq;
  177. int r;
  178. if (fence == NULL) {
  179. WARN(1, "Querying an invalid fence : %p !\n", fence);
  180. return 0;
  181. }
  182. rdev = fence->rdev;
  183. if (radeon_fence_signaled(fence)) {
  184. return 0;
  185. }
  186. timeout = rdev->fence_drv.last_timeout;
  187. retry:
  188. /* save current sequence used to check for GPU lockup */
  189. seq = rdev->fence_drv.last_seq;
  190. if (intr) {
  191. radeon_irq_kms_sw_irq_get(rdev);
  192. r = wait_event_interruptible_timeout(rdev->fence_drv.queue,
  193. radeon_fence_signaled(fence), timeout);
  194. radeon_irq_kms_sw_irq_put(rdev);
  195. if (unlikely(r < 0))
  196. return r;
  197. } else {
  198. radeon_irq_kms_sw_irq_get(rdev);
  199. r = wait_event_timeout(rdev->fence_drv.queue,
  200. radeon_fence_signaled(fence), timeout);
  201. radeon_irq_kms_sw_irq_put(rdev);
  202. }
  203. if (unlikely(!radeon_fence_signaled(fence))) {
  204. /* we were interrupted for some reason and fence isn't
  205. * isn't signaled yet, resume wait
  206. */
  207. if (r) {
  208. timeout = r;
  209. goto retry;
  210. }
  211. /* don't protect read access to rdev->fence_drv.last_seq
  212. * if we experiencing a lockup the value doesn't change
  213. */
  214. if (seq == rdev->fence_drv.last_seq && radeon_gpu_is_lockup(rdev)) {
  215. /* good news we believe it's a lockup */
  216. dev_warn(rdev->dev, "GPU lockup (last fence id 0x%08X)\n", seq);
  217. r = radeon_gpu_reset(rdev);
  218. if (r)
  219. return r;
  220. /* FIXME: what should we do ? marking everyone
  221. * as signaled for now
  222. */
  223. WREG32(rdev->fence_drv.scratch_reg, fence->seq);
  224. }
  225. timeout = RADEON_FENCE_JIFFIES_TIMEOUT;
  226. write_lock_irqsave(&rdev->fence_drv.lock, irq_flags);
  227. rdev->fence_drv.last_timeout = RADEON_FENCE_JIFFIES_TIMEOUT;
  228. rdev->fence_drv.last_jiffies = jiffies;
  229. write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags);
  230. goto retry;
  231. }
  232. return 0;
  233. }
  234. int radeon_fence_wait_next(struct radeon_device *rdev)
  235. {
  236. unsigned long irq_flags;
  237. struct radeon_fence *fence;
  238. int r;
  239. if (rdev->gpu_lockup) {
  240. return 0;
  241. }
  242. write_lock_irqsave(&rdev->fence_drv.lock, irq_flags);
  243. if (list_empty(&rdev->fence_drv.emited)) {
  244. write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags);
  245. return 0;
  246. }
  247. fence = list_entry(rdev->fence_drv.emited.next,
  248. struct radeon_fence, list);
  249. radeon_fence_ref(fence);
  250. write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags);
  251. r = radeon_fence_wait(fence, false);
  252. radeon_fence_unref(&fence);
  253. return r;
  254. }
  255. int radeon_fence_wait_last(struct radeon_device *rdev)
  256. {
  257. unsigned long irq_flags;
  258. struct radeon_fence *fence;
  259. int r;
  260. if (rdev->gpu_lockup) {
  261. return 0;
  262. }
  263. write_lock_irqsave(&rdev->fence_drv.lock, irq_flags);
  264. if (list_empty(&rdev->fence_drv.emited)) {
  265. write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags);
  266. return 0;
  267. }
  268. fence = list_entry(rdev->fence_drv.emited.prev,
  269. struct radeon_fence, list);
  270. radeon_fence_ref(fence);
  271. write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags);
  272. r = radeon_fence_wait(fence, false);
  273. radeon_fence_unref(&fence);
  274. return r;
  275. }
  276. struct radeon_fence *radeon_fence_ref(struct radeon_fence *fence)
  277. {
  278. kref_get(&fence->kref);
  279. return fence;
  280. }
  281. void radeon_fence_unref(struct radeon_fence **fence)
  282. {
  283. struct radeon_fence *tmp = *fence;
  284. *fence = NULL;
  285. if (tmp) {
  286. kref_put(&tmp->kref, &radeon_fence_destroy);
  287. }
  288. }
  289. void radeon_fence_process(struct radeon_device *rdev)
  290. {
  291. unsigned long irq_flags;
  292. bool wake;
  293. write_lock_irqsave(&rdev->fence_drv.lock, irq_flags);
  294. wake = radeon_fence_poll_locked(rdev);
  295. write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags);
  296. if (wake) {
  297. wake_up_all(&rdev->fence_drv.queue);
  298. }
  299. }
  300. int radeon_fence_driver_init(struct radeon_device *rdev)
  301. {
  302. unsigned long irq_flags;
  303. int r;
  304. write_lock_irqsave(&rdev->fence_drv.lock, irq_flags);
  305. r = radeon_scratch_get(rdev, &rdev->fence_drv.scratch_reg);
  306. if (r) {
  307. dev_err(rdev->dev, "fence failed to get scratch register\n");
  308. write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags);
  309. return r;
  310. }
  311. WREG32(rdev->fence_drv.scratch_reg, 0);
  312. atomic_set(&rdev->fence_drv.seq, 0);
  313. INIT_LIST_HEAD(&rdev->fence_drv.created);
  314. INIT_LIST_HEAD(&rdev->fence_drv.emited);
  315. INIT_LIST_HEAD(&rdev->fence_drv.signaled);
  316. init_waitqueue_head(&rdev->fence_drv.queue);
  317. rdev->fence_drv.initialized = true;
  318. write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags);
  319. if (radeon_debugfs_fence_init(rdev)) {
  320. dev_err(rdev->dev, "fence debugfs file creation failed\n");
  321. }
  322. return 0;
  323. }
  324. void radeon_fence_driver_fini(struct radeon_device *rdev)
  325. {
  326. unsigned long irq_flags;
  327. if (!rdev->fence_drv.initialized)
  328. return;
  329. wake_up_all(&rdev->fence_drv.queue);
  330. write_lock_irqsave(&rdev->fence_drv.lock, irq_flags);
  331. radeon_scratch_free(rdev, rdev->fence_drv.scratch_reg);
  332. write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags);
  333. rdev->fence_drv.initialized = false;
  334. }
  335. /*
  336. * Fence debugfs
  337. */
  338. #if defined(CONFIG_DEBUG_FS)
  339. static int radeon_debugfs_fence_info(struct seq_file *m, void *data)
  340. {
  341. struct drm_info_node *node = (struct drm_info_node *)m->private;
  342. struct drm_device *dev = node->minor->dev;
  343. struct radeon_device *rdev = dev->dev_private;
  344. struct radeon_fence *fence;
  345. seq_printf(m, "Last signaled fence 0x%08X\n",
  346. RREG32(rdev->fence_drv.scratch_reg));
  347. if (!list_empty(&rdev->fence_drv.emited)) {
  348. fence = list_entry(rdev->fence_drv.emited.prev,
  349. struct radeon_fence, list);
  350. seq_printf(m, "Last emited fence %p with 0x%08X\n",
  351. fence, fence->seq);
  352. }
  353. return 0;
  354. }
  355. static struct drm_info_list radeon_debugfs_fence_list[] = {
  356. {"radeon_fence_info", &radeon_debugfs_fence_info, 0, NULL},
  357. };
  358. #endif
  359. int radeon_debugfs_fence_init(struct radeon_device *rdev)
  360. {
  361. #if defined(CONFIG_DEBUG_FS)
  362. return radeon_debugfs_add_files(rdev, radeon_debugfs_fence_list, 1);
  363. #else
  364. return 0;
  365. #endif
  366. }