cpu_buffer.c 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454
  1. /**
  2. * @file cpu_buffer.c
  3. *
  4. * @remark Copyright 2002-2009 OProfile authors
  5. * @remark Read the file COPYING
  6. *
  7. * @author John Levon <levon@movementarian.org>
  8. * @author Barry Kasindorf <barry.kasindorf@amd.com>
  9. * @author Robert Richter <robert.richter@amd.com>
  10. *
  11. * Each CPU has a local buffer that stores PC value/event
  12. * pairs. We also log context switches when we notice them.
  13. * Eventually each CPU's buffer is processed into the global
  14. * event buffer by sync_buffer().
  15. *
  16. * We use a local buffer for two reasons: an NMI or similar
  17. * interrupt cannot synchronise, and high sampling rates
  18. * would lead to catastrophic global synchronisation if
  19. * a global buffer was used.
  20. */
  21. #include <linux/sched.h>
  22. #include <linux/oprofile.h>
  23. #include <linux/errno.h>
  24. #include "event_buffer.h"
  25. #include "cpu_buffer.h"
  26. #include "buffer_sync.h"
  27. #include "oprof.h"
  28. #define OP_BUFFER_FLAGS 0
  29. static struct ring_buffer *op_ring_buffer;
  30. DEFINE_PER_CPU(struct oprofile_cpu_buffer, op_cpu_buffer);
  31. static void wq_sync_buffer(struct work_struct *work);
  32. #define DEFAULT_TIMER_EXPIRE (HZ / 10)
  33. static int work_enabled;
  34. unsigned long oprofile_get_cpu_buffer_size(void)
  35. {
  36. return oprofile_cpu_buffer_size;
  37. }
  38. void oprofile_cpu_buffer_inc_smpl_lost(void)
  39. {
  40. struct oprofile_cpu_buffer *cpu_buf = &__get_cpu_var(op_cpu_buffer);
  41. cpu_buf->sample_lost_overflow++;
  42. }
  43. void free_cpu_buffers(void)
  44. {
  45. if (op_ring_buffer)
  46. ring_buffer_free(op_ring_buffer);
  47. op_ring_buffer = NULL;
  48. }
  49. #define RB_EVENT_HDR_SIZE 4
  50. int alloc_cpu_buffers(void)
  51. {
  52. int i;
  53. unsigned long buffer_size = oprofile_cpu_buffer_size;
  54. unsigned long byte_size = buffer_size * (sizeof(struct op_sample) +
  55. RB_EVENT_HDR_SIZE);
  56. op_ring_buffer = ring_buffer_alloc(byte_size, OP_BUFFER_FLAGS);
  57. if (!op_ring_buffer)
  58. goto fail;
  59. for_each_possible_cpu(i) {
  60. struct oprofile_cpu_buffer *b = &per_cpu(op_cpu_buffer, i);
  61. b->last_task = NULL;
  62. b->last_is_kernel = -1;
  63. b->tracing = 0;
  64. b->buffer_size = buffer_size;
  65. b->sample_received = 0;
  66. b->sample_lost_overflow = 0;
  67. b->backtrace_aborted = 0;
  68. b->sample_invalid_eip = 0;
  69. b->cpu = i;
  70. INIT_DELAYED_WORK(&b->work, wq_sync_buffer);
  71. }
  72. return 0;
  73. fail:
  74. free_cpu_buffers();
  75. return -ENOMEM;
  76. }
  77. void start_cpu_work(void)
  78. {
  79. int i;
  80. work_enabled = 1;
  81. for_each_online_cpu(i) {
  82. struct oprofile_cpu_buffer *b = &per_cpu(op_cpu_buffer, i);
  83. /*
  84. * Spread the work by 1 jiffy per cpu so they dont all
  85. * fire at once.
  86. */
  87. schedule_delayed_work_on(i, &b->work, DEFAULT_TIMER_EXPIRE + i);
  88. }
  89. }
  90. void end_cpu_work(void)
  91. {
  92. int i;
  93. work_enabled = 0;
  94. for_each_online_cpu(i) {
  95. struct oprofile_cpu_buffer *b = &per_cpu(op_cpu_buffer, i);
  96. cancel_delayed_work(&b->work);
  97. }
  98. }
  99. /*
  100. * This function prepares the cpu buffer to write a sample.
  101. *
  102. * Struct op_entry is used during operations on the ring buffer while
  103. * struct op_sample contains the data that is stored in the ring
  104. * buffer. Struct entry can be uninitialized. The function reserves a
  105. * data array that is specified by size. Use
  106. * op_cpu_buffer_write_commit() after preparing the sample. In case of
  107. * errors a null pointer is returned, otherwise the pointer to the
  108. * sample.
  109. *
  110. */
  111. struct op_sample
  112. *op_cpu_buffer_write_reserve(struct op_entry *entry, unsigned long size)
  113. {
  114. entry->event = ring_buffer_lock_reserve
  115. (op_ring_buffer, sizeof(struct op_sample) +
  116. size * sizeof(entry->sample->data[0]));
  117. if (!entry->event)
  118. return NULL;
  119. entry->sample = ring_buffer_event_data(entry->event);
  120. entry->size = size;
  121. entry->data = entry->sample->data;
  122. return entry->sample;
  123. }
  124. int op_cpu_buffer_write_commit(struct op_entry *entry)
  125. {
  126. return ring_buffer_unlock_commit(op_ring_buffer, entry->event);
  127. }
  128. struct op_sample *op_cpu_buffer_read_entry(struct op_entry *entry, int cpu)
  129. {
  130. struct ring_buffer_event *e;
  131. e = ring_buffer_consume(op_ring_buffer, cpu, NULL, NULL);
  132. if (!e)
  133. return NULL;
  134. entry->event = e;
  135. entry->sample = ring_buffer_event_data(e);
  136. entry->size = (ring_buffer_event_length(e) - sizeof(struct op_sample))
  137. / sizeof(entry->sample->data[0]);
  138. entry->data = entry->sample->data;
  139. return entry->sample;
  140. }
  141. unsigned long op_cpu_buffer_entries(int cpu)
  142. {
  143. return ring_buffer_entries_cpu(op_ring_buffer, cpu);
  144. }
  145. static int
  146. op_add_code(struct oprofile_cpu_buffer *cpu_buf, unsigned long backtrace,
  147. int is_kernel, struct task_struct *task)
  148. {
  149. struct op_entry entry;
  150. struct op_sample *sample;
  151. unsigned long flags;
  152. int size;
  153. flags = 0;
  154. if (backtrace)
  155. flags |= TRACE_BEGIN;
  156. /* notice a switch from user->kernel or vice versa */
  157. is_kernel = !!is_kernel;
  158. if (cpu_buf->last_is_kernel != is_kernel) {
  159. cpu_buf->last_is_kernel = is_kernel;
  160. flags |= KERNEL_CTX_SWITCH;
  161. if (is_kernel)
  162. flags |= IS_KERNEL;
  163. }
  164. /* notice a task switch */
  165. if (cpu_buf->last_task != task) {
  166. cpu_buf->last_task = task;
  167. flags |= USER_CTX_SWITCH;
  168. }
  169. if (!flags)
  170. /* nothing to do */
  171. return 0;
  172. if (flags & USER_CTX_SWITCH)
  173. size = 1;
  174. else
  175. size = 0;
  176. sample = op_cpu_buffer_write_reserve(&entry, size);
  177. if (!sample)
  178. return -ENOMEM;
  179. sample->eip = ESCAPE_CODE;
  180. sample->event = flags;
  181. if (size)
  182. op_cpu_buffer_add_data(&entry, (unsigned long)task);
  183. op_cpu_buffer_write_commit(&entry);
  184. return 0;
  185. }
  186. static inline int
  187. op_add_sample(struct oprofile_cpu_buffer *cpu_buf,
  188. unsigned long pc, unsigned long event)
  189. {
  190. struct op_entry entry;
  191. struct op_sample *sample;
  192. sample = op_cpu_buffer_write_reserve(&entry, 0);
  193. if (!sample)
  194. return -ENOMEM;
  195. sample->eip = pc;
  196. sample->event = event;
  197. return op_cpu_buffer_write_commit(&entry);
  198. }
  199. /*
  200. * This must be safe from any context.
  201. *
  202. * is_kernel is needed because on some architectures you cannot
  203. * tell if you are in kernel or user space simply by looking at
  204. * pc. We tag this in the buffer by generating kernel enter/exit
  205. * events whenever is_kernel changes
  206. */
  207. static int
  208. log_sample(struct oprofile_cpu_buffer *cpu_buf, unsigned long pc,
  209. unsigned long backtrace, int is_kernel, unsigned long event)
  210. {
  211. cpu_buf->sample_received++;
  212. if (pc == ESCAPE_CODE) {
  213. cpu_buf->sample_invalid_eip++;
  214. return 0;
  215. }
  216. if (op_add_code(cpu_buf, backtrace, is_kernel, current))
  217. goto fail;
  218. if (op_add_sample(cpu_buf, pc, event))
  219. goto fail;
  220. return 1;
  221. fail:
  222. cpu_buf->sample_lost_overflow++;
  223. return 0;
  224. }
  225. static inline void oprofile_begin_trace(struct oprofile_cpu_buffer *cpu_buf)
  226. {
  227. cpu_buf->tracing = 1;
  228. }
  229. static inline void oprofile_end_trace(struct oprofile_cpu_buffer *cpu_buf)
  230. {
  231. cpu_buf->tracing = 0;
  232. }
  233. static inline void
  234. __oprofile_add_ext_sample(unsigned long pc, struct pt_regs * const regs,
  235. unsigned long event, int is_kernel)
  236. {
  237. struct oprofile_cpu_buffer *cpu_buf = &__get_cpu_var(op_cpu_buffer);
  238. unsigned long backtrace = oprofile_backtrace_depth;
  239. /*
  240. * if log_sample() fail we can't backtrace since we lost the
  241. * source of this event
  242. */
  243. if (!log_sample(cpu_buf, pc, backtrace, is_kernel, event))
  244. /* failed */
  245. return;
  246. if (!backtrace)
  247. return;
  248. oprofile_begin_trace(cpu_buf);
  249. oprofile_ops.backtrace(regs, backtrace);
  250. oprofile_end_trace(cpu_buf);
  251. }
  252. void oprofile_add_ext_sample(unsigned long pc, struct pt_regs * const regs,
  253. unsigned long event, int is_kernel)
  254. {
  255. __oprofile_add_ext_sample(pc, regs, event, is_kernel);
  256. }
  257. void oprofile_add_sample(struct pt_regs * const regs, unsigned long event)
  258. {
  259. int is_kernel;
  260. unsigned long pc;
  261. if (likely(regs)) {
  262. is_kernel = !user_mode(regs);
  263. pc = profile_pc(regs);
  264. } else {
  265. is_kernel = 0; /* This value will not be used */
  266. pc = ESCAPE_CODE; /* as this causes an early return. */
  267. }
  268. __oprofile_add_ext_sample(pc, regs, event, is_kernel);
  269. }
  270. /*
  271. * Add samples with data to the ring buffer.
  272. *
  273. * Use oprofile_add_data(&entry, val) to add data and
  274. * oprofile_write_commit(&entry) to commit the sample.
  275. */
  276. void
  277. oprofile_write_reserve(struct op_entry *entry, struct pt_regs * const regs,
  278. unsigned long pc, int code, int size)
  279. {
  280. struct op_sample *sample;
  281. int is_kernel = !user_mode(regs);
  282. struct oprofile_cpu_buffer *cpu_buf = &__get_cpu_var(op_cpu_buffer);
  283. cpu_buf->sample_received++;
  284. /* no backtraces for samples with data */
  285. if (op_add_code(cpu_buf, 0, is_kernel, current))
  286. goto fail;
  287. sample = op_cpu_buffer_write_reserve(entry, size + 2);
  288. if (!sample)
  289. goto fail;
  290. sample->eip = ESCAPE_CODE;
  291. sample->event = 0; /* no flags */
  292. op_cpu_buffer_add_data(entry, code);
  293. op_cpu_buffer_add_data(entry, pc);
  294. return;
  295. fail:
  296. entry->event = NULL;
  297. cpu_buf->sample_lost_overflow++;
  298. }
  299. int oprofile_add_data(struct op_entry *entry, unsigned long val)
  300. {
  301. if (!entry->event)
  302. return 0;
  303. return op_cpu_buffer_add_data(entry, val);
  304. }
  305. int oprofile_add_data64(struct op_entry *entry, u64 val)
  306. {
  307. if (!entry->event)
  308. return 0;
  309. if (op_cpu_buffer_get_size(entry) < 2)
  310. /*
  311. * the function returns 0 to indicate a too small
  312. * buffer, even if there is some space left
  313. */
  314. return 0;
  315. if (!op_cpu_buffer_add_data(entry, (u32)val))
  316. return 0;
  317. return op_cpu_buffer_add_data(entry, (u32)(val >> 32));
  318. }
  319. int oprofile_write_commit(struct op_entry *entry)
  320. {
  321. if (!entry->event)
  322. return -EINVAL;
  323. return op_cpu_buffer_write_commit(entry);
  324. }
  325. void oprofile_add_pc(unsigned long pc, int is_kernel, unsigned long event)
  326. {
  327. struct oprofile_cpu_buffer *cpu_buf = &__get_cpu_var(op_cpu_buffer);
  328. log_sample(cpu_buf, pc, 0, is_kernel, event);
  329. }
  330. void oprofile_add_trace(unsigned long pc)
  331. {
  332. struct oprofile_cpu_buffer *cpu_buf = &__get_cpu_var(op_cpu_buffer);
  333. if (!cpu_buf->tracing)
  334. return;
  335. /*
  336. * broken frame can give an eip with the same value as an
  337. * escape code, abort the trace if we get it
  338. */
  339. if (pc == ESCAPE_CODE)
  340. goto fail;
  341. if (op_add_sample(cpu_buf, pc, 0))
  342. goto fail;
  343. return;
  344. fail:
  345. cpu_buf->tracing = 0;
  346. cpu_buf->backtrace_aborted++;
  347. return;
  348. }
  349. /*
  350. * This serves to avoid cpu buffer overflow, and makes sure
  351. * the task mortuary progresses
  352. *
  353. * By using schedule_delayed_work_on and then schedule_delayed_work
  354. * we guarantee this will stay on the correct cpu
  355. */
  356. static void wq_sync_buffer(struct work_struct *work)
  357. {
  358. struct oprofile_cpu_buffer *b =
  359. container_of(work, struct oprofile_cpu_buffer, work.work);
  360. if (b->cpu != smp_processor_id()) {
  361. printk(KERN_DEBUG "WQ on CPU%d, prefer CPU%d\n",
  362. smp_processor_id(), b->cpu);
  363. if (!cpu_online(b->cpu)) {
  364. cancel_delayed_work(&b->work);
  365. return;
  366. }
  367. }
  368. sync_buffer(b->cpu);
  369. /* don't re-add the work if we're shutting down */
  370. if (work_enabled)
  371. schedule_delayed_work(&b->work, DEFAULT_TIMER_EXPIRE);
  372. }