perf_event.h 32 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223
  1. /*
  2. * Performance events:
  3. *
  4. * Copyright (C) 2008-2009, Thomas Gleixner <tglx@linutronix.de>
  5. * Copyright (C) 2008-2011, Red Hat, Inc., Ingo Molnar
  6. * Copyright (C) 2008-2011, Red Hat, Inc., Peter Zijlstra
  7. *
  8. * Data type definitions, declarations, prototypes.
  9. *
  10. * Started by: Thomas Gleixner and Ingo Molnar
  11. *
  12. * For licencing details see kernel-base/COPYING
  13. */
  14. #ifndef _LINUX_PERF_EVENT_H
  15. #define _LINUX_PERF_EVENT_H
  16. #include <linux/types.h>
  17. #include <linux/ioctl.h>
  18. #include <asm/byteorder.h>
  19. /*
  20. * User-space ABI bits:
  21. */
  22. /*
  23. * attr.type
  24. */
  25. enum perf_type_id {
  26. PERF_TYPE_HARDWARE = 0,
  27. PERF_TYPE_SOFTWARE = 1,
  28. PERF_TYPE_TRACEPOINT = 2,
  29. PERF_TYPE_HW_CACHE = 3,
  30. PERF_TYPE_RAW = 4,
  31. PERF_TYPE_BREAKPOINT = 5,
  32. PERF_TYPE_MAX, /* non-ABI */
  33. };
  34. /*
  35. * Generalized performance event event_id types, used by the
  36. * attr.event_id parameter of the sys_perf_event_open()
  37. * syscall:
  38. */
  39. enum perf_hw_id {
  40. /*
  41. * Common hardware events, generalized by the kernel:
  42. */
  43. PERF_COUNT_HW_CPU_CYCLES = 0,
  44. PERF_COUNT_HW_INSTRUCTIONS = 1,
  45. PERF_COUNT_HW_CACHE_REFERENCES = 2,
  46. PERF_COUNT_HW_CACHE_MISSES = 3,
  47. PERF_COUNT_HW_BRANCH_INSTRUCTIONS = 4,
  48. PERF_COUNT_HW_BRANCH_MISSES = 5,
  49. PERF_COUNT_HW_BUS_CYCLES = 6,
  50. PERF_COUNT_HW_STALLED_CYCLES_FRONTEND = 7,
  51. PERF_COUNT_HW_STALLED_CYCLES_BACKEND = 8,
  52. PERF_COUNT_HW_REF_CPU_CYCLES = 9,
  53. PERF_COUNT_HW_MAX, /* non-ABI */
  54. };
  55. /*
  56. * Generalized hardware cache events:
  57. *
  58. * { L1-D, L1-I, LLC, ITLB, DTLB, BPU, NODE } x
  59. * { read, write, prefetch } x
  60. * { accesses, misses }
  61. */
  62. enum perf_hw_cache_id {
  63. PERF_COUNT_HW_CACHE_L1D = 0,
  64. PERF_COUNT_HW_CACHE_L1I = 1,
  65. PERF_COUNT_HW_CACHE_LL = 2,
  66. PERF_COUNT_HW_CACHE_DTLB = 3,
  67. PERF_COUNT_HW_CACHE_ITLB = 4,
  68. PERF_COUNT_HW_CACHE_BPU = 5,
  69. PERF_COUNT_HW_CACHE_NODE = 6,
  70. PERF_COUNT_HW_CACHE_MAX, /* non-ABI */
  71. };
  72. enum perf_hw_cache_op_id {
  73. PERF_COUNT_HW_CACHE_OP_READ = 0,
  74. PERF_COUNT_HW_CACHE_OP_WRITE = 1,
  75. PERF_COUNT_HW_CACHE_OP_PREFETCH = 2,
  76. PERF_COUNT_HW_CACHE_OP_MAX, /* non-ABI */
  77. };
  78. enum perf_hw_cache_op_result_id {
  79. PERF_COUNT_HW_CACHE_RESULT_ACCESS = 0,
  80. PERF_COUNT_HW_CACHE_RESULT_MISS = 1,
  81. PERF_COUNT_HW_CACHE_RESULT_MAX, /* non-ABI */
  82. };
  83. /*
  84. * Special "software" events provided by the kernel, even if the hardware
  85. * does not support performance events. These events measure various
  86. * physical and sw events of the kernel (and allow the profiling of them as
  87. * well):
  88. */
  89. enum perf_sw_ids {
  90. PERF_COUNT_SW_CPU_CLOCK = 0,
  91. PERF_COUNT_SW_TASK_CLOCK = 1,
  92. PERF_COUNT_SW_PAGE_FAULTS = 2,
  93. PERF_COUNT_SW_CONTEXT_SWITCHES = 3,
  94. PERF_COUNT_SW_CPU_MIGRATIONS = 4,
  95. PERF_COUNT_SW_PAGE_FAULTS_MIN = 5,
  96. PERF_COUNT_SW_PAGE_FAULTS_MAJ = 6,
  97. PERF_COUNT_SW_ALIGNMENT_FAULTS = 7,
  98. PERF_COUNT_SW_EMULATION_FAULTS = 8,
  99. PERF_COUNT_SW_MAX, /* non-ABI */
  100. };
  101. /*
  102. * Bits that can be set in attr.sample_type to request information
  103. * in the overflow packets.
  104. */
  105. enum perf_event_sample_format {
  106. PERF_SAMPLE_IP = 1U << 0,
  107. PERF_SAMPLE_TID = 1U << 1,
  108. PERF_SAMPLE_TIME = 1U << 2,
  109. PERF_SAMPLE_ADDR = 1U << 3,
  110. PERF_SAMPLE_READ = 1U << 4,
  111. PERF_SAMPLE_CALLCHAIN = 1U << 5,
  112. PERF_SAMPLE_ID = 1U << 6,
  113. PERF_SAMPLE_CPU = 1U << 7,
  114. PERF_SAMPLE_PERIOD = 1U << 8,
  115. PERF_SAMPLE_STREAM_ID = 1U << 9,
  116. PERF_SAMPLE_RAW = 1U << 10,
  117. PERF_SAMPLE_MAX = 1U << 11, /* non-ABI */
  118. };
  119. /*
  120. * The format of the data returned by read() on a perf event fd,
  121. * as specified by attr.read_format:
  122. *
  123. * struct read_format {
  124. * { u64 value;
  125. * { u64 time_enabled; } && PERF_FORMAT_TOTAL_TIME_ENABLED
  126. * { u64 time_running; } && PERF_FORMAT_TOTAL_TIME_RUNNING
  127. * { u64 id; } && PERF_FORMAT_ID
  128. * } && !PERF_FORMAT_GROUP
  129. *
  130. * { u64 nr;
  131. * { u64 time_enabled; } && PERF_FORMAT_TOTAL_TIME_ENABLED
  132. * { u64 time_running; } && PERF_FORMAT_TOTAL_TIME_RUNNING
  133. * { u64 value;
  134. * { u64 id; } && PERF_FORMAT_ID
  135. * } cntr[nr];
  136. * } && PERF_FORMAT_GROUP
  137. * };
  138. */
  139. enum perf_event_read_format {
  140. PERF_FORMAT_TOTAL_TIME_ENABLED = 1U << 0,
  141. PERF_FORMAT_TOTAL_TIME_RUNNING = 1U << 1,
  142. PERF_FORMAT_ID = 1U << 2,
  143. PERF_FORMAT_GROUP = 1U << 3,
  144. PERF_FORMAT_MAX = 1U << 4, /* non-ABI */
  145. };
  146. #define PERF_ATTR_SIZE_VER0 64 /* sizeof first published struct */
  147. /*
  148. * Hardware event_id to monitor via a performance monitoring event:
  149. */
  150. struct perf_event_attr {
  151. /*
  152. * Major type: hardware/software/tracepoint/etc.
  153. */
  154. __u32 type;
  155. /*
  156. * Size of the attr structure, for fwd/bwd compat.
  157. */
  158. __u32 size;
  159. /*
  160. * Type specific configuration information.
  161. */
  162. __u64 config;
  163. union {
  164. __u64 sample_period;
  165. __u64 sample_freq;
  166. };
  167. __u64 sample_type;
  168. __u64 read_format;
  169. __u64 disabled : 1, /* off by default */
  170. inherit : 1, /* children inherit it */
  171. pinned : 1, /* must always be on PMU */
  172. exclusive : 1, /* only group on PMU */
  173. exclude_user : 1, /* don't count user */
  174. exclude_kernel : 1, /* ditto kernel */
  175. exclude_hv : 1, /* ditto hypervisor */
  176. exclude_idle : 1, /* don't count when idle */
  177. mmap : 1, /* include mmap data */
  178. comm : 1, /* include comm data */
  179. freq : 1, /* use freq, not period */
  180. inherit_stat : 1, /* per task counts */
  181. enable_on_exec : 1, /* next exec enables */
  182. task : 1, /* trace fork/exit */
  183. watermark : 1, /* wakeup_watermark */
  184. /*
  185. * precise_ip:
  186. *
  187. * 0 - SAMPLE_IP can have arbitrary skid
  188. * 1 - SAMPLE_IP must have constant skid
  189. * 2 - SAMPLE_IP requested to have 0 skid
  190. * 3 - SAMPLE_IP must have 0 skid
  191. *
  192. * See also PERF_RECORD_MISC_EXACT_IP
  193. */
  194. precise_ip : 2, /* skid constraint */
  195. mmap_data : 1, /* non-exec mmap data */
  196. sample_id_all : 1, /* sample_type all events */
  197. exclude_host : 1, /* don't count in host */
  198. exclude_guest : 1, /* don't count in guest */
  199. __reserved_1 : 43;
  200. union {
  201. __u32 wakeup_events; /* wakeup every n events */
  202. __u32 wakeup_watermark; /* bytes before wakeup */
  203. };
  204. __u32 bp_type;
  205. union {
  206. __u64 bp_addr;
  207. __u64 config1; /* extension of config */
  208. };
  209. union {
  210. __u64 bp_len;
  211. __u64 config2; /* extension of config1 */
  212. };
  213. };
  214. /*
  215. * Ioctls that can be done on a perf event fd:
  216. */
  217. #define PERF_EVENT_IOC_ENABLE _IO ('$', 0)
  218. #define PERF_EVENT_IOC_DISABLE _IO ('$', 1)
  219. #define PERF_EVENT_IOC_REFRESH _IO ('$', 2)
  220. #define PERF_EVENT_IOC_RESET _IO ('$', 3)
  221. #define PERF_EVENT_IOC_PERIOD _IOW('$', 4, __u64)
  222. #define PERF_EVENT_IOC_SET_OUTPUT _IO ('$', 5)
  223. #define PERF_EVENT_IOC_SET_FILTER _IOW('$', 6, char *)
  224. enum perf_event_ioc_flags {
  225. PERF_IOC_FLAG_GROUP = 1U << 0,
  226. };
  227. /*
  228. * Structure of the page that can be mapped via mmap
  229. */
  230. struct perf_event_mmap_page {
  231. __u32 version; /* version number of this structure */
  232. __u32 compat_version; /* lowest version this is compat with */
  233. /*
  234. * Bits needed to read the hw events in user-space.
  235. *
  236. * u32 seq;
  237. * s64 count;
  238. *
  239. * do {
  240. * seq = pc->lock;
  241. *
  242. * barrier()
  243. * if (pc->index) {
  244. * count = pmc_read(pc->index - 1);
  245. * count += pc->offset;
  246. * } else
  247. * goto regular_read;
  248. *
  249. * barrier();
  250. * } while (pc->lock != seq);
  251. *
  252. * NOTE: for obvious reason this only works on self-monitoring
  253. * processes.
  254. */
  255. __u32 lock; /* seqlock for synchronization */
  256. __u32 index; /* hardware event identifier */
  257. __s64 offset; /* add to hardware event value */
  258. __u64 time_enabled; /* time event active */
  259. __u64 time_running; /* time event on cpu */
  260. __u32 time_mult, time_shift;
  261. __u64 time_offset;
  262. /*
  263. * Hole for extension of the self monitor capabilities
  264. */
  265. __u64 __reserved[121]; /* align to 1k */
  266. /*
  267. * Control data for the mmap() data buffer.
  268. *
  269. * User-space reading the @data_head value should issue an rmb(), on
  270. * SMP capable platforms, after reading this value -- see
  271. * perf_event_wakeup().
  272. *
  273. * When the mapping is PROT_WRITE the @data_tail value should be
  274. * written by userspace to reflect the last read data. In this case
  275. * the kernel will not over-write unread data.
  276. */
  277. __u64 data_head; /* head in the data section */
  278. __u64 data_tail; /* user-space written tail */
  279. };
  280. #define PERF_RECORD_MISC_CPUMODE_MASK (7 << 0)
  281. #define PERF_RECORD_MISC_CPUMODE_UNKNOWN (0 << 0)
  282. #define PERF_RECORD_MISC_KERNEL (1 << 0)
  283. #define PERF_RECORD_MISC_USER (2 << 0)
  284. #define PERF_RECORD_MISC_HYPERVISOR (3 << 0)
  285. #define PERF_RECORD_MISC_GUEST_KERNEL (4 << 0)
  286. #define PERF_RECORD_MISC_GUEST_USER (5 << 0)
  287. /*
  288. * Indicates that the content of PERF_SAMPLE_IP points to
  289. * the actual instruction that triggered the event. See also
  290. * perf_event_attr::precise_ip.
  291. */
  292. #define PERF_RECORD_MISC_EXACT_IP (1 << 14)
  293. /*
  294. * Reserve the last bit to indicate some extended misc field
  295. */
  296. #define PERF_RECORD_MISC_EXT_RESERVED (1 << 15)
  297. struct perf_event_header {
  298. __u32 type;
  299. __u16 misc;
  300. __u16 size;
  301. };
  302. enum perf_event_type {
  303. /*
  304. * If perf_event_attr.sample_id_all is set then all event types will
  305. * have the sample_type selected fields related to where/when
  306. * (identity) an event took place (TID, TIME, ID, CPU, STREAM_ID)
  307. * described in PERF_RECORD_SAMPLE below, it will be stashed just after
  308. * the perf_event_header and the fields already present for the existing
  309. * fields, i.e. at the end of the payload. That way a newer perf.data
  310. * file will be supported by older perf tools, with these new optional
  311. * fields being ignored.
  312. *
  313. * The MMAP events record the PROT_EXEC mappings so that we can
  314. * correlate userspace IPs to code. They have the following structure:
  315. *
  316. * struct {
  317. * struct perf_event_header header;
  318. *
  319. * u32 pid, tid;
  320. * u64 addr;
  321. * u64 len;
  322. * u64 pgoff;
  323. * char filename[];
  324. * };
  325. */
  326. PERF_RECORD_MMAP = 1,
  327. /*
  328. * struct {
  329. * struct perf_event_header header;
  330. * u64 id;
  331. * u64 lost;
  332. * };
  333. */
  334. PERF_RECORD_LOST = 2,
  335. /*
  336. * struct {
  337. * struct perf_event_header header;
  338. *
  339. * u32 pid, tid;
  340. * char comm[];
  341. * };
  342. */
  343. PERF_RECORD_COMM = 3,
  344. /*
  345. * struct {
  346. * struct perf_event_header header;
  347. * u32 pid, ppid;
  348. * u32 tid, ptid;
  349. * u64 time;
  350. * };
  351. */
  352. PERF_RECORD_EXIT = 4,
  353. /*
  354. * struct {
  355. * struct perf_event_header header;
  356. * u64 time;
  357. * u64 id;
  358. * u64 stream_id;
  359. * };
  360. */
  361. PERF_RECORD_THROTTLE = 5,
  362. PERF_RECORD_UNTHROTTLE = 6,
  363. /*
  364. * struct {
  365. * struct perf_event_header header;
  366. * u32 pid, ppid;
  367. * u32 tid, ptid;
  368. * u64 time;
  369. * };
  370. */
  371. PERF_RECORD_FORK = 7,
  372. /*
  373. * struct {
  374. * struct perf_event_header header;
  375. * u32 pid, tid;
  376. *
  377. * struct read_format values;
  378. * };
  379. */
  380. PERF_RECORD_READ = 8,
  381. /*
  382. * struct {
  383. * struct perf_event_header header;
  384. *
  385. * { u64 ip; } && PERF_SAMPLE_IP
  386. * { u32 pid, tid; } && PERF_SAMPLE_TID
  387. * { u64 time; } && PERF_SAMPLE_TIME
  388. * { u64 addr; } && PERF_SAMPLE_ADDR
  389. * { u64 id; } && PERF_SAMPLE_ID
  390. * { u64 stream_id;} && PERF_SAMPLE_STREAM_ID
  391. * { u32 cpu, res; } && PERF_SAMPLE_CPU
  392. * { u64 period; } && PERF_SAMPLE_PERIOD
  393. *
  394. * { struct read_format values; } && PERF_SAMPLE_READ
  395. *
  396. * { u64 nr,
  397. * u64 ips[nr]; } && PERF_SAMPLE_CALLCHAIN
  398. *
  399. * #
  400. * # The RAW record below is opaque data wrt the ABI
  401. * #
  402. * # That is, the ABI doesn't make any promises wrt to
  403. * # the stability of its content, it may vary depending
  404. * # on event, hardware, kernel version and phase of
  405. * # the moon.
  406. * #
  407. * # In other words, PERF_SAMPLE_RAW contents are not an ABI.
  408. * #
  409. *
  410. * { u32 size;
  411. * char data[size];}&& PERF_SAMPLE_RAW
  412. * };
  413. */
  414. PERF_RECORD_SAMPLE = 9,
  415. PERF_RECORD_MAX, /* non-ABI */
  416. };
  417. enum perf_callchain_context {
  418. PERF_CONTEXT_HV = (__u64)-32,
  419. PERF_CONTEXT_KERNEL = (__u64)-128,
  420. PERF_CONTEXT_USER = (__u64)-512,
  421. PERF_CONTEXT_GUEST = (__u64)-2048,
  422. PERF_CONTEXT_GUEST_KERNEL = (__u64)-2176,
  423. PERF_CONTEXT_GUEST_USER = (__u64)-2560,
  424. PERF_CONTEXT_MAX = (__u64)-4095,
  425. };
  426. #define PERF_FLAG_FD_NO_GROUP (1U << 0)
  427. #define PERF_FLAG_FD_OUTPUT (1U << 1)
  428. #define PERF_FLAG_PID_CGROUP (1U << 2) /* pid=cgroup id, per-cpu mode only */
  429. #ifdef __KERNEL__
  430. /*
  431. * Kernel-internal data types and definitions:
  432. */
  433. #ifdef CONFIG_PERF_EVENTS
  434. # include <linux/cgroup.h>
  435. # include <asm/perf_event.h>
  436. # include <asm/local64.h>
  437. #endif
  438. struct perf_guest_info_callbacks {
  439. int (*is_in_guest)(void);
  440. int (*is_user_mode)(void);
  441. unsigned long (*get_guest_ip)(void);
  442. };
  443. #ifdef CONFIG_HAVE_HW_BREAKPOINT
  444. #include <asm/hw_breakpoint.h>
  445. #endif
  446. #include <linux/list.h>
  447. #include <linux/mutex.h>
  448. #include <linux/rculist.h>
  449. #include <linux/rcupdate.h>
  450. #include <linux/spinlock.h>
  451. #include <linux/hrtimer.h>
  452. #include <linux/fs.h>
  453. #include <linux/pid_namespace.h>
  454. #include <linux/workqueue.h>
  455. #include <linux/ftrace.h>
  456. #include <linux/cpu.h>
  457. #include <linux/irq_work.h>
  458. #include <linux/static_key.h>
  459. #include <linux/atomic.h>
  460. #include <asm/local.h>
  461. #define PERF_MAX_STACK_DEPTH 255
  462. struct perf_callchain_entry {
  463. __u64 nr;
  464. __u64 ip[PERF_MAX_STACK_DEPTH];
  465. };
  466. struct perf_raw_record {
  467. u32 size;
  468. void *data;
  469. };
  470. struct perf_branch_entry {
  471. __u64 from;
  472. __u64 to;
  473. __u64 flags;
  474. };
  475. struct perf_branch_stack {
  476. __u64 nr;
  477. struct perf_branch_entry entries[0];
  478. };
  479. struct task_struct;
  480. /*
  481. * extra PMU register associated with an event
  482. */
  483. struct hw_perf_event_extra {
  484. u64 config; /* register value */
  485. unsigned int reg; /* register address or index */
  486. int alloc; /* extra register already allocated */
  487. int idx; /* index in shared_regs->regs[] */
  488. };
  489. /**
  490. * struct hw_perf_event - performance event hardware details:
  491. */
  492. struct hw_perf_event {
  493. #ifdef CONFIG_PERF_EVENTS
  494. union {
  495. struct { /* hardware */
  496. u64 config;
  497. u64 last_tag;
  498. unsigned long config_base;
  499. unsigned long event_base;
  500. int idx;
  501. int last_cpu;
  502. struct hw_perf_event_extra extra_reg;
  503. };
  504. struct { /* software */
  505. struct hrtimer hrtimer;
  506. };
  507. #ifdef CONFIG_HAVE_HW_BREAKPOINT
  508. struct { /* breakpoint */
  509. struct arch_hw_breakpoint info;
  510. struct list_head bp_list;
  511. /*
  512. * Crufty hack to avoid the chicken and egg
  513. * problem hw_breakpoint has with context
  514. * creation and event initalization.
  515. */
  516. struct task_struct *bp_target;
  517. };
  518. #endif
  519. };
  520. int state;
  521. local64_t prev_count;
  522. u64 sample_period;
  523. u64 last_period;
  524. local64_t period_left;
  525. u64 interrupts_seq;
  526. u64 interrupts;
  527. u64 freq_time_stamp;
  528. u64 freq_count_stamp;
  529. #endif
  530. };
  531. /*
  532. * hw_perf_event::state flags
  533. */
  534. #define PERF_HES_STOPPED 0x01 /* the counter is stopped */
  535. #define PERF_HES_UPTODATE 0x02 /* event->count up-to-date */
  536. #define PERF_HES_ARCH 0x04
  537. struct perf_event;
  538. /*
  539. * Common implementation detail of pmu::{start,commit,cancel}_txn
  540. */
  541. #define PERF_EVENT_TXN 0x1
  542. /**
  543. * struct pmu - generic performance monitoring unit
  544. */
  545. struct pmu {
  546. struct list_head entry;
  547. struct device *dev;
  548. const struct attribute_group **attr_groups;
  549. char *name;
  550. int type;
  551. int * __percpu pmu_disable_count;
  552. struct perf_cpu_context * __percpu pmu_cpu_context;
  553. int task_ctx_nr;
  554. /*
  555. * Fully disable/enable this PMU, can be used to protect from the PMI
  556. * as well as for lazy/batch writing of the MSRs.
  557. */
  558. void (*pmu_enable) (struct pmu *pmu); /* optional */
  559. void (*pmu_disable) (struct pmu *pmu); /* optional */
  560. /*
  561. * Try and initialize the event for this PMU.
  562. * Should return -ENOENT when the @event doesn't match this PMU.
  563. */
  564. int (*event_init) (struct perf_event *event);
  565. #define PERF_EF_START 0x01 /* start the counter when adding */
  566. #define PERF_EF_RELOAD 0x02 /* reload the counter when starting */
  567. #define PERF_EF_UPDATE 0x04 /* update the counter when stopping */
  568. /*
  569. * Adds/Removes a counter to/from the PMU, can be done inside
  570. * a transaction, see the ->*_txn() methods.
  571. */
  572. int (*add) (struct perf_event *event, int flags);
  573. void (*del) (struct perf_event *event, int flags);
  574. /*
  575. * Starts/Stops a counter present on the PMU. The PMI handler
  576. * should stop the counter when perf_event_overflow() returns
  577. * !0. ->start() will be used to continue.
  578. */
  579. void (*start) (struct perf_event *event, int flags);
  580. void (*stop) (struct perf_event *event, int flags);
  581. /*
  582. * Updates the counter value of the event.
  583. */
  584. void (*read) (struct perf_event *event);
  585. /*
  586. * Group events scheduling is treated as a transaction, add
  587. * group events as a whole and perform one schedulability test.
  588. * If the test fails, roll back the whole group
  589. *
  590. * Start the transaction, after this ->add() doesn't need to
  591. * do schedulability tests.
  592. */
  593. void (*start_txn) (struct pmu *pmu); /* optional */
  594. /*
  595. * If ->start_txn() disabled the ->add() schedulability test
  596. * then ->commit_txn() is required to perform one. On success
  597. * the transaction is closed. On error the transaction is kept
  598. * open until ->cancel_txn() is called.
  599. */
  600. int (*commit_txn) (struct pmu *pmu); /* optional */
  601. /*
  602. * Will cancel the transaction, assumes ->del() is called
  603. * for each successful ->add() during the transaction.
  604. */
  605. void (*cancel_txn) (struct pmu *pmu); /* optional */
  606. /*
  607. * Will return the value for perf_event_mmap_page::index for this event,
  608. * if no implementation is provided it will default to: event->hw.idx + 1.
  609. */
  610. int (*event_idx) (struct perf_event *event); /*optional */
  611. };
  612. /**
  613. * enum perf_event_active_state - the states of a event
  614. */
  615. enum perf_event_active_state {
  616. PERF_EVENT_STATE_ERROR = -2,
  617. PERF_EVENT_STATE_OFF = -1,
  618. PERF_EVENT_STATE_INACTIVE = 0,
  619. PERF_EVENT_STATE_ACTIVE = 1,
  620. };
  621. struct file;
  622. struct perf_sample_data;
  623. typedef void (*perf_overflow_handler_t)(struct perf_event *,
  624. struct perf_sample_data *,
  625. struct pt_regs *regs);
  626. enum perf_group_flag {
  627. PERF_GROUP_SOFTWARE = 0x1,
  628. };
  629. #define SWEVENT_HLIST_BITS 8
  630. #define SWEVENT_HLIST_SIZE (1 << SWEVENT_HLIST_BITS)
  631. struct swevent_hlist {
  632. struct hlist_head heads[SWEVENT_HLIST_SIZE];
  633. struct rcu_head rcu_head;
  634. };
  635. #define PERF_ATTACH_CONTEXT 0x01
  636. #define PERF_ATTACH_GROUP 0x02
  637. #define PERF_ATTACH_TASK 0x04
  638. #ifdef CONFIG_CGROUP_PERF
  639. /*
  640. * perf_cgroup_info keeps track of time_enabled for a cgroup.
  641. * This is a per-cpu dynamically allocated data structure.
  642. */
  643. struct perf_cgroup_info {
  644. u64 time;
  645. u64 timestamp;
  646. };
  647. struct perf_cgroup {
  648. struct cgroup_subsys_state css;
  649. struct perf_cgroup_info *info; /* timing info, one per cpu */
  650. };
  651. #endif
  652. struct ring_buffer;
  653. /**
  654. * struct perf_event - performance event kernel representation:
  655. */
  656. struct perf_event {
  657. #ifdef CONFIG_PERF_EVENTS
  658. struct list_head group_entry;
  659. struct list_head event_entry;
  660. struct list_head sibling_list;
  661. struct hlist_node hlist_entry;
  662. int nr_siblings;
  663. int group_flags;
  664. struct perf_event *group_leader;
  665. struct pmu *pmu;
  666. enum perf_event_active_state state;
  667. unsigned int attach_state;
  668. local64_t count;
  669. atomic64_t child_count;
  670. /*
  671. * These are the total time in nanoseconds that the event
  672. * has been enabled (i.e. eligible to run, and the task has
  673. * been scheduled in, if this is a per-task event)
  674. * and running (scheduled onto the CPU), respectively.
  675. *
  676. * They are computed from tstamp_enabled, tstamp_running and
  677. * tstamp_stopped when the event is in INACTIVE or ACTIVE state.
  678. */
  679. u64 total_time_enabled;
  680. u64 total_time_running;
  681. /*
  682. * These are timestamps used for computing total_time_enabled
  683. * and total_time_running when the event is in INACTIVE or
  684. * ACTIVE state, measured in nanoseconds from an arbitrary point
  685. * in time.
  686. * tstamp_enabled: the notional time when the event was enabled
  687. * tstamp_running: the notional time when the event was scheduled on
  688. * tstamp_stopped: in INACTIVE state, the notional time when the
  689. * event was scheduled off.
  690. */
  691. u64 tstamp_enabled;
  692. u64 tstamp_running;
  693. u64 tstamp_stopped;
  694. /*
  695. * timestamp shadows the actual context timing but it can
  696. * be safely used in NMI interrupt context. It reflects the
  697. * context time as it was when the event was last scheduled in.
  698. *
  699. * ctx_time already accounts for ctx->timestamp. Therefore to
  700. * compute ctx_time for a sample, simply add perf_clock().
  701. */
  702. u64 shadow_ctx_time;
  703. struct perf_event_attr attr;
  704. u16 header_size;
  705. u16 id_header_size;
  706. u16 read_size;
  707. struct hw_perf_event hw;
  708. struct perf_event_context *ctx;
  709. struct file *filp;
  710. /*
  711. * These accumulate total time (in nanoseconds) that children
  712. * events have been enabled and running, respectively.
  713. */
  714. atomic64_t child_total_time_enabled;
  715. atomic64_t child_total_time_running;
  716. /*
  717. * Protect attach/detach and child_list:
  718. */
  719. struct mutex child_mutex;
  720. struct list_head child_list;
  721. struct perf_event *parent;
  722. int oncpu;
  723. int cpu;
  724. struct list_head owner_entry;
  725. struct task_struct *owner;
  726. /* mmap bits */
  727. struct mutex mmap_mutex;
  728. atomic_t mmap_count;
  729. int mmap_locked;
  730. struct user_struct *mmap_user;
  731. struct ring_buffer *rb;
  732. struct list_head rb_entry;
  733. /* poll related */
  734. wait_queue_head_t waitq;
  735. struct fasync_struct *fasync;
  736. /* delayed work for NMIs and such */
  737. int pending_wakeup;
  738. int pending_kill;
  739. int pending_disable;
  740. struct irq_work pending;
  741. atomic_t event_limit;
  742. void (*destroy)(struct perf_event *);
  743. struct rcu_head rcu_head;
  744. struct pid_namespace *ns;
  745. u64 id;
  746. perf_overflow_handler_t overflow_handler;
  747. void *overflow_handler_context;
  748. #ifdef CONFIG_EVENT_TRACING
  749. struct ftrace_event_call *tp_event;
  750. struct event_filter *filter;
  751. #ifdef CONFIG_FUNCTION_TRACER
  752. struct ftrace_ops ftrace_ops;
  753. #endif
  754. #endif
  755. #ifdef CONFIG_CGROUP_PERF
  756. struct perf_cgroup *cgrp; /* cgroup event is attach to */
  757. int cgrp_defer_enabled;
  758. #endif
  759. #endif /* CONFIG_PERF_EVENTS */
  760. };
  761. enum perf_event_context_type {
  762. task_context,
  763. cpu_context,
  764. };
  765. /**
  766. * struct perf_event_context - event context structure
  767. *
  768. * Used as a container for task events and CPU events as well:
  769. */
  770. struct perf_event_context {
  771. struct pmu *pmu;
  772. enum perf_event_context_type type;
  773. /*
  774. * Protect the states of the events in the list,
  775. * nr_active, and the list:
  776. */
  777. raw_spinlock_t lock;
  778. /*
  779. * Protect the list of events. Locking either mutex or lock
  780. * is sufficient to ensure the list doesn't change; to change
  781. * the list you need to lock both the mutex and the spinlock.
  782. */
  783. struct mutex mutex;
  784. struct list_head pinned_groups;
  785. struct list_head flexible_groups;
  786. struct list_head event_list;
  787. int nr_events;
  788. int nr_active;
  789. int is_active;
  790. int nr_stat;
  791. int nr_freq;
  792. int rotate_disable;
  793. atomic_t refcount;
  794. struct task_struct *task;
  795. /*
  796. * Context clock, runs when context enabled.
  797. */
  798. u64 time;
  799. u64 timestamp;
  800. /*
  801. * These fields let us detect when two contexts have both
  802. * been cloned (inherited) from a common ancestor.
  803. */
  804. struct perf_event_context *parent_ctx;
  805. u64 parent_gen;
  806. u64 generation;
  807. int pin_count;
  808. int nr_cgroups; /* cgroup events present */
  809. struct rcu_head rcu_head;
  810. };
  811. /*
  812. * Number of contexts where an event can trigger:
  813. * task, softirq, hardirq, nmi.
  814. */
  815. #define PERF_NR_CONTEXTS 4
  816. /**
  817. * struct perf_event_cpu_context - per cpu event context structure
  818. */
  819. struct perf_cpu_context {
  820. struct perf_event_context ctx;
  821. struct perf_event_context *task_ctx;
  822. int active_oncpu;
  823. int exclusive;
  824. struct list_head rotation_list;
  825. int jiffies_interval;
  826. struct pmu *active_pmu;
  827. struct perf_cgroup *cgrp;
  828. };
  829. struct perf_output_handle {
  830. struct perf_event *event;
  831. struct ring_buffer *rb;
  832. unsigned long wakeup;
  833. unsigned long size;
  834. void *addr;
  835. int page;
  836. };
  837. #ifdef CONFIG_PERF_EVENTS
  838. extern int perf_pmu_register(struct pmu *pmu, char *name, int type);
  839. extern void perf_pmu_unregister(struct pmu *pmu);
  840. extern int perf_num_counters(void);
  841. extern const char *perf_pmu_name(void);
  842. extern void __perf_event_task_sched_in(struct task_struct *prev,
  843. struct task_struct *task);
  844. extern void __perf_event_task_sched_out(struct task_struct *prev,
  845. struct task_struct *next);
  846. extern int perf_event_init_task(struct task_struct *child);
  847. extern void perf_event_exit_task(struct task_struct *child);
  848. extern void perf_event_free_task(struct task_struct *task);
  849. extern void perf_event_delayed_put(struct task_struct *task);
  850. extern void perf_event_print_debug(void);
  851. extern void perf_pmu_disable(struct pmu *pmu);
  852. extern void perf_pmu_enable(struct pmu *pmu);
  853. extern int perf_event_task_disable(void);
  854. extern int perf_event_task_enable(void);
  855. extern int perf_event_refresh(struct perf_event *event, int refresh);
  856. extern void perf_event_update_userpage(struct perf_event *event);
  857. extern int perf_event_release_kernel(struct perf_event *event);
  858. extern struct perf_event *
  859. perf_event_create_kernel_counter(struct perf_event_attr *attr,
  860. int cpu,
  861. struct task_struct *task,
  862. perf_overflow_handler_t callback,
  863. void *context);
  864. extern u64 perf_event_read_value(struct perf_event *event,
  865. u64 *enabled, u64 *running);
  866. struct perf_sample_data {
  867. u64 type;
  868. u64 ip;
  869. struct {
  870. u32 pid;
  871. u32 tid;
  872. } tid_entry;
  873. u64 time;
  874. u64 addr;
  875. u64 id;
  876. u64 stream_id;
  877. struct {
  878. u32 cpu;
  879. u32 reserved;
  880. } cpu_entry;
  881. u64 period;
  882. struct perf_callchain_entry *callchain;
  883. struct perf_raw_record *raw;
  884. };
  885. static inline void perf_sample_data_init(struct perf_sample_data *data, u64 addr)
  886. {
  887. data->addr = addr;
  888. data->raw = NULL;
  889. }
  890. extern void perf_output_sample(struct perf_output_handle *handle,
  891. struct perf_event_header *header,
  892. struct perf_sample_data *data,
  893. struct perf_event *event);
  894. extern void perf_prepare_sample(struct perf_event_header *header,
  895. struct perf_sample_data *data,
  896. struct perf_event *event,
  897. struct pt_regs *regs);
  898. extern int perf_event_overflow(struct perf_event *event,
  899. struct perf_sample_data *data,
  900. struct pt_regs *regs);
  901. static inline bool is_sampling_event(struct perf_event *event)
  902. {
  903. return event->attr.sample_period != 0;
  904. }
  905. /*
  906. * Return 1 for a software event, 0 for a hardware event
  907. */
  908. static inline int is_software_event(struct perf_event *event)
  909. {
  910. return event->pmu->task_ctx_nr == perf_sw_context;
  911. }
  912. extern struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
  913. extern void __perf_sw_event(u32, u64, struct pt_regs *, u64);
  914. #ifndef perf_arch_fetch_caller_regs
  915. static inline void perf_arch_fetch_caller_regs(struct pt_regs *regs, unsigned long ip) { }
  916. #endif
  917. /*
  918. * Take a snapshot of the regs. Skip ip and frame pointer to
  919. * the nth caller. We only need a few of the regs:
  920. * - ip for PERF_SAMPLE_IP
  921. * - cs for user_mode() tests
  922. * - bp for callchains
  923. * - eflags, for future purposes, just in case
  924. */
  925. static inline void perf_fetch_caller_regs(struct pt_regs *regs)
  926. {
  927. memset(regs, 0, sizeof(*regs));
  928. perf_arch_fetch_caller_regs(regs, CALLER_ADDR0);
  929. }
  930. static __always_inline void
  931. perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
  932. {
  933. struct pt_regs hot_regs;
  934. if (static_key_false(&perf_swevent_enabled[event_id])) {
  935. if (!regs) {
  936. perf_fetch_caller_regs(&hot_regs);
  937. regs = &hot_regs;
  938. }
  939. __perf_sw_event(event_id, nr, regs, addr);
  940. }
  941. }
  942. extern struct static_key_deferred perf_sched_events;
  943. static inline void perf_event_task_sched_in(struct task_struct *prev,
  944. struct task_struct *task)
  945. {
  946. if (static_key_false(&perf_sched_events.key))
  947. __perf_event_task_sched_in(prev, task);
  948. }
  949. static inline void perf_event_task_sched_out(struct task_struct *prev,
  950. struct task_struct *next)
  951. {
  952. perf_sw_event(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, NULL, 0);
  953. if (static_key_false(&perf_sched_events.key))
  954. __perf_event_task_sched_out(prev, next);
  955. }
  956. extern void perf_event_mmap(struct vm_area_struct *vma);
  957. extern struct perf_guest_info_callbacks *perf_guest_cbs;
  958. extern int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *callbacks);
  959. extern int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *callbacks);
  960. extern void perf_event_comm(struct task_struct *tsk);
  961. extern void perf_event_fork(struct task_struct *tsk);
  962. /* Callchains */
  963. DECLARE_PER_CPU(struct perf_callchain_entry, perf_callchain_entry);
  964. extern void perf_callchain_user(struct perf_callchain_entry *entry, struct pt_regs *regs);
  965. extern void perf_callchain_kernel(struct perf_callchain_entry *entry, struct pt_regs *regs);
  966. static inline void perf_callchain_store(struct perf_callchain_entry *entry, u64 ip)
  967. {
  968. if (entry->nr < PERF_MAX_STACK_DEPTH)
  969. entry->ip[entry->nr++] = ip;
  970. }
  971. extern int sysctl_perf_event_paranoid;
  972. extern int sysctl_perf_event_mlock;
  973. extern int sysctl_perf_event_sample_rate;
  974. extern int perf_proc_update_handler(struct ctl_table *table, int write,
  975. void __user *buffer, size_t *lenp,
  976. loff_t *ppos);
  977. static inline bool perf_paranoid_tracepoint_raw(void)
  978. {
  979. return sysctl_perf_event_paranoid > -1;
  980. }
  981. static inline bool perf_paranoid_cpu(void)
  982. {
  983. return sysctl_perf_event_paranoid > 0;
  984. }
  985. static inline bool perf_paranoid_kernel(void)
  986. {
  987. return sysctl_perf_event_paranoid > 1;
  988. }
  989. extern void perf_event_init(void);
  990. extern void perf_tp_event(u64 addr, u64 count, void *record,
  991. int entry_size, struct pt_regs *regs,
  992. struct hlist_head *head, int rctx);
  993. extern void perf_bp_event(struct perf_event *event, void *data);
  994. #ifndef perf_misc_flags
  995. # define perf_misc_flags(regs) \
  996. (user_mode(regs) ? PERF_RECORD_MISC_USER : PERF_RECORD_MISC_KERNEL)
  997. # define perf_instruction_pointer(regs) instruction_pointer(regs)
  998. #endif
  999. extern int perf_output_begin(struct perf_output_handle *handle,
  1000. struct perf_event *event, unsigned int size);
  1001. extern void perf_output_end(struct perf_output_handle *handle);
  1002. extern void perf_output_copy(struct perf_output_handle *handle,
  1003. const void *buf, unsigned int len);
  1004. extern int perf_swevent_get_recursion_context(void);
  1005. extern void perf_swevent_put_recursion_context(int rctx);
  1006. extern void perf_event_enable(struct perf_event *event);
  1007. extern void perf_event_disable(struct perf_event *event);
  1008. extern void perf_event_task_tick(void);
  1009. #else
  1010. static inline void
  1011. perf_event_task_sched_in(struct task_struct *prev,
  1012. struct task_struct *task) { }
  1013. static inline void
  1014. perf_event_task_sched_out(struct task_struct *prev,
  1015. struct task_struct *next) { }
  1016. static inline int perf_event_init_task(struct task_struct *child) { return 0; }
  1017. static inline void perf_event_exit_task(struct task_struct *child) { }
  1018. static inline void perf_event_free_task(struct task_struct *task) { }
  1019. static inline void perf_event_delayed_put(struct task_struct *task) { }
  1020. static inline void perf_event_print_debug(void) { }
  1021. static inline int perf_event_task_disable(void) { return -EINVAL; }
  1022. static inline int perf_event_task_enable(void) { return -EINVAL; }
  1023. static inline int perf_event_refresh(struct perf_event *event, int refresh)
  1024. {
  1025. return -EINVAL;
  1026. }
  1027. static inline void
  1028. perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr) { }
  1029. static inline void
  1030. perf_bp_event(struct perf_event *event, void *data) { }
  1031. static inline int perf_register_guest_info_callbacks
  1032. (struct perf_guest_info_callbacks *callbacks) { return 0; }
  1033. static inline int perf_unregister_guest_info_callbacks
  1034. (struct perf_guest_info_callbacks *callbacks) { return 0; }
  1035. static inline void perf_event_mmap(struct vm_area_struct *vma) { }
  1036. static inline void perf_event_comm(struct task_struct *tsk) { }
  1037. static inline void perf_event_fork(struct task_struct *tsk) { }
  1038. static inline void perf_event_init(void) { }
  1039. static inline int perf_swevent_get_recursion_context(void) { return -1; }
  1040. static inline void perf_swevent_put_recursion_context(int rctx) { }
  1041. static inline void perf_event_enable(struct perf_event *event) { }
  1042. static inline void perf_event_disable(struct perf_event *event) { }
  1043. static inline void perf_event_task_tick(void) { }
  1044. #endif
  1045. #define perf_output_put(handle, x) perf_output_copy((handle), &(x), sizeof(x))
  1046. /*
  1047. * This has to have a higher priority than migration_notifier in sched.c.
  1048. */
  1049. #define perf_cpu_notifier(fn) \
  1050. do { \
  1051. static struct notifier_block fn##_nb __cpuinitdata = \
  1052. { .notifier_call = fn, .priority = CPU_PRI_PERF }; \
  1053. fn(&fn##_nb, (unsigned long)CPU_UP_PREPARE, \
  1054. (void *)(unsigned long)smp_processor_id()); \
  1055. fn(&fn##_nb, (unsigned long)CPU_STARTING, \
  1056. (void *)(unsigned long)smp_processor_id()); \
  1057. fn(&fn##_nb, (unsigned long)CPU_ONLINE, \
  1058. (void *)(unsigned long)smp_processor_id()); \
  1059. register_cpu_notifier(&fn##_nb); \
  1060. } while (0)
  1061. #endif /* __KERNEL__ */
  1062. #endif /* _LINUX_PERF_EVENT_H */