perf_event.c 43 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905
  1. /*
  2. * Performance events x86 architecture code
  3. *
  4. * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
  5. * Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
  6. * Copyright (C) 2009 Jaswinder Singh Rajput
  7. * Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
  8. * Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
  9. * Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com>
  10. * Copyright (C) 2009 Google, Inc., Stephane Eranian
  11. *
  12. * For licencing details see kernel-base/COPYING
  13. */
  14. #include <linux/perf_event.h>
  15. #include <linux/capability.h>
  16. #include <linux/notifier.h>
  17. #include <linux/hardirq.h>
  18. #include <linux/kprobes.h>
  19. #include <linux/module.h>
  20. #include <linux/kdebug.h>
  21. #include <linux/sched.h>
  22. #include <linux/uaccess.h>
  23. #include <linux/slab.h>
  24. #include <linux/highmem.h>
  25. #include <linux/cpu.h>
  26. #include <linux/bitops.h>
  27. #include <asm/apic.h>
  28. #include <asm/stacktrace.h>
  29. #include <asm/nmi.h>
  30. #include <asm/compat.h>
  31. #include <asm/smp.h>
  32. #if 0
  33. #undef wrmsrl
  34. #define wrmsrl(msr, val) \
  35. do { \
  36. trace_printk("wrmsrl(%lx, %lx)\n", (unsigned long)(msr),\
  37. (unsigned long)(val)); \
  38. native_write_msr((msr), (u32)((u64)(val)), \
  39. (u32)((u64)(val) >> 32)); \
  40. } while (0)
  41. #endif
  42. /*
  43. * best effort, GUP based copy_from_user() that assumes IRQ or NMI context
  44. */
  45. static unsigned long
  46. copy_from_user_nmi(void *to, const void __user *from, unsigned long n)
  47. {
  48. unsigned long offset, addr = (unsigned long)from;
  49. unsigned long size, len = 0;
  50. struct page *page;
  51. void *map;
  52. int ret;
  53. do {
  54. ret = __get_user_pages_fast(addr, 1, 0, &page);
  55. if (!ret)
  56. break;
  57. offset = addr & (PAGE_SIZE - 1);
  58. size = min(PAGE_SIZE - offset, n - len);
  59. map = kmap_atomic(page);
  60. memcpy(to, map+offset, size);
  61. kunmap_atomic(map);
  62. put_page(page);
  63. len += size;
  64. to += size;
  65. addr += size;
  66. } while (len < n);
  67. return len;
  68. }
  69. struct event_constraint {
  70. union {
  71. unsigned long idxmsk[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
  72. u64 idxmsk64;
  73. };
  74. u64 code;
  75. u64 cmask;
  76. int weight;
  77. };
  78. struct amd_nb {
  79. int nb_id; /* NorthBridge id */
  80. int refcnt; /* reference count */
  81. struct perf_event *owners[X86_PMC_IDX_MAX];
  82. struct event_constraint event_constraints[X86_PMC_IDX_MAX];
  83. };
  84. struct intel_percore;
  85. #define MAX_LBR_ENTRIES 16
  86. struct cpu_hw_events {
  87. /*
  88. * Generic x86 PMC bits
  89. */
  90. struct perf_event *events[X86_PMC_IDX_MAX]; /* in counter order */
  91. unsigned long active_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
  92. unsigned long running[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
  93. int enabled;
  94. int n_events;
  95. int n_added;
  96. int n_txn;
  97. int assign[X86_PMC_IDX_MAX]; /* event to counter assignment */
  98. u64 tags[X86_PMC_IDX_MAX];
  99. struct perf_event *event_list[X86_PMC_IDX_MAX]; /* in enabled order */
  100. unsigned int group_flag;
  101. /*
  102. * Intel DebugStore bits
  103. */
  104. struct debug_store *ds;
  105. u64 pebs_enabled;
  106. /*
  107. * Intel LBR bits
  108. */
  109. int lbr_users;
  110. void *lbr_context;
  111. struct perf_branch_stack lbr_stack;
  112. struct perf_branch_entry lbr_entries[MAX_LBR_ENTRIES];
  113. /*
  114. * Intel percore register state.
  115. * Coordinate shared resources between HT threads.
  116. */
  117. int percore_used; /* Used by this CPU? */
  118. struct intel_percore *per_core;
  119. /*
  120. * AMD specific bits
  121. */
  122. struct amd_nb *amd_nb;
  123. };
  124. #define __EVENT_CONSTRAINT(c, n, m, w) {\
  125. { .idxmsk64 = (n) }, \
  126. .code = (c), \
  127. .cmask = (m), \
  128. .weight = (w), \
  129. }
  130. #define EVENT_CONSTRAINT(c, n, m) \
  131. __EVENT_CONSTRAINT(c, n, m, HWEIGHT(n))
  132. /*
  133. * Constraint on the Event code.
  134. */
  135. #define INTEL_EVENT_CONSTRAINT(c, n) \
  136. EVENT_CONSTRAINT(c, n, ARCH_PERFMON_EVENTSEL_EVENT)
  137. /*
  138. * Constraint on the Event code + UMask + fixed-mask
  139. *
  140. * filter mask to validate fixed counter events.
  141. * the following filters disqualify for fixed counters:
  142. * - inv
  143. * - edge
  144. * - cnt-mask
  145. * The other filters are supported by fixed counters.
  146. * The any-thread option is supported starting with v3.
  147. */
  148. #define FIXED_EVENT_CONSTRAINT(c, n) \
  149. EVENT_CONSTRAINT(c, (1ULL << (32+n)), X86_RAW_EVENT_MASK)
  150. /*
  151. * Constraint on the Event code + UMask
  152. */
  153. #define INTEL_UEVENT_CONSTRAINT(c, n) \
  154. EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK)
  155. #define EVENT_CONSTRAINT_END \
  156. EVENT_CONSTRAINT(0, 0, 0)
  157. #define for_each_event_constraint(e, c) \
  158. for ((e) = (c); (e)->weight; (e)++)
  159. /*
  160. * Extra registers for specific events.
  161. * Some events need large masks and require external MSRs.
  162. * Define a mapping to these extra registers.
  163. */
  164. struct extra_reg {
  165. unsigned int event;
  166. unsigned int msr;
  167. u64 config_mask;
  168. u64 valid_mask;
  169. };
  170. #define EVENT_EXTRA_REG(e, ms, m, vm) { \
  171. .event = (e), \
  172. .msr = (ms), \
  173. .config_mask = (m), \
  174. .valid_mask = (vm), \
  175. }
  176. #define INTEL_EVENT_EXTRA_REG(event, msr, vm) \
  177. EVENT_EXTRA_REG(event, msr, ARCH_PERFMON_EVENTSEL_EVENT, vm)
  178. #define EVENT_EXTRA_END EVENT_EXTRA_REG(0, 0, 0, 0)
  179. union perf_capabilities {
  180. struct {
  181. u64 lbr_format : 6;
  182. u64 pebs_trap : 1;
  183. u64 pebs_arch_reg : 1;
  184. u64 pebs_format : 4;
  185. u64 smm_freeze : 1;
  186. };
  187. u64 capabilities;
  188. };
  189. /*
  190. * struct x86_pmu - generic x86 pmu
  191. */
  192. struct x86_pmu {
  193. /*
  194. * Generic x86 PMC bits
  195. */
  196. const char *name;
  197. int version;
  198. int (*handle_irq)(struct pt_regs *);
  199. void (*disable_all)(void);
  200. void (*enable_all)(int added);
  201. void (*enable)(struct perf_event *);
  202. void (*disable)(struct perf_event *);
  203. int (*hw_config)(struct perf_event *event);
  204. int (*schedule_events)(struct cpu_hw_events *cpuc, int n, int *assign);
  205. unsigned eventsel;
  206. unsigned perfctr;
  207. u64 (*event_map)(int);
  208. int max_events;
  209. int num_counters;
  210. int num_counters_fixed;
  211. int cntval_bits;
  212. u64 cntval_mask;
  213. int apic;
  214. u64 max_period;
  215. struct event_constraint *
  216. (*get_event_constraints)(struct cpu_hw_events *cpuc,
  217. struct perf_event *event);
  218. void (*put_event_constraints)(struct cpu_hw_events *cpuc,
  219. struct perf_event *event);
  220. struct event_constraint *event_constraints;
  221. struct event_constraint *percore_constraints;
  222. void (*quirks)(void);
  223. int perfctr_second_write;
  224. int (*cpu_prepare)(int cpu);
  225. void (*cpu_starting)(int cpu);
  226. void (*cpu_dying)(int cpu);
  227. void (*cpu_dead)(int cpu);
  228. /*
  229. * Intel Arch Perfmon v2+
  230. */
  231. u64 intel_ctrl;
  232. union perf_capabilities intel_cap;
  233. /*
  234. * Intel DebugStore bits
  235. */
  236. int bts, pebs;
  237. int bts_active, pebs_active;
  238. int pebs_record_size;
  239. void (*drain_pebs)(struct pt_regs *regs);
  240. struct event_constraint *pebs_constraints;
  241. /*
  242. * Intel LBR
  243. */
  244. unsigned long lbr_tos, lbr_from, lbr_to; /* MSR base regs */
  245. int lbr_nr; /* hardware stack size */
  246. /*
  247. * Extra registers for events
  248. */
  249. struct extra_reg *extra_regs;
  250. };
  251. static struct x86_pmu x86_pmu __read_mostly;
  252. static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = {
  253. .enabled = 1,
  254. };
  255. static int x86_perf_event_set_period(struct perf_event *event);
  256. /*
  257. * Generalized hw caching related hw_event table, filled
  258. * in on a per model basis. A value of 0 means
  259. * 'not supported', -1 means 'hw_event makes no sense on
  260. * this CPU', any other value means the raw hw_event
  261. * ID.
  262. */
  263. #define C(x) PERF_COUNT_HW_CACHE_##x
  264. static u64 __read_mostly hw_cache_event_ids
  265. [PERF_COUNT_HW_CACHE_MAX]
  266. [PERF_COUNT_HW_CACHE_OP_MAX]
  267. [PERF_COUNT_HW_CACHE_RESULT_MAX];
  268. static u64 __read_mostly hw_cache_extra_regs
  269. [PERF_COUNT_HW_CACHE_MAX]
  270. [PERF_COUNT_HW_CACHE_OP_MAX]
  271. [PERF_COUNT_HW_CACHE_RESULT_MAX];
  272. /*
  273. * Propagate event elapsed time into the generic event.
  274. * Can only be executed on the CPU where the event is active.
  275. * Returns the delta events processed.
  276. */
  277. static u64
  278. x86_perf_event_update(struct perf_event *event)
  279. {
  280. struct hw_perf_event *hwc = &event->hw;
  281. int shift = 64 - x86_pmu.cntval_bits;
  282. u64 prev_raw_count, new_raw_count;
  283. int idx = hwc->idx;
  284. s64 delta;
  285. if (idx == X86_PMC_IDX_FIXED_BTS)
  286. return 0;
  287. /*
  288. * Careful: an NMI might modify the previous event value.
  289. *
  290. * Our tactic to handle this is to first atomically read and
  291. * exchange a new raw count - then add that new-prev delta
  292. * count to the generic event atomically:
  293. */
  294. again:
  295. prev_raw_count = local64_read(&hwc->prev_count);
  296. rdmsrl(hwc->event_base, new_raw_count);
  297. if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
  298. new_raw_count) != prev_raw_count)
  299. goto again;
  300. /*
  301. * Now we have the new raw value and have updated the prev
  302. * timestamp already. We can now calculate the elapsed delta
  303. * (event-)time and add that to the generic event.
  304. *
  305. * Careful, not all hw sign-extends above the physical width
  306. * of the count.
  307. */
  308. delta = (new_raw_count << shift) - (prev_raw_count << shift);
  309. delta >>= shift;
  310. local64_add(delta, &event->count);
  311. local64_sub(delta, &hwc->period_left);
  312. return new_raw_count;
  313. }
  314. /* using X86_FEATURE_PERFCTR_CORE to later implement ALTERNATIVE() here */
  315. static inline int x86_pmu_addr_offset(int index)
  316. {
  317. if (boot_cpu_has(X86_FEATURE_PERFCTR_CORE))
  318. return index << 1;
  319. return index;
  320. }
  321. static inline unsigned int x86_pmu_config_addr(int index)
  322. {
  323. return x86_pmu.eventsel + x86_pmu_addr_offset(index);
  324. }
  325. static inline unsigned int x86_pmu_event_addr(int index)
  326. {
  327. return x86_pmu.perfctr + x86_pmu_addr_offset(index);
  328. }
  329. /*
  330. * Find and validate any extra registers to set up.
  331. */
  332. static int x86_pmu_extra_regs(u64 config, struct perf_event *event)
  333. {
  334. struct extra_reg *er;
  335. event->hw.extra_reg = 0;
  336. event->hw.extra_config = 0;
  337. if (!x86_pmu.extra_regs)
  338. return 0;
  339. for (er = x86_pmu.extra_regs; er->msr; er++) {
  340. if (er->event != (config & er->config_mask))
  341. continue;
  342. if (event->attr.config1 & ~er->valid_mask)
  343. return -EINVAL;
  344. event->hw.extra_reg = er->msr;
  345. event->hw.extra_config = event->attr.config1;
  346. break;
  347. }
  348. return 0;
  349. }
  350. static atomic_t active_events;
  351. static DEFINE_MUTEX(pmc_reserve_mutex);
  352. #ifdef CONFIG_X86_LOCAL_APIC
  353. static bool reserve_pmc_hardware(void)
  354. {
  355. int i;
  356. for (i = 0; i < x86_pmu.num_counters; i++) {
  357. if (!reserve_perfctr_nmi(x86_pmu_event_addr(i)))
  358. goto perfctr_fail;
  359. }
  360. for (i = 0; i < x86_pmu.num_counters; i++) {
  361. if (!reserve_evntsel_nmi(x86_pmu_config_addr(i)))
  362. goto eventsel_fail;
  363. }
  364. return true;
  365. eventsel_fail:
  366. for (i--; i >= 0; i--)
  367. release_evntsel_nmi(x86_pmu_config_addr(i));
  368. i = x86_pmu.num_counters;
  369. perfctr_fail:
  370. for (i--; i >= 0; i--)
  371. release_perfctr_nmi(x86_pmu_event_addr(i));
  372. return false;
  373. }
  374. static void release_pmc_hardware(void)
  375. {
  376. int i;
  377. for (i = 0; i < x86_pmu.num_counters; i++) {
  378. release_perfctr_nmi(x86_pmu_event_addr(i));
  379. release_evntsel_nmi(x86_pmu_config_addr(i));
  380. }
  381. }
  382. #else
  383. static bool reserve_pmc_hardware(void) { return true; }
  384. static void release_pmc_hardware(void) {}
  385. #endif
  386. static bool check_hw_exists(void)
  387. {
  388. u64 val, val_new = 0;
  389. int i, reg, ret = 0;
  390. /*
  391. * Check to see if the BIOS enabled any of the counters, if so
  392. * complain and bail.
  393. */
  394. for (i = 0; i < x86_pmu.num_counters; i++) {
  395. reg = x86_pmu_config_addr(i);
  396. ret = rdmsrl_safe(reg, &val);
  397. if (ret)
  398. goto msr_fail;
  399. if (val & ARCH_PERFMON_EVENTSEL_ENABLE)
  400. goto bios_fail;
  401. }
  402. if (x86_pmu.num_counters_fixed) {
  403. reg = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
  404. ret = rdmsrl_safe(reg, &val);
  405. if (ret)
  406. goto msr_fail;
  407. for (i = 0; i < x86_pmu.num_counters_fixed; i++) {
  408. if (val & (0x03 << i*4))
  409. goto bios_fail;
  410. }
  411. }
  412. /*
  413. * Now write a value and read it back to see if it matches,
  414. * this is needed to detect certain hardware emulators (qemu/kvm)
  415. * that don't trap on the MSR access and always return 0s.
  416. */
  417. val = 0xabcdUL;
  418. ret = checking_wrmsrl(x86_pmu_event_addr(0), val);
  419. ret |= rdmsrl_safe(x86_pmu_event_addr(0), &val_new);
  420. if (ret || val != val_new)
  421. goto msr_fail;
  422. return true;
  423. bios_fail:
  424. /*
  425. * We still allow the PMU driver to operate:
  426. */
  427. printk(KERN_CONT "Broken BIOS detected, complain to your hardware vendor.\n");
  428. printk(KERN_ERR FW_BUG "the BIOS has corrupted hw-PMU resources (MSR %x is %Lx)\n", reg, val);
  429. return true;
  430. msr_fail:
  431. printk(KERN_CONT "Broken PMU hardware detected, using software events only.\n");
  432. return false;
  433. }
  434. static void reserve_ds_buffers(void);
  435. static void release_ds_buffers(void);
  436. static void hw_perf_event_destroy(struct perf_event *event)
  437. {
  438. if (atomic_dec_and_mutex_lock(&active_events, &pmc_reserve_mutex)) {
  439. release_pmc_hardware();
  440. release_ds_buffers();
  441. mutex_unlock(&pmc_reserve_mutex);
  442. }
  443. }
  444. static inline int x86_pmu_initialized(void)
  445. {
  446. return x86_pmu.handle_irq != NULL;
  447. }
  448. static inline int
  449. set_ext_hw_attr(struct hw_perf_event *hwc, struct perf_event *event)
  450. {
  451. struct perf_event_attr *attr = &event->attr;
  452. unsigned int cache_type, cache_op, cache_result;
  453. u64 config, val;
  454. config = attr->config;
  455. cache_type = (config >> 0) & 0xff;
  456. if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
  457. return -EINVAL;
  458. cache_op = (config >> 8) & 0xff;
  459. if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
  460. return -EINVAL;
  461. cache_result = (config >> 16) & 0xff;
  462. if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
  463. return -EINVAL;
  464. val = hw_cache_event_ids[cache_type][cache_op][cache_result];
  465. if (val == 0)
  466. return -ENOENT;
  467. if (val == -1)
  468. return -EINVAL;
  469. hwc->config |= val;
  470. attr->config1 = hw_cache_extra_regs[cache_type][cache_op][cache_result];
  471. return x86_pmu_extra_regs(val, event);
  472. }
  473. static int x86_setup_perfctr(struct perf_event *event)
  474. {
  475. struct perf_event_attr *attr = &event->attr;
  476. struct hw_perf_event *hwc = &event->hw;
  477. u64 config;
  478. if (!is_sampling_event(event)) {
  479. hwc->sample_period = x86_pmu.max_period;
  480. hwc->last_period = hwc->sample_period;
  481. local64_set(&hwc->period_left, hwc->sample_period);
  482. } else {
  483. /*
  484. * If we have a PMU initialized but no APIC
  485. * interrupts, we cannot sample hardware
  486. * events (user-space has to fall back and
  487. * sample via a hrtimer based software event):
  488. */
  489. if (!x86_pmu.apic)
  490. return -EOPNOTSUPP;
  491. }
  492. if (attr->type == PERF_TYPE_RAW)
  493. return x86_pmu_extra_regs(event->attr.config, event);
  494. if (attr->type == PERF_TYPE_HW_CACHE)
  495. return set_ext_hw_attr(hwc, event);
  496. if (attr->config >= x86_pmu.max_events)
  497. return -EINVAL;
  498. /*
  499. * The generic map:
  500. */
  501. config = x86_pmu.event_map(attr->config);
  502. if (config == 0)
  503. return -ENOENT;
  504. if (config == -1LL)
  505. return -EINVAL;
  506. /*
  507. * Branch tracing:
  508. */
  509. if ((attr->config == PERF_COUNT_HW_BRANCH_INSTRUCTIONS) &&
  510. (hwc->sample_period == 1)) {
  511. /* BTS is not supported by this architecture. */
  512. if (!x86_pmu.bts_active)
  513. return -EOPNOTSUPP;
  514. /* BTS is currently only allowed for user-mode. */
  515. if (!attr->exclude_kernel)
  516. return -EOPNOTSUPP;
  517. }
  518. hwc->config |= config;
  519. return 0;
  520. }
  521. static int x86_pmu_hw_config(struct perf_event *event)
  522. {
  523. if (event->attr.precise_ip) {
  524. int precise = 0;
  525. /* Support for constant skid */
  526. if (x86_pmu.pebs_active) {
  527. precise++;
  528. /* Support for IP fixup */
  529. if (x86_pmu.lbr_nr)
  530. precise++;
  531. }
  532. if (event->attr.precise_ip > precise)
  533. return -EOPNOTSUPP;
  534. }
  535. /*
  536. * Generate PMC IRQs:
  537. * (keep 'enabled' bit clear for now)
  538. */
  539. event->hw.config = ARCH_PERFMON_EVENTSEL_INT;
  540. /*
  541. * Count user and OS events unless requested not to
  542. */
  543. if (!event->attr.exclude_user)
  544. event->hw.config |= ARCH_PERFMON_EVENTSEL_USR;
  545. if (!event->attr.exclude_kernel)
  546. event->hw.config |= ARCH_PERFMON_EVENTSEL_OS;
  547. if (event->attr.type == PERF_TYPE_RAW)
  548. event->hw.config |= event->attr.config & X86_RAW_EVENT_MASK;
  549. return x86_setup_perfctr(event);
  550. }
  551. /*
  552. * Setup the hardware configuration for a given attr_type
  553. */
  554. static int __x86_pmu_event_init(struct perf_event *event)
  555. {
  556. int err;
  557. if (!x86_pmu_initialized())
  558. return -ENODEV;
  559. err = 0;
  560. if (!atomic_inc_not_zero(&active_events)) {
  561. mutex_lock(&pmc_reserve_mutex);
  562. if (atomic_read(&active_events) == 0) {
  563. if (!reserve_pmc_hardware())
  564. err = -EBUSY;
  565. else
  566. reserve_ds_buffers();
  567. }
  568. if (!err)
  569. atomic_inc(&active_events);
  570. mutex_unlock(&pmc_reserve_mutex);
  571. }
  572. if (err)
  573. return err;
  574. event->destroy = hw_perf_event_destroy;
  575. event->hw.idx = -1;
  576. event->hw.last_cpu = -1;
  577. event->hw.last_tag = ~0ULL;
  578. return x86_pmu.hw_config(event);
  579. }
  580. static void x86_pmu_disable_all(void)
  581. {
  582. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  583. int idx;
  584. for (idx = 0; idx < x86_pmu.num_counters; idx++) {
  585. u64 val;
  586. if (!test_bit(idx, cpuc->active_mask))
  587. continue;
  588. rdmsrl(x86_pmu_config_addr(idx), val);
  589. if (!(val & ARCH_PERFMON_EVENTSEL_ENABLE))
  590. continue;
  591. val &= ~ARCH_PERFMON_EVENTSEL_ENABLE;
  592. wrmsrl(x86_pmu_config_addr(idx), val);
  593. }
  594. }
  595. static void x86_pmu_disable(struct pmu *pmu)
  596. {
  597. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  598. if (!x86_pmu_initialized())
  599. return;
  600. if (!cpuc->enabled)
  601. return;
  602. cpuc->n_added = 0;
  603. cpuc->enabled = 0;
  604. barrier();
  605. x86_pmu.disable_all();
  606. }
  607. static inline void __x86_pmu_enable_event(struct hw_perf_event *hwc,
  608. u64 enable_mask)
  609. {
  610. if (hwc->extra_reg)
  611. wrmsrl(hwc->extra_reg, hwc->extra_config);
  612. wrmsrl(hwc->config_base, hwc->config | enable_mask);
  613. }
  614. static void x86_pmu_enable_all(int added)
  615. {
  616. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  617. int idx;
  618. for (idx = 0; idx < x86_pmu.num_counters; idx++) {
  619. struct hw_perf_event *hwc = &cpuc->events[idx]->hw;
  620. if (!test_bit(idx, cpuc->active_mask))
  621. continue;
  622. __x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE);
  623. }
  624. }
  625. static struct pmu pmu;
  626. static inline int is_x86_event(struct perf_event *event)
  627. {
  628. return event->pmu == &pmu;
  629. }
  630. static int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign)
  631. {
  632. struct event_constraint *c, *constraints[X86_PMC_IDX_MAX];
  633. unsigned long used_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
  634. int i, j, w, wmax, num = 0;
  635. struct hw_perf_event *hwc;
  636. bitmap_zero(used_mask, X86_PMC_IDX_MAX);
  637. for (i = 0; i < n; i++) {
  638. c = x86_pmu.get_event_constraints(cpuc, cpuc->event_list[i]);
  639. constraints[i] = c;
  640. }
  641. /*
  642. * fastpath, try to reuse previous register
  643. */
  644. for (i = 0; i < n; i++) {
  645. hwc = &cpuc->event_list[i]->hw;
  646. c = constraints[i];
  647. /* never assigned */
  648. if (hwc->idx == -1)
  649. break;
  650. /* constraint still honored */
  651. if (!test_bit(hwc->idx, c->idxmsk))
  652. break;
  653. /* not already used */
  654. if (test_bit(hwc->idx, used_mask))
  655. break;
  656. __set_bit(hwc->idx, used_mask);
  657. if (assign)
  658. assign[i] = hwc->idx;
  659. }
  660. if (i == n)
  661. goto done;
  662. /*
  663. * begin slow path
  664. */
  665. bitmap_zero(used_mask, X86_PMC_IDX_MAX);
  666. /*
  667. * weight = number of possible counters
  668. *
  669. * 1 = most constrained, only works on one counter
  670. * wmax = least constrained, works on any counter
  671. *
  672. * assign events to counters starting with most
  673. * constrained events.
  674. */
  675. wmax = x86_pmu.num_counters;
  676. /*
  677. * when fixed event counters are present,
  678. * wmax is incremented by 1 to account
  679. * for one more choice
  680. */
  681. if (x86_pmu.num_counters_fixed)
  682. wmax++;
  683. for (w = 1, num = n; num && w <= wmax; w++) {
  684. /* for each event */
  685. for (i = 0; num && i < n; i++) {
  686. c = constraints[i];
  687. hwc = &cpuc->event_list[i]->hw;
  688. if (c->weight != w)
  689. continue;
  690. for_each_set_bit(j, c->idxmsk, X86_PMC_IDX_MAX) {
  691. if (!test_bit(j, used_mask))
  692. break;
  693. }
  694. if (j == X86_PMC_IDX_MAX)
  695. break;
  696. __set_bit(j, used_mask);
  697. if (assign)
  698. assign[i] = j;
  699. num--;
  700. }
  701. }
  702. done:
  703. /*
  704. * scheduling failed or is just a simulation,
  705. * free resources if necessary
  706. */
  707. if (!assign || num) {
  708. for (i = 0; i < n; i++) {
  709. if (x86_pmu.put_event_constraints)
  710. x86_pmu.put_event_constraints(cpuc, cpuc->event_list[i]);
  711. }
  712. }
  713. return num ? -ENOSPC : 0;
  714. }
  715. /*
  716. * dogrp: true if must collect siblings events (group)
  717. * returns total number of events and error code
  718. */
  719. static int collect_events(struct cpu_hw_events *cpuc, struct perf_event *leader, bool dogrp)
  720. {
  721. struct perf_event *event;
  722. int n, max_count;
  723. max_count = x86_pmu.num_counters + x86_pmu.num_counters_fixed;
  724. /* current number of events already accepted */
  725. n = cpuc->n_events;
  726. if (is_x86_event(leader)) {
  727. if (n >= max_count)
  728. return -ENOSPC;
  729. cpuc->event_list[n] = leader;
  730. n++;
  731. }
  732. if (!dogrp)
  733. return n;
  734. list_for_each_entry(event, &leader->sibling_list, group_entry) {
  735. if (!is_x86_event(event) ||
  736. event->state <= PERF_EVENT_STATE_OFF)
  737. continue;
  738. if (n >= max_count)
  739. return -ENOSPC;
  740. cpuc->event_list[n] = event;
  741. n++;
  742. }
  743. return n;
  744. }
  745. static inline void x86_assign_hw_event(struct perf_event *event,
  746. struct cpu_hw_events *cpuc, int i)
  747. {
  748. struct hw_perf_event *hwc = &event->hw;
  749. hwc->idx = cpuc->assign[i];
  750. hwc->last_cpu = smp_processor_id();
  751. hwc->last_tag = ++cpuc->tags[i];
  752. if (hwc->idx == X86_PMC_IDX_FIXED_BTS) {
  753. hwc->config_base = 0;
  754. hwc->event_base = 0;
  755. } else if (hwc->idx >= X86_PMC_IDX_FIXED) {
  756. hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
  757. hwc->event_base = MSR_ARCH_PERFMON_FIXED_CTR0 + (hwc->idx - X86_PMC_IDX_FIXED);
  758. } else {
  759. hwc->config_base = x86_pmu_config_addr(hwc->idx);
  760. hwc->event_base = x86_pmu_event_addr(hwc->idx);
  761. }
  762. }
  763. static inline int match_prev_assignment(struct hw_perf_event *hwc,
  764. struct cpu_hw_events *cpuc,
  765. int i)
  766. {
  767. return hwc->idx == cpuc->assign[i] &&
  768. hwc->last_cpu == smp_processor_id() &&
  769. hwc->last_tag == cpuc->tags[i];
  770. }
  771. static void x86_pmu_start(struct perf_event *event, int flags);
  772. static void x86_pmu_stop(struct perf_event *event, int flags);
  773. static void x86_pmu_enable(struct pmu *pmu)
  774. {
  775. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  776. struct perf_event *event;
  777. struct hw_perf_event *hwc;
  778. int i, added = cpuc->n_added;
  779. if (!x86_pmu_initialized())
  780. return;
  781. if (cpuc->enabled)
  782. return;
  783. if (cpuc->n_added) {
  784. int n_running = cpuc->n_events - cpuc->n_added;
  785. /*
  786. * apply assignment obtained either from
  787. * hw_perf_group_sched_in() or x86_pmu_enable()
  788. *
  789. * step1: save events moving to new counters
  790. * step2: reprogram moved events into new counters
  791. */
  792. for (i = 0; i < n_running; i++) {
  793. event = cpuc->event_list[i];
  794. hwc = &event->hw;
  795. /*
  796. * we can avoid reprogramming counter if:
  797. * - assigned same counter as last time
  798. * - running on same CPU as last time
  799. * - no other event has used the counter since
  800. */
  801. if (hwc->idx == -1 ||
  802. match_prev_assignment(hwc, cpuc, i))
  803. continue;
  804. /*
  805. * Ensure we don't accidentally enable a stopped
  806. * counter simply because we rescheduled.
  807. */
  808. if (hwc->state & PERF_HES_STOPPED)
  809. hwc->state |= PERF_HES_ARCH;
  810. x86_pmu_stop(event, PERF_EF_UPDATE);
  811. }
  812. for (i = 0; i < cpuc->n_events; i++) {
  813. event = cpuc->event_list[i];
  814. hwc = &event->hw;
  815. if (!match_prev_assignment(hwc, cpuc, i))
  816. x86_assign_hw_event(event, cpuc, i);
  817. else if (i < n_running)
  818. continue;
  819. if (hwc->state & PERF_HES_ARCH)
  820. continue;
  821. x86_pmu_start(event, PERF_EF_RELOAD);
  822. }
  823. cpuc->n_added = 0;
  824. perf_events_lapic_init();
  825. }
  826. cpuc->enabled = 1;
  827. barrier();
  828. x86_pmu.enable_all(added);
  829. }
  830. static inline void x86_pmu_disable_event(struct perf_event *event)
  831. {
  832. struct hw_perf_event *hwc = &event->hw;
  833. wrmsrl(hwc->config_base, hwc->config);
  834. }
  835. static DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left);
  836. /*
  837. * Set the next IRQ period, based on the hwc->period_left value.
  838. * To be called with the event disabled in hw:
  839. */
  840. static int
  841. x86_perf_event_set_period(struct perf_event *event)
  842. {
  843. struct hw_perf_event *hwc = &event->hw;
  844. s64 left = local64_read(&hwc->period_left);
  845. s64 period = hwc->sample_period;
  846. int ret = 0, idx = hwc->idx;
  847. if (idx == X86_PMC_IDX_FIXED_BTS)
  848. return 0;
  849. /*
  850. * If we are way outside a reasonable range then just skip forward:
  851. */
  852. if (unlikely(left <= -period)) {
  853. left = period;
  854. local64_set(&hwc->period_left, left);
  855. hwc->last_period = period;
  856. ret = 1;
  857. }
  858. if (unlikely(left <= 0)) {
  859. left += period;
  860. local64_set(&hwc->period_left, left);
  861. hwc->last_period = period;
  862. ret = 1;
  863. }
  864. /*
  865. * Quirk: certain CPUs dont like it if just 1 hw_event is left:
  866. */
  867. if (unlikely(left < 2))
  868. left = 2;
  869. if (left > x86_pmu.max_period)
  870. left = x86_pmu.max_period;
  871. per_cpu(pmc_prev_left[idx], smp_processor_id()) = left;
  872. /*
  873. * The hw event starts counting from this event offset,
  874. * mark it to be able to extra future deltas:
  875. */
  876. local64_set(&hwc->prev_count, (u64)-left);
  877. wrmsrl(hwc->event_base, (u64)(-left) & x86_pmu.cntval_mask);
  878. /*
  879. * Due to erratum on certan cpu we need
  880. * a second write to be sure the register
  881. * is updated properly
  882. */
  883. if (x86_pmu.perfctr_second_write) {
  884. wrmsrl(hwc->event_base,
  885. (u64)(-left) & x86_pmu.cntval_mask);
  886. }
  887. perf_event_update_userpage(event);
  888. return ret;
  889. }
  890. static void x86_pmu_enable_event(struct perf_event *event)
  891. {
  892. if (__this_cpu_read(cpu_hw_events.enabled))
  893. __x86_pmu_enable_event(&event->hw,
  894. ARCH_PERFMON_EVENTSEL_ENABLE);
  895. }
  896. /*
  897. * Add a single event to the PMU.
  898. *
  899. * The event is added to the group of enabled events
  900. * but only if it can be scehduled with existing events.
  901. */
  902. static int x86_pmu_add(struct perf_event *event, int flags)
  903. {
  904. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  905. struct hw_perf_event *hwc;
  906. int assign[X86_PMC_IDX_MAX];
  907. int n, n0, ret;
  908. hwc = &event->hw;
  909. perf_pmu_disable(event->pmu);
  910. n0 = cpuc->n_events;
  911. ret = n = collect_events(cpuc, event, false);
  912. if (ret < 0)
  913. goto out;
  914. hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
  915. if (!(flags & PERF_EF_START))
  916. hwc->state |= PERF_HES_ARCH;
  917. /*
  918. * If group events scheduling transaction was started,
  919. * skip the schedulability test here, it will be performed
  920. * at commit time (->commit_txn) as a whole
  921. */
  922. if (cpuc->group_flag & PERF_EVENT_TXN)
  923. goto done_collect;
  924. ret = x86_pmu.schedule_events(cpuc, n, assign);
  925. if (ret)
  926. goto out;
  927. /*
  928. * copy new assignment, now we know it is possible
  929. * will be used by hw_perf_enable()
  930. */
  931. memcpy(cpuc->assign, assign, n*sizeof(int));
  932. done_collect:
  933. cpuc->n_events = n;
  934. cpuc->n_added += n - n0;
  935. cpuc->n_txn += n - n0;
  936. ret = 0;
  937. out:
  938. perf_pmu_enable(event->pmu);
  939. return ret;
  940. }
  941. static void x86_pmu_start(struct perf_event *event, int flags)
  942. {
  943. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  944. int idx = event->hw.idx;
  945. if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
  946. return;
  947. if (WARN_ON_ONCE(idx == -1))
  948. return;
  949. if (flags & PERF_EF_RELOAD) {
  950. WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
  951. x86_perf_event_set_period(event);
  952. }
  953. event->hw.state = 0;
  954. cpuc->events[idx] = event;
  955. __set_bit(idx, cpuc->active_mask);
  956. __set_bit(idx, cpuc->running);
  957. x86_pmu.enable(event);
  958. perf_event_update_userpage(event);
  959. }
  960. void perf_event_print_debug(void)
  961. {
  962. u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed;
  963. u64 pebs;
  964. struct cpu_hw_events *cpuc;
  965. unsigned long flags;
  966. int cpu, idx;
  967. if (!x86_pmu.num_counters)
  968. return;
  969. local_irq_save(flags);
  970. cpu = smp_processor_id();
  971. cpuc = &per_cpu(cpu_hw_events, cpu);
  972. if (x86_pmu.version >= 2) {
  973. rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
  974. rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
  975. rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow);
  976. rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed);
  977. rdmsrl(MSR_IA32_PEBS_ENABLE, pebs);
  978. pr_info("\n");
  979. pr_info("CPU#%d: ctrl: %016llx\n", cpu, ctrl);
  980. pr_info("CPU#%d: status: %016llx\n", cpu, status);
  981. pr_info("CPU#%d: overflow: %016llx\n", cpu, overflow);
  982. pr_info("CPU#%d: fixed: %016llx\n", cpu, fixed);
  983. pr_info("CPU#%d: pebs: %016llx\n", cpu, pebs);
  984. }
  985. pr_info("CPU#%d: active: %016llx\n", cpu, *(u64 *)cpuc->active_mask);
  986. for (idx = 0; idx < x86_pmu.num_counters; idx++) {
  987. rdmsrl(x86_pmu_config_addr(idx), pmc_ctrl);
  988. rdmsrl(x86_pmu_event_addr(idx), pmc_count);
  989. prev_left = per_cpu(pmc_prev_left[idx], cpu);
  990. pr_info("CPU#%d: gen-PMC%d ctrl: %016llx\n",
  991. cpu, idx, pmc_ctrl);
  992. pr_info("CPU#%d: gen-PMC%d count: %016llx\n",
  993. cpu, idx, pmc_count);
  994. pr_info("CPU#%d: gen-PMC%d left: %016llx\n",
  995. cpu, idx, prev_left);
  996. }
  997. for (idx = 0; idx < x86_pmu.num_counters_fixed; idx++) {
  998. rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count);
  999. pr_info("CPU#%d: fixed-PMC%d count: %016llx\n",
  1000. cpu, idx, pmc_count);
  1001. }
  1002. local_irq_restore(flags);
  1003. }
  1004. static void x86_pmu_stop(struct perf_event *event, int flags)
  1005. {
  1006. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  1007. struct hw_perf_event *hwc = &event->hw;
  1008. if (__test_and_clear_bit(hwc->idx, cpuc->active_mask)) {
  1009. x86_pmu.disable(event);
  1010. cpuc->events[hwc->idx] = NULL;
  1011. WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
  1012. hwc->state |= PERF_HES_STOPPED;
  1013. }
  1014. if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
  1015. /*
  1016. * Drain the remaining delta count out of a event
  1017. * that we are disabling:
  1018. */
  1019. x86_perf_event_update(event);
  1020. hwc->state |= PERF_HES_UPTODATE;
  1021. }
  1022. }
  1023. static void x86_pmu_del(struct perf_event *event, int flags)
  1024. {
  1025. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  1026. int i;
  1027. /*
  1028. * If we're called during a txn, we don't need to do anything.
  1029. * The events never got scheduled and ->cancel_txn will truncate
  1030. * the event_list.
  1031. */
  1032. if (cpuc->group_flag & PERF_EVENT_TXN)
  1033. return;
  1034. x86_pmu_stop(event, PERF_EF_UPDATE);
  1035. for (i = 0; i < cpuc->n_events; i++) {
  1036. if (event == cpuc->event_list[i]) {
  1037. if (x86_pmu.put_event_constraints)
  1038. x86_pmu.put_event_constraints(cpuc, event);
  1039. while (++i < cpuc->n_events)
  1040. cpuc->event_list[i-1] = cpuc->event_list[i];
  1041. --cpuc->n_events;
  1042. break;
  1043. }
  1044. }
  1045. perf_event_update_userpage(event);
  1046. }
  1047. static int x86_pmu_handle_irq(struct pt_regs *regs)
  1048. {
  1049. struct perf_sample_data data;
  1050. struct cpu_hw_events *cpuc;
  1051. struct perf_event *event;
  1052. int idx, handled = 0;
  1053. u64 val;
  1054. perf_sample_data_init(&data, 0);
  1055. cpuc = &__get_cpu_var(cpu_hw_events);
  1056. for (idx = 0; idx < x86_pmu.num_counters; idx++) {
  1057. if (!test_bit(idx, cpuc->active_mask)) {
  1058. /*
  1059. * Though we deactivated the counter some cpus
  1060. * might still deliver spurious interrupts still
  1061. * in flight. Catch them:
  1062. */
  1063. if (__test_and_clear_bit(idx, cpuc->running))
  1064. handled++;
  1065. continue;
  1066. }
  1067. event = cpuc->events[idx];
  1068. val = x86_perf_event_update(event);
  1069. if (val & (1ULL << (x86_pmu.cntval_bits - 1)))
  1070. continue;
  1071. /*
  1072. * event overflow
  1073. */
  1074. handled++;
  1075. data.period = event->hw.last_period;
  1076. if (!x86_perf_event_set_period(event))
  1077. continue;
  1078. if (perf_event_overflow(event, 1, &data, regs))
  1079. x86_pmu_stop(event, 0);
  1080. }
  1081. if (handled)
  1082. inc_irq_stat(apic_perf_irqs);
  1083. return handled;
  1084. }
  1085. void perf_events_lapic_init(void)
  1086. {
  1087. if (!x86_pmu.apic || !x86_pmu_initialized())
  1088. return;
  1089. /*
  1090. * Always use NMI for PMU
  1091. */
  1092. apic_write(APIC_LVTPC, APIC_DM_NMI);
  1093. }
  1094. struct pmu_nmi_state {
  1095. unsigned int marked;
  1096. int handled;
  1097. };
  1098. static DEFINE_PER_CPU(struct pmu_nmi_state, pmu_nmi);
  1099. static int __kprobes
  1100. perf_event_nmi_handler(struct notifier_block *self,
  1101. unsigned long cmd, void *__args)
  1102. {
  1103. struct die_args *args = __args;
  1104. unsigned int this_nmi;
  1105. int handled;
  1106. if (!atomic_read(&active_events))
  1107. return NOTIFY_DONE;
  1108. switch (cmd) {
  1109. case DIE_NMI:
  1110. break;
  1111. case DIE_NMIUNKNOWN:
  1112. this_nmi = percpu_read(irq_stat.__nmi_count);
  1113. if (this_nmi != __this_cpu_read(pmu_nmi.marked))
  1114. /* let the kernel handle the unknown nmi */
  1115. return NOTIFY_DONE;
  1116. /*
  1117. * This one is a PMU back-to-back nmi. Two events
  1118. * trigger 'simultaneously' raising two back-to-back
  1119. * NMIs. If the first NMI handles both, the latter
  1120. * will be empty and daze the CPU. So, we drop it to
  1121. * avoid false-positive 'unknown nmi' messages.
  1122. */
  1123. return NOTIFY_STOP;
  1124. default:
  1125. return NOTIFY_DONE;
  1126. }
  1127. apic_write(APIC_LVTPC, APIC_DM_NMI);
  1128. handled = x86_pmu.handle_irq(args->regs);
  1129. if (!handled)
  1130. return NOTIFY_DONE;
  1131. this_nmi = percpu_read(irq_stat.__nmi_count);
  1132. if ((handled > 1) ||
  1133. /* the next nmi could be a back-to-back nmi */
  1134. ((__this_cpu_read(pmu_nmi.marked) == this_nmi) &&
  1135. (__this_cpu_read(pmu_nmi.handled) > 1))) {
  1136. /*
  1137. * We could have two subsequent back-to-back nmis: The
  1138. * first handles more than one counter, the 2nd
  1139. * handles only one counter and the 3rd handles no
  1140. * counter.
  1141. *
  1142. * This is the 2nd nmi because the previous was
  1143. * handling more than one counter. We will mark the
  1144. * next (3rd) and then drop it if unhandled.
  1145. */
  1146. __this_cpu_write(pmu_nmi.marked, this_nmi + 1);
  1147. __this_cpu_write(pmu_nmi.handled, handled);
  1148. }
  1149. return NOTIFY_STOP;
  1150. }
  1151. static __read_mostly struct notifier_block perf_event_nmi_notifier = {
  1152. .notifier_call = perf_event_nmi_handler,
  1153. .next = NULL,
  1154. .priority = NMI_LOCAL_LOW_PRIOR,
  1155. };
  1156. static struct event_constraint unconstrained;
  1157. static struct event_constraint emptyconstraint;
  1158. static struct event_constraint *
  1159. x86_get_event_constraints(struct cpu_hw_events *cpuc, struct perf_event *event)
  1160. {
  1161. struct event_constraint *c;
  1162. if (x86_pmu.event_constraints) {
  1163. for_each_event_constraint(c, x86_pmu.event_constraints) {
  1164. if ((event->hw.config & c->cmask) == c->code)
  1165. return c;
  1166. }
  1167. }
  1168. return &unconstrained;
  1169. }
  1170. #include "perf_event_amd.c"
  1171. #include "perf_event_p6.c"
  1172. #include "perf_event_p4.c"
  1173. #include "perf_event_intel_lbr.c"
  1174. #include "perf_event_intel_ds.c"
  1175. #include "perf_event_intel.c"
  1176. static int __cpuinit
  1177. x86_pmu_notifier(struct notifier_block *self, unsigned long action, void *hcpu)
  1178. {
  1179. unsigned int cpu = (long)hcpu;
  1180. int ret = NOTIFY_OK;
  1181. switch (action & ~CPU_TASKS_FROZEN) {
  1182. case CPU_UP_PREPARE:
  1183. if (x86_pmu.cpu_prepare)
  1184. ret = x86_pmu.cpu_prepare(cpu);
  1185. break;
  1186. case CPU_STARTING:
  1187. if (x86_pmu.cpu_starting)
  1188. x86_pmu.cpu_starting(cpu);
  1189. break;
  1190. case CPU_DYING:
  1191. if (x86_pmu.cpu_dying)
  1192. x86_pmu.cpu_dying(cpu);
  1193. break;
  1194. case CPU_UP_CANCELED:
  1195. case CPU_DEAD:
  1196. if (x86_pmu.cpu_dead)
  1197. x86_pmu.cpu_dead(cpu);
  1198. break;
  1199. default:
  1200. break;
  1201. }
  1202. return ret;
  1203. }
  1204. static void __init pmu_check_apic(void)
  1205. {
  1206. if (cpu_has_apic)
  1207. return;
  1208. x86_pmu.apic = 0;
  1209. pr_info("no APIC, boot with the \"lapic\" boot parameter to force-enable it.\n");
  1210. pr_info("no hardware sampling interrupt available.\n");
  1211. }
  1212. static int __init init_hw_perf_events(void)
  1213. {
  1214. struct event_constraint *c;
  1215. int err;
  1216. pr_info("Performance Events: ");
  1217. switch (boot_cpu_data.x86_vendor) {
  1218. case X86_VENDOR_INTEL:
  1219. err = intel_pmu_init();
  1220. break;
  1221. case X86_VENDOR_AMD:
  1222. err = amd_pmu_init();
  1223. break;
  1224. default:
  1225. return 0;
  1226. }
  1227. if (err != 0) {
  1228. pr_cont("no PMU driver, software events only.\n");
  1229. return 0;
  1230. }
  1231. pmu_check_apic();
  1232. /* sanity check that the hardware exists or is emulated */
  1233. if (!check_hw_exists())
  1234. return 0;
  1235. pr_cont("%s PMU driver.\n", x86_pmu.name);
  1236. if (x86_pmu.quirks)
  1237. x86_pmu.quirks();
  1238. if (x86_pmu.num_counters > X86_PMC_MAX_GENERIC) {
  1239. WARN(1, KERN_ERR "hw perf events %d > max(%d), clipping!",
  1240. x86_pmu.num_counters, X86_PMC_MAX_GENERIC);
  1241. x86_pmu.num_counters = X86_PMC_MAX_GENERIC;
  1242. }
  1243. x86_pmu.intel_ctrl = (1 << x86_pmu.num_counters) - 1;
  1244. if (x86_pmu.num_counters_fixed > X86_PMC_MAX_FIXED) {
  1245. WARN(1, KERN_ERR "hw perf events fixed %d > max(%d), clipping!",
  1246. x86_pmu.num_counters_fixed, X86_PMC_MAX_FIXED);
  1247. x86_pmu.num_counters_fixed = X86_PMC_MAX_FIXED;
  1248. }
  1249. x86_pmu.intel_ctrl |=
  1250. ((1LL << x86_pmu.num_counters_fixed)-1) << X86_PMC_IDX_FIXED;
  1251. perf_events_lapic_init();
  1252. register_die_notifier(&perf_event_nmi_notifier);
  1253. unconstrained = (struct event_constraint)
  1254. __EVENT_CONSTRAINT(0, (1ULL << x86_pmu.num_counters) - 1,
  1255. 0, x86_pmu.num_counters);
  1256. if (x86_pmu.event_constraints) {
  1257. for_each_event_constraint(c, x86_pmu.event_constraints) {
  1258. if (c->cmask != X86_RAW_EVENT_MASK)
  1259. continue;
  1260. c->idxmsk64 |= (1ULL << x86_pmu.num_counters) - 1;
  1261. c->weight += x86_pmu.num_counters;
  1262. }
  1263. }
  1264. pr_info("... version: %d\n", x86_pmu.version);
  1265. pr_info("... bit width: %d\n", x86_pmu.cntval_bits);
  1266. pr_info("... generic registers: %d\n", x86_pmu.num_counters);
  1267. pr_info("... value mask: %016Lx\n", x86_pmu.cntval_mask);
  1268. pr_info("... max period: %016Lx\n", x86_pmu.max_period);
  1269. pr_info("... fixed-purpose events: %d\n", x86_pmu.num_counters_fixed);
  1270. pr_info("... event mask: %016Lx\n", x86_pmu.intel_ctrl);
  1271. perf_pmu_register(&pmu, "cpu", PERF_TYPE_RAW);
  1272. perf_cpu_notifier(x86_pmu_notifier);
  1273. return 0;
  1274. }
  1275. early_initcall(init_hw_perf_events);
  1276. static inline void x86_pmu_read(struct perf_event *event)
  1277. {
  1278. x86_perf_event_update(event);
  1279. }
  1280. /*
  1281. * Start group events scheduling transaction
  1282. * Set the flag to make pmu::enable() not perform the
  1283. * schedulability test, it will be performed at commit time
  1284. */
  1285. static void x86_pmu_start_txn(struct pmu *pmu)
  1286. {
  1287. perf_pmu_disable(pmu);
  1288. __this_cpu_or(cpu_hw_events.group_flag, PERF_EVENT_TXN);
  1289. __this_cpu_write(cpu_hw_events.n_txn, 0);
  1290. }
  1291. /*
  1292. * Stop group events scheduling transaction
  1293. * Clear the flag and pmu::enable() will perform the
  1294. * schedulability test.
  1295. */
  1296. static void x86_pmu_cancel_txn(struct pmu *pmu)
  1297. {
  1298. __this_cpu_and(cpu_hw_events.group_flag, ~PERF_EVENT_TXN);
  1299. /*
  1300. * Truncate the collected events.
  1301. */
  1302. __this_cpu_sub(cpu_hw_events.n_added, __this_cpu_read(cpu_hw_events.n_txn));
  1303. __this_cpu_sub(cpu_hw_events.n_events, __this_cpu_read(cpu_hw_events.n_txn));
  1304. perf_pmu_enable(pmu);
  1305. }
  1306. /*
  1307. * Commit group events scheduling transaction
  1308. * Perform the group schedulability test as a whole
  1309. * Return 0 if success
  1310. */
  1311. static int x86_pmu_commit_txn(struct pmu *pmu)
  1312. {
  1313. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  1314. int assign[X86_PMC_IDX_MAX];
  1315. int n, ret;
  1316. n = cpuc->n_events;
  1317. if (!x86_pmu_initialized())
  1318. return -EAGAIN;
  1319. ret = x86_pmu.schedule_events(cpuc, n, assign);
  1320. if (ret)
  1321. return ret;
  1322. /*
  1323. * copy new assignment, now we know it is possible
  1324. * will be used by hw_perf_enable()
  1325. */
  1326. memcpy(cpuc->assign, assign, n*sizeof(int));
  1327. cpuc->group_flag &= ~PERF_EVENT_TXN;
  1328. perf_pmu_enable(pmu);
  1329. return 0;
  1330. }
  1331. /*
  1332. * validate that we can schedule this event
  1333. */
  1334. static int validate_event(struct perf_event *event)
  1335. {
  1336. struct cpu_hw_events *fake_cpuc;
  1337. struct event_constraint *c;
  1338. int ret = 0;
  1339. fake_cpuc = kmalloc(sizeof(*fake_cpuc), GFP_KERNEL | __GFP_ZERO);
  1340. if (!fake_cpuc)
  1341. return -ENOMEM;
  1342. c = x86_pmu.get_event_constraints(fake_cpuc, event);
  1343. if (!c || !c->weight)
  1344. ret = -ENOSPC;
  1345. if (x86_pmu.put_event_constraints)
  1346. x86_pmu.put_event_constraints(fake_cpuc, event);
  1347. kfree(fake_cpuc);
  1348. return ret;
  1349. }
  1350. /*
  1351. * validate a single event group
  1352. *
  1353. * validation include:
  1354. * - check events are compatible which each other
  1355. * - events do not compete for the same counter
  1356. * - number of events <= number of counters
  1357. *
  1358. * validation ensures the group can be loaded onto the
  1359. * PMU if it was the only group available.
  1360. */
  1361. static int validate_group(struct perf_event *event)
  1362. {
  1363. struct perf_event *leader = event->group_leader;
  1364. struct cpu_hw_events *fake_cpuc;
  1365. int ret, n;
  1366. ret = -ENOMEM;
  1367. fake_cpuc = kmalloc(sizeof(*fake_cpuc), GFP_KERNEL | __GFP_ZERO);
  1368. if (!fake_cpuc)
  1369. goto out;
  1370. /*
  1371. * the event is not yet connected with its
  1372. * siblings therefore we must first collect
  1373. * existing siblings, then add the new event
  1374. * before we can simulate the scheduling
  1375. */
  1376. ret = -ENOSPC;
  1377. n = collect_events(fake_cpuc, leader, true);
  1378. if (n < 0)
  1379. goto out_free;
  1380. fake_cpuc->n_events = n;
  1381. n = collect_events(fake_cpuc, event, false);
  1382. if (n < 0)
  1383. goto out_free;
  1384. fake_cpuc->n_events = n;
  1385. ret = x86_pmu.schedule_events(fake_cpuc, n, NULL);
  1386. out_free:
  1387. kfree(fake_cpuc);
  1388. out:
  1389. return ret;
  1390. }
  1391. static int x86_pmu_event_init(struct perf_event *event)
  1392. {
  1393. struct pmu *tmp;
  1394. int err;
  1395. switch (event->attr.type) {
  1396. case PERF_TYPE_RAW:
  1397. case PERF_TYPE_HARDWARE:
  1398. case PERF_TYPE_HW_CACHE:
  1399. break;
  1400. default:
  1401. return -ENOENT;
  1402. }
  1403. err = __x86_pmu_event_init(event);
  1404. if (!err) {
  1405. /*
  1406. * we temporarily connect event to its pmu
  1407. * such that validate_group() can classify
  1408. * it as an x86 event using is_x86_event()
  1409. */
  1410. tmp = event->pmu;
  1411. event->pmu = &pmu;
  1412. if (event->group_leader != event)
  1413. err = validate_group(event);
  1414. else
  1415. err = validate_event(event);
  1416. event->pmu = tmp;
  1417. }
  1418. if (err) {
  1419. if (event->destroy)
  1420. event->destroy(event);
  1421. }
  1422. return err;
  1423. }
  1424. static struct pmu pmu = {
  1425. .pmu_enable = x86_pmu_enable,
  1426. .pmu_disable = x86_pmu_disable,
  1427. .event_init = x86_pmu_event_init,
  1428. .add = x86_pmu_add,
  1429. .del = x86_pmu_del,
  1430. .start = x86_pmu_start,
  1431. .stop = x86_pmu_stop,
  1432. .read = x86_pmu_read,
  1433. .start_txn = x86_pmu_start_txn,
  1434. .cancel_txn = x86_pmu_cancel_txn,
  1435. .commit_txn = x86_pmu_commit_txn,
  1436. };
  1437. /*
  1438. * callchain support
  1439. */
  1440. static void
  1441. backtrace_warning_symbol(void *data, char *msg, unsigned long symbol)
  1442. {
  1443. /* Ignore warnings */
  1444. }
  1445. static void backtrace_warning(void *data, char *msg)
  1446. {
  1447. /* Ignore warnings */
  1448. }
  1449. static int backtrace_stack(void *data, char *name)
  1450. {
  1451. return 0;
  1452. }
  1453. static void backtrace_address(void *data, unsigned long addr, int reliable)
  1454. {
  1455. struct perf_callchain_entry *entry = data;
  1456. perf_callchain_store(entry, addr);
  1457. }
  1458. static const struct stacktrace_ops backtrace_ops = {
  1459. .warning = backtrace_warning,
  1460. .warning_symbol = backtrace_warning_symbol,
  1461. .stack = backtrace_stack,
  1462. .address = backtrace_address,
  1463. .walk_stack = print_context_stack_bp,
  1464. };
  1465. void
  1466. perf_callchain_kernel(struct perf_callchain_entry *entry, struct pt_regs *regs)
  1467. {
  1468. if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
  1469. /* TODO: We don't support guest os callchain now */
  1470. return;
  1471. }
  1472. perf_callchain_store(entry, regs->ip);
  1473. dump_trace(NULL, regs, NULL, 0, &backtrace_ops, entry);
  1474. }
  1475. #ifdef CONFIG_COMPAT
  1476. static inline int
  1477. perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry *entry)
  1478. {
  1479. /* 32-bit process in 64-bit kernel. */
  1480. struct stack_frame_ia32 frame;
  1481. const void __user *fp;
  1482. if (!test_thread_flag(TIF_IA32))
  1483. return 0;
  1484. fp = compat_ptr(regs->bp);
  1485. while (entry->nr < PERF_MAX_STACK_DEPTH) {
  1486. unsigned long bytes;
  1487. frame.next_frame = 0;
  1488. frame.return_address = 0;
  1489. bytes = copy_from_user_nmi(&frame, fp, sizeof(frame));
  1490. if (bytes != sizeof(frame))
  1491. break;
  1492. if (fp < compat_ptr(regs->sp))
  1493. break;
  1494. perf_callchain_store(entry, frame.return_address);
  1495. fp = compat_ptr(frame.next_frame);
  1496. }
  1497. return 1;
  1498. }
  1499. #else
  1500. static inline int
  1501. perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry *entry)
  1502. {
  1503. return 0;
  1504. }
  1505. #endif
  1506. void
  1507. perf_callchain_user(struct perf_callchain_entry *entry, struct pt_regs *regs)
  1508. {
  1509. struct stack_frame frame;
  1510. const void __user *fp;
  1511. if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
  1512. /* TODO: We don't support guest os callchain now */
  1513. return;
  1514. }
  1515. fp = (void __user *)regs->bp;
  1516. perf_callchain_store(entry, regs->ip);
  1517. if (perf_callchain_user32(regs, entry))
  1518. return;
  1519. while (entry->nr < PERF_MAX_STACK_DEPTH) {
  1520. unsigned long bytes;
  1521. frame.next_frame = NULL;
  1522. frame.return_address = 0;
  1523. bytes = copy_from_user_nmi(&frame, fp, sizeof(frame));
  1524. if (bytes != sizeof(frame))
  1525. break;
  1526. if ((unsigned long)fp < regs->sp)
  1527. break;
  1528. perf_callchain_store(entry, frame.return_address);
  1529. fp = frame.next_frame;
  1530. }
  1531. }
  1532. unsigned long perf_instruction_pointer(struct pt_regs *regs)
  1533. {
  1534. unsigned long ip;
  1535. if (perf_guest_cbs && perf_guest_cbs->is_in_guest())
  1536. ip = perf_guest_cbs->get_guest_ip();
  1537. else
  1538. ip = instruction_pointer(regs);
  1539. return ip;
  1540. }
  1541. unsigned long perf_misc_flags(struct pt_regs *regs)
  1542. {
  1543. int misc = 0;
  1544. if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
  1545. if (perf_guest_cbs->is_user_mode())
  1546. misc |= PERF_RECORD_MISC_GUEST_USER;
  1547. else
  1548. misc |= PERF_RECORD_MISC_GUEST_KERNEL;
  1549. } else {
  1550. if (user_mode(regs))
  1551. misc |= PERF_RECORD_MISC_USER;
  1552. else
  1553. misc |= PERF_RECORD_MISC_KERNEL;
  1554. }
  1555. if (regs->flags & PERF_EFLAGS_EXACT)
  1556. misc |= PERF_RECORD_MISC_EXACT_IP;
  1557. return misc;
  1558. }