op_model_amd.c 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543
  1. /*
  2. * @file op_model_amd.c
  3. * athlon / K7 / K8 / Family 10h model-specific MSR operations
  4. *
  5. * @remark Copyright 2002-2008 OProfile authors
  6. * @remark Read the file COPYING
  7. *
  8. * @author John Levon
  9. * @author Philippe Elie
  10. * @author Graydon Hoare
  11. * @author Robert Richter <robert.richter@amd.com>
  12. * @author Barry Kasindorf
  13. */
  14. #include <linux/oprofile.h>
  15. #include <linux/device.h>
  16. #include <linux/pci.h>
  17. #include <asm/ptrace.h>
  18. #include <asm/msr.h>
  19. #include <asm/nmi.h>
  20. #include "op_x86_model.h"
  21. #include "op_counter.h"
  22. #define NUM_COUNTERS 4
  23. #define NUM_CONTROLS 4
  24. #define CTR_IS_RESERVED(msrs, c) (msrs->counters[(c)].addr ? 1 : 0)
  25. #define CTR_READ(l, h, msrs, c) do {rdmsr(msrs->counters[(c)].addr, (l), (h)); } while (0)
  26. #define CTR_WRITE(l, msrs, c) do {wrmsr(msrs->counters[(c)].addr, -(unsigned int)(l), -1); } while (0)
  27. #define CTR_OVERFLOWED(n) (!((n) & (1U<<31)))
  28. #define CTRL_IS_RESERVED(msrs, c) (msrs->controls[(c)].addr ? 1 : 0)
  29. #define CTRL_READ(l, h, msrs, c) do {rdmsr(msrs->controls[(c)].addr, (l), (h)); } while (0)
  30. #define CTRL_WRITE(l, h, msrs, c) do {wrmsr(msrs->controls[(c)].addr, (l), (h)); } while (0)
  31. #define CTRL_SET_ACTIVE(n) (n |= (1<<22))
  32. #define CTRL_SET_INACTIVE(n) (n &= ~(1<<22))
  33. #define CTRL_CLEAR_LO(x) (x &= (1<<21))
  34. #define CTRL_CLEAR_HI(x) (x &= 0xfffffcf0)
  35. #define CTRL_SET_ENABLE(val) (val |= 1<<20)
  36. #define CTRL_SET_USR(val, u) (val |= ((u & 1) << 16))
  37. #define CTRL_SET_KERN(val, k) (val |= ((k & 1) << 17))
  38. #define CTRL_SET_UM(val, m) (val |= (m << 8))
  39. #define CTRL_SET_EVENT_LOW(val, e) (val |= (e & 0xff))
  40. #define CTRL_SET_EVENT_HIGH(val, e) (val |= ((e >> 8) & 0xf))
  41. #define CTRL_SET_HOST_ONLY(val, h) (val |= ((h & 1) << 9))
  42. #define CTRL_SET_GUEST_ONLY(val, h) (val |= ((h & 1) << 8))
  43. static unsigned long reset_value[NUM_COUNTERS];
  44. #ifdef CONFIG_OPROFILE_IBS
  45. /* IbsFetchCtl bits/masks */
  46. #define IBS_FETCH_HIGH_VALID_BIT (1UL << 17) /* bit 49 */
  47. #define IBS_FETCH_HIGH_ENABLE (1UL << 16) /* bit 48 */
  48. #define IBS_FETCH_LOW_MAX_CNT_MASK 0x0000FFFFUL /* MaxCnt mask */
  49. /*IbsOpCtl bits */
  50. #define IBS_OP_LOW_VALID_BIT (1ULL<<18) /* bit 18 */
  51. #define IBS_OP_LOW_ENABLE (1ULL<<17) /* bit 17 */
  52. /* Codes used in cpu_buffer.c */
  53. /* This produces duplicate code, need to be fixed */
  54. #define IBS_FETCH_BEGIN 3
  55. #define IBS_OP_BEGIN 4
  56. /* The function interface needs to be fixed, something like add
  57. data. Should then be added to linux/oprofile.h. */
  58. extern void oprofile_add_ibs_sample(struct pt_regs *const regs,
  59. unsigned int * const ibs_sample, u8 code);
  60. struct ibs_fetch_sample {
  61. /* MSRC001_1031 IBS Fetch Linear Address Register */
  62. unsigned int ibs_fetch_lin_addr_low;
  63. unsigned int ibs_fetch_lin_addr_high;
  64. /* MSRC001_1030 IBS Fetch Control Register */
  65. unsigned int ibs_fetch_ctl_low;
  66. unsigned int ibs_fetch_ctl_high;
  67. /* MSRC001_1032 IBS Fetch Physical Address Register */
  68. unsigned int ibs_fetch_phys_addr_low;
  69. unsigned int ibs_fetch_phys_addr_high;
  70. };
  71. struct ibs_op_sample {
  72. /* MSRC001_1034 IBS Op Logical Address Register (IbsRIP) */
  73. unsigned int ibs_op_rip_low;
  74. unsigned int ibs_op_rip_high;
  75. /* MSRC001_1035 IBS Op Data Register */
  76. unsigned int ibs_op_data1_low;
  77. unsigned int ibs_op_data1_high;
  78. /* MSRC001_1036 IBS Op Data 2 Register */
  79. unsigned int ibs_op_data2_low;
  80. unsigned int ibs_op_data2_high;
  81. /* MSRC001_1037 IBS Op Data 3 Register */
  82. unsigned int ibs_op_data3_low;
  83. unsigned int ibs_op_data3_high;
  84. /* MSRC001_1038 IBS DC Linear Address Register (IbsDcLinAd) */
  85. unsigned int ibs_dc_linear_low;
  86. unsigned int ibs_dc_linear_high;
  87. /* MSRC001_1039 IBS DC Physical Address Register (IbsDcPhysAd) */
  88. unsigned int ibs_dc_phys_low;
  89. unsigned int ibs_dc_phys_high;
  90. };
  91. /*
  92. * unitialize the APIC for the IBS interrupts if needed on AMD Family10h+
  93. */
  94. static void clear_ibs_nmi(void);
  95. static int ibs_allowed; /* AMD Family10h and later */
  96. struct op_ibs_config {
  97. unsigned long op_enabled;
  98. unsigned long fetch_enabled;
  99. unsigned long max_cnt_fetch;
  100. unsigned long max_cnt_op;
  101. unsigned long rand_en;
  102. unsigned long dispatched_ops;
  103. };
  104. static struct op_ibs_config ibs_config;
  105. #endif
  106. /* functions for op_amd_spec */
  107. static void op_amd_fill_in_addresses(struct op_msrs * const msrs)
  108. {
  109. int i;
  110. for (i = 0; i < NUM_COUNTERS; i++) {
  111. if (reserve_perfctr_nmi(MSR_K7_PERFCTR0 + i))
  112. msrs->counters[i].addr = MSR_K7_PERFCTR0 + i;
  113. else
  114. msrs->counters[i].addr = 0;
  115. }
  116. for (i = 0; i < NUM_CONTROLS; i++) {
  117. if (reserve_evntsel_nmi(MSR_K7_EVNTSEL0 + i))
  118. msrs->controls[i].addr = MSR_K7_EVNTSEL0 + i;
  119. else
  120. msrs->controls[i].addr = 0;
  121. }
  122. }
  123. static void op_amd_setup_ctrs(struct op_msrs const * const msrs)
  124. {
  125. unsigned int low, high;
  126. int i;
  127. /* clear all counters */
  128. for (i = 0 ; i < NUM_CONTROLS; ++i) {
  129. if (unlikely(!CTRL_IS_RESERVED(msrs, i)))
  130. continue;
  131. CTRL_READ(low, high, msrs, i);
  132. CTRL_CLEAR_LO(low);
  133. CTRL_CLEAR_HI(high);
  134. CTRL_WRITE(low, high, msrs, i);
  135. }
  136. /* avoid a false detection of ctr overflows in NMI handler */
  137. for (i = 0; i < NUM_COUNTERS; ++i) {
  138. if (unlikely(!CTR_IS_RESERVED(msrs, i)))
  139. continue;
  140. CTR_WRITE(1, msrs, i);
  141. }
  142. /* enable active counters */
  143. for (i = 0; i < NUM_COUNTERS; ++i) {
  144. if ((counter_config[i].enabled) && (CTR_IS_RESERVED(msrs, i))) {
  145. reset_value[i] = counter_config[i].count;
  146. CTR_WRITE(counter_config[i].count, msrs, i);
  147. CTRL_READ(low, high, msrs, i);
  148. CTRL_CLEAR_LO(low);
  149. CTRL_CLEAR_HI(high);
  150. CTRL_SET_ENABLE(low);
  151. CTRL_SET_USR(low, counter_config[i].user);
  152. CTRL_SET_KERN(low, counter_config[i].kernel);
  153. CTRL_SET_UM(low, counter_config[i].unit_mask);
  154. CTRL_SET_EVENT_LOW(low, counter_config[i].event);
  155. CTRL_SET_EVENT_HIGH(high, counter_config[i].event);
  156. CTRL_SET_HOST_ONLY(high, 0);
  157. CTRL_SET_GUEST_ONLY(high, 0);
  158. CTRL_WRITE(low, high, msrs, i);
  159. } else {
  160. reset_value[i] = 0;
  161. }
  162. }
  163. }
  164. #ifdef CONFIG_OPROFILE_IBS
  165. static inline int
  166. op_amd_handle_ibs(struct pt_regs * const regs,
  167. struct op_msrs const * const msrs)
  168. {
  169. unsigned int low, high;
  170. struct ibs_fetch_sample ibs_fetch;
  171. struct ibs_op_sample ibs_op;
  172. if (!ibs_allowed)
  173. return 1;
  174. if (ibs_config.fetch_enabled) {
  175. rdmsr(MSR_AMD64_IBSFETCHCTL, low, high);
  176. if (high & IBS_FETCH_HIGH_VALID_BIT) {
  177. ibs_fetch.ibs_fetch_ctl_high = high;
  178. ibs_fetch.ibs_fetch_ctl_low = low;
  179. rdmsr(MSR_AMD64_IBSFETCHLINAD, low, high);
  180. ibs_fetch.ibs_fetch_lin_addr_high = high;
  181. ibs_fetch.ibs_fetch_lin_addr_low = low;
  182. rdmsr(MSR_AMD64_IBSFETCHPHYSAD, low, high);
  183. ibs_fetch.ibs_fetch_phys_addr_high = high;
  184. ibs_fetch.ibs_fetch_phys_addr_low = low;
  185. oprofile_add_ibs_sample(regs,
  186. (unsigned int *)&ibs_fetch,
  187. IBS_FETCH_BEGIN);
  188. /*reenable the IRQ */
  189. rdmsr(MSR_AMD64_IBSFETCHCTL, low, high);
  190. high &= ~IBS_FETCH_HIGH_VALID_BIT;
  191. high |= IBS_FETCH_HIGH_ENABLE;
  192. low &= IBS_FETCH_LOW_MAX_CNT_MASK;
  193. wrmsr(MSR_AMD64_IBSFETCHCTL, low, high);
  194. }
  195. }
  196. if (ibs_config.op_enabled) {
  197. rdmsr(MSR_AMD64_IBSOPCTL, low, high);
  198. if (low & IBS_OP_LOW_VALID_BIT) {
  199. rdmsr(MSR_AMD64_IBSOPRIP, low, high);
  200. ibs_op.ibs_op_rip_low = low;
  201. ibs_op.ibs_op_rip_high = high;
  202. rdmsr(MSR_AMD64_IBSOPDATA, low, high);
  203. ibs_op.ibs_op_data1_low = low;
  204. ibs_op.ibs_op_data1_high = high;
  205. rdmsr(MSR_AMD64_IBSOPDATA2, low, high);
  206. ibs_op.ibs_op_data2_low = low;
  207. ibs_op.ibs_op_data2_high = high;
  208. rdmsr(MSR_AMD64_IBSOPDATA3, low, high);
  209. ibs_op.ibs_op_data3_low = low;
  210. ibs_op.ibs_op_data3_high = high;
  211. rdmsr(MSR_AMD64_IBSDCLINAD, low, high);
  212. ibs_op.ibs_dc_linear_low = low;
  213. ibs_op.ibs_dc_linear_high = high;
  214. rdmsr(MSR_AMD64_IBSDCPHYSAD, low, high);
  215. ibs_op.ibs_dc_phys_low = low;
  216. ibs_op.ibs_dc_phys_high = high;
  217. /* reenable the IRQ */
  218. oprofile_add_ibs_sample(regs,
  219. (unsigned int *)&ibs_op,
  220. IBS_OP_BEGIN);
  221. rdmsr(MSR_AMD64_IBSOPCTL, low, high);
  222. high = 0;
  223. low &= ~IBS_OP_LOW_VALID_BIT;
  224. low |= IBS_OP_LOW_ENABLE;
  225. wrmsr(MSR_AMD64_IBSOPCTL, low, high);
  226. }
  227. }
  228. return 1;
  229. }
  230. #endif
  231. static int op_amd_check_ctrs(struct pt_regs * const regs,
  232. struct op_msrs const * const msrs)
  233. {
  234. unsigned int low, high;
  235. int i;
  236. for (i = 0 ; i < NUM_COUNTERS; ++i) {
  237. if (!reset_value[i])
  238. continue;
  239. CTR_READ(low, high, msrs, i);
  240. if (CTR_OVERFLOWED(low)) {
  241. oprofile_add_sample(regs, i);
  242. CTR_WRITE(reset_value[i], msrs, i);
  243. }
  244. }
  245. #ifdef CONFIG_OPROFILE_IBS
  246. op_amd_handle_ibs(regs, msrs);
  247. #endif
  248. /* See op_model_ppro.c */
  249. return 1;
  250. }
  251. static void op_amd_start(struct op_msrs const * const msrs)
  252. {
  253. unsigned int low, high;
  254. int i;
  255. for (i = 0 ; i < NUM_COUNTERS ; ++i) {
  256. if (reset_value[i]) {
  257. CTRL_READ(low, high, msrs, i);
  258. CTRL_SET_ACTIVE(low);
  259. CTRL_WRITE(low, high, msrs, i);
  260. }
  261. }
  262. #ifdef CONFIG_OPROFILE_IBS
  263. if (ibs_allowed && ibs_config.fetch_enabled) {
  264. low = (ibs_config.max_cnt_fetch >> 4) & 0xFFFF;
  265. high = IBS_FETCH_HIGH_ENABLE;
  266. wrmsr(MSR_AMD64_IBSFETCHCTL, low, high);
  267. }
  268. if (ibs_allowed && ibs_config.op_enabled) {
  269. low = ((ibs_config.max_cnt_op >> 4) & 0xFFFF) + IBS_OP_LOW_ENABLE;
  270. high = 0;
  271. wrmsr(MSR_AMD64_IBSOPCTL, low, high);
  272. }
  273. #endif
  274. }
  275. static void op_amd_stop(struct op_msrs const * const msrs)
  276. {
  277. unsigned int low, high;
  278. int i;
  279. /* Subtle: stop on all counters to avoid race with
  280. * setting our pm callback */
  281. for (i = 0 ; i < NUM_COUNTERS ; ++i) {
  282. if (!reset_value[i])
  283. continue;
  284. CTRL_READ(low, high, msrs, i);
  285. CTRL_SET_INACTIVE(low);
  286. CTRL_WRITE(low, high, msrs, i);
  287. }
  288. #ifdef CONFIG_OPROFILE_IBS
  289. if (ibs_allowed && ibs_config.fetch_enabled) {
  290. low = 0; /* clear max count and enable */
  291. high = 0;
  292. wrmsr(MSR_AMD64_IBSFETCHCTL, low, high);
  293. }
  294. if (ibs_allowed && ibs_config.op_enabled) {
  295. low = 0; /* clear max count and enable */
  296. high = 0;
  297. wrmsr(MSR_AMD64_IBSOPCTL, low, high);
  298. }
  299. #endif
  300. }
  301. static void op_amd_shutdown(struct op_msrs const * const msrs)
  302. {
  303. int i;
  304. for (i = 0 ; i < NUM_COUNTERS ; ++i) {
  305. if (CTR_IS_RESERVED(msrs, i))
  306. release_perfctr_nmi(MSR_K7_PERFCTR0 + i);
  307. }
  308. for (i = 0 ; i < NUM_CONTROLS ; ++i) {
  309. if (CTRL_IS_RESERVED(msrs, i))
  310. release_evntsel_nmi(MSR_K7_EVNTSEL0 + i);
  311. }
  312. }
  313. #ifndef CONFIG_OPROFILE_IBS
  314. /* no IBS support */
  315. static int op_amd_init(struct oprofile_operations *ops)
  316. {
  317. return 0;
  318. }
  319. static void op_amd_exit(void) {}
  320. #else
  321. static u8 ibs_eilvt_off;
  322. static inline void apic_init_ibs_nmi_per_cpu(void *arg)
  323. {
  324. ibs_eilvt_off = setup_APIC_eilvt_ibs(0, APIC_EILVT_MSG_NMI, 0);
  325. }
  326. static inline void apic_clear_ibs_nmi_per_cpu(void *arg)
  327. {
  328. setup_APIC_eilvt_ibs(0, APIC_EILVT_MSG_FIX, 1);
  329. }
  330. static int pfm_amd64_setup_eilvt(void)
  331. {
  332. #define IBSCTL_LVTOFFSETVAL (1 << 8)
  333. #define IBSCTL 0x1cc
  334. struct pci_dev *cpu_cfg;
  335. int nodes;
  336. u32 value = 0;
  337. /* per CPU setup */
  338. on_each_cpu(apic_init_ibs_nmi_per_cpu, NULL, 1);
  339. nodes = 0;
  340. cpu_cfg = NULL;
  341. do {
  342. cpu_cfg = pci_get_device(PCI_VENDOR_ID_AMD,
  343. PCI_DEVICE_ID_AMD_10H_NB_MISC,
  344. cpu_cfg);
  345. if (!cpu_cfg)
  346. break;
  347. ++nodes;
  348. pci_write_config_dword(cpu_cfg, IBSCTL, ibs_eilvt_off
  349. | IBSCTL_LVTOFFSETVAL);
  350. pci_read_config_dword(cpu_cfg, IBSCTL, &value);
  351. if (value != (ibs_eilvt_off | IBSCTL_LVTOFFSETVAL)) {
  352. printk(KERN_DEBUG "Failed to setup IBS LVT offset, "
  353. "IBSCTL = 0x%08x", value);
  354. return 1;
  355. }
  356. } while (1);
  357. if (!nodes) {
  358. printk(KERN_DEBUG "No CPU node configured for IBS");
  359. return 1;
  360. }
  361. #ifdef CONFIG_NUMA
  362. /* Sanity check */
  363. /* Works only for 64bit with proper numa implementation. */
  364. if (nodes != num_possible_nodes()) {
  365. printk(KERN_DEBUG "Failed to setup CPU node(s) for IBS, "
  366. "found: %d, expected %d",
  367. nodes, num_possible_nodes());
  368. return 1;
  369. }
  370. #endif
  371. return 0;
  372. }
  373. /*
  374. * initialize the APIC for the IBS interrupts
  375. * if available (AMD Family10h rev B0 and later)
  376. */
  377. static void setup_ibs(void)
  378. {
  379. ibs_allowed = boot_cpu_has(X86_FEATURE_IBS);
  380. if (!ibs_allowed)
  381. return;
  382. if (pfm_amd64_setup_eilvt()) {
  383. ibs_allowed = 0;
  384. return;
  385. }
  386. printk(KERN_INFO "oprofile: AMD IBS detected\n");
  387. }
  388. /*
  389. * unitialize the APIC for the IBS interrupts if needed on AMD Family10h
  390. * rev B0 and later */
  391. static void clear_ibs_nmi(void)
  392. {
  393. if (ibs_allowed)
  394. on_each_cpu(apic_clear_ibs_nmi_per_cpu, NULL, 1);
  395. }
  396. static int (*create_arch_files)(struct super_block * sb, struct dentry * root);
  397. static int setup_ibs_files(struct super_block * sb, struct dentry * root)
  398. {
  399. char buf[12];
  400. struct dentry *dir;
  401. int ret = 0;
  402. /* architecture specific files */
  403. if (create_arch_files)
  404. ret = create_arch_files(sb, root);
  405. if (ret)
  406. return ret;
  407. if (!ibs_allowed)
  408. return ret;
  409. /* model specific files */
  410. /* setup some reasonable defaults */
  411. ibs_config.max_cnt_fetch = 250000;
  412. ibs_config.fetch_enabled = 0;
  413. ibs_config.max_cnt_op = 250000;
  414. ibs_config.op_enabled = 0;
  415. ibs_config.dispatched_ops = 1;
  416. snprintf(buf, sizeof(buf), "ibs_fetch");
  417. dir = oprofilefs_mkdir(sb, root, buf);
  418. oprofilefs_create_ulong(sb, dir, "rand_enable",
  419. &ibs_config.rand_en);
  420. oprofilefs_create_ulong(sb, dir, "enable",
  421. &ibs_config.fetch_enabled);
  422. oprofilefs_create_ulong(sb, dir, "max_count",
  423. &ibs_config.max_cnt_fetch);
  424. snprintf(buf, sizeof(buf), "ibs_uops");
  425. dir = oprofilefs_mkdir(sb, root, buf);
  426. oprofilefs_create_ulong(sb, dir, "enable",
  427. &ibs_config.op_enabled);
  428. oprofilefs_create_ulong(sb, dir, "max_count",
  429. &ibs_config.max_cnt_op);
  430. oprofilefs_create_ulong(sb, dir, "dispatched_ops",
  431. &ibs_config.dispatched_ops);
  432. return 0;
  433. }
  434. static int op_amd_init(struct oprofile_operations *ops)
  435. {
  436. setup_ibs();
  437. create_arch_files = ops->create_files;
  438. ops->create_files = setup_ibs_files;
  439. return 0;
  440. }
  441. static void op_amd_exit(void)
  442. {
  443. clear_ibs_nmi();
  444. }
  445. #endif
  446. struct op_x86_model_spec const op_amd_spec = {
  447. .init = op_amd_init,
  448. .exit = op_amd_exit,
  449. .num_counters = NUM_COUNTERS,
  450. .num_controls = NUM_CONTROLS,
  451. .fill_in_addresses = &op_amd_fill_in_addresses,
  452. .setup_ctrs = &op_amd_setup_ctrs,
  453. .check_ctrs = &op_amd_check_ctrs,
  454. .start = &op_amd_start,
  455. .stop = &op_amd_stop,
  456. .shutdown = &op_amd_shutdown
  457. };