|
@@ -13,11 +13,13 @@
|
|
|
#include <linux/perf_event.h>
|
|
|
#include <linux/percpu.h>
|
|
|
#include <linux/hardirq.h>
|
|
|
+#include <linux/uaccess.h>
|
|
|
#include <asm/reg.h>
|
|
|
#include <asm/pmc.h>
|
|
|
#include <asm/machdep.h>
|
|
|
#include <asm/firmware.h>
|
|
|
#include <asm/ptrace.h>
|
|
|
+#include <asm/code-patching.h>
|
|
|
|
|
|
#define BHRB_MAX_ENTRIES 32
|
|
|
#define BHRB_TARGET 0x0000000000000002
|
|
@@ -100,6 +102,10 @@ static inline int siar_valid(struct pt_regs *regs)
|
|
|
return 1;
|
|
|
}
|
|
|
|
|
|
+static inline void power_pmu_bhrb_enable(struct perf_event *event) {}
|
|
|
+static inline void power_pmu_bhrb_disable(struct perf_event *event) {}
|
|
|
+void power_pmu_flush_branch_stack(void) {}
|
|
|
+static inline void power_pmu_bhrb_read(struct cpu_hw_events *cpuhw) {}
|
|
|
#endif /* CONFIG_PPC32 */
|
|
|
|
|
|
static bool regs_use_siar(struct pt_regs *regs)
|
|
@@ -308,6 +314,159 @@ static inline int siar_valid(struct pt_regs *regs)
|
|
|
return 1;
|
|
|
}
|
|
|
|
|
|
+
|
|
|
+/* Reset all possible BHRB entries */
|
|
|
+static void power_pmu_bhrb_reset(void)
|
|
|
+{
|
|
|
+ asm volatile(PPC_CLRBHRB);
|
|
|
+}
|
|
|
+
|
|
|
+static void power_pmu_bhrb_enable(struct perf_event *event)
|
|
|
+{
|
|
|
+ struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
|
|
|
+
|
|
|
+ if (!ppmu->bhrb_nr)
|
|
|
+ return;
|
|
|
+
|
|
|
+ /* Clear BHRB if we changed task context to avoid data leaks */
|
|
|
+ if (event->ctx->task && cpuhw->bhrb_context != event->ctx) {
|
|
|
+ power_pmu_bhrb_reset();
|
|
|
+ cpuhw->bhrb_context = event->ctx;
|
|
|
+ }
|
|
|
+ cpuhw->bhrb_users++;
|
|
|
+}
|
|
|
+
|
|
|
+static void power_pmu_bhrb_disable(struct perf_event *event)
|
|
|
+{
|
|
|
+ struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
|
|
|
+
|
|
|
+ if (!ppmu->bhrb_nr)
|
|
|
+ return;
|
|
|
+
|
|
|
+ cpuhw->bhrb_users--;
|
|
|
+ WARN_ON_ONCE(cpuhw->bhrb_users < 0);
|
|
|
+
|
|
|
+ if (!cpuhw->disabled && !cpuhw->bhrb_users) {
|
|
|
+ /* BHRB cannot be turned off when other
|
|
|
+ * events are active on the PMU.
|
|
|
+ */
|
|
|
+
|
|
|
+ /* avoid stale pointer */
|
|
|
+ cpuhw->bhrb_context = NULL;
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+/* Called from ctxsw to prevent one process's branch entries to
|
|
|
+ * mingle with the other process's entries during context switch.
|
|
|
+ */
|
|
|
+void power_pmu_flush_branch_stack(void)
|
|
|
+{
|
|
|
+ if (ppmu->bhrb_nr)
|
|
|
+ power_pmu_bhrb_reset();
|
|
|
+}
|
|
|
+/* Calculate the to address for a branch */
|
|
|
+static __u64 power_pmu_bhrb_to(u64 addr)
|
|
|
+{
|
|
|
+ unsigned int instr;
|
|
|
+ int ret;
|
|
|
+ __u64 target;
|
|
|
+
|
|
|
+ if (is_kernel_addr(addr))
|
|
|
+ return branch_target((unsigned int *)addr);
|
|
|
+
|
|
|
+ /* Userspace: need copy instruction here then translate it */
|
|
|
+ pagefault_disable();
|
|
|
+ ret = __get_user_inatomic(instr, (unsigned int __user *)addr);
|
|
|
+ if (ret) {
|
|
|
+ pagefault_enable();
|
|
|
+ return 0;
|
|
|
+ }
|
|
|
+ pagefault_enable();
|
|
|
+
|
|
|
+ target = branch_target(&instr);
|
|
|
+ if ((!target) || (instr & BRANCH_ABSOLUTE))
|
|
|
+ return target;
|
|
|
+
|
|
|
+ /* Translate relative branch target from kernel to user address */
|
|
|
+ return target - (unsigned long)&instr + addr;
|
|
|
+}
|
|
|
+
|
|
|
+/* Processing BHRB entries */
|
|
|
+void power_pmu_bhrb_read(struct cpu_hw_events *cpuhw)
|
|
|
+{
|
|
|
+ u64 val;
|
|
|
+ u64 addr;
|
|
|
+ int r_index, u_index, pred;
|
|
|
+
|
|
|
+ r_index = 0;
|
|
|
+ u_index = 0;
|
|
|
+ while (r_index < ppmu->bhrb_nr) {
|
|
|
+ /* Assembly read function */
|
|
|
+ val = read_bhrb(r_index++);
|
|
|
+ if (!val)
|
|
|
+ /* Terminal marker: End of valid BHRB entries */
|
|
|
+ break;
|
|
|
+ else {
|
|
|
+ addr = val & BHRB_EA;
|
|
|
+ pred = val & BHRB_PREDICTION;
|
|
|
+
|
|
|
+ if (!addr)
|
|
|
+ /* invalid entry */
|
|
|
+ continue;
|
|
|
+
|
|
|
+ /* Branches are read most recent first (ie. mfbhrb 0 is
|
|
|
+ * the most recent branch).
|
|
|
+ * There are two types of valid entries:
|
|
|
+ * 1) a target entry which is the to address of a
|
|
|
+ * computed goto like a blr,bctr,btar. The next
|
|
|
+ * entry read from the bhrb will be branch
|
|
|
+ * corresponding to this target (ie. the actual
|
|
|
+ * blr/bctr/btar instruction).
|
|
|
+ * 2) a from address which is an actual branch. If a
|
|
|
+ * target entry proceeds this, then this is the
|
|
|
+ * matching branch for that target. If this is not
|
|
|
+ * following a target entry, then this is a branch
|
|
|
+ * where the target is given as an immediate field
|
|
|
+ * in the instruction (ie. an i or b form branch).
|
|
|
+ * In this case we need to read the instruction from
|
|
|
+ * memory to determine the target/to address.
|
|
|
+ */
|
|
|
+
|
|
|
+ if (val & BHRB_TARGET) {
|
|
|
+ /* Target branches use two entries
|
|
|
+ * (ie. computed gotos/XL form)
|
|
|
+ */
|
|
|
+ cpuhw->bhrb_entries[u_index].to = addr;
|
|
|
+ cpuhw->bhrb_entries[u_index].mispred = pred;
|
|
|
+ cpuhw->bhrb_entries[u_index].predicted = ~pred;
|
|
|
+
|
|
|
+ /* Get from address in next entry */
|
|
|
+ val = read_bhrb(r_index++);
|
|
|
+ addr = val & BHRB_EA;
|
|
|
+ if (val & BHRB_TARGET) {
|
|
|
+ /* Shouldn't have two targets in a
|
|
|
+ row.. Reset index and try again */
|
|
|
+ r_index--;
|
|
|
+ addr = 0;
|
|
|
+ }
|
|
|
+ cpuhw->bhrb_entries[u_index].from = addr;
|
|
|
+ } else {
|
|
|
+ /* Branches to immediate field
|
|
|
+ (ie I or B form) */
|
|
|
+ cpuhw->bhrb_entries[u_index].from = addr;
|
|
|
+ cpuhw->bhrb_entries[u_index].to =
|
|
|
+ power_pmu_bhrb_to(addr);
|
|
|
+ cpuhw->bhrb_entries[u_index].mispred = pred;
|
|
|
+ cpuhw->bhrb_entries[u_index].predicted = ~pred;
|
|
|
+ }
|
|
|
+ u_index++;
|
|
|
+
|
|
|
+ }
|
|
|
+ }
|
|
|
+ cpuhw->bhrb_stack.nr = u_index;
|
|
|
+ return;
|
|
|
+}
|
|
|
+
|
|
|
#endif /* CONFIG_PPC64 */
|
|
|
|
|
|
static void perf_event_interrupt(struct pt_regs *regs);
|
|
@@ -904,47 +1063,6 @@ static int collect_events(struct perf_event *group, int max_count,
|
|
|
return n;
|
|
|
}
|
|
|
|
|
|
-/* Reset all possible BHRB entries */
|
|
|
-static void power_pmu_bhrb_reset(void)
|
|
|
-{
|
|
|
- asm volatile(PPC_CLRBHRB);
|
|
|
-}
|
|
|
-
|
|
|
-void power_pmu_bhrb_enable(struct perf_event *event)
|
|
|
-{
|
|
|
- struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
|
|
|
-
|
|
|
- if (!ppmu->bhrb_nr)
|
|
|
- return;
|
|
|
-
|
|
|
- /* Clear BHRB if we changed task context to avoid data leaks */
|
|
|
- if (event->ctx->task && cpuhw->bhrb_context != event->ctx) {
|
|
|
- power_pmu_bhrb_reset();
|
|
|
- cpuhw->bhrb_context = event->ctx;
|
|
|
- }
|
|
|
- cpuhw->bhrb_users++;
|
|
|
-}
|
|
|
-
|
|
|
-void power_pmu_bhrb_disable(struct perf_event *event)
|
|
|
-{
|
|
|
- struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
|
|
|
-
|
|
|
- if (!ppmu->bhrb_nr)
|
|
|
- return;
|
|
|
-
|
|
|
- cpuhw->bhrb_users--;
|
|
|
- WARN_ON_ONCE(cpuhw->bhrb_users < 0);
|
|
|
-
|
|
|
- if (!cpuhw->disabled && !cpuhw->bhrb_users) {
|
|
|
- /* BHRB cannot be turned off when other
|
|
|
- * events are active on the PMU.
|
|
|
- */
|
|
|
-
|
|
|
- /* avoid stale pointer */
|
|
|
- cpuhw->bhrb_context = NULL;
|
|
|
- }
|
|
|
-}
|
|
|
-
|
|
|
/*
|
|
|
* Add a event to the PMU.
|
|
|
* If all events are not already frozen, then we disable and
|
|
@@ -1180,15 +1298,6 @@ int power_pmu_commit_txn(struct pmu *pmu)
|
|
|
return 0;
|
|
|
}
|
|
|
|
|
|
-/* Called from ctxsw to prevent one process's branch entries to
|
|
|
- * mingle with the other process's entries during context switch.
|
|
|
- */
|
|
|
-void power_pmu_flush_branch_stack(void)
|
|
|
-{
|
|
|
- if (ppmu->bhrb_nr)
|
|
|
- power_pmu_bhrb_reset();
|
|
|
-}
|
|
|
-
|
|
|
/*
|
|
|
* Return 1 if we might be able to put event on a limited PMC,
|
|
|
* or 0 if not.
|
|
@@ -1458,77 +1567,6 @@ struct pmu power_pmu = {
|
|
|
.flush_branch_stack = power_pmu_flush_branch_stack,
|
|
|
};
|
|
|
|
|
|
-/* Processing BHRB entries */
|
|
|
-void power_pmu_bhrb_read(struct cpu_hw_events *cpuhw)
|
|
|
-{
|
|
|
- u64 val;
|
|
|
- u64 addr;
|
|
|
- int r_index, u_index, target, pred;
|
|
|
-
|
|
|
- r_index = 0;
|
|
|
- u_index = 0;
|
|
|
- while (r_index < ppmu->bhrb_nr) {
|
|
|
- /* Assembly read function */
|
|
|
- val = read_bhrb(r_index);
|
|
|
-
|
|
|
- /* Terminal marker: End of valid BHRB entries */
|
|
|
- if (val == 0) {
|
|
|
- break;
|
|
|
- } else {
|
|
|
- /* BHRB field break up */
|
|
|
- addr = val & BHRB_EA;
|
|
|
- pred = val & BHRB_PREDICTION;
|
|
|
- target = val & BHRB_TARGET;
|
|
|
-
|
|
|
- /* Probable Missed entry: Not applicable for POWER8 */
|
|
|
- if ((addr == 0) && (target == 0) && (pred == 1)) {
|
|
|
- r_index++;
|
|
|
- continue;
|
|
|
- }
|
|
|
-
|
|
|
- /* Real Missed entry: Power8 based missed entry */
|
|
|
- if ((addr == 0) && (target == 1) && (pred == 1)) {
|
|
|
- r_index++;
|
|
|
- continue;
|
|
|
- }
|
|
|
-
|
|
|
- /* Reserved condition: Not a valid entry */
|
|
|
- if ((addr == 0) && (target == 1) && (pred == 0)) {
|
|
|
- r_index++;
|
|
|
- continue;
|
|
|
- }
|
|
|
-
|
|
|
- /* Is a target address */
|
|
|
- if (val & BHRB_TARGET) {
|
|
|
- /* First address cannot be a target address */
|
|
|
- if (r_index == 0) {
|
|
|
- r_index++;
|
|
|
- continue;
|
|
|
- }
|
|
|
-
|
|
|
- /* Update target address for the previous entry */
|
|
|
- cpuhw->bhrb_entries[u_index - 1].to = addr;
|
|
|
- cpuhw->bhrb_entries[u_index - 1].mispred = pred;
|
|
|
- cpuhw->bhrb_entries[u_index - 1].predicted = ~pred;
|
|
|
-
|
|
|
- /* Dont increment u_index */
|
|
|
- r_index++;
|
|
|
- } else {
|
|
|
- /* Update address, flags for current entry */
|
|
|
- cpuhw->bhrb_entries[u_index].from = addr;
|
|
|
- cpuhw->bhrb_entries[u_index].mispred = pred;
|
|
|
- cpuhw->bhrb_entries[u_index].predicted = ~pred;
|
|
|
-
|
|
|
- /* Successfully popullated one entry */
|
|
|
- u_index++;
|
|
|
- r_index++;
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- cpuhw->bhrb_stack.nr = u_index;
|
|
|
- return;
|
|
|
-}
|
|
|
-
|
|
|
/*
|
|
|
* A counter has overflowed; update its count and record
|
|
|
* things if requested. Note that interrupts are hard-disabled
|