|
@@ -334,7 +334,7 @@ ftrace_record_ip(unsigned long ip)
|
|
|
{
|
|
|
struct dyn_ftrace *rec;
|
|
|
|
|
|
- if (!ftrace_enabled || ftrace_disabled)
|
|
|
+ if (ftrace_disabled)
|
|
|
return NULL;
|
|
|
|
|
|
rec = ftrace_alloc_dyn_node(ip);
|
|
@@ -348,107 +348,129 @@ ftrace_record_ip(unsigned long ip)
|
|
|
return rec;
|
|
|
}
|
|
|
|
|
|
+static void print_ip_ins(const char *fmt, unsigned char *p)
|
|
|
+{
|
|
|
+ int i;
|
|
|
+
|
|
|
+ printk(KERN_CONT "%s", fmt);
|
|
|
+
|
|
|
+ for (i = 0; i < MCOUNT_INSN_SIZE; i++)
|
|
|
+ printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
|
|
|
+}
|
|
|
+
|
|
|
+static void ftrace_bug(int failed, unsigned long ip)
|
|
|
+{
|
|
|
+ switch (failed) {
|
|
|
+ case -EFAULT:
|
|
|
+ FTRACE_WARN_ON_ONCE(1);
|
|
|
+ pr_info("ftrace faulted on modifying ");
|
|
|
+ print_ip_sym(ip);
|
|
|
+ break;
|
|
|
+ case -EINVAL:
|
|
|
+ FTRACE_WARN_ON_ONCE(1);
|
|
|
+ pr_info("ftrace failed to modify ");
|
|
|
+ print_ip_sym(ip);
|
|
|
+ print_ip_ins(" actual: ", (unsigned char *)ip);
|
|
|
+ printk(KERN_CONT "\n");
|
|
|
+ break;
|
|
|
+ case -EPERM:
|
|
|
+ FTRACE_WARN_ON_ONCE(1);
|
|
|
+ pr_info("ftrace faulted on writing ");
|
|
|
+ print_ip_sym(ip);
|
|
|
+ break;
|
|
|
+ default:
|
|
|
+ FTRACE_WARN_ON_ONCE(1);
|
|
|
+ pr_info("ftrace faulted on unknown error ");
|
|
|
+ print_ip_sym(ip);
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
#define FTRACE_ADDR ((long)(ftrace_caller))
|
|
|
|
|
|
static int
|
|
|
-__ftrace_replace_code(struct dyn_ftrace *rec,
|
|
|
- unsigned char *old, unsigned char *new, int enable)
|
|
|
+__ftrace_replace_code(struct dyn_ftrace *rec, int enable)
|
|
|
{
|
|
|
unsigned long ip, fl;
|
|
|
|
|
|
ip = rec->ip;
|
|
|
|
|
|
- if (ftrace_filtered && enable) {
|
|
|
+ /*
|
|
|
+ * If this record is not to be traced and
|
|
|
+ * it is not enabled then do nothing.
|
|
|
+ *
|
|
|
+ * If this record is not to be traced and
|
|
|
+ * it is enabled then disabled it.
|
|
|
+ *
|
|
|
+ */
|
|
|
+ if (rec->flags & FTRACE_FL_NOTRACE) {
|
|
|
+ if (rec->flags & FTRACE_FL_ENABLED)
|
|
|
+ rec->flags &= ~FTRACE_FL_ENABLED;
|
|
|
+ else
|
|
|
+ return 0;
|
|
|
+
|
|
|
+ } else if (ftrace_filtered && enable) {
|
|
|
/*
|
|
|
- * If filtering is on:
|
|
|
- *
|
|
|
- * If this record is set to be filtered and
|
|
|
- * is enabled then do nothing.
|
|
|
- *
|
|
|
- * If this record is set to be filtered and
|
|
|
- * it is not enabled, enable it.
|
|
|
- *
|
|
|
- * If this record is not set to be filtered
|
|
|
- * and it is not enabled do nothing.
|
|
|
- *
|
|
|
- * If this record is set not to trace then
|
|
|
- * do nothing.
|
|
|
- *
|
|
|
- * If this record is set not to trace and
|
|
|
- * it is enabled then disable it.
|
|
|
- *
|
|
|
- * If this record is not set to be filtered and
|
|
|
- * it is enabled, disable it.
|
|
|
+ * Filtering is on:
|
|
|
*/
|
|
|
|
|
|
- fl = rec->flags & (FTRACE_FL_FILTER | FTRACE_FL_NOTRACE |
|
|
|
- FTRACE_FL_ENABLED);
|
|
|
+ fl = rec->flags & (FTRACE_FL_FILTER | FTRACE_FL_ENABLED);
|
|
|
|
|
|
- if ((fl == (FTRACE_FL_FILTER | FTRACE_FL_ENABLED)) ||
|
|
|
- (fl == (FTRACE_FL_FILTER | FTRACE_FL_NOTRACE)) ||
|
|
|
- !fl || (fl == FTRACE_FL_NOTRACE))
|
|
|
+ /* Record is filtered and enabled, do nothing */
|
|
|
+ if (fl == (FTRACE_FL_FILTER | FTRACE_FL_ENABLED))
|
|
|
return 0;
|
|
|
|
|
|
- /*
|
|
|
- * If it is enabled disable it,
|
|
|
- * otherwise enable it!
|
|
|
- */
|
|
|
- if (fl & FTRACE_FL_ENABLED) {
|
|
|
- /* swap new and old */
|
|
|
- new = old;
|
|
|
- old = ftrace_call_replace(ip, FTRACE_ADDR);
|
|
|
+ /* Record is not filtered and is not enabled do nothing */
|
|
|
+ if (!fl)
|
|
|
+ return 0;
|
|
|
+
|
|
|
+ /* Record is not filtered but enabled, disable it */
|
|
|
+ if (fl == FTRACE_FL_ENABLED)
|
|
|
rec->flags &= ~FTRACE_FL_ENABLED;
|
|
|
- } else {
|
|
|
- new = ftrace_call_replace(ip, FTRACE_ADDR);
|
|
|
+ else
|
|
|
+ /* Otherwise record is filtered but not enabled, enable it */
|
|
|
rec->flags |= FTRACE_FL_ENABLED;
|
|
|
- }
|
|
|
} else {
|
|
|
+ /* Disable or not filtered */
|
|
|
|
|
|
if (enable) {
|
|
|
- /*
|
|
|
- * If this record is set not to trace and is
|
|
|
- * not enabled, do nothing.
|
|
|
- */
|
|
|
- fl = rec->flags & (FTRACE_FL_NOTRACE | FTRACE_FL_ENABLED);
|
|
|
- if (fl == FTRACE_FL_NOTRACE)
|
|
|
- return 0;
|
|
|
-
|
|
|
- new = ftrace_call_replace(ip, FTRACE_ADDR);
|
|
|
- } else
|
|
|
- old = ftrace_call_replace(ip, FTRACE_ADDR);
|
|
|
-
|
|
|
- if (enable) {
|
|
|
+ /* if record is enabled, do nothing */
|
|
|
if (rec->flags & FTRACE_FL_ENABLED)
|
|
|
return 0;
|
|
|
+
|
|
|
rec->flags |= FTRACE_FL_ENABLED;
|
|
|
+
|
|
|
} else {
|
|
|
+
|
|
|
+ /* if record is not enabled do nothing */
|
|
|
if (!(rec->flags & FTRACE_FL_ENABLED))
|
|
|
return 0;
|
|
|
+
|
|
|
rec->flags &= ~FTRACE_FL_ENABLED;
|
|
|
}
|
|
|
}
|
|
|
|
|
|
- return ftrace_modify_code(ip, old, new);
|
|
|
+ if (rec->flags & FTRACE_FL_ENABLED)
|
|
|
+ return ftrace_make_call(rec, FTRACE_ADDR);
|
|
|
+ else
|
|
|
+ return ftrace_make_nop(NULL, rec, FTRACE_ADDR);
|
|
|
}
|
|
|
|
|
|
static void ftrace_replace_code(int enable)
|
|
|
{
|
|
|
int i, failed;
|
|
|
- unsigned char *new = NULL, *old = NULL;
|
|
|
struct dyn_ftrace *rec;
|
|
|
struct ftrace_page *pg;
|
|
|
|
|
|
- if (enable)
|
|
|
- old = ftrace_nop_replace();
|
|
|
- else
|
|
|
- new = ftrace_nop_replace();
|
|
|
-
|
|
|
for (pg = ftrace_pages_start; pg; pg = pg->next) {
|
|
|
for (i = 0; i < pg->index; i++) {
|
|
|
rec = &pg->records[i];
|
|
|
|
|
|
- /* don't modify code that has already faulted */
|
|
|
- if (rec->flags & FTRACE_FL_FAILED)
|
|
|
+ /*
|
|
|
+ * Skip over free records and records that have
|
|
|
+ * failed.
|
|
|
+ */
|
|
|
+ if (rec->flags & FTRACE_FL_FREE ||
|
|
|
+ rec->flags & FTRACE_FL_FAILED)
|
|
|
continue;
|
|
|
|
|
|
/* ignore updates to this record's mcount site */
|
|
@@ -459,68 +481,30 @@ static void ftrace_replace_code(int enable)
|
|
|
unfreeze_record(rec);
|
|
|
}
|
|
|
|
|
|
- failed = __ftrace_replace_code(rec, old, new, enable);
|
|
|
+ failed = __ftrace_replace_code(rec, enable);
|
|
|
if (failed && (rec->flags & FTRACE_FL_CONVERTED)) {
|
|
|
rec->flags |= FTRACE_FL_FAILED;
|
|
|
if ((system_state == SYSTEM_BOOTING) ||
|
|
|
!core_kernel_text(rec->ip)) {
|
|
|
ftrace_free_rec(rec);
|
|
|
- }
|
|
|
+ } else
|
|
|
+ ftrace_bug(failed, rec->ip);
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
|
|
|
-static void print_ip_ins(const char *fmt, unsigned char *p)
|
|
|
-{
|
|
|
- int i;
|
|
|
-
|
|
|
- printk(KERN_CONT "%s", fmt);
|
|
|
-
|
|
|
- for (i = 0; i < MCOUNT_INSN_SIZE; i++)
|
|
|
- printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
|
|
|
-}
|
|
|
-
|
|
|
static int
|
|
|
-ftrace_code_disable(struct dyn_ftrace *rec)
|
|
|
+ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
|
|
|
{
|
|
|
unsigned long ip;
|
|
|
- unsigned char *nop, *call;
|
|
|
int ret;
|
|
|
|
|
|
ip = rec->ip;
|
|
|
|
|
|
- nop = ftrace_nop_replace();
|
|
|
- call = ftrace_call_replace(ip, mcount_addr);
|
|
|
-
|
|
|
- ret = ftrace_modify_code(ip, call, nop);
|
|
|
+ ret = ftrace_make_nop(mod, rec, mcount_addr);
|
|
|
if (ret) {
|
|
|
- switch (ret) {
|
|
|
- case -EFAULT:
|
|
|
- FTRACE_WARN_ON_ONCE(1);
|
|
|
- pr_info("ftrace faulted on modifying ");
|
|
|
- print_ip_sym(ip);
|
|
|
- break;
|
|
|
- case -EINVAL:
|
|
|
- FTRACE_WARN_ON_ONCE(1);
|
|
|
- pr_info("ftrace failed to modify ");
|
|
|
- print_ip_sym(ip);
|
|
|
- print_ip_ins(" expected: ", call);
|
|
|
- print_ip_ins(" actual: ", (unsigned char *)ip);
|
|
|
- print_ip_ins(" replace: ", nop);
|
|
|
- printk(KERN_CONT "\n");
|
|
|
- break;
|
|
|
- case -EPERM:
|
|
|
- FTRACE_WARN_ON_ONCE(1);
|
|
|
- pr_info("ftrace faulted on writing ");
|
|
|
- print_ip_sym(ip);
|
|
|
- break;
|
|
|
- default:
|
|
|
- FTRACE_WARN_ON_ONCE(1);
|
|
|
- pr_info("ftrace faulted on unknown error ");
|
|
|
- print_ip_sym(ip);
|
|
|
- }
|
|
|
-
|
|
|
+ ftrace_bug(ret, ip);
|
|
|
rec->flags |= FTRACE_FL_FAILED;
|
|
|
return 0;
|
|
|
}
|
|
@@ -560,8 +544,7 @@ static void ftrace_startup(void)
|
|
|
|
|
|
mutex_lock(&ftrace_start_lock);
|
|
|
ftrace_start_up++;
|
|
|
- if (ftrace_start_up == 1)
|
|
|
- command |= FTRACE_ENABLE_CALLS;
|
|
|
+ command |= FTRACE_ENABLE_CALLS;
|
|
|
|
|
|
if (saved_ftrace_func != ftrace_trace_function) {
|
|
|
saved_ftrace_func = ftrace_trace_function;
|
|
@@ -639,7 +622,7 @@ static cycle_t ftrace_update_time;
|
|
|
static unsigned long ftrace_update_cnt;
|
|
|
unsigned long ftrace_update_tot_cnt;
|
|
|
|
|
|
-static int ftrace_update_code(void)
|
|
|
+static int ftrace_update_code(struct module *mod)
|
|
|
{
|
|
|
struct dyn_ftrace *p, *t;
|
|
|
cycle_t start, stop;
|
|
@@ -656,7 +639,7 @@ static int ftrace_update_code(void)
|
|
|
list_del_init(&p->list);
|
|
|
|
|
|
/* convert record (i.e, patch mcount-call with NOP) */
|
|
|
- if (ftrace_code_disable(p)) {
|
|
|
+ if (ftrace_code_disable(mod, p)) {
|
|
|
p->flags |= FTRACE_FL_CONVERTED;
|
|
|
ftrace_update_cnt++;
|
|
|
} else
|
|
@@ -1211,7 +1194,7 @@ ftrace_regex_release(struct inode *inode, struct file *file, int enable)
|
|
|
|
|
|
mutex_lock(&ftrace_sysctl_lock);
|
|
|
mutex_lock(&ftrace_start_lock);
|
|
|
- if (iter->filtered && ftrace_start_up && ftrace_enabled)
|
|
|
+ if (ftrace_start_up && ftrace_enabled)
|
|
|
ftrace_run_update_code(FTRACE_ENABLE_CALLS);
|
|
|
mutex_unlock(&ftrace_start_lock);
|
|
|
mutex_unlock(&ftrace_sysctl_lock);
|
|
@@ -1298,7 +1281,8 @@ static __init int ftrace_init_debugfs(void)
|
|
|
|
|
|
fs_initcall(ftrace_init_debugfs);
|
|
|
|
|
|
-static int ftrace_convert_nops(unsigned long *start,
|
|
|
+static int ftrace_convert_nops(struct module *mod,
|
|
|
+ unsigned long *start,
|
|
|
unsigned long *end)
|
|
|
{
|
|
|
unsigned long *p;
|
|
@@ -1309,23 +1293,32 @@ static int ftrace_convert_nops(unsigned long *start,
|
|
|
p = start;
|
|
|
while (p < end) {
|
|
|
addr = ftrace_call_adjust(*p++);
|
|
|
+ /*
|
|
|
+ * Some architecture linkers will pad between
|
|
|
+ * the different mcount_loc sections of different
|
|
|
+ * object files to satisfy alignments.
|
|
|
+ * Skip any NULL pointers.
|
|
|
+ */
|
|
|
+ if (!addr)
|
|
|
+ continue;
|
|
|
ftrace_record_ip(addr);
|
|
|
}
|
|
|
|
|
|
/* disable interrupts to prevent kstop machine */
|
|
|
local_irq_save(flags);
|
|
|
- ftrace_update_code();
|
|
|
+ ftrace_update_code(mod);
|
|
|
local_irq_restore(flags);
|
|
|
mutex_unlock(&ftrace_start_lock);
|
|
|
|
|
|
return 0;
|
|
|
}
|
|
|
|
|
|
-void ftrace_init_module(unsigned long *start, unsigned long *end)
|
|
|
+void ftrace_init_module(struct module *mod,
|
|
|
+ unsigned long *start, unsigned long *end)
|
|
|
{
|
|
|
if (ftrace_disabled || start == end)
|
|
|
return;
|
|
|
- ftrace_convert_nops(start, end);
|
|
|
+ ftrace_convert_nops(mod, start, end);
|
|
|
}
|
|
|
|
|
|
extern unsigned long __start_mcount_loc[];
|
|
@@ -1355,7 +1348,8 @@ void __init ftrace_init(void)
|
|
|
|
|
|
last_ftrace_enabled = ftrace_enabled = 1;
|
|
|
|
|
|
- ret = ftrace_convert_nops(__start_mcount_loc,
|
|
|
+ ret = ftrace_convert_nops(NULL,
|
|
|
+ __start_mcount_loc,
|
|
|
__stop_mcount_loc);
|
|
|
|
|
|
return;
|