|
|
@@ -1,930 +0,0 @@
|
|
|
-/*
|
|
|
- * Copyright (C) 2007 Mathieu Desnoyers
|
|
|
- *
|
|
|
- * This program is free software; you can redistribute it and/or modify
|
|
|
- * it under the terms of the GNU General Public License as published by
|
|
|
- * the Free Software Foundation; either version 2 of the License, or
|
|
|
- * (at your option) any later version.
|
|
|
- *
|
|
|
- * This program is distributed in the hope that it will be useful,
|
|
|
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
- * GNU General Public License for more details.
|
|
|
- *
|
|
|
- * You should have received a copy of the GNU General Public License
|
|
|
- * along with this program; if not, write to the Free Software
|
|
|
- * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
|
|
- */
|
|
|
-#include <linux/module.h>
|
|
|
-#include <linux/mutex.h>
|
|
|
-#include <linux/types.h>
|
|
|
-#include <linux/jhash.h>
|
|
|
-#include <linux/list.h>
|
|
|
-#include <linux/rcupdate.h>
|
|
|
-#include <linux/marker.h>
|
|
|
-#include <linux/err.h>
|
|
|
-#include <linux/slab.h>
|
|
|
-
|
|
|
-extern struct marker __start___markers[];
|
|
|
-extern struct marker __stop___markers[];
|
|
|
-
|
|
|
-/* Set to 1 to enable marker debug output */
|
|
|
-static const int marker_debug;
|
|
|
-
|
|
|
-/*
|
|
|
- * markers_mutex nests inside module_mutex. Markers mutex protects the builtin
|
|
|
- * and module markers and the hash table.
|
|
|
- */
|
|
|
-static DEFINE_MUTEX(markers_mutex);
|
|
|
-
|
|
|
-/*
|
|
|
- * Marker hash table, containing the active markers.
|
|
|
- * Protected by module_mutex.
|
|
|
- */
|
|
|
-#define MARKER_HASH_BITS 6
|
|
|
-#define MARKER_TABLE_SIZE (1 << MARKER_HASH_BITS)
|
|
|
-static struct hlist_head marker_table[MARKER_TABLE_SIZE];
|
|
|
-
|
|
|
-/*
|
|
|
- * Note about RCU :
|
|
|
- * It is used to make sure every handler has finished using its private data
|
|
|
- * between two consecutive operation (add or remove) on a given marker. It is
|
|
|
- * also used to delay the free of multiple probes array until a quiescent state
|
|
|
- * is reached.
|
|
|
- * marker entries modifications are protected by the markers_mutex.
|
|
|
- */
|
|
|
-struct marker_entry {
|
|
|
- struct hlist_node hlist;
|
|
|
- char *format;
|
|
|
- /* Probe wrapper */
|
|
|
- void (*call)(const struct marker *mdata, void *call_private, ...);
|
|
|
- struct marker_probe_closure single;
|
|
|
- struct marker_probe_closure *multi;
|
|
|
- int refcount; /* Number of times armed. 0 if disarmed. */
|
|
|
- struct rcu_head rcu;
|
|
|
- void *oldptr;
|
|
|
- int rcu_pending;
|
|
|
- unsigned char ptype:1;
|
|
|
- unsigned char format_allocated:1;
|
|
|
- char name[0]; /* Contains name'\0'format'\0' */
|
|
|
-};
|
|
|
-
|
|
|
-/**
|
|
|
- * __mark_empty_function - Empty probe callback
|
|
|
- * @probe_private: probe private data
|
|
|
- * @call_private: call site private data
|
|
|
- * @fmt: format string
|
|
|
- * @...: variable argument list
|
|
|
- *
|
|
|
- * Empty callback provided as a probe to the markers. By providing this to a
|
|
|
- * disabled marker, we make sure the execution flow is always valid even
|
|
|
- * though the function pointer change and the marker enabling are two distinct
|
|
|
- * operations that modifies the execution flow of preemptible code.
|
|
|
- */
|
|
|
-notrace void __mark_empty_function(void *probe_private, void *call_private,
|
|
|
- const char *fmt, va_list *args)
|
|
|
-{
|
|
|
-}
|
|
|
-EXPORT_SYMBOL_GPL(__mark_empty_function);
|
|
|
-
|
|
|
-/*
|
|
|
- * marker_probe_cb Callback that prepares the variable argument list for probes.
|
|
|
- * @mdata: pointer of type struct marker
|
|
|
- * @call_private: caller site private data
|
|
|
- * @...: Variable argument list.
|
|
|
- *
|
|
|
- * Since we do not use "typical" pointer based RCU in the 1 argument case, we
|
|
|
- * need to put a full smp_rmb() in this branch. This is why we do not use
|
|
|
- * rcu_dereference() for the pointer read.
|
|
|
- */
|
|
|
-notrace void marker_probe_cb(const struct marker *mdata,
|
|
|
- void *call_private, ...)
|
|
|
-{
|
|
|
- va_list args;
|
|
|
- char ptype;
|
|
|
-
|
|
|
- /*
|
|
|
- * rcu_read_lock_sched does two things : disabling preemption to make
|
|
|
- * sure the teardown of the callbacks can be done correctly when they
|
|
|
- * are in modules and they insure RCU read coherency.
|
|
|
- */
|
|
|
- rcu_read_lock_sched_notrace();
|
|
|
- ptype = mdata->ptype;
|
|
|
- if (likely(!ptype)) {
|
|
|
- marker_probe_func *func;
|
|
|
- /* Must read the ptype before ptr. They are not data dependant,
|
|
|
- * so we put an explicit smp_rmb() here. */
|
|
|
- smp_rmb();
|
|
|
- func = mdata->single.func;
|
|
|
- /* Must read the ptr before private data. They are not data
|
|
|
- * dependant, so we put an explicit smp_rmb() here. */
|
|
|
- smp_rmb();
|
|
|
- va_start(args, call_private);
|
|
|
- func(mdata->single.probe_private, call_private, mdata->format,
|
|
|
- &args);
|
|
|
- va_end(args);
|
|
|
- } else {
|
|
|
- struct marker_probe_closure *multi;
|
|
|
- int i;
|
|
|
- /*
|
|
|
- * Read mdata->ptype before mdata->multi.
|
|
|
- */
|
|
|
- smp_rmb();
|
|
|
- multi = mdata->multi;
|
|
|
- /*
|
|
|
- * multi points to an array, therefore accessing the array
|
|
|
- * depends on reading multi. However, even in this case,
|
|
|
- * we must insure that the pointer is read _before_ the array
|
|
|
- * data. Same as rcu_dereference, but we need a full smp_rmb()
|
|
|
- * in the fast path, so put the explicit barrier here.
|
|
|
- */
|
|
|
- smp_read_barrier_depends();
|
|
|
- for (i = 0; multi[i].func; i++) {
|
|
|
- va_start(args, call_private);
|
|
|
- multi[i].func(multi[i].probe_private, call_private,
|
|
|
- mdata->format, &args);
|
|
|
- va_end(args);
|
|
|
- }
|
|
|
- }
|
|
|
- rcu_read_unlock_sched_notrace();
|
|
|
-}
|
|
|
-EXPORT_SYMBOL_GPL(marker_probe_cb);
|
|
|
-
|
|
|
-/*
|
|
|
- * marker_probe_cb Callback that does not prepare the variable argument list.
|
|
|
- * @mdata: pointer of type struct marker
|
|
|
- * @call_private: caller site private data
|
|
|
- * @...: Variable argument list.
|
|
|
- *
|
|
|
- * Should be connected to markers "MARK_NOARGS".
|
|
|
- */
|
|
|
-static notrace void marker_probe_cb_noarg(const struct marker *mdata,
|
|
|
- void *call_private, ...)
|
|
|
-{
|
|
|
- va_list args; /* not initialized */
|
|
|
- char ptype;
|
|
|
-
|
|
|
- rcu_read_lock_sched_notrace();
|
|
|
- ptype = mdata->ptype;
|
|
|
- if (likely(!ptype)) {
|
|
|
- marker_probe_func *func;
|
|
|
- /* Must read the ptype before ptr. They are not data dependant,
|
|
|
- * so we put an explicit smp_rmb() here. */
|
|
|
- smp_rmb();
|
|
|
- func = mdata->single.func;
|
|
|
- /* Must read the ptr before private data. They are not data
|
|
|
- * dependant, so we put an explicit smp_rmb() here. */
|
|
|
- smp_rmb();
|
|
|
- func(mdata->single.probe_private, call_private, mdata->format,
|
|
|
- &args);
|
|
|
- } else {
|
|
|
- struct marker_probe_closure *multi;
|
|
|
- int i;
|
|
|
- /*
|
|
|
- * Read mdata->ptype before mdata->multi.
|
|
|
- */
|
|
|
- smp_rmb();
|
|
|
- multi = mdata->multi;
|
|
|
- /*
|
|
|
- * multi points to an array, therefore accessing the array
|
|
|
- * depends on reading multi. However, even in this case,
|
|
|
- * we must insure that the pointer is read _before_ the array
|
|
|
- * data. Same as rcu_dereference, but we need a full smp_rmb()
|
|
|
- * in the fast path, so put the explicit barrier here.
|
|
|
- */
|
|
|
- smp_read_barrier_depends();
|
|
|
- for (i = 0; multi[i].func; i++)
|
|
|
- multi[i].func(multi[i].probe_private, call_private,
|
|
|
- mdata->format, &args);
|
|
|
- }
|
|
|
- rcu_read_unlock_sched_notrace();
|
|
|
-}
|
|
|
-
|
|
|
-static void free_old_closure(struct rcu_head *head)
|
|
|
-{
|
|
|
- struct marker_entry *entry = container_of(head,
|
|
|
- struct marker_entry, rcu);
|
|
|
- kfree(entry->oldptr);
|
|
|
- /* Make sure we free the data before setting the pending flag to 0 */
|
|
|
- smp_wmb();
|
|
|
- entry->rcu_pending = 0;
|
|
|
-}
|
|
|
-
|
|
|
-static void debug_print_probes(struct marker_entry *entry)
|
|
|
-{
|
|
|
- int i;
|
|
|
-
|
|
|
- if (!marker_debug)
|
|
|
- return;
|
|
|
-
|
|
|
- if (!entry->ptype) {
|
|
|
- printk(KERN_DEBUG "Single probe : %p %p\n",
|
|
|
- entry->single.func,
|
|
|
- entry->single.probe_private);
|
|
|
- } else {
|
|
|
- for (i = 0; entry->multi[i].func; i++)
|
|
|
- printk(KERN_DEBUG "Multi probe %d : %p %p\n", i,
|
|
|
- entry->multi[i].func,
|
|
|
- entry->multi[i].probe_private);
|
|
|
- }
|
|
|
-}
|
|
|
-
|
|
|
-static struct marker_probe_closure *
|
|
|
-marker_entry_add_probe(struct marker_entry *entry,
|
|
|
- marker_probe_func *probe, void *probe_private)
|
|
|
-{
|
|
|
- int nr_probes = 0;
|
|
|
- struct marker_probe_closure *old, *new;
|
|
|
-
|
|
|
- WARN_ON(!probe);
|
|
|
-
|
|
|
- debug_print_probes(entry);
|
|
|
- old = entry->multi;
|
|
|
- if (!entry->ptype) {
|
|
|
- if (entry->single.func == probe &&
|
|
|
- entry->single.probe_private == probe_private)
|
|
|
- return ERR_PTR(-EBUSY);
|
|
|
- if (entry->single.func == __mark_empty_function) {
|
|
|
- /* 0 -> 1 probes */
|
|
|
- entry->single.func = probe;
|
|
|
- entry->single.probe_private = probe_private;
|
|
|
- entry->refcount = 1;
|
|
|
- entry->ptype = 0;
|
|
|
- debug_print_probes(entry);
|
|
|
- return NULL;
|
|
|
- } else {
|
|
|
- /* 1 -> 2 probes */
|
|
|
- nr_probes = 1;
|
|
|
- old = NULL;
|
|
|
- }
|
|
|
- } else {
|
|
|
- /* (N -> N+1), (N != 0, 1) probes */
|
|
|
- for (nr_probes = 0; old[nr_probes].func; nr_probes++)
|
|
|
- if (old[nr_probes].func == probe
|
|
|
- && old[nr_probes].probe_private
|
|
|
- == probe_private)
|
|
|
- return ERR_PTR(-EBUSY);
|
|
|
- }
|
|
|
- /* + 2 : one for new probe, one for NULL func */
|
|
|
- new = kzalloc((nr_probes + 2) * sizeof(struct marker_probe_closure),
|
|
|
- GFP_KERNEL);
|
|
|
- if (new == NULL)
|
|
|
- return ERR_PTR(-ENOMEM);
|
|
|
- if (!old)
|
|
|
- new[0] = entry->single;
|
|
|
- else
|
|
|
- memcpy(new, old,
|
|
|
- nr_probes * sizeof(struct marker_probe_closure));
|
|
|
- new[nr_probes].func = probe;
|
|
|
- new[nr_probes].probe_private = probe_private;
|
|
|
- entry->refcount = nr_probes + 1;
|
|
|
- entry->multi = new;
|
|
|
- entry->ptype = 1;
|
|
|
- debug_print_probes(entry);
|
|
|
- return old;
|
|
|
-}
|
|
|
-
|
|
|
-static struct marker_probe_closure *
|
|
|
-marker_entry_remove_probe(struct marker_entry *entry,
|
|
|
- marker_probe_func *probe, void *probe_private)
|
|
|
-{
|
|
|
- int nr_probes = 0, nr_del = 0, i;
|
|
|
- struct marker_probe_closure *old, *new;
|
|
|
-
|
|
|
- old = entry->multi;
|
|
|
-
|
|
|
- debug_print_probes(entry);
|
|
|
- if (!entry->ptype) {
|
|
|
- /* 0 -> N is an error */
|
|
|
- WARN_ON(entry->single.func == __mark_empty_function);
|
|
|
- /* 1 -> 0 probes */
|
|
|
- WARN_ON(probe && entry->single.func != probe);
|
|
|
- WARN_ON(entry->single.probe_private != probe_private);
|
|
|
- entry->single.func = __mark_empty_function;
|
|
|
- entry->refcount = 0;
|
|
|
- entry->ptype = 0;
|
|
|
- debug_print_probes(entry);
|
|
|
- return NULL;
|
|
|
- } else {
|
|
|
- /* (N -> M), (N > 1, M >= 0) probes */
|
|
|
- for (nr_probes = 0; old[nr_probes].func; nr_probes++) {
|
|
|
- if ((!probe || old[nr_probes].func == probe)
|
|
|
- && old[nr_probes].probe_private
|
|
|
- == probe_private)
|
|
|
- nr_del++;
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- if (nr_probes - nr_del == 0) {
|
|
|
- /* N -> 0, (N > 1) */
|
|
|
- entry->single.func = __mark_empty_function;
|
|
|
- entry->refcount = 0;
|
|
|
- entry->ptype = 0;
|
|
|
- } else if (nr_probes - nr_del == 1) {
|
|
|
- /* N -> 1, (N > 1) */
|
|
|
- for (i = 0; old[i].func; i++)
|
|
|
- if ((probe && old[i].func != probe) ||
|
|
|
- old[i].probe_private != probe_private)
|
|
|
- entry->single = old[i];
|
|
|
- entry->refcount = 1;
|
|
|
- entry->ptype = 0;
|
|
|
- } else {
|
|
|
- int j = 0;
|
|
|
- /* N -> M, (N > 1, M > 1) */
|
|
|
- /* + 1 for NULL */
|
|
|
- new = kzalloc((nr_probes - nr_del + 1)
|
|
|
- * sizeof(struct marker_probe_closure), GFP_KERNEL);
|
|
|
- if (new == NULL)
|
|
|
- return ERR_PTR(-ENOMEM);
|
|
|
- for (i = 0; old[i].func; i++)
|
|
|
- if ((probe && old[i].func != probe) ||
|
|
|
- old[i].probe_private != probe_private)
|
|
|
- new[j++] = old[i];
|
|
|
- entry->refcount = nr_probes - nr_del;
|
|
|
- entry->ptype = 1;
|
|
|
- entry->multi = new;
|
|
|
- }
|
|
|
- debug_print_probes(entry);
|
|
|
- return old;
|
|
|
-}
|
|
|
-
|
|
|
-/*
|
|
|
- * Get marker if the marker is present in the marker hash table.
|
|
|
- * Must be called with markers_mutex held.
|
|
|
- * Returns NULL if not present.
|
|
|
- */
|
|
|
-static struct marker_entry *get_marker(const char *name)
|
|
|
-{
|
|
|
- struct hlist_head *head;
|
|
|
- struct hlist_node *node;
|
|
|
- struct marker_entry *e;
|
|
|
- u32 hash = jhash(name, strlen(name), 0);
|
|
|
-
|
|
|
- head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)];
|
|
|
- hlist_for_each_entry(e, node, head, hlist) {
|
|
|
- if (!strcmp(name, e->name))
|
|
|
- return e;
|
|
|
- }
|
|
|
- return NULL;
|
|
|
-}
|
|
|
-
|
|
|
-/*
|
|
|
- * Add the marker to the marker hash table. Must be called with markers_mutex
|
|
|
- * held.
|
|
|
- */
|
|
|
-static struct marker_entry *add_marker(const char *name, const char *format)
|
|
|
-{
|
|
|
- struct hlist_head *head;
|
|
|
- struct hlist_node *node;
|
|
|
- struct marker_entry *e;
|
|
|
- size_t name_len = strlen(name) + 1;
|
|
|
- size_t format_len = 0;
|
|
|
- u32 hash = jhash(name, name_len-1, 0);
|
|
|
-
|
|
|
- if (format)
|
|
|
- format_len = strlen(format) + 1;
|
|
|
- head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)];
|
|
|
- hlist_for_each_entry(e, node, head, hlist) {
|
|
|
- if (!strcmp(name, e->name)) {
|
|
|
- printk(KERN_NOTICE
|
|
|
- "Marker %s busy\n", name);
|
|
|
- return ERR_PTR(-EBUSY); /* Already there */
|
|
|
- }
|
|
|
- }
|
|
|
- /*
|
|
|
- * Using kmalloc here to allocate a variable length element. Could
|
|
|
- * cause some memory fragmentation if overused.
|
|
|
- */
|
|
|
- e = kmalloc(sizeof(struct marker_entry) + name_len + format_len,
|
|
|
- GFP_KERNEL);
|
|
|
- if (!e)
|
|
|
- return ERR_PTR(-ENOMEM);
|
|
|
- memcpy(&e->name[0], name, name_len);
|
|
|
- if (format) {
|
|
|
- e->format = &e->name[name_len];
|
|
|
- memcpy(e->format, format, format_len);
|
|
|
- if (strcmp(e->format, MARK_NOARGS) == 0)
|
|
|
- e->call = marker_probe_cb_noarg;
|
|
|
- else
|
|
|
- e->call = marker_probe_cb;
|
|
|
- trace_mark(core_marker_format, "name %s format %s",
|
|
|
- e->name, e->format);
|
|
|
- } else {
|
|
|
- e->format = NULL;
|
|
|
- e->call = marker_probe_cb;
|
|
|
- }
|
|
|
- e->single.func = __mark_empty_function;
|
|
|
- e->single.probe_private = NULL;
|
|
|
- e->multi = NULL;
|
|
|
- e->ptype = 0;
|
|
|
- e->format_allocated = 0;
|
|
|
- e->refcount = 0;
|
|
|
- e->rcu_pending = 0;
|
|
|
- hlist_add_head(&e->hlist, head);
|
|
|
- return e;
|
|
|
-}
|
|
|
-
|
|
|
-/*
|
|
|
- * Remove the marker from the marker hash table. Must be called with mutex_lock
|
|
|
- * held.
|
|
|
- */
|
|
|
-static int remove_marker(const char *name)
|
|
|
-{
|
|
|
- struct hlist_head *head;
|
|
|
- struct hlist_node *node;
|
|
|
- struct marker_entry *e;
|
|
|
- int found = 0;
|
|
|
- size_t len = strlen(name) + 1;
|
|
|
- u32 hash = jhash(name, len-1, 0);
|
|
|
-
|
|
|
- head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)];
|
|
|
- hlist_for_each_entry(e, node, head, hlist) {
|
|
|
- if (!strcmp(name, e->name)) {
|
|
|
- found = 1;
|
|
|
- break;
|
|
|
- }
|
|
|
- }
|
|
|
- if (!found)
|
|
|
- return -ENOENT;
|
|
|
- if (e->single.func != __mark_empty_function)
|
|
|
- return -EBUSY;
|
|
|
- hlist_del(&e->hlist);
|
|
|
- if (e->format_allocated)
|
|
|
- kfree(e->format);
|
|
|
- /* Make sure the call_rcu has been executed */
|
|
|
- if (e->rcu_pending)
|
|
|
- rcu_barrier_sched();
|
|
|
- kfree(e);
|
|
|
- return 0;
|
|
|
-}
|
|
|
-
|
|
|
-/*
|
|
|
- * Set the mark_entry format to the format found in the element.
|
|
|
- */
|
|
|
-static int marker_set_format(struct marker_entry *entry, const char *format)
|
|
|
-{
|
|
|
- entry->format = kstrdup(format, GFP_KERNEL);
|
|
|
- if (!entry->format)
|
|
|
- return -ENOMEM;
|
|
|
- entry->format_allocated = 1;
|
|
|
-
|
|
|
- trace_mark(core_marker_format, "name %s format %s",
|
|
|
- entry->name, entry->format);
|
|
|
- return 0;
|
|
|
-}
|
|
|
-
|
|
|
-/*
|
|
|
- * Sets the probe callback corresponding to one marker.
|
|
|
- */
|
|
|
-static int set_marker(struct marker_entry *entry, struct marker *elem,
|
|
|
- int active)
|
|
|
-{
|
|
|
- int ret = 0;
|
|
|
- WARN_ON(strcmp(entry->name, elem->name) != 0);
|
|
|
-
|
|
|
- if (entry->format) {
|
|
|
- if (strcmp(entry->format, elem->format) != 0) {
|
|
|
- printk(KERN_NOTICE
|
|
|
- "Format mismatch for probe %s "
|
|
|
- "(%s), marker (%s)\n",
|
|
|
- entry->name,
|
|
|
- entry->format,
|
|
|
- elem->format);
|
|
|
- return -EPERM;
|
|
|
- }
|
|
|
- } else {
|
|
|
- ret = marker_set_format(entry, elem->format);
|
|
|
- if (ret)
|
|
|
- return ret;
|
|
|
- }
|
|
|
-
|
|
|
- /*
|
|
|
- * probe_cb setup (statically known) is done here. It is
|
|
|
- * asynchronous with the rest of execution, therefore we only
|
|
|
- * pass from a "safe" callback (with argument) to an "unsafe"
|
|
|
- * callback (does not set arguments).
|
|
|
- */
|
|
|
- elem->call = entry->call;
|
|
|
- /*
|
|
|
- * Sanity check :
|
|
|
- * We only update the single probe private data when the ptr is
|
|
|
- * set to a _non_ single probe! (0 -> 1 and N -> 1, N != 1)
|
|
|
- */
|
|
|
- WARN_ON(elem->single.func != __mark_empty_function
|
|
|
- && elem->single.probe_private != entry->single.probe_private
|
|
|
- && !elem->ptype);
|
|
|
- elem->single.probe_private = entry->single.probe_private;
|
|
|
- /*
|
|
|
- * Make sure the private data is valid when we update the
|
|
|
- * single probe ptr.
|
|
|
- */
|
|
|
- smp_wmb();
|
|
|
- elem->single.func = entry->single.func;
|
|
|
- /*
|
|
|
- * We also make sure that the new probe callbacks array is consistent
|
|
|
- * before setting a pointer to it.
|
|
|
- */
|
|
|
- rcu_assign_pointer(elem->multi, entry->multi);
|
|
|
- /*
|
|
|
- * Update the function or multi probe array pointer before setting the
|
|
|
- * ptype.
|
|
|
- */
|
|
|
- smp_wmb();
|
|
|
- elem->ptype = entry->ptype;
|
|
|
-
|
|
|
- if (elem->tp_name && (active ^ elem->state)) {
|
|
|
- WARN_ON(!elem->tp_cb);
|
|
|
- /*
|
|
|
- * It is ok to directly call the probe registration because type
|
|
|
- * checking has been done in the __trace_mark_tp() macro.
|
|
|
- */
|
|
|
-
|
|
|
- if (active) {
|
|
|
- /*
|
|
|
- * try_module_get should always succeed because we hold
|
|
|
- * lock_module() to get the tp_cb address.
|
|
|
- */
|
|
|
- ret = try_module_get(__module_text_address(
|
|
|
- (unsigned long)elem->tp_cb));
|
|
|
- BUG_ON(!ret);
|
|
|
- ret = tracepoint_probe_register_noupdate(
|
|
|
- elem->tp_name,
|
|
|
- elem->tp_cb);
|
|
|
- } else {
|
|
|
- ret = tracepoint_probe_unregister_noupdate(
|
|
|
- elem->tp_name,
|
|
|
- elem->tp_cb);
|
|
|
- /*
|
|
|
- * tracepoint_probe_update_all() must be called
|
|
|
- * before the module containing tp_cb is unloaded.
|
|
|
- */
|
|
|
- module_put(__module_text_address(
|
|
|
- (unsigned long)elem->tp_cb));
|
|
|
- }
|
|
|
- }
|
|
|
- elem->state = active;
|
|
|
-
|
|
|
- return ret;
|
|
|
-}
|
|
|
-
|
|
|
-/*
|
|
|
- * Disable a marker and its probe callback.
|
|
|
- * Note: only waiting an RCU period after setting elem->call to the empty
|
|
|
- * function insures that the original callback is not used anymore. This insured
|
|
|
- * by rcu_read_lock_sched around the call site.
|
|
|
- */
|
|
|
-static void disable_marker(struct marker *elem)
|
|
|
-{
|
|
|
- int ret;
|
|
|
-
|
|
|
- /* leave "call" as is. It is known statically. */
|
|
|
- if (elem->tp_name && elem->state) {
|
|
|
- WARN_ON(!elem->tp_cb);
|
|
|
- /*
|
|
|
- * It is ok to directly call the probe registration because type
|
|
|
- * checking has been done in the __trace_mark_tp() macro.
|
|
|
- */
|
|
|
- ret = tracepoint_probe_unregister_noupdate(elem->tp_name,
|
|
|
- elem->tp_cb);
|
|
|
- WARN_ON(ret);
|
|
|
- /*
|
|
|
- * tracepoint_probe_update_all() must be called
|
|
|
- * before the module containing tp_cb is unloaded.
|
|
|
- */
|
|
|
- module_put(__module_text_address((unsigned long)elem->tp_cb));
|
|
|
- }
|
|
|
- elem->state = 0;
|
|
|
- elem->single.func = __mark_empty_function;
|
|
|
- /* Update the function before setting the ptype */
|
|
|
- smp_wmb();
|
|
|
- elem->ptype = 0; /* single probe */
|
|
|
- /*
|
|
|
- * Leave the private data and id there, because removal is racy and
|
|
|
- * should be done only after an RCU period. These are never used until
|
|
|
- * the next initialization anyway.
|
|
|
- */
|
|
|
-}
|
|
|
-
|
|
|
-/**
|
|
|
- * marker_update_probe_range - Update a probe range
|
|
|
- * @begin: beginning of the range
|
|
|
- * @end: end of the range
|
|
|
- *
|
|
|
- * Updates the probe callback corresponding to a range of markers.
|
|
|
- */
|
|
|
-void marker_update_probe_range(struct marker *begin,
|
|
|
- struct marker *end)
|
|
|
-{
|
|
|
- struct marker *iter;
|
|
|
- struct marker_entry *mark_entry;
|
|
|
-
|
|
|
- mutex_lock(&markers_mutex);
|
|
|
- for (iter = begin; iter < end; iter++) {
|
|
|
- mark_entry = get_marker(iter->name);
|
|
|
- if (mark_entry) {
|
|
|
- set_marker(mark_entry, iter, !!mark_entry->refcount);
|
|
|
- /*
|
|
|
- * ignore error, continue
|
|
|
- */
|
|
|
- } else {
|
|
|
- disable_marker(iter);
|
|
|
- }
|
|
|
- }
|
|
|
- mutex_unlock(&markers_mutex);
|
|
|
-}
|
|
|
-
|
|
|
-/*
|
|
|
- * Update probes, removing the faulty probes.
|
|
|
- *
|
|
|
- * Internal callback only changed before the first probe is connected to it.
|
|
|
- * Single probe private data can only be changed on 0 -> 1 and 2 -> 1
|
|
|
- * transitions. All other transitions will leave the old private data valid.
|
|
|
- * This makes the non-atomicity of the callback/private data updates valid.
|
|
|
- *
|
|
|
- * "special case" updates :
|
|
|
- * 0 -> 1 callback
|
|
|
- * 1 -> 0 callback
|
|
|
- * 1 -> 2 callbacks
|
|
|
- * 2 -> 1 callbacks
|
|
|
- * Other updates all behave the same, just like the 2 -> 3 or 3 -> 2 updates.
|
|
|
- * Site effect : marker_set_format may delete the marker entry (creating a
|
|
|
- * replacement).
|
|
|
- */
|
|
|
-static void marker_update_probes(void)
|
|
|
-{
|
|
|
- /* Core kernel markers */
|
|
|
- marker_update_probe_range(__start___markers, __stop___markers);
|
|
|
- /* Markers in modules. */
|
|
|
- module_update_markers();
|
|
|
- tracepoint_probe_update_all();
|
|
|
-}
|
|
|
-
|
|
|
-/**
|
|
|
- * marker_probe_register - Connect a probe to a marker
|
|
|
- * @name: marker name
|
|
|
- * @format: format string
|
|
|
- * @probe: probe handler
|
|
|
- * @probe_private: probe private data
|
|
|
- *
|
|
|
- * private data must be a valid allocated memory address, or NULL.
|
|
|
- * Returns 0 if ok, error value on error.
|
|
|
- * The probe address must at least be aligned on the architecture pointer size.
|
|
|
- */
|
|
|
-int marker_probe_register(const char *name, const char *format,
|
|
|
- marker_probe_func *probe, void *probe_private)
|
|
|
-{
|
|
|
- struct marker_entry *entry;
|
|
|
- int ret = 0;
|
|
|
- struct marker_probe_closure *old;
|
|
|
-
|
|
|
- mutex_lock(&markers_mutex);
|
|
|
- entry = get_marker(name);
|
|
|
- if (!entry) {
|
|
|
- entry = add_marker(name, format);
|
|
|
- if (IS_ERR(entry))
|
|
|
- ret = PTR_ERR(entry);
|
|
|
- } else if (format) {
|
|
|
- if (!entry->format)
|
|
|
- ret = marker_set_format(entry, format);
|
|
|
- else if (strcmp(entry->format, format))
|
|
|
- ret = -EPERM;
|
|
|
- }
|
|
|
- if (ret)
|
|
|
- goto end;
|
|
|
-
|
|
|
- /*
|
|
|
- * If we detect that a call_rcu is pending for this marker,
|
|
|
- * make sure it's executed now.
|
|
|
- */
|
|
|
- if (entry->rcu_pending)
|
|
|
- rcu_barrier_sched();
|
|
|
- old = marker_entry_add_probe(entry, probe, probe_private);
|
|
|
- if (IS_ERR(old)) {
|
|
|
- ret = PTR_ERR(old);
|
|
|
- goto end;
|
|
|
- }
|
|
|
- mutex_unlock(&markers_mutex);
|
|
|
- marker_update_probes();
|
|
|
- mutex_lock(&markers_mutex);
|
|
|
- entry = get_marker(name);
|
|
|
- if (!entry)
|
|
|
- goto end;
|
|
|
- if (entry->rcu_pending)
|
|
|
- rcu_barrier_sched();
|
|
|
- entry->oldptr = old;
|
|
|
- entry->rcu_pending = 1;
|
|
|
- /* write rcu_pending before calling the RCU callback */
|
|
|
- smp_wmb();
|
|
|
- call_rcu_sched(&entry->rcu, free_old_closure);
|
|
|
-end:
|
|
|
- mutex_unlock(&markers_mutex);
|
|
|
- return ret;
|
|
|
-}
|
|
|
-EXPORT_SYMBOL_GPL(marker_probe_register);
|
|
|
-
|
|
|
-/**
|
|
|
- * marker_probe_unregister - Disconnect a probe from a marker
|
|
|
- * @name: marker name
|
|
|
- * @probe: probe function pointer
|
|
|
- * @probe_private: probe private data
|
|
|
- *
|
|
|
- * Returns the private data given to marker_probe_register, or an ERR_PTR().
|
|
|
- * We do not need to call a synchronize_sched to make sure the probes have
|
|
|
- * finished running before doing a module unload, because the module unload
|
|
|
- * itself uses stop_machine(), which insures that every preempt disabled section
|
|
|
- * have finished.
|
|
|
- */
|
|
|
-int marker_probe_unregister(const char *name,
|
|
|
- marker_probe_func *probe, void *probe_private)
|
|
|
-{
|
|
|
- struct marker_entry *entry;
|
|
|
- struct marker_probe_closure *old;
|
|
|
- int ret = -ENOENT;
|
|
|
-
|
|
|
- mutex_lock(&markers_mutex);
|
|
|
- entry = get_marker(name);
|
|
|
- if (!entry)
|
|
|
- goto end;
|
|
|
- if (entry->rcu_pending)
|
|
|
- rcu_barrier_sched();
|
|
|
- old = marker_entry_remove_probe(entry, probe, probe_private);
|
|
|
- mutex_unlock(&markers_mutex);
|
|
|
- marker_update_probes();
|
|
|
- mutex_lock(&markers_mutex);
|
|
|
- entry = get_marker(name);
|
|
|
- if (!entry)
|
|
|
- goto end;
|
|
|
- if (entry->rcu_pending)
|
|
|
- rcu_barrier_sched();
|
|
|
- entry->oldptr = old;
|
|
|
- entry->rcu_pending = 1;
|
|
|
- /* write rcu_pending before calling the RCU callback */
|
|
|
- smp_wmb();
|
|
|
- call_rcu_sched(&entry->rcu, free_old_closure);
|
|
|
- remove_marker(name); /* Ignore busy error message */
|
|
|
- ret = 0;
|
|
|
-end:
|
|
|
- mutex_unlock(&markers_mutex);
|
|
|
- return ret;
|
|
|
-}
|
|
|
-EXPORT_SYMBOL_GPL(marker_probe_unregister);
|
|
|
-
|
|
|
-static struct marker_entry *
|
|
|
-get_marker_from_private_data(marker_probe_func *probe, void *probe_private)
|
|
|
-{
|
|
|
- struct marker_entry *entry;
|
|
|
- unsigned int i;
|
|
|
- struct hlist_head *head;
|
|
|
- struct hlist_node *node;
|
|
|
-
|
|
|
- for (i = 0; i < MARKER_TABLE_SIZE; i++) {
|
|
|
- head = &marker_table[i];
|
|
|
- hlist_for_each_entry(entry, node, head, hlist) {
|
|
|
- if (!entry->ptype) {
|
|
|
- if (entry->single.func == probe
|
|
|
- && entry->single.probe_private
|
|
|
- == probe_private)
|
|
|
- return entry;
|
|
|
- } else {
|
|
|
- struct marker_probe_closure *closure;
|
|
|
- closure = entry->multi;
|
|
|
- for (i = 0; closure[i].func; i++) {
|
|
|
- if (closure[i].func == probe &&
|
|
|
- closure[i].probe_private
|
|
|
- == probe_private)
|
|
|
- return entry;
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- return NULL;
|
|
|
-}
|
|
|
-
|
|
|
-/**
|
|
|
- * marker_probe_unregister_private_data - Disconnect a probe from a marker
|
|
|
- * @probe: probe function
|
|
|
- * @probe_private: probe private data
|
|
|
- *
|
|
|
- * Unregister a probe by providing the registered private data.
|
|
|
- * Only removes the first marker found in hash table.
|
|
|
- * Return 0 on success or error value.
|
|
|
- * We do not need to call a synchronize_sched to make sure the probes have
|
|
|
- * finished running before doing a module unload, because the module unload
|
|
|
- * itself uses stop_machine(), which insures that every preempt disabled section
|
|
|
- * have finished.
|
|
|
- */
|
|
|
-int marker_probe_unregister_private_data(marker_probe_func *probe,
|
|
|
- void *probe_private)
|
|
|
-{
|
|
|
- struct marker_entry *entry;
|
|
|
- int ret = 0;
|
|
|
- struct marker_probe_closure *old;
|
|
|
-
|
|
|
- mutex_lock(&markers_mutex);
|
|
|
- entry = get_marker_from_private_data(probe, probe_private);
|
|
|
- if (!entry) {
|
|
|
- ret = -ENOENT;
|
|
|
- goto end;
|
|
|
- }
|
|
|
- if (entry->rcu_pending)
|
|
|
- rcu_barrier_sched();
|
|
|
- old = marker_entry_remove_probe(entry, NULL, probe_private);
|
|
|
- mutex_unlock(&markers_mutex);
|
|
|
- marker_update_probes();
|
|
|
- mutex_lock(&markers_mutex);
|
|
|
- entry = get_marker_from_private_data(probe, probe_private);
|
|
|
- if (!entry)
|
|
|
- goto end;
|
|
|
- if (entry->rcu_pending)
|
|
|
- rcu_barrier_sched();
|
|
|
- entry->oldptr = old;
|
|
|
- entry->rcu_pending = 1;
|
|
|
- /* write rcu_pending before calling the RCU callback */
|
|
|
- smp_wmb();
|
|
|
- call_rcu_sched(&entry->rcu, free_old_closure);
|
|
|
- remove_marker(entry->name); /* Ignore busy error message */
|
|
|
-end:
|
|
|
- mutex_unlock(&markers_mutex);
|
|
|
- return ret;
|
|
|
-}
|
|
|
-EXPORT_SYMBOL_GPL(marker_probe_unregister_private_data);
|
|
|
-
|
|
|
-/**
|
|
|
- * marker_get_private_data - Get a marker's probe private data
|
|
|
- * @name: marker name
|
|
|
- * @probe: probe to match
|
|
|
- * @num: get the nth matching probe's private data
|
|
|
- *
|
|
|
- * Returns the nth private data pointer (starting from 0) matching, or an
|
|
|
- * ERR_PTR.
|
|
|
- * Returns the private data pointer, or an ERR_PTR.
|
|
|
- * The private data pointer should _only_ be dereferenced if the caller is the
|
|
|
- * owner of the data, or its content could vanish. This is mostly used to
|
|
|
- * confirm that a caller is the owner of a registered probe.
|
|
|
- */
|
|
|
-void *marker_get_private_data(const char *name, marker_probe_func *probe,
|
|
|
- int num)
|
|
|
-{
|
|
|
- struct hlist_head *head;
|
|
|
- struct hlist_node *node;
|
|
|
- struct marker_entry *e;
|
|
|
- size_t name_len = strlen(name) + 1;
|
|
|
- u32 hash = jhash(name, name_len-1, 0);
|
|
|
- int i;
|
|
|
-
|
|
|
- head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)];
|
|
|
- hlist_for_each_entry(e, node, head, hlist) {
|
|
|
- if (!strcmp(name, e->name)) {
|
|
|
- if (!e->ptype) {
|
|
|
- if (num == 0 && e->single.func == probe)
|
|
|
- return e->single.probe_private;
|
|
|
- } else {
|
|
|
- struct marker_probe_closure *closure;
|
|
|
- int match = 0;
|
|
|
- closure = e->multi;
|
|
|
- for (i = 0; closure[i].func; i++) {
|
|
|
- if (closure[i].func != probe)
|
|
|
- continue;
|
|
|
- if (match++ == num)
|
|
|
- return closure[i].probe_private;
|
|
|
- }
|
|
|
- }
|
|
|
- break;
|
|
|
- }
|
|
|
- }
|
|
|
- return ERR_PTR(-ENOENT);
|
|
|
-}
|
|
|
-EXPORT_SYMBOL_GPL(marker_get_private_data);
|
|
|
-
|
|
|
-#ifdef CONFIG_MODULES
|
|
|
-
|
|
|
-int marker_module_notify(struct notifier_block *self,
|
|
|
- unsigned long val, void *data)
|
|
|
-{
|
|
|
- struct module *mod = data;
|
|
|
-
|
|
|
- switch (val) {
|
|
|
- case MODULE_STATE_COMING:
|
|
|
- marker_update_probe_range(mod->markers,
|
|
|
- mod->markers + mod->num_markers);
|
|
|
- break;
|
|
|
- case MODULE_STATE_GOING:
|
|
|
- marker_update_probe_range(mod->markers,
|
|
|
- mod->markers + mod->num_markers);
|
|
|
- break;
|
|
|
- }
|
|
|
- return 0;
|
|
|
-}
|
|
|
-
|
|
|
-struct notifier_block marker_module_nb = {
|
|
|
- .notifier_call = marker_module_notify,
|
|
|
- .priority = 0,
|
|
|
-};
|
|
|
-
|
|
|
-static int init_markers(void)
|
|
|
-{
|
|
|
- return register_module_notifier(&marker_module_nb);
|
|
|
-}
|
|
|
-__initcall(init_markers);
|
|
|
-
|
|
|
-#endif /* CONFIG_MODULES */
|
Christoph Hellwig