lockdep.c 101 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167
  1. /*
  2. * kernel/lockdep.c
  3. *
  4. * Runtime locking correctness validator
  5. *
  6. * Started by Ingo Molnar:
  7. *
  8. * Copyright (C) 2006,2007 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
  9. * Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
  10. *
  11. * this code maps all the lock dependencies as they occur in a live kernel
  12. * and will warn about the following classes of locking bugs:
  13. *
  14. * - lock inversion scenarios
  15. * - circular lock dependencies
  16. * - hardirq/softirq safe/unsafe locking bugs
  17. *
  18. * Bugs are reported even if the current locking scenario does not cause
  19. * any deadlock at this point.
  20. *
  21. * I.e. if anytime in the past two locks were taken in a different order,
  22. * even if it happened for another task, even if those were different
  23. * locks (but of the same class as this lock), this code will detect it.
  24. *
  25. * Thanks to Arjan van de Ven for coming up with the initial idea of
  26. * mapping lock dependencies runtime.
  27. */
  28. #define DISABLE_BRANCH_PROFILING
  29. #include <linux/mutex.h>
  30. #include <linux/sched.h>
  31. #include <linux/delay.h>
  32. #include <linux/module.h>
  33. #include <linux/proc_fs.h>
  34. #include <linux/seq_file.h>
  35. #include <linux/spinlock.h>
  36. #include <linux/kallsyms.h>
  37. #include <linux/interrupt.h>
  38. #include <linux/stacktrace.h>
  39. #include <linux/debug_locks.h>
  40. #include <linux/irqflags.h>
  41. #include <linux/utsname.h>
  42. #include <linux/hash.h>
  43. #include <linux/ftrace.h>
  44. #include <linux/stringify.h>
  45. #include <linux/bitops.h>
  46. #include <linux/gfp.h>
  47. #include <asm/sections.h>
  48. #include "lockdep_internals.h"
  49. #define CREATE_TRACE_POINTS
  50. #include <trace/events/lock.h>
  51. #ifdef CONFIG_PROVE_LOCKING
  52. int prove_locking = 1;
  53. module_param(prove_locking, int, 0644);
  54. #else
  55. #define prove_locking 0
  56. #endif
  57. #ifdef CONFIG_LOCK_STAT
  58. int lock_stat = 1;
  59. module_param(lock_stat, int, 0644);
  60. #else
  61. #define lock_stat 0
  62. #endif
  63. /*
  64. * lockdep_lock: protects the lockdep graph, the hashes and the
  65. * class/list/hash allocators.
  66. *
  67. * This is one of the rare exceptions where it's justified
  68. * to use a raw spinlock - we really dont want the spinlock
  69. * code to recurse back into the lockdep code...
  70. */
  71. static arch_spinlock_t lockdep_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
  72. static int graph_lock(void)
  73. {
  74. arch_spin_lock(&lockdep_lock);
  75. /*
  76. * Make sure that if another CPU detected a bug while
  77. * walking the graph we dont change it (while the other
  78. * CPU is busy printing out stuff with the graph lock
  79. * dropped already)
  80. */
  81. if (!debug_locks) {
  82. arch_spin_unlock(&lockdep_lock);
  83. return 0;
  84. }
  85. /* prevent any recursions within lockdep from causing deadlocks */
  86. current->lockdep_recursion++;
  87. return 1;
  88. }
  89. static inline int graph_unlock(void)
  90. {
  91. if (debug_locks && !arch_spin_is_locked(&lockdep_lock)) {
  92. /*
  93. * The lockdep graph lock isn't locked while we expect it to
  94. * be, we're confused now, bye!
  95. */
  96. return DEBUG_LOCKS_WARN_ON(1);
  97. }
  98. current->lockdep_recursion--;
  99. arch_spin_unlock(&lockdep_lock);
  100. return 0;
  101. }
  102. /*
  103. * Turn lock debugging off and return with 0 if it was off already,
  104. * and also release the graph lock:
  105. */
  106. static inline int debug_locks_off_graph_unlock(void)
  107. {
  108. int ret = debug_locks_off();
  109. arch_spin_unlock(&lockdep_lock);
  110. return ret;
  111. }
  112. static int lockdep_initialized;
  113. unsigned long nr_list_entries;
  114. static struct lock_list list_entries[MAX_LOCKDEP_ENTRIES];
  115. /*
  116. * All data structures here are protected by the global debug_lock.
  117. *
  118. * Mutex key structs only get allocated, once during bootup, and never
  119. * get freed - this significantly simplifies the debugging code.
  120. */
  121. unsigned long nr_lock_classes;
  122. static struct lock_class lock_classes[MAX_LOCKDEP_KEYS];
  123. static inline struct lock_class *hlock_class(struct held_lock *hlock)
  124. {
  125. if (!hlock->class_idx) {
  126. /*
  127. * Someone passed in garbage, we give up.
  128. */
  129. DEBUG_LOCKS_WARN_ON(1);
  130. return NULL;
  131. }
  132. return lock_classes + hlock->class_idx - 1;
  133. }
  134. #ifdef CONFIG_LOCK_STAT
  135. static DEFINE_PER_CPU(struct lock_class_stats[MAX_LOCKDEP_KEYS],
  136. cpu_lock_stats);
  137. static inline u64 lockstat_clock(void)
  138. {
  139. return local_clock();
  140. }
  141. static int lock_point(unsigned long points[], unsigned long ip)
  142. {
  143. int i;
  144. for (i = 0; i < LOCKSTAT_POINTS; i++) {
  145. if (points[i] == 0) {
  146. points[i] = ip;
  147. break;
  148. }
  149. if (points[i] == ip)
  150. break;
  151. }
  152. return i;
  153. }
  154. static void lock_time_inc(struct lock_time *lt, u64 time)
  155. {
  156. if (time > lt->max)
  157. lt->max = time;
  158. if (time < lt->min || !lt->nr)
  159. lt->min = time;
  160. lt->total += time;
  161. lt->nr++;
  162. }
  163. static inline void lock_time_add(struct lock_time *src, struct lock_time *dst)
  164. {
  165. if (!src->nr)
  166. return;
  167. if (src->max > dst->max)
  168. dst->max = src->max;
  169. if (src->min < dst->min || !dst->nr)
  170. dst->min = src->min;
  171. dst->total += src->total;
  172. dst->nr += src->nr;
  173. }
  174. struct lock_class_stats lock_stats(struct lock_class *class)
  175. {
  176. struct lock_class_stats stats;
  177. int cpu, i;
  178. memset(&stats, 0, sizeof(struct lock_class_stats));
  179. for_each_possible_cpu(cpu) {
  180. struct lock_class_stats *pcs =
  181. &per_cpu(cpu_lock_stats, cpu)[class - lock_classes];
  182. for (i = 0; i < ARRAY_SIZE(stats.contention_point); i++)
  183. stats.contention_point[i] += pcs->contention_point[i];
  184. for (i = 0; i < ARRAY_SIZE(stats.contending_point); i++)
  185. stats.contending_point[i] += pcs->contending_point[i];
  186. lock_time_add(&pcs->read_waittime, &stats.read_waittime);
  187. lock_time_add(&pcs->write_waittime, &stats.write_waittime);
  188. lock_time_add(&pcs->read_holdtime, &stats.read_holdtime);
  189. lock_time_add(&pcs->write_holdtime, &stats.write_holdtime);
  190. for (i = 0; i < ARRAY_SIZE(stats.bounces); i++)
  191. stats.bounces[i] += pcs->bounces[i];
  192. }
  193. return stats;
  194. }
  195. void clear_lock_stats(struct lock_class *class)
  196. {
  197. int cpu;
  198. for_each_possible_cpu(cpu) {
  199. struct lock_class_stats *cpu_stats =
  200. &per_cpu(cpu_lock_stats, cpu)[class - lock_classes];
  201. memset(cpu_stats, 0, sizeof(struct lock_class_stats));
  202. }
  203. memset(class->contention_point, 0, sizeof(class->contention_point));
  204. memset(class->contending_point, 0, sizeof(class->contending_point));
  205. }
  206. static struct lock_class_stats *get_lock_stats(struct lock_class *class)
  207. {
  208. return &get_cpu_var(cpu_lock_stats)[class - lock_classes];
  209. }
  210. static void put_lock_stats(struct lock_class_stats *stats)
  211. {
  212. put_cpu_var(cpu_lock_stats);
  213. }
  214. static void lock_release_holdtime(struct held_lock *hlock)
  215. {
  216. struct lock_class_stats *stats;
  217. u64 holdtime;
  218. if (!lock_stat)
  219. return;
  220. holdtime = lockstat_clock() - hlock->holdtime_stamp;
  221. stats = get_lock_stats(hlock_class(hlock));
  222. if (hlock->read)
  223. lock_time_inc(&stats->read_holdtime, holdtime);
  224. else
  225. lock_time_inc(&stats->write_holdtime, holdtime);
  226. put_lock_stats(stats);
  227. }
  228. #else
  229. static inline void lock_release_holdtime(struct held_lock *hlock)
  230. {
  231. }
  232. #endif
  233. /*
  234. * We keep a global list of all lock classes. The list only grows,
  235. * never shrinks. The list is only accessed with the lockdep
  236. * spinlock lock held.
  237. */
  238. LIST_HEAD(all_lock_classes);
  239. /*
  240. * The lockdep classes are in a hash-table as well, for fast lookup:
  241. */
  242. #define CLASSHASH_BITS (MAX_LOCKDEP_KEYS_BITS - 1)
  243. #define CLASSHASH_SIZE (1UL << CLASSHASH_BITS)
  244. #define __classhashfn(key) hash_long((unsigned long)key, CLASSHASH_BITS)
  245. #define classhashentry(key) (classhash_table + __classhashfn((key)))
  246. static struct list_head classhash_table[CLASSHASH_SIZE];
  247. /*
  248. * We put the lock dependency chains into a hash-table as well, to cache
  249. * their existence:
  250. */
  251. #define CHAINHASH_BITS (MAX_LOCKDEP_CHAINS_BITS-1)
  252. #define CHAINHASH_SIZE (1UL << CHAINHASH_BITS)
  253. #define __chainhashfn(chain) hash_long(chain, CHAINHASH_BITS)
  254. #define chainhashentry(chain) (chainhash_table + __chainhashfn((chain)))
  255. static struct list_head chainhash_table[CHAINHASH_SIZE];
  256. /*
  257. * The hash key of the lock dependency chains is a hash itself too:
  258. * it's a hash of all locks taken up to that lock, including that lock.
  259. * It's a 64-bit hash, because it's important for the keys to be
  260. * unique.
  261. */
  262. #define iterate_chain_key(key1, key2) \
  263. (((key1) << MAX_LOCKDEP_KEYS_BITS) ^ \
  264. ((key1) >> (64-MAX_LOCKDEP_KEYS_BITS)) ^ \
  265. (key2))
  266. void lockdep_off(void)
  267. {
  268. current->lockdep_recursion++;
  269. }
  270. EXPORT_SYMBOL(lockdep_off);
  271. void lockdep_on(void)
  272. {
  273. current->lockdep_recursion--;
  274. }
  275. EXPORT_SYMBOL(lockdep_on);
  276. /*
  277. * Debugging switches:
  278. */
  279. #define VERBOSE 0
  280. #define VERY_VERBOSE 0
  281. #if VERBOSE
  282. # define HARDIRQ_VERBOSE 1
  283. # define SOFTIRQ_VERBOSE 1
  284. # define RECLAIM_VERBOSE 1
  285. #else
  286. # define HARDIRQ_VERBOSE 0
  287. # define SOFTIRQ_VERBOSE 0
  288. # define RECLAIM_VERBOSE 0
  289. #endif
  290. #if VERBOSE || HARDIRQ_VERBOSE || SOFTIRQ_VERBOSE || RECLAIM_VERBOSE
  291. /*
  292. * Quick filtering for interesting events:
  293. */
  294. static int class_filter(struct lock_class *class)
  295. {
  296. #if 0
  297. /* Example */
  298. if (class->name_version == 1 &&
  299. !strcmp(class->name, "lockname"))
  300. return 1;
  301. if (class->name_version == 1 &&
  302. !strcmp(class->name, "&struct->lockfield"))
  303. return 1;
  304. #endif
  305. /* Filter everything else. 1 would be to allow everything else */
  306. return 0;
  307. }
  308. #endif
  309. static int verbose(struct lock_class *class)
  310. {
  311. #if VERBOSE
  312. return class_filter(class);
  313. #endif
  314. return 0;
  315. }
  316. /*
  317. * Stack-trace: tightly packed array of stack backtrace
  318. * addresses. Protected by the graph_lock.
  319. */
  320. unsigned long nr_stack_trace_entries;
  321. static unsigned long stack_trace[MAX_STACK_TRACE_ENTRIES];
  322. static int save_trace(struct stack_trace *trace)
  323. {
  324. trace->nr_entries = 0;
  325. trace->max_entries = MAX_STACK_TRACE_ENTRIES - nr_stack_trace_entries;
  326. trace->entries = stack_trace + nr_stack_trace_entries;
  327. trace->skip = 3;
  328. save_stack_trace(trace);
  329. /*
  330. * Some daft arches put -1 at the end to indicate its a full trace.
  331. *
  332. * <rant> this is buggy anyway, since it takes a whole extra entry so a
  333. * complete trace that maxes out the entries provided will be reported
  334. * as incomplete, friggin useless </rant>
  335. */
  336. if (trace->nr_entries != 0 &&
  337. trace->entries[trace->nr_entries-1] == ULONG_MAX)
  338. trace->nr_entries--;
  339. trace->max_entries = trace->nr_entries;
  340. nr_stack_trace_entries += trace->nr_entries;
  341. if (nr_stack_trace_entries >= MAX_STACK_TRACE_ENTRIES-1) {
  342. if (!debug_locks_off_graph_unlock())
  343. return 0;
  344. printk("BUG: MAX_STACK_TRACE_ENTRIES too low!\n");
  345. printk("turning off the locking correctness validator.\n");
  346. dump_stack();
  347. return 0;
  348. }
  349. return 1;
  350. }
  351. unsigned int nr_hardirq_chains;
  352. unsigned int nr_softirq_chains;
  353. unsigned int nr_process_chains;
  354. unsigned int max_lockdep_depth;
  355. #ifdef CONFIG_DEBUG_LOCKDEP
  356. /*
  357. * We cannot printk in early bootup code. Not even early_printk()
  358. * might work. So we mark any initialization errors and printk
  359. * about it later on, in lockdep_info().
  360. */
  361. static int lockdep_init_error;
  362. static unsigned long lockdep_init_trace_data[20];
  363. static struct stack_trace lockdep_init_trace = {
  364. .max_entries = ARRAY_SIZE(lockdep_init_trace_data),
  365. .entries = lockdep_init_trace_data,
  366. };
  367. /*
  368. * Various lockdep statistics:
  369. */
  370. DEFINE_PER_CPU(struct lockdep_stats, lockdep_stats);
  371. #endif
  372. /*
  373. * Locking printouts:
  374. */
  375. #define __USAGE(__STATE) \
  376. [LOCK_USED_IN_##__STATE] = "IN-"__stringify(__STATE)"-W", \
  377. [LOCK_ENABLED_##__STATE] = __stringify(__STATE)"-ON-W", \
  378. [LOCK_USED_IN_##__STATE##_READ] = "IN-"__stringify(__STATE)"-R",\
  379. [LOCK_ENABLED_##__STATE##_READ] = __stringify(__STATE)"-ON-R",
  380. static const char *usage_str[] =
  381. {
  382. #define LOCKDEP_STATE(__STATE) __USAGE(__STATE)
  383. #include "lockdep_states.h"
  384. #undef LOCKDEP_STATE
  385. [LOCK_USED] = "INITIAL USE",
  386. };
  387. const char * __get_key_name(struct lockdep_subclass_key *key, char *str)
  388. {
  389. return kallsyms_lookup((unsigned long)key, NULL, NULL, NULL, str);
  390. }
  391. static inline unsigned long lock_flag(enum lock_usage_bit bit)
  392. {
  393. return 1UL << bit;
  394. }
  395. static char get_usage_char(struct lock_class *class, enum lock_usage_bit bit)
  396. {
  397. char c = '.';
  398. if (class->usage_mask & lock_flag(bit + 2))
  399. c = '+';
  400. if (class->usage_mask & lock_flag(bit)) {
  401. c = '-';
  402. if (class->usage_mask & lock_flag(bit + 2))
  403. c = '?';
  404. }
  405. return c;
  406. }
  407. void get_usage_chars(struct lock_class *class, char usage[LOCK_USAGE_CHARS])
  408. {
  409. int i = 0;
  410. #define LOCKDEP_STATE(__STATE) \
  411. usage[i++] = get_usage_char(class, LOCK_USED_IN_##__STATE); \
  412. usage[i++] = get_usage_char(class, LOCK_USED_IN_##__STATE##_READ);
  413. #include "lockdep_states.h"
  414. #undef LOCKDEP_STATE
  415. usage[i] = '\0';
  416. }
  417. static void __print_lock_name(struct lock_class *class)
  418. {
  419. char str[KSYM_NAME_LEN];
  420. const char *name;
  421. name = class->name;
  422. if (!name) {
  423. name = __get_key_name(class->key, str);
  424. printk("%s", name);
  425. } else {
  426. printk("%s", name);
  427. if (class->name_version > 1)
  428. printk("#%d", class->name_version);
  429. if (class->subclass)
  430. printk("/%d", class->subclass);
  431. }
  432. }
  433. static void print_lock_name(struct lock_class *class)
  434. {
  435. char usage[LOCK_USAGE_CHARS];
  436. get_usage_chars(class, usage);
  437. printk(" (");
  438. __print_lock_name(class);
  439. printk("){%s}", usage);
  440. }
  441. static void print_lockdep_cache(struct lockdep_map *lock)
  442. {
  443. const char *name;
  444. char str[KSYM_NAME_LEN];
  445. name = lock->name;
  446. if (!name)
  447. name = __get_key_name(lock->key->subkeys, str);
  448. printk("%s", name);
  449. }
  450. static void print_lock(struct held_lock *hlock)
  451. {
  452. print_lock_name(hlock_class(hlock));
  453. printk(", at: ");
  454. print_ip_sym(hlock->acquire_ip);
  455. }
  456. static void lockdep_print_held_locks(struct task_struct *curr)
  457. {
  458. int i, depth = curr->lockdep_depth;
  459. if (!depth) {
  460. printk("no locks held by %s/%d.\n", curr->comm, task_pid_nr(curr));
  461. return;
  462. }
  463. printk("%d lock%s held by %s/%d:\n",
  464. depth, depth > 1 ? "s" : "", curr->comm, task_pid_nr(curr));
  465. for (i = 0; i < depth; i++) {
  466. printk(" #%d: ", i);
  467. print_lock(curr->held_locks + i);
  468. }
  469. }
  470. static void print_kernel_version(void)
  471. {
  472. printk("%s %.*s\n", init_utsname()->release,
  473. (int)strcspn(init_utsname()->version, " "),
  474. init_utsname()->version);
  475. }
  476. static int very_verbose(struct lock_class *class)
  477. {
  478. #if VERY_VERBOSE
  479. return class_filter(class);
  480. #endif
  481. return 0;
  482. }
  483. /*
  484. * Is this the address of a static object:
  485. */
  486. static int static_obj(void *obj)
  487. {
  488. unsigned long start = (unsigned long) &_stext,
  489. end = (unsigned long) &_end,
  490. addr = (unsigned long) obj;
  491. /*
  492. * static variable?
  493. */
  494. if ((addr >= start) && (addr < end))
  495. return 1;
  496. if (arch_is_kernel_data(addr))
  497. return 1;
  498. /*
  499. * in-kernel percpu var?
  500. */
  501. if (is_kernel_percpu_address(addr))
  502. return 1;
  503. /*
  504. * module static or percpu var?
  505. */
  506. return is_module_address(addr) || is_module_percpu_address(addr);
  507. }
  508. /*
  509. * To make lock name printouts unique, we calculate a unique
  510. * class->name_version generation counter:
  511. */
  512. static int count_matching_names(struct lock_class *new_class)
  513. {
  514. struct lock_class *class;
  515. int count = 0;
  516. if (!new_class->name)
  517. return 0;
  518. list_for_each_entry(class, &all_lock_classes, lock_entry) {
  519. if (new_class->key - new_class->subclass == class->key)
  520. return class->name_version;
  521. if (class->name && !strcmp(class->name, new_class->name))
  522. count = max(count, class->name_version);
  523. }
  524. return count + 1;
  525. }
  526. /*
  527. * Register a lock's class in the hash-table, if the class is not present
  528. * yet. Otherwise we look it up. We cache the result in the lock object
  529. * itself, so actual lookup of the hash should be once per lock object.
  530. */
  531. static inline struct lock_class *
  532. look_up_lock_class(struct lockdep_map *lock, unsigned int subclass)
  533. {
  534. struct lockdep_subclass_key *key;
  535. struct list_head *hash_head;
  536. struct lock_class *class;
  537. #ifdef CONFIG_DEBUG_LOCKDEP
  538. /*
  539. * If the architecture calls into lockdep before initializing
  540. * the hashes then we'll warn about it later. (we cannot printk
  541. * right now)
  542. */
  543. if (unlikely(!lockdep_initialized)) {
  544. lockdep_init();
  545. lockdep_init_error = 1;
  546. save_stack_trace(&lockdep_init_trace);
  547. }
  548. #endif
  549. if (unlikely(subclass >= MAX_LOCKDEP_SUBCLASSES)) {
  550. debug_locks_off();
  551. printk(KERN_ERR
  552. "BUG: looking up invalid subclass: %u\n", subclass);
  553. printk(KERN_ERR
  554. "turning off the locking correctness validator.\n");
  555. dump_stack();
  556. return NULL;
  557. }
  558. /*
  559. * Static locks do not have their class-keys yet - for them the key
  560. * is the lock object itself:
  561. */
  562. if (unlikely(!lock->key))
  563. lock->key = (void *)lock;
  564. /*
  565. * NOTE: the class-key must be unique. For dynamic locks, a static
  566. * lock_class_key variable is passed in through the mutex_init()
  567. * (or spin_lock_init()) call - which acts as the key. For static
  568. * locks we use the lock object itself as the key.
  569. */
  570. BUILD_BUG_ON(sizeof(struct lock_class_key) >
  571. sizeof(struct lockdep_map));
  572. key = lock->key->subkeys + subclass;
  573. hash_head = classhashentry(key);
  574. /*
  575. * We can walk the hash lockfree, because the hash only
  576. * grows, and we are careful when adding entries to the end:
  577. */
  578. list_for_each_entry(class, hash_head, hash_entry) {
  579. if (class->key == key) {
  580. /*
  581. * Huh! same key, different name? Did someone trample
  582. * on some memory? We're most confused.
  583. */
  584. WARN_ON_ONCE(class->name != lock->name);
  585. return class;
  586. }
  587. }
  588. return NULL;
  589. }
  590. /*
  591. * Register a lock's class in the hash-table, if the class is not present
  592. * yet. Otherwise we look it up. We cache the result in the lock object
  593. * itself, so actual lookup of the hash should be once per lock object.
  594. */
  595. static inline struct lock_class *
  596. register_lock_class(struct lockdep_map *lock, unsigned int subclass, int force)
  597. {
  598. struct lockdep_subclass_key *key;
  599. struct list_head *hash_head;
  600. struct lock_class *class;
  601. unsigned long flags;
  602. class = look_up_lock_class(lock, subclass);
  603. if (likely(class))
  604. return class;
  605. /*
  606. * Debug-check: all keys must be persistent!
  607. */
  608. if (!static_obj(lock->key)) {
  609. debug_locks_off();
  610. printk("INFO: trying to register non-static key.\n");
  611. printk("the code is fine but needs lockdep annotation.\n");
  612. printk("turning off the locking correctness validator.\n");
  613. dump_stack();
  614. return NULL;
  615. }
  616. key = lock->key->subkeys + subclass;
  617. hash_head = classhashentry(key);
  618. raw_local_irq_save(flags);
  619. if (!graph_lock()) {
  620. raw_local_irq_restore(flags);
  621. return NULL;
  622. }
  623. /*
  624. * We have to do the hash-walk again, to avoid races
  625. * with another CPU:
  626. */
  627. list_for_each_entry(class, hash_head, hash_entry)
  628. if (class->key == key)
  629. goto out_unlock_set;
  630. /*
  631. * Allocate a new key from the static array, and add it to
  632. * the hash:
  633. */
  634. if (nr_lock_classes >= MAX_LOCKDEP_KEYS) {
  635. if (!debug_locks_off_graph_unlock()) {
  636. raw_local_irq_restore(flags);
  637. return NULL;
  638. }
  639. raw_local_irq_restore(flags);
  640. printk("BUG: MAX_LOCKDEP_KEYS too low!\n");
  641. printk("turning off the locking correctness validator.\n");
  642. dump_stack();
  643. return NULL;
  644. }
  645. class = lock_classes + nr_lock_classes++;
  646. debug_atomic_inc(nr_unused_locks);
  647. class->key = key;
  648. class->name = lock->name;
  649. class->subclass = subclass;
  650. INIT_LIST_HEAD(&class->lock_entry);
  651. INIT_LIST_HEAD(&class->locks_before);
  652. INIT_LIST_HEAD(&class->locks_after);
  653. class->name_version = count_matching_names(class);
  654. /*
  655. * We use RCU's safe list-add method to make
  656. * parallel walking of the hash-list safe:
  657. */
  658. list_add_tail_rcu(&class->hash_entry, hash_head);
  659. /*
  660. * Add it to the global list of classes:
  661. */
  662. list_add_tail_rcu(&class->lock_entry, &all_lock_classes);
  663. if (verbose(class)) {
  664. graph_unlock();
  665. raw_local_irq_restore(flags);
  666. printk("\nnew class %p: %s", class->key, class->name);
  667. if (class->name_version > 1)
  668. printk("#%d", class->name_version);
  669. printk("\n");
  670. dump_stack();
  671. raw_local_irq_save(flags);
  672. if (!graph_lock()) {
  673. raw_local_irq_restore(flags);
  674. return NULL;
  675. }
  676. }
  677. out_unlock_set:
  678. graph_unlock();
  679. raw_local_irq_restore(flags);
  680. if (!subclass || force)
  681. lock->class_cache[0] = class;
  682. else if (subclass < NR_LOCKDEP_CACHING_CLASSES)
  683. lock->class_cache[subclass] = class;
  684. /*
  685. * Hash collision, did we smoke some? We found a class with a matching
  686. * hash but the subclass -- which is hashed in -- didn't match.
  687. */
  688. if (DEBUG_LOCKS_WARN_ON(class->subclass != subclass))
  689. return NULL;
  690. return class;
  691. }
  692. #ifdef CONFIG_PROVE_LOCKING
  693. /*
  694. * Allocate a lockdep entry. (assumes the graph_lock held, returns
  695. * with NULL on failure)
  696. */
  697. static struct lock_list *alloc_list_entry(void)
  698. {
  699. if (nr_list_entries >= MAX_LOCKDEP_ENTRIES) {
  700. if (!debug_locks_off_graph_unlock())
  701. return NULL;
  702. printk("BUG: MAX_LOCKDEP_ENTRIES too low!\n");
  703. printk("turning off the locking correctness validator.\n");
  704. dump_stack();
  705. return NULL;
  706. }
  707. return list_entries + nr_list_entries++;
  708. }
  709. /*
  710. * Add a new dependency to the head of the list:
  711. */
  712. static int add_lock_to_list(struct lock_class *class, struct lock_class *this,
  713. struct list_head *head, unsigned long ip,
  714. int distance, struct stack_trace *trace)
  715. {
  716. struct lock_list *entry;
  717. /*
  718. * Lock not present yet - get a new dependency struct and
  719. * add it to the list:
  720. */
  721. entry = alloc_list_entry();
  722. if (!entry)
  723. return 0;
  724. entry->class = this;
  725. entry->distance = distance;
  726. entry->trace = *trace;
  727. /*
  728. * Since we never remove from the dependency list, the list can
  729. * be walked lockless by other CPUs, it's only allocation
  730. * that must be protected by the spinlock. But this also means
  731. * we must make new entries visible only once writes to the
  732. * entry become visible - hence the RCU op:
  733. */
  734. list_add_tail_rcu(&entry->entry, head);
  735. return 1;
  736. }
  737. /*
  738. * For good efficiency of modular, we use power of 2
  739. */
  740. #define MAX_CIRCULAR_QUEUE_SIZE 4096UL
  741. #define CQ_MASK (MAX_CIRCULAR_QUEUE_SIZE-1)
  742. /*
  743. * The circular_queue and helpers is used to implement the
  744. * breadth-first search(BFS)algorithem, by which we can build
  745. * the shortest path from the next lock to be acquired to the
  746. * previous held lock if there is a circular between them.
  747. */
  748. struct circular_queue {
  749. unsigned long element[MAX_CIRCULAR_QUEUE_SIZE];
  750. unsigned int front, rear;
  751. };
  752. static struct circular_queue lock_cq;
  753. unsigned int max_bfs_queue_depth;
  754. static unsigned int lockdep_dependency_gen_id;
  755. static inline void __cq_init(struct circular_queue *cq)
  756. {
  757. cq->front = cq->rear = 0;
  758. lockdep_dependency_gen_id++;
  759. }
  760. static inline int __cq_empty(struct circular_queue *cq)
  761. {
  762. return (cq->front == cq->rear);
  763. }
  764. static inline int __cq_full(struct circular_queue *cq)
  765. {
  766. return ((cq->rear + 1) & CQ_MASK) == cq->front;
  767. }
  768. static inline int __cq_enqueue(struct circular_queue *cq, unsigned long elem)
  769. {
  770. if (__cq_full(cq))
  771. return -1;
  772. cq->element[cq->rear] = elem;
  773. cq->rear = (cq->rear + 1) & CQ_MASK;
  774. return 0;
  775. }
  776. static inline int __cq_dequeue(struct circular_queue *cq, unsigned long *elem)
  777. {
  778. if (__cq_empty(cq))
  779. return -1;
  780. *elem = cq->element[cq->front];
  781. cq->front = (cq->front + 1) & CQ_MASK;
  782. return 0;
  783. }
  784. static inline unsigned int __cq_get_elem_count(struct circular_queue *cq)
  785. {
  786. return (cq->rear - cq->front) & CQ_MASK;
  787. }
  788. static inline void mark_lock_accessed(struct lock_list *lock,
  789. struct lock_list *parent)
  790. {
  791. unsigned long nr;
  792. nr = lock - list_entries;
  793. WARN_ON(nr >= nr_list_entries); /* Out-of-bounds, input fail */
  794. lock->parent = parent;
  795. lock->class->dep_gen_id = lockdep_dependency_gen_id;
  796. }
  797. static inline unsigned long lock_accessed(struct lock_list *lock)
  798. {
  799. unsigned long nr;
  800. nr = lock - list_entries;
  801. WARN_ON(nr >= nr_list_entries); /* Out-of-bounds, input fail */
  802. return lock->class->dep_gen_id == lockdep_dependency_gen_id;
  803. }
  804. static inline struct lock_list *get_lock_parent(struct lock_list *child)
  805. {
  806. return child->parent;
  807. }
  808. static inline int get_lock_depth(struct lock_list *child)
  809. {
  810. int depth = 0;
  811. struct lock_list *parent;
  812. while ((parent = get_lock_parent(child))) {
  813. child = parent;
  814. depth++;
  815. }
  816. return depth;
  817. }
  818. static int __bfs(struct lock_list *source_entry,
  819. void *data,
  820. int (*match)(struct lock_list *entry, void *data),
  821. struct lock_list **target_entry,
  822. int forward)
  823. {
  824. struct lock_list *entry;
  825. struct list_head *head;
  826. struct circular_queue *cq = &lock_cq;
  827. int ret = 1;
  828. if (match(source_entry, data)) {
  829. *target_entry = source_entry;
  830. ret = 0;
  831. goto exit;
  832. }
  833. if (forward)
  834. head = &source_entry->class->locks_after;
  835. else
  836. head = &source_entry->class->locks_before;
  837. if (list_empty(head))
  838. goto exit;
  839. __cq_init(cq);
  840. __cq_enqueue(cq, (unsigned long)source_entry);
  841. while (!__cq_empty(cq)) {
  842. struct lock_list *lock;
  843. __cq_dequeue(cq, (unsigned long *)&lock);
  844. if (!lock->class) {
  845. ret = -2;
  846. goto exit;
  847. }
  848. if (forward)
  849. head = &lock->class->locks_after;
  850. else
  851. head = &lock->class->locks_before;
  852. list_for_each_entry(entry, head, entry) {
  853. if (!lock_accessed(entry)) {
  854. unsigned int cq_depth;
  855. mark_lock_accessed(entry, lock);
  856. if (match(entry, data)) {
  857. *target_entry = entry;
  858. ret = 0;
  859. goto exit;
  860. }
  861. if (__cq_enqueue(cq, (unsigned long)entry)) {
  862. ret = -1;
  863. goto exit;
  864. }
  865. cq_depth = __cq_get_elem_count(cq);
  866. if (max_bfs_queue_depth < cq_depth)
  867. max_bfs_queue_depth = cq_depth;
  868. }
  869. }
  870. }
  871. exit:
  872. return ret;
  873. }
  874. static inline int __bfs_forwards(struct lock_list *src_entry,
  875. void *data,
  876. int (*match)(struct lock_list *entry, void *data),
  877. struct lock_list **target_entry)
  878. {
  879. return __bfs(src_entry, data, match, target_entry, 1);
  880. }
  881. static inline int __bfs_backwards(struct lock_list *src_entry,
  882. void *data,
  883. int (*match)(struct lock_list *entry, void *data),
  884. struct lock_list **target_entry)
  885. {
  886. return __bfs(src_entry, data, match, target_entry, 0);
  887. }
  888. /*
  889. * Recursive, forwards-direction lock-dependency checking, used for
  890. * both noncyclic checking and for hardirq-unsafe/softirq-unsafe
  891. * checking.
  892. */
  893. /*
  894. * Print a dependency chain entry (this is only done when a deadlock
  895. * has been detected):
  896. */
  897. static noinline int
  898. print_circular_bug_entry(struct lock_list *target, int depth)
  899. {
  900. if (debug_locks_silent)
  901. return 0;
  902. printk("\n-> #%u", depth);
  903. print_lock_name(target->class);
  904. printk(":\n");
  905. print_stack_trace(&target->trace, 6);
  906. return 0;
  907. }
  908. static void
  909. print_circular_lock_scenario(struct held_lock *src,
  910. struct held_lock *tgt,
  911. struct lock_list *prt)
  912. {
  913. struct lock_class *source = hlock_class(src);
  914. struct lock_class *target = hlock_class(tgt);
  915. struct lock_class *parent = prt->class;
  916. /*
  917. * A direct locking problem where unsafe_class lock is taken
  918. * directly by safe_class lock, then all we need to show
  919. * is the deadlock scenario, as it is obvious that the
  920. * unsafe lock is taken under the safe lock.
  921. *
  922. * But if there is a chain instead, where the safe lock takes
  923. * an intermediate lock (middle_class) where this lock is
  924. * not the same as the safe lock, then the lock chain is
  925. * used to describe the problem. Otherwise we would need
  926. * to show a different CPU case for each link in the chain
  927. * from the safe_class lock to the unsafe_class lock.
  928. */
  929. if (parent != source) {
  930. printk("Chain exists of:\n ");
  931. __print_lock_name(source);
  932. printk(" --> ");
  933. __print_lock_name(parent);
  934. printk(" --> ");
  935. __print_lock_name(target);
  936. printk("\n\n");
  937. }
  938. printk(" Possible unsafe locking scenario:\n\n");
  939. printk(" CPU0 CPU1\n");
  940. printk(" ---- ----\n");
  941. printk(" lock(");
  942. __print_lock_name(target);
  943. printk(");\n");
  944. printk(" lock(");
  945. __print_lock_name(parent);
  946. printk(");\n");
  947. printk(" lock(");
  948. __print_lock_name(target);
  949. printk(");\n");
  950. printk(" lock(");
  951. __print_lock_name(source);
  952. printk(");\n");
  953. printk("\n *** DEADLOCK ***\n\n");
  954. }
  955. /*
  956. * When a circular dependency is detected, print the
  957. * header first:
  958. */
  959. static noinline int
  960. print_circular_bug_header(struct lock_list *entry, unsigned int depth,
  961. struct held_lock *check_src,
  962. struct held_lock *check_tgt)
  963. {
  964. struct task_struct *curr = current;
  965. if (debug_locks_silent)
  966. return 0;
  967. printk("\n");
  968. printk("======================================================\n");
  969. printk("[ INFO: possible circular locking dependency detected ]\n");
  970. print_kernel_version();
  971. printk("-------------------------------------------------------\n");
  972. printk("%s/%d is trying to acquire lock:\n",
  973. curr->comm, task_pid_nr(curr));
  974. print_lock(check_src);
  975. printk("\nbut task is already holding lock:\n");
  976. print_lock(check_tgt);
  977. printk("\nwhich lock already depends on the new lock.\n\n");
  978. printk("\nthe existing dependency chain (in reverse order) is:\n");
  979. print_circular_bug_entry(entry, depth);
  980. return 0;
  981. }
  982. static inline int class_equal(struct lock_list *entry, void *data)
  983. {
  984. return entry->class == data;
  985. }
  986. static noinline int print_circular_bug(struct lock_list *this,
  987. struct lock_list *target,
  988. struct held_lock *check_src,
  989. struct held_lock *check_tgt)
  990. {
  991. struct task_struct *curr = current;
  992. struct lock_list *parent;
  993. struct lock_list *first_parent;
  994. int depth;
  995. if (!debug_locks_off_graph_unlock() || debug_locks_silent)
  996. return 0;
  997. if (!save_trace(&this->trace))
  998. return 0;
  999. depth = get_lock_depth(target);
  1000. print_circular_bug_header(target, depth, check_src, check_tgt);
  1001. parent = get_lock_parent(target);
  1002. first_parent = parent;
  1003. while (parent) {
  1004. print_circular_bug_entry(parent, --depth);
  1005. parent = get_lock_parent(parent);
  1006. }
  1007. printk("\nother info that might help us debug this:\n\n");
  1008. print_circular_lock_scenario(check_src, check_tgt,
  1009. first_parent);
  1010. lockdep_print_held_locks(curr);
  1011. printk("\nstack backtrace:\n");
  1012. dump_stack();
  1013. return 0;
  1014. }
  1015. static noinline int print_bfs_bug(int ret)
  1016. {
  1017. if (!debug_locks_off_graph_unlock())
  1018. return 0;
  1019. /*
  1020. * Breadth-first-search failed, graph got corrupted?
  1021. */
  1022. WARN(1, "lockdep bfs error:%d\n", ret);
  1023. return 0;
  1024. }
  1025. static int noop_count(struct lock_list *entry, void *data)
  1026. {
  1027. (*(unsigned long *)data)++;
  1028. return 0;
  1029. }
  1030. unsigned long __lockdep_count_forward_deps(struct lock_list *this)
  1031. {
  1032. unsigned long count = 0;
  1033. struct lock_list *uninitialized_var(target_entry);
  1034. __bfs_forwards(this, (void *)&count, noop_count, &target_entry);
  1035. return count;
  1036. }
  1037. unsigned long lockdep_count_forward_deps(struct lock_class *class)
  1038. {
  1039. unsigned long ret, flags;
  1040. struct lock_list this;
  1041. this.parent = NULL;
  1042. this.class = class;
  1043. local_irq_save(flags);
  1044. arch_spin_lock(&lockdep_lock);
  1045. ret = __lockdep_count_forward_deps(&this);
  1046. arch_spin_unlock(&lockdep_lock);
  1047. local_irq_restore(flags);
  1048. return ret;
  1049. }
  1050. unsigned long __lockdep_count_backward_deps(struct lock_list *this)
  1051. {
  1052. unsigned long count = 0;
  1053. struct lock_list *uninitialized_var(target_entry);
  1054. __bfs_backwards(this, (void *)&count, noop_count, &target_entry);
  1055. return count;
  1056. }
  1057. unsigned long lockdep_count_backward_deps(struct lock_class *class)
  1058. {
  1059. unsigned long ret, flags;
  1060. struct lock_list this;
  1061. this.parent = NULL;
  1062. this.class = class;
  1063. local_irq_save(flags);
  1064. arch_spin_lock(&lockdep_lock);
  1065. ret = __lockdep_count_backward_deps(&this);
  1066. arch_spin_unlock(&lockdep_lock);
  1067. local_irq_restore(flags);
  1068. return ret;
  1069. }
  1070. /*
  1071. * Prove that the dependency graph starting at <entry> can not
  1072. * lead to <target>. Print an error and return 0 if it does.
  1073. */
  1074. static noinline int
  1075. check_noncircular(struct lock_list *root, struct lock_class *target,
  1076. struct lock_list **target_entry)
  1077. {
  1078. int result;
  1079. debug_atomic_inc(nr_cyclic_checks);
  1080. result = __bfs_forwards(root, target, class_equal, target_entry);
  1081. return result;
  1082. }
  1083. #if defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_PROVE_LOCKING)
  1084. /*
  1085. * Forwards and backwards subgraph searching, for the purposes of
  1086. * proving that two subgraphs can be connected by a new dependency
  1087. * without creating any illegal irq-safe -> irq-unsafe lock dependency.
  1088. */
  1089. static inline int usage_match(struct lock_list *entry, void *bit)
  1090. {
  1091. return entry->class->usage_mask & (1 << (enum lock_usage_bit)bit);
  1092. }
  1093. /*
  1094. * Find a node in the forwards-direction dependency sub-graph starting
  1095. * at @root->class that matches @bit.
  1096. *
  1097. * Return 0 if such a node exists in the subgraph, and put that node
  1098. * into *@target_entry.
  1099. *
  1100. * Return 1 otherwise and keep *@target_entry unchanged.
  1101. * Return <0 on error.
  1102. */
  1103. static int
  1104. find_usage_forwards(struct lock_list *root, enum lock_usage_bit bit,
  1105. struct lock_list **target_entry)
  1106. {
  1107. int result;
  1108. debug_atomic_inc(nr_find_usage_forwards_checks);
  1109. result = __bfs_forwards(root, (void *)bit, usage_match, target_entry);
  1110. return result;
  1111. }
  1112. /*
  1113. * Find a node in the backwards-direction dependency sub-graph starting
  1114. * at @root->class that matches @bit.
  1115. *
  1116. * Return 0 if such a node exists in the subgraph, and put that node
  1117. * into *@target_entry.
  1118. *
  1119. * Return 1 otherwise and keep *@target_entry unchanged.
  1120. * Return <0 on error.
  1121. */
  1122. static int
  1123. find_usage_backwards(struct lock_list *root, enum lock_usage_bit bit,
  1124. struct lock_list **target_entry)
  1125. {
  1126. int result;
  1127. debug_atomic_inc(nr_find_usage_backwards_checks);
  1128. result = __bfs_backwards(root, (void *)bit, usage_match, target_entry);
  1129. return result;
  1130. }
  1131. static void print_lock_class_header(struct lock_class *class, int depth)
  1132. {
  1133. int bit;
  1134. printk("%*s->", depth, "");
  1135. print_lock_name(class);
  1136. printk(" ops: %lu", class->ops);
  1137. printk(" {\n");
  1138. for (bit = 0; bit < LOCK_USAGE_STATES; bit++) {
  1139. if (class->usage_mask & (1 << bit)) {
  1140. int len = depth;
  1141. len += printk("%*s %s", depth, "", usage_str[bit]);
  1142. len += printk(" at:\n");
  1143. print_stack_trace(class->usage_traces + bit, len);
  1144. }
  1145. }
  1146. printk("%*s }\n", depth, "");
  1147. printk("%*s ... key at: ",depth,"");
  1148. print_ip_sym((unsigned long)class->key);
  1149. }
  1150. /*
  1151. * printk the shortest lock dependencies from @start to @end in reverse order:
  1152. */
  1153. static void __used
  1154. print_shortest_lock_dependencies(struct lock_list *leaf,
  1155. struct lock_list *root)
  1156. {
  1157. struct lock_list *entry = leaf;
  1158. int depth;
  1159. /*compute depth from generated tree by BFS*/
  1160. depth = get_lock_depth(leaf);
  1161. do {
  1162. print_lock_class_header(entry->class, depth);
  1163. printk("%*s ... acquired at:\n", depth, "");
  1164. print_stack_trace(&entry->trace, 2);
  1165. printk("\n");
  1166. if (depth == 0 && (entry != root)) {
  1167. printk("lockdep:%s bad path found in chain graph\n", __func__);
  1168. break;
  1169. }
  1170. entry = get_lock_parent(entry);
  1171. depth--;
  1172. } while (entry && (depth >= 0));
  1173. return;
  1174. }
  1175. static void
  1176. print_irq_lock_scenario(struct lock_list *safe_entry,
  1177. struct lock_list *unsafe_entry,
  1178. struct lock_class *prev_class,
  1179. struct lock_class *next_class)
  1180. {
  1181. struct lock_class *safe_class = safe_entry->class;
  1182. struct lock_class *unsafe_class = unsafe_entry->class;
  1183. struct lock_class *middle_class = prev_class;
  1184. if (middle_class == safe_class)
  1185. middle_class = next_class;
  1186. /*
  1187. * A direct locking problem where unsafe_class lock is taken
  1188. * directly by safe_class lock, then all we need to show
  1189. * is the deadlock scenario, as it is obvious that the
  1190. * unsafe lock is taken under the safe lock.
  1191. *
  1192. * But if there is a chain instead, where the safe lock takes
  1193. * an intermediate lock (middle_class) where this lock is
  1194. * not the same as the safe lock, then the lock chain is
  1195. * used to describe the problem. Otherwise we would need
  1196. * to show a different CPU case for each link in the chain
  1197. * from the safe_class lock to the unsafe_class lock.
  1198. */
  1199. if (middle_class != unsafe_class) {
  1200. printk("Chain exists of:\n ");
  1201. __print_lock_name(safe_class);
  1202. printk(" --> ");
  1203. __print_lock_name(middle_class);
  1204. printk(" --> ");
  1205. __print_lock_name(unsafe_class);
  1206. printk("\n\n");
  1207. }
  1208. printk(" Possible interrupt unsafe locking scenario:\n\n");
  1209. printk(" CPU0 CPU1\n");
  1210. printk(" ---- ----\n");
  1211. printk(" lock(");
  1212. __print_lock_name(unsafe_class);
  1213. printk(");\n");
  1214. printk(" local_irq_disable();\n");
  1215. printk(" lock(");
  1216. __print_lock_name(safe_class);
  1217. printk(");\n");
  1218. printk(" lock(");
  1219. __print_lock_name(middle_class);
  1220. printk(");\n");
  1221. printk(" <Interrupt>\n");
  1222. printk(" lock(");
  1223. __print_lock_name(safe_class);
  1224. printk(");\n");
  1225. printk("\n *** DEADLOCK ***\n\n");
  1226. }
  1227. static int
  1228. print_bad_irq_dependency(struct task_struct *curr,
  1229. struct lock_list *prev_root,
  1230. struct lock_list *next_root,
  1231. struct lock_list *backwards_entry,
  1232. struct lock_list *forwards_entry,
  1233. struct held_lock *prev,
  1234. struct held_lock *next,
  1235. enum lock_usage_bit bit1,
  1236. enum lock_usage_bit bit2,
  1237. const char *irqclass)
  1238. {
  1239. if (!debug_locks_off_graph_unlock() || debug_locks_silent)
  1240. return 0;
  1241. printk("\n");
  1242. printk("======================================================\n");
  1243. printk("[ INFO: %s-safe -> %s-unsafe lock order detected ]\n",
  1244. irqclass, irqclass);
  1245. print_kernel_version();
  1246. printk("------------------------------------------------------\n");
  1247. printk("%s/%d [HC%u[%lu]:SC%u[%lu]:HE%u:SE%u] is trying to acquire:\n",
  1248. curr->comm, task_pid_nr(curr),
  1249. curr->hardirq_context, hardirq_count() >> HARDIRQ_SHIFT,
  1250. curr->softirq_context, softirq_count() >> SOFTIRQ_SHIFT,
  1251. curr->hardirqs_enabled,
  1252. curr->softirqs_enabled);
  1253. print_lock(next);
  1254. printk("\nand this task is already holding:\n");
  1255. print_lock(prev);
  1256. printk("which would create a new lock dependency:\n");
  1257. print_lock_name(hlock_class(prev));
  1258. printk(" ->");
  1259. print_lock_name(hlock_class(next));
  1260. printk("\n");
  1261. printk("\nbut this new dependency connects a %s-irq-safe lock:\n",
  1262. irqclass);
  1263. print_lock_name(backwards_entry->class);
  1264. printk("\n... which became %s-irq-safe at:\n", irqclass);
  1265. print_stack_trace(backwards_entry->class->usage_traces + bit1, 1);
  1266. printk("\nto a %s-irq-unsafe lock:\n", irqclass);
  1267. print_lock_name(forwards_entry->class);
  1268. printk("\n... which became %s-irq-unsafe at:\n", irqclass);
  1269. printk("...");
  1270. print_stack_trace(forwards_entry->class->usage_traces + bit2, 1);
  1271. printk("\nother info that might help us debug this:\n\n");
  1272. print_irq_lock_scenario(backwards_entry, forwards_entry,
  1273. hlock_class(prev), hlock_class(next));
  1274. lockdep_print_held_locks(curr);
  1275. printk("\nthe dependencies between %s-irq-safe lock", irqclass);
  1276. printk(" and the holding lock:\n");
  1277. if (!save_trace(&prev_root->trace))
  1278. return 0;
  1279. print_shortest_lock_dependencies(backwards_entry, prev_root);
  1280. printk("\nthe dependencies between the lock to be acquired");
  1281. printk(" and %s-irq-unsafe lock:\n", irqclass);
  1282. if (!save_trace(&next_root->trace))
  1283. return 0;
  1284. print_shortest_lock_dependencies(forwards_entry, next_root);
  1285. printk("\nstack backtrace:\n");
  1286. dump_stack();
  1287. return 0;
  1288. }
  1289. static int
  1290. check_usage(struct task_struct *curr, struct held_lock *prev,
  1291. struct held_lock *next, enum lock_usage_bit bit_backwards,
  1292. enum lock_usage_bit bit_forwards, const char *irqclass)
  1293. {
  1294. int ret;
  1295. struct lock_list this, that;
  1296. struct lock_list *uninitialized_var(target_entry);
  1297. struct lock_list *uninitialized_var(target_entry1);
  1298. this.parent = NULL;
  1299. this.class = hlock_class(prev);
  1300. ret = find_usage_backwards(&this, bit_backwards, &target_entry);
  1301. if (ret < 0)
  1302. return print_bfs_bug(ret);
  1303. if (ret == 1)
  1304. return ret;
  1305. that.parent = NULL;
  1306. that.class = hlock_class(next);
  1307. ret = find_usage_forwards(&that, bit_forwards, &target_entry1);
  1308. if (ret < 0)
  1309. return print_bfs_bug(ret);
  1310. if (ret == 1)
  1311. return ret;
  1312. return print_bad_irq_dependency(curr, &this, &that,
  1313. target_entry, target_entry1,
  1314. prev, next,
  1315. bit_backwards, bit_forwards, irqclass);
  1316. }
  1317. static const char *state_names[] = {
  1318. #define LOCKDEP_STATE(__STATE) \
  1319. __stringify(__STATE),
  1320. #include "lockdep_states.h"
  1321. #undef LOCKDEP_STATE
  1322. };
  1323. static const char *state_rnames[] = {
  1324. #define LOCKDEP_STATE(__STATE) \
  1325. __stringify(__STATE)"-READ",
  1326. #include "lockdep_states.h"
  1327. #undef LOCKDEP_STATE
  1328. };
  1329. static inline const char *state_name(enum lock_usage_bit bit)
  1330. {
  1331. return (bit & 1) ? state_rnames[bit >> 2] : state_names[bit >> 2];
  1332. }
  1333. static int exclusive_bit(int new_bit)
  1334. {
  1335. /*
  1336. * USED_IN
  1337. * USED_IN_READ
  1338. * ENABLED
  1339. * ENABLED_READ
  1340. *
  1341. * bit 0 - write/read
  1342. * bit 1 - used_in/enabled
  1343. * bit 2+ state
  1344. */
  1345. int state = new_bit & ~3;
  1346. int dir = new_bit & 2;
  1347. /*
  1348. * keep state, bit flip the direction and strip read.
  1349. */
  1350. return state | (dir ^ 2);
  1351. }
  1352. static int check_irq_usage(struct task_struct *curr, struct held_lock *prev,
  1353. struct held_lock *next, enum lock_usage_bit bit)
  1354. {
  1355. /*
  1356. * Prove that the new dependency does not connect a hardirq-safe
  1357. * lock with a hardirq-unsafe lock - to achieve this we search
  1358. * the backwards-subgraph starting at <prev>, and the
  1359. * forwards-subgraph starting at <next>:
  1360. */
  1361. if (!check_usage(curr, prev, next, bit,
  1362. exclusive_bit(bit), state_name(bit)))
  1363. return 0;
  1364. bit++; /* _READ */
  1365. /*
  1366. * Prove that the new dependency does not connect a hardirq-safe-read
  1367. * lock with a hardirq-unsafe lock - to achieve this we search
  1368. * the backwards-subgraph starting at <prev>, and the
  1369. * forwards-subgraph starting at <next>:
  1370. */
  1371. if (!check_usage(curr, prev, next, bit,
  1372. exclusive_bit(bit), state_name(bit)))
  1373. return 0;
  1374. return 1;
  1375. }
  1376. static int
  1377. check_prev_add_irq(struct task_struct *curr, struct held_lock *prev,
  1378. struct held_lock *next)
  1379. {
  1380. #define LOCKDEP_STATE(__STATE) \
  1381. if (!check_irq_usage(curr, prev, next, LOCK_USED_IN_##__STATE)) \
  1382. return 0;
  1383. #include "lockdep_states.h"
  1384. #undef LOCKDEP_STATE
  1385. return 1;
  1386. }
  1387. static void inc_chains(void)
  1388. {
  1389. if (current->hardirq_context)
  1390. nr_hardirq_chains++;
  1391. else {
  1392. if (current->softirq_context)
  1393. nr_softirq_chains++;
  1394. else
  1395. nr_process_chains++;
  1396. }
  1397. }
  1398. #else
  1399. static inline int
  1400. check_prev_add_irq(struct task_struct *curr, struct held_lock *prev,
  1401. struct held_lock *next)
  1402. {
  1403. return 1;
  1404. }
  1405. static inline void inc_chains(void)
  1406. {
  1407. nr_process_chains++;
  1408. }
  1409. #endif
  1410. static void
  1411. print_deadlock_scenario(struct held_lock *nxt,
  1412. struct held_lock *prv)
  1413. {
  1414. struct lock_class *next = hlock_class(nxt);
  1415. struct lock_class *prev = hlock_class(prv);
  1416. printk(" Possible unsafe locking scenario:\n\n");
  1417. printk(" CPU0\n");
  1418. printk(" ----\n");
  1419. printk(" lock(");
  1420. __print_lock_name(prev);
  1421. printk(");\n");
  1422. printk(" lock(");
  1423. __print_lock_name(next);
  1424. printk(");\n");
  1425. printk("\n *** DEADLOCK ***\n\n");
  1426. printk(" May be due to missing lock nesting notation\n\n");
  1427. }
  1428. static int
  1429. print_deadlock_bug(struct task_struct *curr, struct held_lock *prev,
  1430. struct held_lock *next)
  1431. {
  1432. if (!debug_locks_off_graph_unlock() || debug_locks_silent)
  1433. return 0;
  1434. printk("\n");
  1435. printk("=============================================\n");
  1436. printk("[ INFO: possible recursive locking detected ]\n");
  1437. print_kernel_version();
  1438. printk("---------------------------------------------\n");
  1439. printk("%s/%d is trying to acquire lock:\n",
  1440. curr->comm, task_pid_nr(curr));
  1441. print_lock(next);
  1442. printk("\nbut task is already holding lock:\n");
  1443. print_lock(prev);
  1444. printk("\nother info that might help us debug this:\n");
  1445. print_deadlock_scenario(next, prev);
  1446. lockdep_print_held_locks(curr);
  1447. printk("\nstack backtrace:\n");
  1448. dump_stack();
  1449. return 0;
  1450. }
  1451. /*
  1452. * Check whether we are holding such a class already.
  1453. *
  1454. * (Note that this has to be done separately, because the graph cannot
  1455. * detect such classes of deadlocks.)
  1456. *
  1457. * Returns: 0 on deadlock detected, 1 on OK, 2 on recursive read
  1458. */
  1459. static int
  1460. check_deadlock(struct task_struct *curr, struct held_lock *next,
  1461. struct lockdep_map *next_instance, int read)
  1462. {
  1463. struct held_lock *prev;
  1464. struct held_lock *nest = NULL;
  1465. int i;
  1466. for (i = 0; i < curr->lockdep_depth; i++) {
  1467. prev = curr->held_locks + i;
  1468. if (prev->instance == next->nest_lock)
  1469. nest = prev;
  1470. if (hlock_class(prev) != hlock_class(next))
  1471. continue;
  1472. /*
  1473. * Allow read-after-read recursion of the same
  1474. * lock class (i.e. read_lock(lock)+read_lock(lock)):
  1475. */
  1476. if ((read == 2) && prev->read)
  1477. return 2;
  1478. /*
  1479. * We're holding the nest_lock, which serializes this lock's
  1480. * nesting behaviour.
  1481. */
  1482. if (nest)
  1483. return 2;
  1484. return print_deadlock_bug(curr, prev, next);
  1485. }
  1486. return 1;
  1487. }
  1488. /*
  1489. * There was a chain-cache miss, and we are about to add a new dependency
  1490. * to a previous lock. We recursively validate the following rules:
  1491. *
  1492. * - would the adding of the <prev> -> <next> dependency create a
  1493. * circular dependency in the graph? [== circular deadlock]
  1494. *
  1495. * - does the new prev->next dependency connect any hardirq-safe lock
  1496. * (in the full backwards-subgraph starting at <prev>) with any
  1497. * hardirq-unsafe lock (in the full forwards-subgraph starting at
  1498. * <next>)? [== illegal lock inversion with hardirq contexts]
  1499. *
  1500. * - does the new prev->next dependency connect any softirq-safe lock
  1501. * (in the full backwards-subgraph starting at <prev>) with any
  1502. * softirq-unsafe lock (in the full forwards-subgraph starting at
  1503. * <next>)? [== illegal lock inversion with softirq contexts]
  1504. *
  1505. * any of these scenarios could lead to a deadlock.
  1506. *
  1507. * Then if all the validations pass, we add the forwards and backwards
  1508. * dependency.
  1509. */
  1510. static int
  1511. check_prev_add(struct task_struct *curr, struct held_lock *prev,
  1512. struct held_lock *next, int distance, int trylock_loop)
  1513. {
  1514. struct lock_list *entry;
  1515. int ret;
  1516. struct lock_list this;
  1517. struct lock_list *uninitialized_var(target_entry);
  1518. /*
  1519. * Static variable, serialized by the graph_lock().
  1520. *
  1521. * We use this static variable to save the stack trace in case
  1522. * we call into this function multiple times due to encountering
  1523. * trylocks in the held lock stack.
  1524. */
  1525. static struct stack_trace trace;
  1526. /*
  1527. * Prove that the new <prev> -> <next> dependency would not
  1528. * create a circular dependency in the graph. (We do this by
  1529. * forward-recursing into the graph starting at <next>, and
  1530. * checking whether we can reach <prev>.)
  1531. *
  1532. * We are using global variables to control the recursion, to
  1533. * keep the stackframe size of the recursive functions low:
  1534. */
  1535. this.class = hlock_class(next);
  1536. this.parent = NULL;
  1537. ret = check_noncircular(&this, hlock_class(prev), &target_entry);
  1538. if (unlikely(!ret))
  1539. return print_circular_bug(&this, target_entry, next, prev);
  1540. else if (unlikely(ret < 0))
  1541. return print_bfs_bug(ret);
  1542. if (!check_prev_add_irq(curr, prev, next))
  1543. return 0;
  1544. /*
  1545. * For recursive read-locks we do all the dependency checks,
  1546. * but we dont store read-triggered dependencies (only
  1547. * write-triggered dependencies). This ensures that only the
  1548. * write-side dependencies matter, and that if for example a
  1549. * write-lock never takes any other locks, then the reads are
  1550. * equivalent to a NOP.
  1551. */
  1552. if (next->read == 2 || prev->read == 2)
  1553. return 1;
  1554. /*
  1555. * Is the <prev> -> <next> dependency already present?
  1556. *
  1557. * (this may occur even though this is a new chain: consider
  1558. * e.g. the L1 -> L2 -> L3 -> L4 and the L5 -> L1 -> L2 -> L3
  1559. * chains - the second one will be new, but L1 already has
  1560. * L2 added to its dependency list, due to the first chain.)
  1561. */
  1562. list_for_each_entry(entry, &hlock_class(prev)->locks_after, entry) {
  1563. if (entry->class == hlock_class(next)) {
  1564. if (distance == 1)
  1565. entry->distance = 1;
  1566. return 2;
  1567. }
  1568. }
  1569. if (!trylock_loop && !save_trace(&trace))
  1570. return 0;
  1571. /*
  1572. * Ok, all validations passed, add the new lock
  1573. * to the previous lock's dependency list:
  1574. */
  1575. ret = add_lock_to_list(hlock_class(prev), hlock_class(next),
  1576. &hlock_class(prev)->locks_after,
  1577. next->acquire_ip, distance, &trace);
  1578. if (!ret)
  1579. return 0;
  1580. ret = add_lock_to_list(hlock_class(next), hlock_class(prev),
  1581. &hlock_class(next)->locks_before,
  1582. next->acquire_ip, distance, &trace);
  1583. if (!ret)
  1584. return 0;
  1585. /*
  1586. * Debugging printouts:
  1587. */
  1588. if (verbose(hlock_class(prev)) || verbose(hlock_class(next))) {
  1589. graph_unlock();
  1590. printk("\n new dependency: ");
  1591. print_lock_name(hlock_class(prev));
  1592. printk(" => ");
  1593. print_lock_name(hlock_class(next));
  1594. printk("\n");
  1595. dump_stack();
  1596. return graph_lock();
  1597. }
  1598. return 1;
  1599. }
  1600. /*
  1601. * Add the dependency to all directly-previous locks that are 'relevant'.
  1602. * The ones that are relevant are (in increasing distance from curr):
  1603. * all consecutive trylock entries and the final non-trylock entry - or
  1604. * the end of this context's lock-chain - whichever comes first.
  1605. */
  1606. static int
  1607. check_prevs_add(struct task_struct *curr, struct held_lock *next)
  1608. {
  1609. int depth = curr->lockdep_depth;
  1610. int trylock_loop = 0;
  1611. struct held_lock *hlock;
  1612. /*
  1613. * Debugging checks.
  1614. *
  1615. * Depth must not be zero for a non-head lock:
  1616. */
  1617. if (!depth)
  1618. goto out_bug;
  1619. /*
  1620. * At least two relevant locks must exist for this
  1621. * to be a head:
  1622. */
  1623. if (curr->held_locks[depth].irq_context !=
  1624. curr->held_locks[depth-1].irq_context)
  1625. goto out_bug;
  1626. for (;;) {
  1627. int distance = curr->lockdep_depth - depth + 1;
  1628. hlock = curr->held_locks + depth-1;
  1629. /*
  1630. * Only non-recursive-read entries get new dependencies
  1631. * added:
  1632. */
  1633. if (hlock->read != 2) {
  1634. if (!check_prev_add(curr, hlock, next,
  1635. distance, trylock_loop))
  1636. return 0;
  1637. /*
  1638. * Stop after the first non-trylock entry,
  1639. * as non-trylock entries have added their
  1640. * own direct dependencies already, so this
  1641. * lock is connected to them indirectly:
  1642. */
  1643. if (!hlock->trylock)
  1644. break;
  1645. }
  1646. depth--;
  1647. /*
  1648. * End of lock-stack?
  1649. */
  1650. if (!depth)
  1651. break;
  1652. /*
  1653. * Stop the search if we cross into another context:
  1654. */
  1655. if (curr->held_locks[depth].irq_context !=
  1656. curr->held_locks[depth-1].irq_context)
  1657. break;
  1658. trylock_loop = 1;
  1659. }
  1660. return 1;
  1661. out_bug:
  1662. if (!debug_locks_off_graph_unlock())
  1663. return 0;
  1664. /*
  1665. * Clearly we all shouldn't be here, but since we made it we
  1666. * can reliable say we messed up our state. See the above two
  1667. * gotos for reasons why we could possibly end up here.
  1668. */
  1669. WARN_ON(1);
  1670. return 0;
  1671. }
  1672. unsigned long nr_lock_chains;
  1673. struct lock_chain lock_chains[MAX_LOCKDEP_CHAINS];
  1674. int nr_chain_hlocks;
  1675. static u16 chain_hlocks[MAX_LOCKDEP_CHAIN_HLOCKS];
  1676. struct lock_class *lock_chain_get_class(struct lock_chain *chain, int i)
  1677. {
  1678. return lock_classes + chain_hlocks[chain->base + i];
  1679. }
  1680. /*
  1681. * Look up a dependency chain. If the key is not present yet then
  1682. * add it and return 1 - in this case the new dependency chain is
  1683. * validated. If the key is already hashed, return 0.
  1684. * (On return with 1 graph_lock is held.)
  1685. */
  1686. static inline int lookup_chain_cache(struct task_struct *curr,
  1687. struct held_lock *hlock,
  1688. u64 chain_key)
  1689. {
  1690. struct lock_class *class = hlock_class(hlock);
  1691. struct list_head *hash_head = chainhashentry(chain_key);
  1692. struct lock_chain *chain;
  1693. struct held_lock *hlock_curr, *hlock_next;
  1694. int i, j;
  1695. /*
  1696. * We might need to take the graph lock, ensure we've got IRQs
  1697. * disabled to make this an IRQ-safe lock.. for recursion reasons
  1698. * lockdep won't complain about its own locking errors.
  1699. */
  1700. if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
  1701. return 0;
  1702. /*
  1703. * We can walk it lock-free, because entries only get added
  1704. * to the hash:
  1705. */
  1706. list_for_each_entry(chain, hash_head, entry) {
  1707. if (chain->chain_key == chain_key) {
  1708. cache_hit:
  1709. debug_atomic_inc(chain_lookup_hits);
  1710. if (very_verbose(class))
  1711. printk("\nhash chain already cached, key: "
  1712. "%016Lx tail class: [%p] %s\n",
  1713. (unsigned long long)chain_key,
  1714. class->key, class->name);
  1715. return 0;
  1716. }
  1717. }
  1718. if (very_verbose(class))
  1719. printk("\nnew hash chain, key: %016Lx tail class: [%p] %s\n",
  1720. (unsigned long long)chain_key, class->key, class->name);
  1721. /*
  1722. * Allocate a new chain entry from the static array, and add
  1723. * it to the hash:
  1724. */
  1725. if (!graph_lock())
  1726. return 0;
  1727. /*
  1728. * We have to walk the chain again locked - to avoid duplicates:
  1729. */
  1730. list_for_each_entry(chain, hash_head, entry) {
  1731. if (chain->chain_key == chain_key) {
  1732. graph_unlock();
  1733. goto cache_hit;
  1734. }
  1735. }
  1736. if (unlikely(nr_lock_chains >= MAX_LOCKDEP_CHAINS)) {
  1737. if (!debug_locks_off_graph_unlock())
  1738. return 0;
  1739. printk("BUG: MAX_LOCKDEP_CHAINS too low!\n");
  1740. printk("turning off the locking correctness validator.\n");
  1741. dump_stack();
  1742. return 0;
  1743. }
  1744. chain = lock_chains + nr_lock_chains++;
  1745. chain->chain_key = chain_key;
  1746. chain->irq_context = hlock->irq_context;
  1747. /* Find the first held_lock of current chain */
  1748. hlock_next = hlock;
  1749. for (i = curr->lockdep_depth - 1; i >= 0; i--) {
  1750. hlock_curr = curr->held_locks + i;
  1751. if (hlock_curr->irq_context != hlock_next->irq_context)
  1752. break;
  1753. hlock_next = hlock;
  1754. }
  1755. i++;
  1756. chain->depth = curr->lockdep_depth + 1 - i;
  1757. if (likely(nr_chain_hlocks + chain->depth <= MAX_LOCKDEP_CHAIN_HLOCKS)) {
  1758. chain->base = nr_chain_hlocks;
  1759. nr_chain_hlocks += chain->depth;
  1760. for (j = 0; j < chain->depth - 1; j++, i++) {
  1761. int lock_id = curr->held_locks[i].class_idx - 1;
  1762. chain_hlocks[chain->base + j] = lock_id;
  1763. }
  1764. chain_hlocks[chain->base + j] = class - lock_classes;
  1765. }
  1766. list_add_tail_rcu(&chain->entry, hash_head);
  1767. debug_atomic_inc(chain_lookup_misses);
  1768. inc_chains();
  1769. return 1;
  1770. }
  1771. static int validate_chain(struct task_struct *curr, struct lockdep_map *lock,
  1772. struct held_lock *hlock, int chain_head, u64 chain_key)
  1773. {
  1774. /*
  1775. * Trylock needs to maintain the stack of held locks, but it
  1776. * does not add new dependencies, because trylock can be done
  1777. * in any order.
  1778. *
  1779. * We look up the chain_key and do the O(N^2) check and update of
  1780. * the dependencies only if this is a new dependency chain.
  1781. * (If lookup_chain_cache() returns with 1 it acquires
  1782. * graph_lock for us)
  1783. */
  1784. if (!hlock->trylock && (hlock->check == 2) &&
  1785. lookup_chain_cache(curr, hlock, chain_key)) {
  1786. /*
  1787. * Check whether last held lock:
  1788. *
  1789. * - is irq-safe, if this lock is irq-unsafe
  1790. * - is softirq-safe, if this lock is hardirq-unsafe
  1791. *
  1792. * And check whether the new lock's dependency graph
  1793. * could lead back to the previous lock.
  1794. *
  1795. * any of these scenarios could lead to a deadlock. If
  1796. * All validations
  1797. */
  1798. int ret = check_deadlock(curr, hlock, lock, hlock->read);
  1799. if (!ret)
  1800. return 0;
  1801. /*
  1802. * Mark recursive read, as we jump over it when
  1803. * building dependencies (just like we jump over
  1804. * trylock entries):
  1805. */
  1806. if (ret == 2)
  1807. hlock->read = 2;
  1808. /*
  1809. * Add dependency only if this lock is not the head
  1810. * of the chain, and if it's not a secondary read-lock:
  1811. */
  1812. if (!chain_head && ret != 2)
  1813. if (!check_prevs_add(curr, hlock))
  1814. return 0;
  1815. graph_unlock();
  1816. } else
  1817. /* after lookup_chain_cache(): */
  1818. if (unlikely(!debug_locks))
  1819. return 0;
  1820. return 1;
  1821. }
  1822. #else
  1823. static inline int validate_chain(struct task_struct *curr,
  1824. struct lockdep_map *lock, struct held_lock *hlock,
  1825. int chain_head, u64 chain_key)
  1826. {
  1827. return 1;
  1828. }
  1829. #endif
  1830. /*
  1831. * We are building curr_chain_key incrementally, so double-check
  1832. * it from scratch, to make sure that it's done correctly:
  1833. */
  1834. static void check_chain_key(struct task_struct *curr)
  1835. {
  1836. #ifdef CONFIG_DEBUG_LOCKDEP
  1837. struct held_lock *hlock, *prev_hlock = NULL;
  1838. unsigned int i, id;
  1839. u64 chain_key = 0;
  1840. for (i = 0; i < curr->lockdep_depth; i++) {
  1841. hlock = curr->held_locks + i;
  1842. if (chain_key != hlock->prev_chain_key) {
  1843. debug_locks_off();
  1844. /*
  1845. * We got mighty confused, our chain keys don't match
  1846. * with what we expect, someone trample on our task state?
  1847. */
  1848. WARN(1, "hm#1, depth: %u [%u], %016Lx != %016Lx\n",
  1849. curr->lockdep_depth, i,
  1850. (unsigned long long)chain_key,
  1851. (unsigned long long)hlock->prev_chain_key);
  1852. return;
  1853. }
  1854. id = hlock->class_idx - 1;
  1855. /*
  1856. * Whoops ran out of static storage again?
  1857. */
  1858. if (DEBUG_LOCKS_WARN_ON(id >= MAX_LOCKDEP_KEYS))
  1859. return;
  1860. if (prev_hlock && (prev_hlock->irq_context !=
  1861. hlock->irq_context))
  1862. chain_key = 0;
  1863. chain_key = iterate_chain_key(chain_key, id);
  1864. prev_hlock = hlock;
  1865. }
  1866. if (chain_key != curr->curr_chain_key) {
  1867. debug_locks_off();
  1868. /*
  1869. * More smoking hash instead of calculating it, damn see these
  1870. * numbers float.. I bet that a pink elephant stepped on my memory.
  1871. */
  1872. WARN(1, "hm#2, depth: %u [%u], %016Lx != %016Lx\n",
  1873. curr->lockdep_depth, i,
  1874. (unsigned long long)chain_key,
  1875. (unsigned long long)curr->curr_chain_key);
  1876. }
  1877. #endif
  1878. }
  1879. static void
  1880. print_usage_bug_scenario(struct held_lock *lock)
  1881. {
  1882. struct lock_class *class = hlock_class(lock);
  1883. printk(" Possible unsafe locking scenario:\n\n");
  1884. printk(" CPU0\n");
  1885. printk(" ----\n");
  1886. printk(" lock(");
  1887. __print_lock_name(class);
  1888. printk(");\n");
  1889. printk(" <Interrupt>\n");
  1890. printk(" lock(");
  1891. __print_lock_name(class);
  1892. printk(");\n");
  1893. printk("\n *** DEADLOCK ***\n\n");
  1894. }
  1895. static int
  1896. print_usage_bug(struct task_struct *curr, struct held_lock *this,
  1897. enum lock_usage_bit prev_bit, enum lock_usage_bit new_bit)
  1898. {
  1899. if (!debug_locks_off_graph_unlock() || debug_locks_silent)
  1900. return 0;
  1901. printk("\n");
  1902. printk("=================================\n");
  1903. printk("[ INFO: inconsistent lock state ]\n");
  1904. print_kernel_version();
  1905. printk("---------------------------------\n");
  1906. printk("inconsistent {%s} -> {%s} usage.\n",
  1907. usage_str[prev_bit], usage_str[new_bit]);
  1908. printk("%s/%d [HC%u[%lu]:SC%u[%lu]:HE%u:SE%u] takes:\n",
  1909. curr->comm, task_pid_nr(curr),
  1910. trace_hardirq_context(curr), hardirq_count() >> HARDIRQ_SHIFT,
  1911. trace_softirq_context(curr), softirq_count() >> SOFTIRQ_SHIFT,
  1912. trace_hardirqs_enabled(curr),
  1913. trace_softirqs_enabled(curr));
  1914. print_lock(this);
  1915. printk("{%s} state was registered at:\n", usage_str[prev_bit]);
  1916. print_stack_trace(hlock_class(this)->usage_traces + prev_bit, 1);
  1917. print_irqtrace_events(curr);
  1918. printk("\nother info that might help us debug this:\n");
  1919. print_usage_bug_scenario(this);
  1920. lockdep_print_held_locks(curr);
  1921. printk("\nstack backtrace:\n");
  1922. dump_stack();
  1923. return 0;
  1924. }
  1925. /*
  1926. * Print out an error if an invalid bit is set:
  1927. */
  1928. static inline int
  1929. valid_state(struct task_struct *curr, struct held_lock *this,
  1930. enum lock_usage_bit new_bit, enum lock_usage_bit bad_bit)
  1931. {
  1932. if (unlikely(hlock_class(this)->usage_mask & (1 << bad_bit)))
  1933. return print_usage_bug(curr, this, bad_bit, new_bit);
  1934. return 1;
  1935. }
  1936. static int mark_lock(struct task_struct *curr, struct held_lock *this,
  1937. enum lock_usage_bit new_bit);
  1938. #if defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_PROVE_LOCKING)
  1939. /*
  1940. * print irq inversion bug:
  1941. */
  1942. static int
  1943. print_irq_inversion_bug(struct task_struct *curr,
  1944. struct lock_list *root, struct lock_list *other,
  1945. struct held_lock *this, int forwards,
  1946. const char *irqclass)
  1947. {
  1948. struct lock_list *entry = other;
  1949. struct lock_list *middle = NULL;
  1950. int depth;
  1951. if (!debug_locks_off_graph_unlock() || debug_locks_silent)
  1952. return 0;
  1953. printk("\n");
  1954. printk("=========================================================\n");
  1955. printk("[ INFO: possible irq lock inversion dependency detected ]\n");
  1956. print_kernel_version();
  1957. printk("---------------------------------------------------------\n");
  1958. printk("%s/%d just changed the state of lock:\n",
  1959. curr->comm, task_pid_nr(curr));
  1960. print_lock(this);
  1961. if (forwards)
  1962. printk("but this lock took another, %s-unsafe lock in the past:\n", irqclass);
  1963. else
  1964. printk("but this lock was taken by another, %s-safe lock in the past:\n", irqclass);
  1965. print_lock_name(other->class);
  1966. printk("\n\nand interrupts could create inverse lock ordering between them.\n\n");
  1967. printk("\nother info that might help us debug this:\n");
  1968. /* Find a middle lock (if one exists) */
  1969. depth = get_lock_depth(other);
  1970. do {
  1971. if (depth == 0 && (entry != root)) {
  1972. printk("lockdep:%s bad path found in chain graph\n", __func__);
  1973. break;
  1974. }
  1975. middle = entry;
  1976. entry = get_lock_parent(entry);
  1977. depth--;
  1978. } while (entry && entry != root && (depth >= 0));
  1979. if (forwards)
  1980. print_irq_lock_scenario(root, other,
  1981. middle ? middle->class : root->class, other->class);
  1982. else
  1983. print_irq_lock_scenario(other, root,
  1984. middle ? middle->class : other->class, root->class);
  1985. lockdep_print_held_locks(curr);
  1986. printk("\nthe shortest dependencies between 2nd lock and 1st lock:\n");
  1987. if (!save_trace(&root->trace))
  1988. return 0;
  1989. print_shortest_lock_dependencies(other, root);
  1990. printk("\nstack backtrace:\n");
  1991. dump_stack();
  1992. return 0;
  1993. }
  1994. /*
  1995. * Prove that in the forwards-direction subgraph starting at <this>
  1996. * there is no lock matching <mask>:
  1997. */
  1998. static int
  1999. check_usage_forwards(struct task_struct *curr, struct held_lock *this,
  2000. enum lock_usage_bit bit, const char *irqclass)
  2001. {
  2002. int ret;
  2003. struct lock_list root;
  2004. struct lock_list *uninitialized_var(target_entry);
  2005. root.parent = NULL;
  2006. root.class = hlock_class(this);
  2007. ret = find_usage_forwards(&root, bit, &target_entry);
  2008. if (ret < 0)
  2009. return print_bfs_bug(ret);
  2010. if (ret == 1)
  2011. return ret;
  2012. return print_irq_inversion_bug(curr, &root, target_entry,
  2013. this, 1, irqclass);
  2014. }
  2015. /*
  2016. * Prove that in the backwards-direction subgraph starting at <this>
  2017. * there is no lock matching <mask>:
  2018. */
  2019. static int
  2020. check_usage_backwards(struct task_struct *curr, struct held_lock *this,
  2021. enum lock_usage_bit bit, const char *irqclass)
  2022. {
  2023. int ret;
  2024. struct lock_list root;
  2025. struct lock_list *uninitialized_var(target_entry);
  2026. root.parent = NULL;
  2027. root.class = hlock_class(this);
  2028. ret = find_usage_backwards(&root, bit, &target_entry);
  2029. if (ret < 0)
  2030. return print_bfs_bug(ret);
  2031. if (ret == 1)
  2032. return ret;
  2033. return print_irq_inversion_bug(curr, &root, target_entry,
  2034. this, 0, irqclass);
  2035. }
  2036. void print_irqtrace_events(struct task_struct *curr)
  2037. {
  2038. printk("irq event stamp: %u\n", curr->irq_events);
  2039. printk("hardirqs last enabled at (%u): ", curr->hardirq_enable_event);
  2040. print_ip_sym(curr->hardirq_enable_ip);
  2041. printk("hardirqs last disabled at (%u): ", curr->hardirq_disable_event);
  2042. print_ip_sym(curr->hardirq_disable_ip);
  2043. printk("softirqs last enabled at (%u): ", curr->softirq_enable_event);
  2044. print_ip_sym(curr->softirq_enable_ip);
  2045. printk("softirqs last disabled at (%u): ", curr->softirq_disable_event);
  2046. print_ip_sym(curr->softirq_disable_ip);
  2047. }
  2048. static int HARDIRQ_verbose(struct lock_class *class)
  2049. {
  2050. #if HARDIRQ_VERBOSE
  2051. return class_filter(class);
  2052. #endif
  2053. return 0;
  2054. }
  2055. static int SOFTIRQ_verbose(struct lock_class *class)
  2056. {
  2057. #if SOFTIRQ_VERBOSE
  2058. return class_filter(class);
  2059. #endif
  2060. return 0;
  2061. }
  2062. static int RECLAIM_FS_verbose(struct lock_class *class)
  2063. {
  2064. #if RECLAIM_VERBOSE
  2065. return class_filter(class);
  2066. #endif
  2067. return 0;
  2068. }
  2069. #define STRICT_READ_CHECKS 1
  2070. static int (*state_verbose_f[])(struct lock_class *class) = {
  2071. #define LOCKDEP_STATE(__STATE) \
  2072. __STATE##_verbose,
  2073. #include "lockdep_states.h"
  2074. #undef LOCKDEP_STATE
  2075. };
  2076. static inline int state_verbose(enum lock_usage_bit bit,
  2077. struct lock_class *class)
  2078. {
  2079. return state_verbose_f[bit >> 2](class);
  2080. }
  2081. typedef int (*check_usage_f)(struct task_struct *, struct held_lock *,
  2082. enum lock_usage_bit bit, const char *name);
  2083. static int
  2084. mark_lock_irq(struct task_struct *curr, struct held_lock *this,
  2085. enum lock_usage_bit new_bit)
  2086. {
  2087. int excl_bit = exclusive_bit(new_bit);
  2088. int read = new_bit & 1;
  2089. int dir = new_bit & 2;
  2090. /*
  2091. * mark USED_IN has to look forwards -- to ensure no dependency
  2092. * has ENABLED state, which would allow recursion deadlocks.
  2093. *
  2094. * mark ENABLED has to look backwards -- to ensure no dependee
  2095. * has USED_IN state, which, again, would allow recursion deadlocks.
  2096. */
  2097. check_usage_f usage = dir ?
  2098. check_usage_backwards : check_usage_forwards;
  2099. /*
  2100. * Validate that this particular lock does not have conflicting
  2101. * usage states.
  2102. */
  2103. if (!valid_state(curr, this, new_bit, excl_bit))
  2104. return 0;
  2105. /*
  2106. * Validate that the lock dependencies don't have conflicting usage
  2107. * states.
  2108. */
  2109. if ((!read || !dir || STRICT_READ_CHECKS) &&
  2110. !usage(curr, this, excl_bit, state_name(new_bit & ~1)))
  2111. return 0;
  2112. /*
  2113. * Check for read in write conflicts
  2114. */
  2115. if (!read) {
  2116. if (!valid_state(curr, this, new_bit, excl_bit + 1))
  2117. return 0;
  2118. if (STRICT_READ_CHECKS &&
  2119. !usage(curr, this, excl_bit + 1,
  2120. state_name(new_bit + 1)))
  2121. return 0;
  2122. }
  2123. if (state_verbose(new_bit, hlock_class(this)))
  2124. return 2;
  2125. return 1;
  2126. }
  2127. enum mark_type {
  2128. #define LOCKDEP_STATE(__STATE) __STATE,
  2129. #include "lockdep_states.h"
  2130. #undef LOCKDEP_STATE
  2131. };
  2132. /*
  2133. * Mark all held locks with a usage bit:
  2134. */
  2135. static int
  2136. mark_held_locks(struct task_struct *curr, enum mark_type mark)
  2137. {
  2138. enum lock_usage_bit usage_bit;
  2139. struct held_lock *hlock;
  2140. int i;
  2141. for (i = 0; i < curr->lockdep_depth; i++) {
  2142. hlock = curr->held_locks + i;
  2143. usage_bit = 2 + (mark << 2); /* ENABLED */
  2144. if (hlock->read)
  2145. usage_bit += 1; /* READ */
  2146. BUG_ON(usage_bit >= LOCK_USAGE_STATES);
  2147. if (hlock_class(hlock)->key == __lockdep_no_validate__.subkeys)
  2148. continue;
  2149. if (!mark_lock(curr, hlock, usage_bit))
  2150. return 0;
  2151. }
  2152. return 1;
  2153. }
  2154. /*
  2155. * Hardirqs will be enabled:
  2156. */
  2157. static void __trace_hardirqs_on_caller(unsigned long ip)
  2158. {
  2159. struct task_struct *curr = current;
  2160. /* we'll do an OFF -> ON transition: */
  2161. curr->hardirqs_enabled = 1;
  2162. /*
  2163. * We are going to turn hardirqs on, so set the
  2164. * usage bit for all held locks:
  2165. */
  2166. if (!mark_held_locks(curr, HARDIRQ))
  2167. return;
  2168. /*
  2169. * If we have softirqs enabled, then set the usage
  2170. * bit for all held locks. (disabled hardirqs prevented
  2171. * this bit from being set before)
  2172. */
  2173. if (curr->softirqs_enabled)
  2174. if (!mark_held_locks(curr, SOFTIRQ))
  2175. return;
  2176. curr->hardirq_enable_ip = ip;
  2177. curr->hardirq_enable_event = ++curr->irq_events;
  2178. debug_atomic_inc(hardirqs_on_events);
  2179. }
  2180. void trace_hardirqs_on_caller(unsigned long ip)
  2181. {
  2182. time_hardirqs_on(CALLER_ADDR0, ip);
  2183. if (unlikely(!debug_locks || current->lockdep_recursion))
  2184. return;
  2185. if (unlikely(current->hardirqs_enabled)) {
  2186. /*
  2187. * Neither irq nor preemption are disabled here
  2188. * so this is racy by nature but losing one hit
  2189. * in a stat is not a big deal.
  2190. */
  2191. __debug_atomic_inc(redundant_hardirqs_on);
  2192. return;
  2193. }
  2194. /*
  2195. * We're enabling irqs and according to our state above irqs weren't
  2196. * already enabled, yet we find the hardware thinks they are in fact
  2197. * enabled.. someone messed up their IRQ state tracing.
  2198. */
  2199. if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
  2200. return;
  2201. /*
  2202. * See the fine text that goes along with this variable definition.
  2203. */
  2204. if (DEBUG_LOCKS_WARN_ON(unlikely(early_boot_irqs_disabled)))
  2205. return;
  2206. /*
  2207. * Can't allow enabling interrupts while in an interrupt handler,
  2208. * that's general bad form and such. Recursion, limited stack etc..
  2209. */
  2210. if (DEBUG_LOCKS_WARN_ON(current->hardirq_context))
  2211. return;
  2212. current->lockdep_recursion = 1;
  2213. __trace_hardirqs_on_caller(ip);
  2214. current->lockdep_recursion = 0;
  2215. }
  2216. EXPORT_SYMBOL(trace_hardirqs_on_caller);
  2217. void trace_hardirqs_on(void)
  2218. {
  2219. trace_hardirqs_on_caller(CALLER_ADDR0);
  2220. }
  2221. EXPORT_SYMBOL(trace_hardirqs_on);
  2222. /*
  2223. * Hardirqs were disabled:
  2224. */
  2225. void trace_hardirqs_off_caller(unsigned long ip)
  2226. {
  2227. struct task_struct *curr = current;
  2228. time_hardirqs_off(CALLER_ADDR0, ip);
  2229. if (unlikely(!debug_locks || current->lockdep_recursion))
  2230. return;
  2231. /*
  2232. * So we're supposed to get called after you mask local IRQs, but for
  2233. * some reason the hardware doesn't quite think you did a proper job.
  2234. */
  2235. if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
  2236. return;
  2237. if (curr->hardirqs_enabled) {
  2238. /*
  2239. * We have done an ON -> OFF transition:
  2240. */
  2241. curr->hardirqs_enabled = 0;
  2242. curr->hardirq_disable_ip = ip;
  2243. curr->hardirq_disable_event = ++curr->irq_events;
  2244. debug_atomic_inc(hardirqs_off_events);
  2245. } else
  2246. debug_atomic_inc(redundant_hardirqs_off);
  2247. }
  2248. EXPORT_SYMBOL(trace_hardirqs_off_caller);
  2249. void trace_hardirqs_off(void)
  2250. {
  2251. trace_hardirqs_off_caller(CALLER_ADDR0);
  2252. }
  2253. EXPORT_SYMBOL(trace_hardirqs_off);
  2254. /*
  2255. * Softirqs will be enabled:
  2256. */
  2257. void trace_softirqs_on(unsigned long ip)
  2258. {
  2259. struct task_struct *curr = current;
  2260. if (unlikely(!debug_locks || current->lockdep_recursion))
  2261. return;
  2262. /*
  2263. * We fancy IRQs being disabled here, see softirq.c, avoids
  2264. * funny state and nesting things.
  2265. */
  2266. if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
  2267. return;
  2268. if (curr->softirqs_enabled) {
  2269. debug_atomic_inc(redundant_softirqs_on);
  2270. return;
  2271. }
  2272. current->lockdep_recursion = 1;
  2273. /*
  2274. * We'll do an OFF -> ON transition:
  2275. */
  2276. curr->softirqs_enabled = 1;
  2277. curr->softirq_enable_ip = ip;
  2278. curr->softirq_enable_event = ++curr->irq_events;
  2279. debug_atomic_inc(softirqs_on_events);
  2280. /*
  2281. * We are going to turn softirqs on, so set the
  2282. * usage bit for all held locks, if hardirqs are
  2283. * enabled too:
  2284. */
  2285. if (curr->hardirqs_enabled)
  2286. mark_held_locks(curr, SOFTIRQ);
  2287. current->lockdep_recursion = 0;
  2288. }
  2289. /*
  2290. * Softirqs were disabled:
  2291. */
  2292. void trace_softirqs_off(unsigned long ip)
  2293. {
  2294. struct task_struct *curr = current;
  2295. if (unlikely(!debug_locks || current->lockdep_recursion))
  2296. return;
  2297. /*
  2298. * We fancy IRQs being disabled here, see softirq.c
  2299. */
  2300. if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
  2301. return;
  2302. if (curr->softirqs_enabled) {
  2303. /*
  2304. * We have done an ON -> OFF transition:
  2305. */
  2306. curr->softirqs_enabled = 0;
  2307. curr->softirq_disable_ip = ip;
  2308. curr->softirq_disable_event = ++curr->irq_events;
  2309. debug_atomic_inc(softirqs_off_events);
  2310. /*
  2311. * Whoops, we wanted softirqs off, so why aren't they?
  2312. */
  2313. DEBUG_LOCKS_WARN_ON(!softirq_count());
  2314. } else
  2315. debug_atomic_inc(redundant_softirqs_off);
  2316. }
  2317. static void __lockdep_trace_alloc(gfp_t gfp_mask, unsigned long flags)
  2318. {
  2319. struct task_struct *curr = current;
  2320. if (unlikely(!debug_locks))
  2321. return;
  2322. /* no reclaim without waiting on it */
  2323. if (!(gfp_mask & __GFP_WAIT))
  2324. return;
  2325. /* this guy won't enter reclaim */
  2326. if ((curr->flags & PF_MEMALLOC) && !(gfp_mask & __GFP_NOMEMALLOC))
  2327. return;
  2328. /* We're only interested __GFP_FS allocations for now */
  2329. if (!(gfp_mask & __GFP_FS))
  2330. return;
  2331. /*
  2332. * Oi! Can't be having __GFP_FS allocations with IRQs disabled.
  2333. */
  2334. if (DEBUG_LOCKS_WARN_ON(irqs_disabled_flags(flags)))
  2335. return;
  2336. mark_held_locks(curr, RECLAIM_FS);
  2337. }
  2338. static void check_flags(unsigned long flags);
  2339. void lockdep_trace_alloc(gfp_t gfp_mask)
  2340. {
  2341. unsigned long flags;
  2342. if (unlikely(current->lockdep_recursion))
  2343. return;
  2344. raw_local_irq_save(flags);
  2345. check_flags(flags);
  2346. current->lockdep_recursion = 1;
  2347. __lockdep_trace_alloc(gfp_mask, flags);
  2348. current->lockdep_recursion = 0;
  2349. raw_local_irq_restore(flags);
  2350. }
  2351. static int mark_irqflags(struct task_struct *curr, struct held_lock *hlock)
  2352. {
  2353. /*
  2354. * If non-trylock use in a hardirq or softirq context, then
  2355. * mark the lock as used in these contexts:
  2356. */
  2357. if (!hlock->trylock) {
  2358. if (hlock->read) {
  2359. if (curr->hardirq_context)
  2360. if (!mark_lock(curr, hlock,
  2361. LOCK_USED_IN_HARDIRQ_READ))
  2362. return 0;
  2363. if (curr->softirq_context)
  2364. if (!mark_lock(curr, hlock,
  2365. LOCK_USED_IN_SOFTIRQ_READ))
  2366. return 0;
  2367. } else {
  2368. if (curr->hardirq_context)
  2369. if (!mark_lock(curr, hlock, LOCK_USED_IN_HARDIRQ))
  2370. return 0;
  2371. if (curr->softirq_context)
  2372. if (!mark_lock(curr, hlock, LOCK_USED_IN_SOFTIRQ))
  2373. return 0;
  2374. }
  2375. }
  2376. if (!hlock->hardirqs_off) {
  2377. if (hlock->read) {
  2378. if (!mark_lock(curr, hlock,
  2379. LOCK_ENABLED_HARDIRQ_READ))
  2380. return 0;
  2381. if (curr->softirqs_enabled)
  2382. if (!mark_lock(curr, hlock,
  2383. LOCK_ENABLED_SOFTIRQ_READ))
  2384. return 0;
  2385. } else {
  2386. if (!mark_lock(curr, hlock,
  2387. LOCK_ENABLED_HARDIRQ))
  2388. return 0;
  2389. if (curr->softirqs_enabled)
  2390. if (!mark_lock(curr, hlock,
  2391. LOCK_ENABLED_SOFTIRQ))
  2392. return 0;
  2393. }
  2394. }
  2395. /*
  2396. * We reuse the irq context infrastructure more broadly as a general
  2397. * context checking code. This tests GFP_FS recursion (a lock taken
  2398. * during reclaim for a GFP_FS allocation is held over a GFP_FS
  2399. * allocation).
  2400. */
  2401. if (!hlock->trylock && (curr->lockdep_reclaim_gfp & __GFP_FS)) {
  2402. if (hlock->read) {
  2403. if (!mark_lock(curr, hlock, LOCK_USED_IN_RECLAIM_FS_READ))
  2404. return 0;
  2405. } else {
  2406. if (!mark_lock(curr, hlock, LOCK_USED_IN_RECLAIM_FS))
  2407. return 0;
  2408. }
  2409. }
  2410. return 1;
  2411. }
  2412. static int separate_irq_context(struct task_struct *curr,
  2413. struct held_lock *hlock)
  2414. {
  2415. unsigned int depth = curr->lockdep_depth;
  2416. /*
  2417. * Keep track of points where we cross into an interrupt context:
  2418. */
  2419. hlock->irq_context = 2*(curr->hardirq_context ? 1 : 0) +
  2420. curr->softirq_context;
  2421. if (depth) {
  2422. struct held_lock *prev_hlock;
  2423. prev_hlock = curr->held_locks + depth-1;
  2424. /*
  2425. * If we cross into another context, reset the
  2426. * hash key (this also prevents the checking and the
  2427. * adding of the dependency to 'prev'):
  2428. */
  2429. if (prev_hlock->irq_context != hlock->irq_context)
  2430. return 1;
  2431. }
  2432. return 0;
  2433. }
  2434. #else /* defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_PROVE_LOCKING) */
  2435. static inline
  2436. int mark_lock_irq(struct task_struct *curr, struct held_lock *this,
  2437. enum lock_usage_bit new_bit)
  2438. {
  2439. WARN_ON(1); /* Impossible innit? when we don't have TRACE_IRQFLAG */
  2440. return 1;
  2441. }
  2442. static inline int mark_irqflags(struct task_struct *curr,
  2443. struct held_lock *hlock)
  2444. {
  2445. return 1;
  2446. }
  2447. static inline int separate_irq_context(struct task_struct *curr,
  2448. struct held_lock *hlock)
  2449. {
  2450. return 0;
  2451. }
  2452. void lockdep_trace_alloc(gfp_t gfp_mask)
  2453. {
  2454. }
  2455. #endif /* defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_PROVE_LOCKING) */
  2456. /*
  2457. * Mark a lock with a usage bit, and validate the state transition:
  2458. */
  2459. static int mark_lock(struct task_struct *curr, struct held_lock *this,
  2460. enum lock_usage_bit new_bit)
  2461. {
  2462. unsigned int new_mask = 1 << new_bit, ret = 1;
  2463. /*
  2464. * If already set then do not dirty the cacheline,
  2465. * nor do any checks:
  2466. */
  2467. if (likely(hlock_class(this)->usage_mask & new_mask))
  2468. return 1;
  2469. if (!graph_lock())
  2470. return 0;
  2471. /*
  2472. * Make sure we didn't race:
  2473. */
  2474. if (unlikely(hlock_class(this)->usage_mask & new_mask)) {
  2475. graph_unlock();
  2476. return 1;
  2477. }
  2478. hlock_class(this)->usage_mask |= new_mask;
  2479. if (!save_trace(hlock_class(this)->usage_traces + new_bit))
  2480. return 0;
  2481. switch (new_bit) {
  2482. #define LOCKDEP_STATE(__STATE) \
  2483. case LOCK_USED_IN_##__STATE: \
  2484. case LOCK_USED_IN_##__STATE##_READ: \
  2485. case LOCK_ENABLED_##__STATE: \
  2486. case LOCK_ENABLED_##__STATE##_READ:
  2487. #include "lockdep_states.h"
  2488. #undef LOCKDEP_STATE
  2489. ret = mark_lock_irq(curr, this, new_bit);
  2490. if (!ret)
  2491. return 0;
  2492. break;
  2493. case LOCK_USED:
  2494. debug_atomic_dec(nr_unused_locks);
  2495. break;
  2496. default:
  2497. if (!debug_locks_off_graph_unlock())
  2498. return 0;
  2499. WARN_ON(1);
  2500. return 0;
  2501. }
  2502. graph_unlock();
  2503. /*
  2504. * We must printk outside of the graph_lock:
  2505. */
  2506. if (ret == 2) {
  2507. printk("\nmarked lock as {%s}:\n", usage_str[new_bit]);
  2508. print_lock(this);
  2509. print_irqtrace_events(curr);
  2510. dump_stack();
  2511. }
  2512. return ret;
  2513. }
  2514. /*
  2515. * Initialize a lock instance's lock-class mapping info:
  2516. */
  2517. void lockdep_init_map(struct lockdep_map *lock, const char *name,
  2518. struct lock_class_key *key, int subclass)
  2519. {
  2520. memset(lock, 0, sizeof(*lock));
  2521. #ifdef CONFIG_LOCK_STAT
  2522. lock->cpu = raw_smp_processor_id();
  2523. #endif
  2524. /*
  2525. * Can't be having no nameless bastards around this place!
  2526. */
  2527. if (DEBUG_LOCKS_WARN_ON(!name)) {
  2528. lock->name = "NULL";
  2529. return;
  2530. }
  2531. lock->name = name;
  2532. /*
  2533. * No key, no joy, we need to hash something.
  2534. */
  2535. if (DEBUG_LOCKS_WARN_ON(!key))
  2536. return;
  2537. /*
  2538. * Sanity check, the lock-class key must be persistent:
  2539. */
  2540. if (!static_obj(key)) {
  2541. printk("BUG: key %p not in .data!\n", key);
  2542. /*
  2543. * What it says above ^^^^^, I suggest you read it.
  2544. */
  2545. DEBUG_LOCKS_WARN_ON(1);
  2546. return;
  2547. }
  2548. lock->key = key;
  2549. if (unlikely(!debug_locks))
  2550. return;
  2551. if (subclass)
  2552. register_lock_class(lock, subclass, 1);
  2553. }
  2554. EXPORT_SYMBOL_GPL(lockdep_init_map);
  2555. struct lock_class_key __lockdep_no_validate__;
  2556. /*
  2557. * This gets called for every mutex_lock*()/spin_lock*() operation.
  2558. * We maintain the dependency maps and validate the locking attempt:
  2559. */
  2560. static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass,
  2561. int trylock, int read, int check, int hardirqs_off,
  2562. struct lockdep_map *nest_lock, unsigned long ip,
  2563. int references)
  2564. {
  2565. struct task_struct *curr = current;
  2566. struct lock_class *class = NULL;
  2567. struct held_lock *hlock;
  2568. unsigned int depth, id;
  2569. int chain_head = 0;
  2570. int class_idx;
  2571. u64 chain_key;
  2572. if (!prove_locking)
  2573. check = 1;
  2574. if (unlikely(!debug_locks))
  2575. return 0;
  2576. /*
  2577. * Lockdep should run with IRQs disabled, otherwise we could
  2578. * get an interrupt which would want to take locks, which would
  2579. * end up in lockdep and have you got a head-ache already?
  2580. */
  2581. if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
  2582. return 0;
  2583. if (lock->key == &__lockdep_no_validate__)
  2584. check = 1;
  2585. if (subclass < NR_LOCKDEP_CACHING_CLASSES)
  2586. class = lock->class_cache[subclass];
  2587. /*
  2588. * Not cached?
  2589. */
  2590. if (unlikely(!class)) {
  2591. class = register_lock_class(lock, subclass, 0);
  2592. if (!class)
  2593. return 0;
  2594. }
  2595. atomic_inc((atomic_t *)&class->ops);
  2596. if (very_verbose(class)) {
  2597. printk("\nacquire class [%p] %s", class->key, class->name);
  2598. if (class->name_version > 1)
  2599. printk("#%d", class->name_version);
  2600. printk("\n");
  2601. dump_stack();
  2602. }
  2603. /*
  2604. * Add the lock to the list of currently held locks.
  2605. * (we dont increase the depth just yet, up until the
  2606. * dependency checks are done)
  2607. */
  2608. depth = curr->lockdep_depth;
  2609. /*
  2610. * Ran out of static storage for our per-task lock stack again have we?
  2611. */
  2612. if (DEBUG_LOCKS_WARN_ON(depth >= MAX_LOCK_DEPTH))
  2613. return 0;
  2614. class_idx = class - lock_classes + 1;
  2615. if (depth) {
  2616. hlock = curr->held_locks + depth - 1;
  2617. if (hlock->class_idx == class_idx && nest_lock) {
  2618. if (hlock->references)
  2619. hlock->references++;
  2620. else
  2621. hlock->references = 2;
  2622. return 1;
  2623. }
  2624. }
  2625. hlock = curr->held_locks + depth;
  2626. /*
  2627. * Plain impossible, we just registered it and checked it weren't no
  2628. * NULL like.. I bet this mushroom I ate was good!
  2629. */
  2630. if (DEBUG_LOCKS_WARN_ON(!class))
  2631. return 0;
  2632. hlock->class_idx = class_idx;
  2633. hlock->acquire_ip = ip;
  2634. hlock->instance = lock;
  2635. hlock->nest_lock = nest_lock;
  2636. hlock->trylock = trylock;
  2637. hlock->read = read;
  2638. hlock->check = check;
  2639. hlock->hardirqs_off = !!hardirqs_off;
  2640. hlock->references = references;
  2641. #ifdef CONFIG_LOCK_STAT
  2642. hlock->waittime_stamp = 0;
  2643. hlock->holdtime_stamp = lockstat_clock();
  2644. #endif
  2645. if (check == 2 && !mark_irqflags(curr, hlock))
  2646. return 0;
  2647. /* mark it as used: */
  2648. if (!mark_lock(curr, hlock, LOCK_USED))
  2649. return 0;
  2650. /*
  2651. * Calculate the chain hash: it's the combined hash of all the
  2652. * lock keys along the dependency chain. We save the hash value
  2653. * at every step so that we can get the current hash easily
  2654. * after unlock. The chain hash is then used to cache dependency
  2655. * results.
  2656. *
  2657. * The 'key ID' is what is the most compact key value to drive
  2658. * the hash, not class->key.
  2659. */
  2660. id = class - lock_classes;
  2661. /*
  2662. * Whoops, we did it again.. ran straight out of our static allocation.
  2663. */
  2664. if (DEBUG_LOCKS_WARN_ON(id >= MAX_LOCKDEP_KEYS))
  2665. return 0;
  2666. chain_key = curr->curr_chain_key;
  2667. if (!depth) {
  2668. /*
  2669. * How can we have a chain hash when we ain't got no keys?!
  2670. */
  2671. if (DEBUG_LOCKS_WARN_ON(chain_key != 0))
  2672. return 0;
  2673. chain_head = 1;
  2674. }
  2675. hlock->prev_chain_key = chain_key;
  2676. if (separate_irq_context(curr, hlock)) {
  2677. chain_key = 0;
  2678. chain_head = 1;
  2679. }
  2680. chain_key = iterate_chain_key(chain_key, id);
  2681. if (!validate_chain(curr, lock, hlock, chain_head, chain_key))
  2682. return 0;
  2683. curr->curr_chain_key = chain_key;
  2684. curr->lockdep_depth++;
  2685. check_chain_key(curr);
  2686. #ifdef CONFIG_DEBUG_LOCKDEP
  2687. if (unlikely(!debug_locks))
  2688. return 0;
  2689. #endif
  2690. if (unlikely(curr->lockdep_depth >= MAX_LOCK_DEPTH)) {
  2691. debug_locks_off();
  2692. printk("BUG: MAX_LOCK_DEPTH too low!\n");
  2693. printk("turning off the locking correctness validator.\n");
  2694. dump_stack();
  2695. return 0;
  2696. }
  2697. if (unlikely(curr->lockdep_depth > max_lockdep_depth))
  2698. max_lockdep_depth = curr->lockdep_depth;
  2699. return 1;
  2700. }
  2701. static int
  2702. print_unlock_inbalance_bug(struct task_struct *curr, struct lockdep_map *lock,
  2703. unsigned long ip)
  2704. {
  2705. if (!debug_locks_off())
  2706. return 0;
  2707. if (debug_locks_silent)
  2708. return 0;
  2709. printk("\n");
  2710. printk("=====================================\n");
  2711. printk("[ BUG: bad unlock balance detected! ]\n");
  2712. printk("-------------------------------------\n");
  2713. printk("%s/%d is trying to release lock (",
  2714. curr->comm, task_pid_nr(curr));
  2715. print_lockdep_cache(lock);
  2716. printk(") at:\n");
  2717. print_ip_sym(ip);
  2718. printk("but there are no more locks to release!\n");
  2719. printk("\nother info that might help us debug this:\n");
  2720. lockdep_print_held_locks(curr);
  2721. printk("\nstack backtrace:\n");
  2722. dump_stack();
  2723. return 0;
  2724. }
  2725. /*
  2726. * Common debugging checks for both nested and non-nested unlock:
  2727. */
  2728. static int check_unlock(struct task_struct *curr, struct lockdep_map *lock,
  2729. unsigned long ip)
  2730. {
  2731. if (unlikely(!debug_locks))
  2732. return 0;
  2733. /*
  2734. * Lockdep should run with IRQs disabled, recursion, head-ache, etc..
  2735. */
  2736. if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
  2737. return 0;
  2738. if (curr->lockdep_depth <= 0)
  2739. return print_unlock_inbalance_bug(curr, lock, ip);
  2740. return 1;
  2741. }
  2742. static int match_held_lock(struct held_lock *hlock, struct lockdep_map *lock)
  2743. {
  2744. if (hlock->instance == lock)
  2745. return 1;
  2746. if (hlock->references) {
  2747. struct lock_class *class = lock->class_cache[0];
  2748. if (!class)
  2749. class = look_up_lock_class(lock, 0);
  2750. /*
  2751. * If look_up_lock_class() failed to find a class, we're trying
  2752. * to test if we hold a lock that has never yet been acquired.
  2753. * Clearly if the lock hasn't been acquired _ever_, we're not
  2754. * holding it either, so report failure.
  2755. */
  2756. if (!class)
  2757. return 0;
  2758. /*
  2759. * References, but not a lock we're actually ref-counting?
  2760. * State got messed up, follow the sites that change ->references
  2761. * and try to make sense of it.
  2762. */
  2763. if (DEBUG_LOCKS_WARN_ON(!hlock->nest_lock))
  2764. return 0;
  2765. if (hlock->class_idx == class - lock_classes + 1)
  2766. return 1;
  2767. }
  2768. return 0;
  2769. }
  2770. static int
  2771. __lock_set_class(struct lockdep_map *lock, const char *name,
  2772. struct lock_class_key *key, unsigned int subclass,
  2773. unsigned long ip)
  2774. {
  2775. struct task_struct *curr = current;
  2776. struct held_lock *hlock, *prev_hlock;
  2777. struct lock_class *class;
  2778. unsigned int depth;
  2779. int i;
  2780. depth = curr->lockdep_depth;
  2781. /*
  2782. * This function is about (re)setting the class of a held lock,
  2783. * yet we're not actually holding any locks. Naughty user!
  2784. */
  2785. if (DEBUG_LOCKS_WARN_ON(!depth))
  2786. return 0;
  2787. prev_hlock = NULL;
  2788. for (i = depth-1; i >= 0; i--) {
  2789. hlock = curr->held_locks + i;
  2790. /*
  2791. * We must not cross into another context:
  2792. */
  2793. if (prev_hlock && prev_hlock->irq_context != hlock->irq_context)
  2794. break;
  2795. if (match_held_lock(hlock, lock))
  2796. goto found_it;
  2797. prev_hlock = hlock;
  2798. }
  2799. return print_unlock_inbalance_bug(curr, lock, ip);
  2800. found_it:
  2801. lockdep_init_map(lock, name, key, 0);
  2802. class = register_lock_class(lock, subclass, 0);
  2803. hlock->class_idx = class - lock_classes + 1;
  2804. curr->lockdep_depth = i;
  2805. curr->curr_chain_key = hlock->prev_chain_key;
  2806. for (; i < depth; i++) {
  2807. hlock = curr->held_locks + i;
  2808. if (!__lock_acquire(hlock->instance,
  2809. hlock_class(hlock)->subclass, hlock->trylock,
  2810. hlock->read, hlock->check, hlock->hardirqs_off,
  2811. hlock->nest_lock, hlock->acquire_ip,
  2812. hlock->references))
  2813. return 0;
  2814. }
  2815. /*
  2816. * I took it apart and put it back together again, except now I have
  2817. * these 'spare' parts.. where shall I put them.
  2818. */
  2819. if (DEBUG_LOCKS_WARN_ON(curr->lockdep_depth != depth))
  2820. return 0;
  2821. return 1;
  2822. }
  2823. /*
  2824. * Remove the lock to the list of currently held locks in a
  2825. * potentially non-nested (out of order) manner. This is a
  2826. * relatively rare operation, as all the unlock APIs default
  2827. * to nested mode (which uses lock_release()):
  2828. */
  2829. static int
  2830. lock_release_non_nested(struct task_struct *curr,
  2831. struct lockdep_map *lock, unsigned long ip)
  2832. {
  2833. struct held_lock *hlock, *prev_hlock;
  2834. unsigned int depth;
  2835. int i;
  2836. /*
  2837. * Check whether the lock exists in the current stack
  2838. * of held locks:
  2839. */
  2840. depth = curr->lockdep_depth;
  2841. /*
  2842. * So we're all set to release this lock.. wait what lock? We don't
  2843. * own any locks, you've been drinking again?
  2844. */
  2845. if (DEBUG_LOCKS_WARN_ON(!depth))
  2846. return 0;
  2847. prev_hlock = NULL;
  2848. for (i = depth-1; i >= 0; i--) {
  2849. hlock = curr->held_locks + i;
  2850. /*
  2851. * We must not cross into another context:
  2852. */
  2853. if (prev_hlock && prev_hlock->irq_context != hlock->irq_context)
  2854. break;
  2855. if (match_held_lock(hlock, lock))
  2856. goto found_it;
  2857. prev_hlock = hlock;
  2858. }
  2859. return print_unlock_inbalance_bug(curr, lock, ip);
  2860. found_it:
  2861. if (hlock->instance == lock)
  2862. lock_release_holdtime(hlock);
  2863. if (hlock->references) {
  2864. hlock->references--;
  2865. if (hlock->references) {
  2866. /*
  2867. * We had, and after removing one, still have
  2868. * references, the current lock stack is still
  2869. * valid. We're done!
  2870. */
  2871. return 1;
  2872. }
  2873. }
  2874. /*
  2875. * We have the right lock to unlock, 'hlock' points to it.
  2876. * Now we remove it from the stack, and add back the other
  2877. * entries (if any), recalculating the hash along the way:
  2878. */
  2879. curr->lockdep_depth = i;
  2880. curr->curr_chain_key = hlock->prev_chain_key;
  2881. for (i++; i < depth; i++) {
  2882. hlock = curr->held_locks + i;
  2883. if (!__lock_acquire(hlock->instance,
  2884. hlock_class(hlock)->subclass, hlock->trylock,
  2885. hlock->read, hlock->check, hlock->hardirqs_off,
  2886. hlock->nest_lock, hlock->acquire_ip,
  2887. hlock->references))
  2888. return 0;
  2889. }
  2890. /*
  2891. * We had N bottles of beer on the wall, we drank one, but now
  2892. * there's not N-1 bottles of beer left on the wall...
  2893. */
  2894. if (DEBUG_LOCKS_WARN_ON(curr->lockdep_depth != depth - 1))
  2895. return 0;
  2896. return 1;
  2897. }
  2898. /*
  2899. * Remove the lock to the list of currently held locks - this gets
  2900. * called on mutex_unlock()/spin_unlock*() (or on a failed
  2901. * mutex_lock_interruptible()). This is done for unlocks that nest
  2902. * perfectly. (i.e. the current top of the lock-stack is unlocked)
  2903. */
  2904. static int lock_release_nested(struct task_struct *curr,
  2905. struct lockdep_map *lock, unsigned long ip)
  2906. {
  2907. struct held_lock *hlock;
  2908. unsigned int depth;
  2909. /*
  2910. * Pop off the top of the lock stack:
  2911. */
  2912. depth = curr->lockdep_depth - 1;
  2913. hlock = curr->held_locks + depth;
  2914. /*
  2915. * Is the unlock non-nested:
  2916. */
  2917. if (hlock->instance != lock || hlock->references)
  2918. return lock_release_non_nested(curr, lock, ip);
  2919. curr->lockdep_depth--;
  2920. /*
  2921. * No more locks, but somehow we've got hash left over, who left it?
  2922. */
  2923. if (DEBUG_LOCKS_WARN_ON(!depth && (hlock->prev_chain_key != 0)))
  2924. return 0;
  2925. curr->curr_chain_key = hlock->prev_chain_key;
  2926. lock_release_holdtime(hlock);
  2927. #ifdef CONFIG_DEBUG_LOCKDEP
  2928. hlock->prev_chain_key = 0;
  2929. hlock->class_idx = 0;
  2930. hlock->acquire_ip = 0;
  2931. hlock->irq_context = 0;
  2932. #endif
  2933. return 1;
  2934. }
  2935. /*
  2936. * Remove the lock to the list of currently held locks - this gets
  2937. * called on mutex_unlock()/spin_unlock*() (or on a failed
  2938. * mutex_lock_interruptible()). This is done for unlocks that nest
  2939. * perfectly. (i.e. the current top of the lock-stack is unlocked)
  2940. */
  2941. static void
  2942. __lock_release(struct lockdep_map *lock, int nested, unsigned long ip)
  2943. {
  2944. struct task_struct *curr = current;
  2945. if (!check_unlock(curr, lock, ip))
  2946. return;
  2947. if (nested) {
  2948. if (!lock_release_nested(curr, lock, ip))
  2949. return;
  2950. } else {
  2951. if (!lock_release_non_nested(curr, lock, ip))
  2952. return;
  2953. }
  2954. check_chain_key(curr);
  2955. }
  2956. static int __lock_is_held(struct lockdep_map *lock)
  2957. {
  2958. struct task_struct *curr = current;
  2959. int i;
  2960. for (i = 0; i < curr->lockdep_depth; i++) {
  2961. struct held_lock *hlock = curr->held_locks + i;
  2962. if (match_held_lock(hlock, lock))
  2963. return 1;
  2964. }
  2965. return 0;
  2966. }
  2967. /*
  2968. * Check whether we follow the irq-flags state precisely:
  2969. */
  2970. static void check_flags(unsigned long flags)
  2971. {
  2972. #if defined(CONFIG_PROVE_LOCKING) && defined(CONFIG_DEBUG_LOCKDEP) && \
  2973. defined(CONFIG_TRACE_IRQFLAGS)
  2974. if (!debug_locks)
  2975. return;
  2976. if (irqs_disabled_flags(flags)) {
  2977. if (DEBUG_LOCKS_WARN_ON(current->hardirqs_enabled)) {
  2978. printk("possible reason: unannotated irqs-off.\n");
  2979. }
  2980. } else {
  2981. if (DEBUG_LOCKS_WARN_ON(!current->hardirqs_enabled)) {
  2982. printk("possible reason: unannotated irqs-on.\n");
  2983. }
  2984. }
  2985. /*
  2986. * We dont accurately track softirq state in e.g.
  2987. * hardirq contexts (such as on 4KSTACKS), so only
  2988. * check if not in hardirq contexts:
  2989. */
  2990. if (!hardirq_count()) {
  2991. if (softirq_count()) {
  2992. /* like the above, but with softirqs */
  2993. DEBUG_LOCKS_WARN_ON(current->softirqs_enabled);
  2994. } else {
  2995. /* lick the above, does it taste good? */
  2996. DEBUG_LOCKS_WARN_ON(!current->softirqs_enabled);
  2997. }
  2998. }
  2999. if (!debug_locks)
  3000. print_irqtrace_events(current);
  3001. #endif
  3002. }
  3003. void lock_set_class(struct lockdep_map *lock, const char *name,
  3004. struct lock_class_key *key, unsigned int subclass,
  3005. unsigned long ip)
  3006. {
  3007. unsigned long flags;
  3008. if (unlikely(current->lockdep_recursion))
  3009. return;
  3010. raw_local_irq_save(flags);
  3011. current->lockdep_recursion = 1;
  3012. check_flags(flags);
  3013. if (__lock_set_class(lock, name, key, subclass, ip))
  3014. check_chain_key(current);
  3015. current->lockdep_recursion = 0;
  3016. raw_local_irq_restore(flags);
  3017. }
  3018. EXPORT_SYMBOL_GPL(lock_set_class);
  3019. /*
  3020. * We are not always called with irqs disabled - do that here,
  3021. * and also avoid lockdep recursion:
  3022. */
  3023. void lock_acquire(struct lockdep_map *lock, unsigned int subclass,
  3024. int trylock, int read, int check,
  3025. struct lockdep_map *nest_lock, unsigned long ip)
  3026. {
  3027. unsigned long flags;
  3028. if (unlikely(current->lockdep_recursion))
  3029. return;
  3030. raw_local_irq_save(flags);
  3031. check_flags(flags);
  3032. current->lockdep_recursion = 1;
  3033. trace_lock_acquire(lock, subclass, trylock, read, check, nest_lock, ip);
  3034. __lock_acquire(lock, subclass, trylock, read, check,
  3035. irqs_disabled_flags(flags), nest_lock, ip, 0);
  3036. current->lockdep_recursion = 0;
  3037. raw_local_irq_restore(flags);
  3038. }
  3039. EXPORT_SYMBOL_GPL(lock_acquire);
  3040. void lock_release(struct lockdep_map *lock, int nested,
  3041. unsigned long ip)
  3042. {
  3043. unsigned long flags;
  3044. if (unlikely(current->lockdep_recursion))
  3045. return;
  3046. raw_local_irq_save(flags);
  3047. check_flags(flags);
  3048. current->lockdep_recursion = 1;
  3049. trace_lock_release(lock, ip);
  3050. __lock_release(lock, nested, ip);
  3051. current->lockdep_recursion = 0;
  3052. raw_local_irq_restore(flags);
  3053. }
  3054. EXPORT_SYMBOL_GPL(lock_release);
  3055. int lock_is_held(struct lockdep_map *lock)
  3056. {
  3057. unsigned long flags;
  3058. int ret = 0;
  3059. if (unlikely(current->lockdep_recursion))
  3060. return 1; /* avoid false negative lockdep_assert_held() */
  3061. raw_local_irq_save(flags);
  3062. check_flags(flags);
  3063. current->lockdep_recursion = 1;
  3064. ret = __lock_is_held(lock);
  3065. current->lockdep_recursion = 0;
  3066. raw_local_irq_restore(flags);
  3067. return ret;
  3068. }
  3069. EXPORT_SYMBOL_GPL(lock_is_held);
  3070. void lockdep_set_current_reclaim_state(gfp_t gfp_mask)
  3071. {
  3072. current->lockdep_reclaim_gfp = gfp_mask;
  3073. }
  3074. void lockdep_clear_current_reclaim_state(void)
  3075. {
  3076. current->lockdep_reclaim_gfp = 0;
  3077. }
  3078. #ifdef CONFIG_LOCK_STAT
  3079. static int
  3080. print_lock_contention_bug(struct task_struct *curr, struct lockdep_map *lock,
  3081. unsigned long ip)
  3082. {
  3083. if (!debug_locks_off())
  3084. return 0;
  3085. if (debug_locks_silent)
  3086. return 0;
  3087. printk("\n");
  3088. printk("=================================\n");
  3089. printk("[ BUG: bad contention detected! ]\n");
  3090. printk("---------------------------------\n");
  3091. printk("%s/%d is trying to contend lock (",
  3092. curr->comm, task_pid_nr(curr));
  3093. print_lockdep_cache(lock);
  3094. printk(") at:\n");
  3095. print_ip_sym(ip);
  3096. printk("but there are no locks held!\n");
  3097. printk("\nother info that might help us debug this:\n");
  3098. lockdep_print_held_locks(curr);
  3099. printk("\nstack backtrace:\n");
  3100. dump_stack();
  3101. return 0;
  3102. }
  3103. static void
  3104. __lock_contended(struct lockdep_map *lock, unsigned long ip)
  3105. {
  3106. struct task_struct *curr = current;
  3107. struct held_lock *hlock, *prev_hlock;
  3108. struct lock_class_stats *stats;
  3109. unsigned int depth;
  3110. int i, contention_point, contending_point;
  3111. depth = curr->lockdep_depth;
  3112. /*
  3113. * Whee, we contended on this lock, except it seems we're not
  3114. * actually trying to acquire anything much at all..
  3115. */
  3116. if (DEBUG_LOCKS_WARN_ON(!depth))
  3117. return;
  3118. prev_hlock = NULL;
  3119. for (i = depth-1; i >= 0; i--) {
  3120. hlock = curr->held_locks + i;
  3121. /*
  3122. * We must not cross into another context:
  3123. */
  3124. if (prev_hlock && prev_hlock->irq_context != hlock->irq_context)
  3125. break;
  3126. if (match_held_lock(hlock, lock))
  3127. goto found_it;
  3128. prev_hlock = hlock;
  3129. }
  3130. print_lock_contention_bug(curr, lock, ip);
  3131. return;
  3132. found_it:
  3133. if (hlock->instance != lock)
  3134. return;
  3135. hlock->waittime_stamp = lockstat_clock();
  3136. contention_point = lock_point(hlock_class(hlock)->contention_point, ip);
  3137. contending_point = lock_point(hlock_class(hlock)->contending_point,
  3138. lock->ip);
  3139. stats = get_lock_stats(hlock_class(hlock));
  3140. if (contention_point < LOCKSTAT_POINTS)
  3141. stats->contention_point[contention_point]++;
  3142. if (contending_point < LOCKSTAT_POINTS)
  3143. stats->contending_point[contending_point]++;
  3144. if (lock->cpu != smp_processor_id())
  3145. stats->bounces[bounce_contended + !!hlock->read]++;
  3146. put_lock_stats(stats);
  3147. }
  3148. static void
  3149. __lock_acquired(struct lockdep_map *lock, unsigned long ip)
  3150. {
  3151. struct task_struct *curr = current;
  3152. struct held_lock *hlock, *prev_hlock;
  3153. struct lock_class_stats *stats;
  3154. unsigned int depth;
  3155. u64 now, waittime = 0;
  3156. int i, cpu;
  3157. depth = curr->lockdep_depth;
  3158. /*
  3159. * Yay, we acquired ownership of this lock we didn't try to
  3160. * acquire, how the heck did that happen?
  3161. */
  3162. if (DEBUG_LOCKS_WARN_ON(!depth))
  3163. return;
  3164. prev_hlock = NULL;
  3165. for (i = depth-1; i >= 0; i--) {
  3166. hlock = curr->held_locks + i;
  3167. /*
  3168. * We must not cross into another context:
  3169. */
  3170. if (prev_hlock && prev_hlock->irq_context != hlock->irq_context)
  3171. break;
  3172. if (match_held_lock(hlock, lock))
  3173. goto found_it;
  3174. prev_hlock = hlock;
  3175. }
  3176. print_lock_contention_bug(curr, lock, _RET_IP_);
  3177. return;
  3178. found_it:
  3179. if (hlock->instance != lock)
  3180. return;
  3181. cpu = smp_processor_id();
  3182. if (hlock->waittime_stamp) {
  3183. now = lockstat_clock();
  3184. waittime = now - hlock->waittime_stamp;
  3185. hlock->holdtime_stamp = now;
  3186. }
  3187. trace_lock_acquired(lock, ip);
  3188. stats = get_lock_stats(hlock_class(hlock));
  3189. if (waittime) {
  3190. if (hlock->read)
  3191. lock_time_inc(&stats->read_waittime, waittime);
  3192. else
  3193. lock_time_inc(&stats->write_waittime, waittime);
  3194. }
  3195. if (lock->cpu != cpu)
  3196. stats->bounces[bounce_acquired + !!hlock->read]++;
  3197. put_lock_stats(stats);
  3198. lock->cpu = cpu;
  3199. lock->ip = ip;
  3200. }
  3201. void lock_contended(struct lockdep_map *lock, unsigned long ip)
  3202. {
  3203. unsigned long flags;
  3204. if (unlikely(!lock_stat))
  3205. return;
  3206. if (unlikely(current->lockdep_recursion))
  3207. return;
  3208. raw_local_irq_save(flags);
  3209. check_flags(flags);
  3210. current->lockdep_recursion = 1;
  3211. trace_lock_contended(lock, ip);
  3212. __lock_contended(lock, ip);
  3213. current->lockdep_recursion = 0;
  3214. raw_local_irq_restore(flags);
  3215. }
  3216. EXPORT_SYMBOL_GPL(lock_contended);
  3217. void lock_acquired(struct lockdep_map *lock, unsigned long ip)
  3218. {
  3219. unsigned long flags;
  3220. if (unlikely(!lock_stat))
  3221. return;
  3222. if (unlikely(current->lockdep_recursion))
  3223. return;
  3224. raw_local_irq_save(flags);
  3225. check_flags(flags);
  3226. current->lockdep_recursion = 1;
  3227. __lock_acquired(lock, ip);
  3228. current->lockdep_recursion = 0;
  3229. raw_local_irq_restore(flags);
  3230. }
  3231. EXPORT_SYMBOL_GPL(lock_acquired);
  3232. #endif
  3233. /*
  3234. * Used by the testsuite, sanitize the validator state
  3235. * after a simulated failure:
  3236. */
  3237. void lockdep_reset(void)
  3238. {
  3239. unsigned long flags;
  3240. int i;
  3241. raw_local_irq_save(flags);
  3242. current->curr_chain_key = 0;
  3243. current->lockdep_depth = 0;
  3244. current->lockdep_recursion = 0;
  3245. memset(current->held_locks, 0, MAX_LOCK_DEPTH*sizeof(struct held_lock));
  3246. nr_hardirq_chains = 0;
  3247. nr_softirq_chains = 0;
  3248. nr_process_chains = 0;
  3249. debug_locks = 1;
  3250. for (i = 0; i < CHAINHASH_SIZE; i++)
  3251. INIT_LIST_HEAD(chainhash_table + i);
  3252. raw_local_irq_restore(flags);
  3253. }
  3254. static void zap_class(struct lock_class *class)
  3255. {
  3256. int i;
  3257. /*
  3258. * Remove all dependencies this lock is
  3259. * involved in:
  3260. */
  3261. for (i = 0; i < nr_list_entries; i++) {
  3262. if (list_entries[i].class == class)
  3263. list_del_rcu(&list_entries[i].entry);
  3264. }
  3265. /*
  3266. * Unhash the class and remove it from the all_lock_classes list:
  3267. */
  3268. list_del_rcu(&class->hash_entry);
  3269. list_del_rcu(&class->lock_entry);
  3270. class->key = NULL;
  3271. }
  3272. static inline int within(const void *addr, void *start, unsigned long size)
  3273. {
  3274. return addr >= start && addr < start + size;
  3275. }
  3276. void lockdep_free_key_range(void *start, unsigned long size)
  3277. {
  3278. struct lock_class *class, *next;
  3279. struct list_head *head;
  3280. unsigned long flags;
  3281. int i;
  3282. int locked;
  3283. raw_local_irq_save(flags);
  3284. locked = graph_lock();
  3285. /*
  3286. * Unhash all classes that were created by this module:
  3287. */
  3288. for (i = 0; i < CLASSHASH_SIZE; i++) {
  3289. head = classhash_table + i;
  3290. if (list_empty(head))
  3291. continue;
  3292. list_for_each_entry_safe(class, next, head, hash_entry) {
  3293. if (within(class->key, start, size))
  3294. zap_class(class);
  3295. else if (within(class->name, start, size))
  3296. zap_class(class);
  3297. }
  3298. }
  3299. if (locked)
  3300. graph_unlock();
  3301. raw_local_irq_restore(flags);
  3302. }
  3303. void lockdep_reset_lock(struct lockdep_map *lock)
  3304. {
  3305. struct lock_class *class, *next;
  3306. struct list_head *head;
  3307. unsigned long flags;
  3308. int i, j;
  3309. int locked;
  3310. raw_local_irq_save(flags);
  3311. /*
  3312. * Remove all classes this lock might have:
  3313. */
  3314. for (j = 0; j < MAX_LOCKDEP_SUBCLASSES; j++) {
  3315. /*
  3316. * If the class exists we look it up and zap it:
  3317. */
  3318. class = look_up_lock_class(lock, j);
  3319. if (class)
  3320. zap_class(class);
  3321. }
  3322. /*
  3323. * Debug check: in the end all mapped classes should
  3324. * be gone.
  3325. */
  3326. locked = graph_lock();
  3327. for (i = 0; i < CLASSHASH_SIZE; i++) {
  3328. head = classhash_table + i;
  3329. if (list_empty(head))
  3330. continue;
  3331. list_for_each_entry_safe(class, next, head, hash_entry) {
  3332. int match = 0;
  3333. for (j = 0; j < NR_LOCKDEP_CACHING_CLASSES; j++)
  3334. match |= class == lock->class_cache[j];
  3335. if (unlikely(match)) {
  3336. if (debug_locks_off_graph_unlock()) {
  3337. /*
  3338. * We all just reset everything, how did it match?
  3339. */
  3340. WARN_ON(1);
  3341. }
  3342. goto out_restore;
  3343. }
  3344. }
  3345. }
  3346. if (locked)
  3347. graph_unlock();
  3348. out_restore:
  3349. raw_local_irq_restore(flags);
  3350. }
  3351. void lockdep_init(void)
  3352. {
  3353. int i;
  3354. /*
  3355. * Some architectures have their own start_kernel()
  3356. * code which calls lockdep_init(), while we also
  3357. * call lockdep_init() from the start_kernel() itself,
  3358. * and we want to initialize the hashes only once:
  3359. */
  3360. if (lockdep_initialized)
  3361. return;
  3362. for (i = 0; i < CLASSHASH_SIZE; i++)
  3363. INIT_LIST_HEAD(classhash_table + i);
  3364. for (i = 0; i < CHAINHASH_SIZE; i++)
  3365. INIT_LIST_HEAD(chainhash_table + i);
  3366. lockdep_initialized = 1;
  3367. }
  3368. void __init lockdep_info(void)
  3369. {
  3370. printk("Lock dependency validator: Copyright (c) 2006 Red Hat, Inc., Ingo Molnar\n");
  3371. printk("... MAX_LOCKDEP_SUBCLASSES: %lu\n", MAX_LOCKDEP_SUBCLASSES);
  3372. printk("... MAX_LOCK_DEPTH: %lu\n", MAX_LOCK_DEPTH);
  3373. printk("... MAX_LOCKDEP_KEYS: %lu\n", MAX_LOCKDEP_KEYS);
  3374. printk("... CLASSHASH_SIZE: %lu\n", CLASSHASH_SIZE);
  3375. printk("... MAX_LOCKDEP_ENTRIES: %lu\n", MAX_LOCKDEP_ENTRIES);
  3376. printk("... MAX_LOCKDEP_CHAINS: %lu\n", MAX_LOCKDEP_CHAINS);
  3377. printk("... CHAINHASH_SIZE: %lu\n", CHAINHASH_SIZE);
  3378. printk(" memory used by lock dependency info: %lu kB\n",
  3379. (sizeof(struct lock_class) * MAX_LOCKDEP_KEYS +
  3380. sizeof(struct list_head) * CLASSHASH_SIZE +
  3381. sizeof(struct lock_list) * MAX_LOCKDEP_ENTRIES +
  3382. sizeof(struct lock_chain) * MAX_LOCKDEP_CHAINS +
  3383. sizeof(struct list_head) * CHAINHASH_SIZE
  3384. #ifdef CONFIG_PROVE_LOCKING
  3385. + sizeof(struct circular_queue)
  3386. #endif
  3387. ) / 1024
  3388. );
  3389. printk(" per task-struct memory footprint: %lu bytes\n",
  3390. sizeof(struct held_lock) * MAX_LOCK_DEPTH);
  3391. #ifdef CONFIG_DEBUG_LOCKDEP
  3392. if (lockdep_init_error) {
  3393. printk("WARNING: lockdep init error! Arch code didn't call lockdep_init() early enough?\n");
  3394. printk("Call stack leading to lockdep invocation was:\n");
  3395. print_stack_trace(&lockdep_init_trace, 0);
  3396. }
  3397. #endif
  3398. }
  3399. static void
  3400. print_freed_lock_bug(struct task_struct *curr, const void *mem_from,
  3401. const void *mem_to, struct held_lock *hlock)
  3402. {
  3403. if (!debug_locks_off())
  3404. return;
  3405. if (debug_locks_silent)
  3406. return;
  3407. printk("\n");
  3408. printk("=========================\n");
  3409. printk("[ BUG: held lock freed! ]\n");
  3410. printk("-------------------------\n");
  3411. printk("%s/%d is freeing memory %p-%p, with a lock still held there!\n",
  3412. curr->comm, task_pid_nr(curr), mem_from, mem_to-1);
  3413. print_lock(hlock);
  3414. lockdep_print_held_locks(curr);
  3415. printk("\nstack backtrace:\n");
  3416. dump_stack();
  3417. }
  3418. static inline int not_in_range(const void* mem_from, unsigned long mem_len,
  3419. const void* lock_from, unsigned long lock_len)
  3420. {
  3421. return lock_from + lock_len <= mem_from ||
  3422. mem_from + mem_len <= lock_from;
  3423. }
  3424. /*
  3425. * Called when kernel memory is freed (or unmapped), or if a lock
  3426. * is destroyed or reinitialized - this code checks whether there is
  3427. * any held lock in the memory range of <from> to <to>:
  3428. */
  3429. void debug_check_no_locks_freed(const void *mem_from, unsigned long mem_len)
  3430. {
  3431. struct task_struct *curr = current;
  3432. struct held_lock *hlock;
  3433. unsigned long flags;
  3434. int i;
  3435. if (unlikely(!debug_locks))
  3436. return;
  3437. local_irq_save(flags);
  3438. for (i = 0; i < curr->lockdep_depth; i++) {
  3439. hlock = curr->held_locks + i;
  3440. if (not_in_range(mem_from, mem_len, hlock->instance,
  3441. sizeof(*hlock->instance)))
  3442. continue;
  3443. print_freed_lock_bug(curr, mem_from, mem_from + mem_len, hlock);
  3444. break;
  3445. }
  3446. local_irq_restore(flags);
  3447. }
  3448. EXPORT_SYMBOL_GPL(debug_check_no_locks_freed);
  3449. static void print_held_locks_bug(struct task_struct *curr)
  3450. {
  3451. if (!debug_locks_off())
  3452. return;
  3453. if (debug_locks_silent)
  3454. return;
  3455. printk("\n");
  3456. printk("=====================================\n");
  3457. printk("[ BUG: lock held at task exit time! ]\n");
  3458. printk("-------------------------------------\n");
  3459. printk("%s/%d is exiting with locks still held!\n",
  3460. curr->comm, task_pid_nr(curr));
  3461. lockdep_print_held_locks(curr);
  3462. printk("\nstack backtrace:\n");
  3463. dump_stack();
  3464. }
  3465. void debug_check_no_locks_held(struct task_struct *task)
  3466. {
  3467. if (unlikely(task->lockdep_depth > 0))
  3468. print_held_locks_bug(task);
  3469. }
  3470. void debug_show_all_locks(void)
  3471. {
  3472. struct task_struct *g, *p;
  3473. int count = 10;
  3474. int unlock = 1;
  3475. if (unlikely(!debug_locks)) {
  3476. printk("INFO: lockdep is turned off.\n");
  3477. return;
  3478. }
  3479. printk("\nShowing all locks held in the system:\n");
  3480. /*
  3481. * Here we try to get the tasklist_lock as hard as possible,
  3482. * if not successful after 2 seconds we ignore it (but keep
  3483. * trying). This is to enable a debug printout even if a
  3484. * tasklist_lock-holding task deadlocks or crashes.
  3485. */
  3486. retry:
  3487. if (!read_trylock(&tasklist_lock)) {
  3488. if (count == 10)
  3489. printk("hm, tasklist_lock locked, retrying... ");
  3490. if (count) {
  3491. count--;
  3492. printk(" #%d", 10-count);
  3493. mdelay(200);
  3494. goto retry;
  3495. }
  3496. printk(" ignoring it.\n");
  3497. unlock = 0;
  3498. } else {
  3499. if (count != 10)
  3500. printk(KERN_CONT " locked it.\n");
  3501. }
  3502. do_each_thread(g, p) {
  3503. /*
  3504. * It's not reliable to print a task's held locks
  3505. * if it's not sleeping (or if it's not the current
  3506. * task):
  3507. */
  3508. if (p->state == TASK_RUNNING && p != current)
  3509. continue;
  3510. if (p->lockdep_depth)
  3511. lockdep_print_held_locks(p);
  3512. if (!unlock)
  3513. if (read_trylock(&tasklist_lock))
  3514. unlock = 1;
  3515. } while_each_thread(g, p);
  3516. printk("\n");
  3517. printk("=============================================\n\n");
  3518. if (unlock)
  3519. read_unlock(&tasklist_lock);
  3520. }
  3521. EXPORT_SYMBOL_GPL(debug_show_all_locks);
  3522. /*
  3523. * Careful: only use this function if you are sure that
  3524. * the task cannot run in parallel!
  3525. */
  3526. void debug_show_held_locks(struct task_struct *task)
  3527. {
  3528. if (unlikely(!debug_locks)) {
  3529. printk("INFO: lockdep is turned off.\n");
  3530. return;
  3531. }
  3532. lockdep_print_held_locks(task);
  3533. }
  3534. EXPORT_SYMBOL_GPL(debug_show_held_locks);
  3535. void lockdep_sys_exit(void)
  3536. {
  3537. struct task_struct *curr = current;
  3538. if (unlikely(curr->lockdep_depth)) {
  3539. if (!debug_locks_off())
  3540. return;
  3541. printk("\n");
  3542. printk("================================================\n");
  3543. printk("[ BUG: lock held when returning to user space! ]\n");
  3544. printk("------------------------------------------------\n");
  3545. printk("%s/%d is leaving the kernel with locks still held!\n",
  3546. curr->comm, curr->pid);
  3547. lockdep_print_held_locks(curr);
  3548. }
  3549. }
  3550. void lockdep_rcu_suspicious(const char *file, const int line, const char *s)
  3551. {
  3552. struct task_struct *curr = current;
  3553. #ifndef CONFIG_PROVE_RCU_REPEATEDLY
  3554. if (!debug_locks_off())
  3555. return;
  3556. #endif /* #ifdef CONFIG_PROVE_RCU_REPEATEDLY */
  3557. /* Note: the following can be executed concurrently, so be careful. */
  3558. printk("\n");
  3559. printk("===============================\n");
  3560. printk("[ INFO: suspicious RCU usage. ]\n");
  3561. printk("-------------------------------\n");
  3562. printk("%s:%d %s!\n", file, line, s);
  3563. printk("\nother info that might help us debug this:\n\n");
  3564. printk("\nrcu_scheduler_active = %d, debug_locks = %d\n", rcu_scheduler_active, debug_locks);
  3565. lockdep_print_held_locks(curr);
  3566. printk("\nstack backtrace:\n");
  3567. dump_stack();
  3568. }
  3569. EXPORT_SYMBOL_GPL(lockdep_rcu_suspicious);