dcache.c 87 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435
  1. /*
  2. * fs/dcache.c
  3. *
  4. * Complete reimplementation
  5. * (C) 1997 Thomas Schoebel-Theuer,
  6. * with heavy changes by Linus Torvalds
  7. */
  8. /*
  9. * Notes on the allocation strategy:
  10. *
  11. * The dcache is a master of the icache - whenever a dcache entry
  12. * exists, the inode will always exist. "iput()" is done either when
  13. * the dcache entry is deleted or garbage collected.
  14. */
  15. #include <linux/syscalls.h>
  16. #include <linux/string.h>
  17. #include <linux/mm.h>
  18. #include <linux/fs.h>
  19. #include <linux/fsnotify.h>
  20. #include <linux/slab.h>
  21. #include <linux/init.h>
  22. #include <linux/hash.h>
  23. #include <linux/cache.h>
  24. #include <linux/export.h>
  25. #include <linux/mount.h>
  26. #include <linux/file.h>
  27. #include <asm/uaccess.h>
  28. #include <linux/security.h>
  29. #include <linux/seqlock.h>
  30. #include <linux/swap.h>
  31. #include <linux/bootmem.h>
  32. #include <linux/fs_struct.h>
  33. #include <linux/hardirq.h>
  34. #include <linux/bit_spinlock.h>
  35. #include <linux/rculist_bl.h>
  36. #include <linux/prefetch.h>
  37. #include <linux/ratelimit.h>
  38. #include <linux/list_lru.h>
  39. #include "internal.h"
  40. #include "mount.h"
  41. /*
  42. * Usage:
  43. * dcache->d_inode->i_lock protects:
  44. * - i_dentry, d_alias, d_inode of aliases
  45. * dcache_hash_bucket lock protects:
  46. * - the dcache hash table
  47. * s_anon bl list spinlock protects:
  48. * - the s_anon list (see __d_drop)
  49. * dentry->d_sb->s_dentry_lru_lock protects:
  50. * - the dcache lru lists and counters
  51. * d_lock protects:
  52. * - d_flags
  53. * - d_name
  54. * - d_lru
  55. * - d_count
  56. * - d_unhashed()
  57. * - d_parent and d_subdirs
  58. * - childrens' d_child and d_parent
  59. * - d_alias, d_inode
  60. *
  61. * Ordering:
  62. * dentry->d_inode->i_lock
  63. * dentry->d_lock
  64. * dentry->d_sb->s_dentry_lru_lock
  65. * dcache_hash_bucket lock
  66. * s_anon lock
  67. *
  68. * If there is an ancestor relationship:
  69. * dentry->d_parent->...->d_parent->d_lock
  70. * ...
  71. * dentry->d_parent->d_lock
  72. * dentry->d_lock
  73. *
  74. * If no ancestor relationship:
  75. * if (dentry1 < dentry2)
  76. * dentry1->d_lock
  77. * dentry2->d_lock
  78. */
  79. int sysctl_vfs_cache_pressure __read_mostly = 100;
  80. EXPORT_SYMBOL_GPL(sysctl_vfs_cache_pressure);
  81. __cacheline_aligned_in_smp DEFINE_SEQLOCK(rename_lock);
  82. EXPORT_SYMBOL(rename_lock);
  83. static struct kmem_cache *dentry_cache __read_mostly;
  84. /*
  85. * This is the single most critical data structure when it comes
  86. * to the dcache: the hashtable for lookups. Somebody should try
  87. * to make this good - I've just made it work.
  88. *
  89. * This hash-function tries to avoid losing too many bits of hash
  90. * information, yet avoid using a prime hash-size or similar.
  91. */
  92. static unsigned int d_hash_mask __read_mostly;
  93. static unsigned int d_hash_shift __read_mostly;
  94. static struct hlist_bl_head *dentry_hashtable __read_mostly;
  95. static inline struct hlist_bl_head *d_hash(const struct dentry *parent,
  96. unsigned int hash)
  97. {
  98. hash += (unsigned long) parent / L1_CACHE_BYTES;
  99. hash = hash + (hash >> d_hash_shift);
  100. return dentry_hashtable + (hash & d_hash_mask);
  101. }
  102. /* Statistics gathering. */
  103. struct dentry_stat_t dentry_stat = {
  104. .age_limit = 45,
  105. };
  106. static DEFINE_PER_CPU(long, nr_dentry);
  107. static DEFINE_PER_CPU(long, nr_dentry_unused);
  108. #if defined(CONFIG_SYSCTL) && defined(CONFIG_PROC_FS)
  109. /*
  110. * Here we resort to our own counters instead of using generic per-cpu counters
  111. * for consistency with what the vfs inode code does. We are expected to harvest
  112. * better code and performance by having our own specialized counters.
  113. *
  114. * Please note that the loop is done over all possible CPUs, not over all online
  115. * CPUs. The reason for this is that we don't want to play games with CPUs going
  116. * on and off. If one of them goes off, we will just keep their counters.
  117. *
  118. * glommer: See cffbc8a for details, and if you ever intend to change this,
  119. * please update all vfs counters to match.
  120. */
  121. static long get_nr_dentry(void)
  122. {
  123. int i;
  124. long sum = 0;
  125. for_each_possible_cpu(i)
  126. sum += per_cpu(nr_dentry, i);
  127. return sum < 0 ? 0 : sum;
  128. }
  129. static long get_nr_dentry_unused(void)
  130. {
  131. int i;
  132. long sum = 0;
  133. for_each_possible_cpu(i)
  134. sum += per_cpu(nr_dentry_unused, i);
  135. return sum < 0 ? 0 : sum;
  136. }
  137. int proc_nr_dentry(ctl_table *table, int write, void __user *buffer,
  138. size_t *lenp, loff_t *ppos)
  139. {
  140. dentry_stat.nr_dentry = get_nr_dentry();
  141. dentry_stat.nr_unused = get_nr_dentry_unused();
  142. return proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
  143. }
  144. #endif
  145. /*
  146. * Compare 2 name strings, return 0 if they match, otherwise non-zero.
  147. * The strings are both count bytes long, and count is non-zero.
  148. */
  149. #ifdef CONFIG_DCACHE_WORD_ACCESS
  150. #include <asm/word-at-a-time.h>
  151. /*
  152. * NOTE! 'cs' and 'scount' come from a dentry, so it has a
  153. * aligned allocation for this particular component. We don't
  154. * strictly need the load_unaligned_zeropad() safety, but it
  155. * doesn't hurt either.
  156. *
  157. * In contrast, 'ct' and 'tcount' can be from a pathname, and do
  158. * need the careful unaligned handling.
  159. */
  160. static inline int dentry_string_cmp(const unsigned char *cs, const unsigned char *ct, unsigned tcount)
  161. {
  162. unsigned long a,b,mask;
  163. for (;;) {
  164. a = *(unsigned long *)cs;
  165. b = load_unaligned_zeropad(ct);
  166. if (tcount < sizeof(unsigned long))
  167. break;
  168. if (unlikely(a != b))
  169. return 1;
  170. cs += sizeof(unsigned long);
  171. ct += sizeof(unsigned long);
  172. tcount -= sizeof(unsigned long);
  173. if (!tcount)
  174. return 0;
  175. }
  176. mask = ~(~0ul << tcount*8);
  177. return unlikely(!!((a ^ b) & mask));
  178. }
  179. #else
  180. static inline int dentry_string_cmp(const unsigned char *cs, const unsigned char *ct, unsigned tcount)
  181. {
  182. do {
  183. if (*cs != *ct)
  184. return 1;
  185. cs++;
  186. ct++;
  187. tcount--;
  188. } while (tcount);
  189. return 0;
  190. }
  191. #endif
  192. static inline int dentry_cmp(const struct dentry *dentry, const unsigned char *ct, unsigned tcount)
  193. {
  194. const unsigned char *cs;
  195. /*
  196. * Be careful about RCU walk racing with rename:
  197. * use ACCESS_ONCE to fetch the name pointer.
  198. *
  199. * NOTE! Even if a rename will mean that the length
  200. * was not loaded atomically, we don't care. The
  201. * RCU walk will check the sequence count eventually,
  202. * and catch it. And we won't overrun the buffer,
  203. * because we're reading the name pointer atomically,
  204. * and a dentry name is guaranteed to be properly
  205. * terminated with a NUL byte.
  206. *
  207. * End result: even if 'len' is wrong, we'll exit
  208. * early because the data cannot match (there can
  209. * be no NUL in the ct/tcount data)
  210. */
  211. cs = ACCESS_ONCE(dentry->d_name.name);
  212. smp_read_barrier_depends();
  213. return dentry_string_cmp(cs, ct, tcount);
  214. }
  215. static void __d_free(struct rcu_head *head)
  216. {
  217. struct dentry *dentry = container_of(head, struct dentry, d_u.d_rcu);
  218. WARN_ON(!hlist_unhashed(&dentry->d_alias));
  219. if (dname_external(dentry))
  220. kfree(dentry->d_name.name);
  221. kmem_cache_free(dentry_cache, dentry);
  222. }
  223. /*
  224. * no locks, please.
  225. */
  226. static void d_free(struct dentry *dentry)
  227. {
  228. BUG_ON((int)dentry->d_lockref.count > 0);
  229. this_cpu_dec(nr_dentry);
  230. if (dentry->d_op && dentry->d_op->d_release)
  231. dentry->d_op->d_release(dentry);
  232. /* if dentry was never visible to RCU, immediate free is OK */
  233. if (!(dentry->d_flags & DCACHE_RCUACCESS))
  234. __d_free(&dentry->d_u.d_rcu);
  235. else
  236. call_rcu(&dentry->d_u.d_rcu, __d_free);
  237. }
  238. /**
  239. * dentry_rcuwalk_barrier - invalidate in-progress rcu-walk lookups
  240. * @dentry: the target dentry
  241. * After this call, in-progress rcu-walk path lookup will fail. This
  242. * should be called after unhashing, and after changing d_inode (if
  243. * the dentry has not already been unhashed).
  244. */
  245. static inline void dentry_rcuwalk_barrier(struct dentry *dentry)
  246. {
  247. assert_spin_locked(&dentry->d_lock);
  248. /* Go through a barrier */
  249. write_seqcount_barrier(&dentry->d_seq);
  250. }
  251. /*
  252. * Release the dentry's inode, using the filesystem
  253. * d_iput() operation if defined. Dentry has no refcount
  254. * and is unhashed.
  255. */
  256. static void dentry_iput(struct dentry * dentry)
  257. __releases(dentry->d_lock)
  258. __releases(dentry->d_inode->i_lock)
  259. {
  260. struct inode *inode = dentry->d_inode;
  261. if (inode) {
  262. dentry->d_inode = NULL;
  263. hlist_del_init(&dentry->d_alias);
  264. spin_unlock(&dentry->d_lock);
  265. spin_unlock(&inode->i_lock);
  266. if (!inode->i_nlink)
  267. fsnotify_inoderemove(inode);
  268. if (dentry->d_op && dentry->d_op->d_iput)
  269. dentry->d_op->d_iput(dentry, inode);
  270. else
  271. iput(inode);
  272. } else {
  273. spin_unlock(&dentry->d_lock);
  274. }
  275. }
  276. /*
  277. * Release the dentry's inode, using the filesystem
  278. * d_iput() operation if defined. dentry remains in-use.
  279. */
  280. static void dentry_unlink_inode(struct dentry * dentry)
  281. __releases(dentry->d_lock)
  282. __releases(dentry->d_inode->i_lock)
  283. {
  284. struct inode *inode = dentry->d_inode;
  285. __d_clear_type(dentry);
  286. dentry->d_inode = NULL;
  287. hlist_del_init(&dentry->d_alias);
  288. dentry_rcuwalk_barrier(dentry);
  289. spin_unlock(&dentry->d_lock);
  290. spin_unlock(&inode->i_lock);
  291. if (!inode->i_nlink)
  292. fsnotify_inoderemove(inode);
  293. if (dentry->d_op && dentry->d_op->d_iput)
  294. dentry->d_op->d_iput(dentry, inode);
  295. else
  296. iput(inode);
  297. }
  298. /*
  299. * The DCACHE_LRU_LIST bit is set whenever the 'd_lru' entry
  300. * is in use - which includes both the "real" per-superblock
  301. * LRU list _and_ the DCACHE_SHRINK_LIST use.
  302. *
  303. * The DCACHE_SHRINK_LIST bit is set whenever the dentry is
  304. * on the shrink list (ie not on the superblock LRU list).
  305. *
  306. * The per-cpu "nr_dentry_unused" counters are updated with
  307. * the DCACHE_LRU_LIST bit.
  308. *
  309. * These helper functions make sure we always follow the
  310. * rules. d_lock must be held by the caller.
  311. */
  312. #define D_FLAG_VERIFY(dentry,x) WARN_ON_ONCE(((dentry)->d_flags & (DCACHE_LRU_LIST | DCACHE_SHRINK_LIST)) != (x))
  313. static void d_lru_add(struct dentry *dentry)
  314. {
  315. D_FLAG_VERIFY(dentry, 0);
  316. dentry->d_flags |= DCACHE_LRU_LIST;
  317. this_cpu_inc(nr_dentry_unused);
  318. WARN_ON_ONCE(!list_lru_add(&dentry->d_sb->s_dentry_lru, &dentry->d_lru));
  319. }
  320. static void d_lru_del(struct dentry *dentry)
  321. {
  322. D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);
  323. dentry->d_flags &= ~DCACHE_LRU_LIST;
  324. this_cpu_dec(nr_dentry_unused);
  325. WARN_ON_ONCE(!list_lru_del(&dentry->d_sb->s_dentry_lru, &dentry->d_lru));
  326. }
  327. static void d_shrink_del(struct dentry *dentry)
  328. {
  329. D_FLAG_VERIFY(dentry, DCACHE_SHRINK_LIST | DCACHE_LRU_LIST);
  330. list_del_init(&dentry->d_lru);
  331. dentry->d_flags &= ~(DCACHE_SHRINK_LIST | DCACHE_LRU_LIST);
  332. this_cpu_dec(nr_dentry_unused);
  333. }
  334. static void d_shrink_add(struct dentry *dentry, struct list_head *list)
  335. {
  336. D_FLAG_VERIFY(dentry, 0);
  337. list_add(&dentry->d_lru, list);
  338. dentry->d_flags |= DCACHE_SHRINK_LIST | DCACHE_LRU_LIST;
  339. this_cpu_inc(nr_dentry_unused);
  340. }
  341. /*
  342. * These can only be called under the global LRU lock, ie during the
  343. * callback for freeing the LRU list. "isolate" removes it from the
  344. * LRU lists entirely, while shrink_move moves it to the indicated
  345. * private list.
  346. */
  347. static void d_lru_isolate(struct dentry *dentry)
  348. {
  349. D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);
  350. dentry->d_flags &= ~DCACHE_LRU_LIST;
  351. this_cpu_dec(nr_dentry_unused);
  352. list_del_init(&dentry->d_lru);
  353. }
  354. static void d_lru_shrink_move(struct dentry *dentry, struct list_head *list)
  355. {
  356. D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);
  357. dentry->d_flags |= DCACHE_SHRINK_LIST;
  358. list_move_tail(&dentry->d_lru, list);
  359. }
  360. /*
  361. * dentry_lru_(add|del)_list) must be called with d_lock held.
  362. */
  363. static void dentry_lru_add(struct dentry *dentry)
  364. {
  365. if (unlikely(!(dentry->d_flags & DCACHE_LRU_LIST)))
  366. d_lru_add(dentry);
  367. }
  368. /*
  369. * Remove a dentry with references from the LRU.
  370. *
  371. * If we are on the shrink list, then we can get to try_prune_one_dentry() and
  372. * lose our last reference through the parent walk. In this case, we need to
  373. * remove ourselves from the shrink list, not the LRU.
  374. */
  375. static void dentry_lru_del(struct dentry *dentry)
  376. {
  377. if (dentry->d_flags & DCACHE_LRU_LIST) {
  378. if (dentry->d_flags & DCACHE_SHRINK_LIST)
  379. return d_shrink_del(dentry);
  380. d_lru_del(dentry);
  381. }
  382. }
  383. /**
  384. * d_kill - kill dentry and return parent
  385. * @dentry: dentry to kill
  386. * @parent: parent dentry
  387. *
  388. * The dentry must already be unhashed and removed from the LRU.
  389. *
  390. * If this is the root of the dentry tree, return NULL.
  391. *
  392. * dentry->d_lock and parent->d_lock must be held by caller, and are dropped by
  393. * d_kill.
  394. */
  395. static struct dentry *d_kill(struct dentry *dentry, struct dentry *parent)
  396. __releases(dentry->d_lock)
  397. __releases(parent->d_lock)
  398. __releases(dentry->d_inode->i_lock)
  399. {
  400. list_del(&dentry->d_u.d_child);
  401. /*
  402. * Inform d_walk() that we are no longer attached to the
  403. * dentry tree
  404. */
  405. dentry->d_flags |= DCACHE_DENTRY_KILLED;
  406. if (parent)
  407. spin_unlock(&parent->d_lock);
  408. dentry_iput(dentry);
  409. /*
  410. * dentry_iput drops the locks, at which point nobody (except
  411. * transient RCU lookups) can reach this dentry.
  412. */
  413. d_free(dentry);
  414. return parent;
  415. }
  416. /**
  417. * d_drop - drop a dentry
  418. * @dentry: dentry to drop
  419. *
  420. * d_drop() unhashes the entry from the parent dentry hashes, so that it won't
  421. * be found through a VFS lookup any more. Note that this is different from
  422. * deleting the dentry - d_delete will try to mark the dentry negative if
  423. * possible, giving a successful _negative_ lookup, while d_drop will
  424. * just make the cache lookup fail.
  425. *
  426. * d_drop() is used mainly for stuff that wants to invalidate a dentry for some
  427. * reason (NFS timeouts or autofs deletes).
  428. *
  429. * __d_drop requires dentry->d_lock.
  430. */
  431. void __d_drop(struct dentry *dentry)
  432. {
  433. if (!d_unhashed(dentry)) {
  434. struct hlist_bl_head *b;
  435. /*
  436. * Hashed dentries are normally on the dentry hashtable,
  437. * with the exception of those newly allocated by
  438. * d_obtain_alias, which are always IS_ROOT:
  439. */
  440. if (unlikely(IS_ROOT(dentry)))
  441. b = &dentry->d_sb->s_anon;
  442. else
  443. b = d_hash(dentry->d_parent, dentry->d_name.hash);
  444. hlist_bl_lock(b);
  445. __hlist_bl_del(&dentry->d_hash);
  446. dentry->d_hash.pprev = NULL;
  447. hlist_bl_unlock(b);
  448. dentry_rcuwalk_barrier(dentry);
  449. }
  450. }
  451. EXPORT_SYMBOL(__d_drop);
  452. void d_drop(struct dentry *dentry)
  453. {
  454. spin_lock(&dentry->d_lock);
  455. __d_drop(dentry);
  456. spin_unlock(&dentry->d_lock);
  457. }
  458. EXPORT_SYMBOL(d_drop);
  459. /*
  460. * Finish off a dentry we've decided to kill.
  461. * dentry->d_lock must be held, returns with it unlocked.
  462. * If ref is non-zero, then decrement the refcount too.
  463. * Returns dentry requiring refcount drop, or NULL if we're done.
  464. */
  465. static struct dentry *
  466. dentry_kill(struct dentry *dentry, int unlock_on_failure)
  467. __releases(dentry->d_lock)
  468. {
  469. struct inode *inode;
  470. struct dentry *parent;
  471. inode = dentry->d_inode;
  472. if (inode && !spin_trylock(&inode->i_lock)) {
  473. relock:
  474. if (unlock_on_failure) {
  475. spin_unlock(&dentry->d_lock);
  476. cpu_relax();
  477. }
  478. return dentry; /* try again with same dentry */
  479. }
  480. if (IS_ROOT(dentry))
  481. parent = NULL;
  482. else
  483. parent = dentry->d_parent;
  484. if (parent && !spin_trylock(&parent->d_lock)) {
  485. if (inode)
  486. spin_unlock(&inode->i_lock);
  487. goto relock;
  488. }
  489. /*
  490. * The dentry is now unrecoverably dead to the world.
  491. */
  492. lockref_mark_dead(&dentry->d_lockref);
  493. /*
  494. * inform the fs via d_prune that this dentry is about to be
  495. * unhashed and destroyed.
  496. */
  497. if ((dentry->d_flags & DCACHE_OP_PRUNE) && !d_unhashed(dentry))
  498. dentry->d_op->d_prune(dentry);
  499. dentry_lru_del(dentry);
  500. /* if it was on the hash then remove it */
  501. __d_drop(dentry);
  502. return d_kill(dentry, parent);
  503. }
  504. /*
  505. * This is dput
  506. *
  507. * This is complicated by the fact that we do not want to put
  508. * dentries that are no longer on any hash chain on the unused
  509. * list: we'd much rather just get rid of them immediately.
  510. *
  511. * However, that implies that we have to traverse the dentry
  512. * tree upwards to the parents which might _also_ now be
  513. * scheduled for deletion (it may have been only waiting for
  514. * its last child to go away).
  515. *
  516. * This tail recursion is done by hand as we don't want to depend
  517. * on the compiler to always get this right (gcc generally doesn't).
  518. * Real recursion would eat up our stack space.
  519. */
  520. /*
  521. * dput - release a dentry
  522. * @dentry: dentry to release
  523. *
  524. * Release a dentry. This will drop the usage count and if appropriate
  525. * call the dentry unlink method as well as removing it from the queues and
  526. * releasing its resources. If the parent dentries were scheduled for release
  527. * they too may now get deleted.
  528. */
  529. void dput(struct dentry *dentry)
  530. {
  531. if (unlikely(!dentry))
  532. return;
  533. repeat:
  534. if (lockref_put_or_lock(&dentry->d_lockref))
  535. return;
  536. /* Unreachable? Get rid of it */
  537. if (unlikely(d_unhashed(dentry)))
  538. goto kill_it;
  539. if (unlikely(dentry->d_flags & DCACHE_OP_DELETE)) {
  540. if (dentry->d_op->d_delete(dentry))
  541. goto kill_it;
  542. }
  543. if (!(dentry->d_flags & DCACHE_REFERENCED))
  544. dentry->d_flags |= DCACHE_REFERENCED;
  545. dentry_lru_add(dentry);
  546. dentry->d_lockref.count--;
  547. spin_unlock(&dentry->d_lock);
  548. return;
  549. kill_it:
  550. dentry = dentry_kill(dentry, 1);
  551. if (dentry)
  552. goto repeat;
  553. }
  554. EXPORT_SYMBOL(dput);
  555. /**
  556. * d_invalidate - invalidate a dentry
  557. * @dentry: dentry to invalidate
  558. *
  559. * Try to invalidate the dentry if it turns out to be
  560. * possible. If there are other dentries that can be
  561. * reached through this one we can't delete it and we
  562. * return -EBUSY. On success we return 0.
  563. *
  564. * no dcache lock.
  565. */
  566. int d_invalidate(struct dentry * dentry)
  567. {
  568. /*
  569. * If it's already been dropped, return OK.
  570. */
  571. spin_lock(&dentry->d_lock);
  572. if (d_unhashed(dentry)) {
  573. spin_unlock(&dentry->d_lock);
  574. return 0;
  575. }
  576. /*
  577. * Check whether to do a partial shrink_dcache
  578. * to get rid of unused child entries.
  579. */
  580. if (!list_empty(&dentry->d_subdirs)) {
  581. spin_unlock(&dentry->d_lock);
  582. shrink_dcache_parent(dentry);
  583. spin_lock(&dentry->d_lock);
  584. }
  585. /*
  586. * Somebody else still using it?
  587. *
  588. * If it's a directory, we can't drop it
  589. * for fear of somebody re-populating it
  590. * with children (even though dropping it
  591. * would make it unreachable from the root,
  592. * we might still populate it if it was a
  593. * working directory or similar).
  594. * We also need to leave mountpoints alone,
  595. * directory or not.
  596. */
  597. if (dentry->d_lockref.count > 1 && dentry->d_inode) {
  598. if (S_ISDIR(dentry->d_inode->i_mode) || d_mountpoint(dentry)) {
  599. spin_unlock(&dentry->d_lock);
  600. return -EBUSY;
  601. }
  602. }
  603. __d_drop(dentry);
  604. spin_unlock(&dentry->d_lock);
  605. return 0;
  606. }
  607. EXPORT_SYMBOL(d_invalidate);
  608. /* This must be called with d_lock held */
  609. static inline void __dget_dlock(struct dentry *dentry)
  610. {
  611. dentry->d_lockref.count++;
  612. }
  613. static inline void __dget(struct dentry *dentry)
  614. {
  615. lockref_get(&dentry->d_lockref);
  616. }
  617. struct dentry *dget_parent(struct dentry *dentry)
  618. {
  619. int gotref;
  620. struct dentry *ret;
  621. /*
  622. * Do optimistic parent lookup without any
  623. * locking.
  624. */
  625. rcu_read_lock();
  626. ret = ACCESS_ONCE(dentry->d_parent);
  627. gotref = lockref_get_not_zero(&ret->d_lockref);
  628. rcu_read_unlock();
  629. if (likely(gotref)) {
  630. if (likely(ret == ACCESS_ONCE(dentry->d_parent)))
  631. return ret;
  632. dput(ret);
  633. }
  634. repeat:
  635. /*
  636. * Don't need rcu_dereference because we re-check it was correct under
  637. * the lock.
  638. */
  639. rcu_read_lock();
  640. ret = dentry->d_parent;
  641. spin_lock(&ret->d_lock);
  642. if (unlikely(ret != dentry->d_parent)) {
  643. spin_unlock(&ret->d_lock);
  644. rcu_read_unlock();
  645. goto repeat;
  646. }
  647. rcu_read_unlock();
  648. BUG_ON(!ret->d_lockref.count);
  649. ret->d_lockref.count++;
  650. spin_unlock(&ret->d_lock);
  651. return ret;
  652. }
  653. EXPORT_SYMBOL(dget_parent);
  654. /**
  655. * d_find_alias - grab a hashed alias of inode
  656. * @inode: inode in question
  657. * @want_discon: flag, used by d_splice_alias, to request
  658. * that only a DISCONNECTED alias be returned.
  659. *
  660. * If inode has a hashed alias, or is a directory and has any alias,
  661. * acquire the reference to alias and return it. Otherwise return NULL.
  662. * Notice that if inode is a directory there can be only one alias and
  663. * it can be unhashed only if it has no children, or if it is the root
  664. * of a filesystem.
  665. *
  666. * If the inode has an IS_ROOT, DCACHE_DISCONNECTED alias, then prefer
  667. * any other hashed alias over that one unless @want_discon is set,
  668. * in which case only return an IS_ROOT, DCACHE_DISCONNECTED alias.
  669. */
  670. static struct dentry *__d_find_alias(struct inode *inode, int want_discon)
  671. {
  672. struct dentry *alias, *discon_alias;
  673. again:
  674. discon_alias = NULL;
  675. hlist_for_each_entry(alias, &inode->i_dentry, d_alias) {
  676. spin_lock(&alias->d_lock);
  677. if (S_ISDIR(inode->i_mode) || !d_unhashed(alias)) {
  678. if (IS_ROOT(alias) &&
  679. (alias->d_flags & DCACHE_DISCONNECTED)) {
  680. discon_alias = alias;
  681. } else if (!want_discon) {
  682. __dget_dlock(alias);
  683. spin_unlock(&alias->d_lock);
  684. return alias;
  685. }
  686. }
  687. spin_unlock(&alias->d_lock);
  688. }
  689. if (discon_alias) {
  690. alias = discon_alias;
  691. spin_lock(&alias->d_lock);
  692. if (S_ISDIR(inode->i_mode) || !d_unhashed(alias)) {
  693. if (IS_ROOT(alias) &&
  694. (alias->d_flags & DCACHE_DISCONNECTED)) {
  695. __dget_dlock(alias);
  696. spin_unlock(&alias->d_lock);
  697. return alias;
  698. }
  699. }
  700. spin_unlock(&alias->d_lock);
  701. goto again;
  702. }
  703. return NULL;
  704. }
  705. struct dentry *d_find_alias(struct inode *inode)
  706. {
  707. struct dentry *de = NULL;
  708. if (!hlist_empty(&inode->i_dentry)) {
  709. spin_lock(&inode->i_lock);
  710. de = __d_find_alias(inode, 0);
  711. spin_unlock(&inode->i_lock);
  712. }
  713. return de;
  714. }
  715. EXPORT_SYMBOL(d_find_alias);
  716. /*
  717. * Try to kill dentries associated with this inode.
  718. * WARNING: you must own a reference to inode.
  719. */
  720. void d_prune_aliases(struct inode *inode)
  721. {
  722. struct dentry *dentry;
  723. restart:
  724. spin_lock(&inode->i_lock);
  725. hlist_for_each_entry(dentry, &inode->i_dentry, d_alias) {
  726. spin_lock(&dentry->d_lock);
  727. if (!dentry->d_lockref.count) {
  728. /*
  729. * inform the fs via d_prune that this dentry
  730. * is about to be unhashed and destroyed.
  731. */
  732. if ((dentry->d_flags & DCACHE_OP_PRUNE) &&
  733. !d_unhashed(dentry))
  734. dentry->d_op->d_prune(dentry);
  735. __dget_dlock(dentry);
  736. __d_drop(dentry);
  737. spin_unlock(&dentry->d_lock);
  738. spin_unlock(&inode->i_lock);
  739. dput(dentry);
  740. goto restart;
  741. }
  742. spin_unlock(&dentry->d_lock);
  743. }
  744. spin_unlock(&inode->i_lock);
  745. }
  746. EXPORT_SYMBOL(d_prune_aliases);
  747. /*
  748. * Try to throw away a dentry - free the inode, dput the parent.
  749. * Requires dentry->d_lock is held, and dentry->d_count == 0.
  750. * Releases dentry->d_lock.
  751. *
  752. * This may fail if locks cannot be acquired no problem, just try again.
  753. */
  754. static struct dentry * try_prune_one_dentry(struct dentry *dentry)
  755. __releases(dentry->d_lock)
  756. {
  757. struct dentry *parent;
  758. parent = dentry_kill(dentry, 0);
  759. /*
  760. * If dentry_kill returns NULL, we have nothing more to do.
  761. * if it returns the same dentry, trylocks failed. In either
  762. * case, just loop again.
  763. *
  764. * Otherwise, we need to prune ancestors too. This is necessary
  765. * to prevent quadratic behavior of shrink_dcache_parent(), but
  766. * is also expected to be beneficial in reducing dentry cache
  767. * fragmentation.
  768. */
  769. if (!parent)
  770. return NULL;
  771. if (parent == dentry)
  772. return dentry;
  773. /* Prune ancestors. */
  774. dentry = parent;
  775. while (dentry) {
  776. if (lockref_put_or_lock(&dentry->d_lockref))
  777. return NULL;
  778. dentry = dentry_kill(dentry, 1);
  779. }
  780. return NULL;
  781. }
  782. static void shrink_dentry_list(struct list_head *list)
  783. {
  784. struct dentry *dentry;
  785. rcu_read_lock();
  786. for (;;) {
  787. dentry = list_entry_rcu(list->prev, struct dentry, d_lru);
  788. if (&dentry->d_lru == list)
  789. break; /* empty */
  790. /*
  791. * Get the dentry lock, and re-verify that the dentry is
  792. * this on the shrinking list. If it is, we know that
  793. * DCACHE_SHRINK_LIST and DCACHE_LRU_LIST are set.
  794. */
  795. spin_lock(&dentry->d_lock);
  796. if (dentry != list_entry(list->prev, struct dentry, d_lru)) {
  797. spin_unlock(&dentry->d_lock);
  798. continue;
  799. }
  800. /*
  801. * The dispose list is isolated and dentries are not accounted
  802. * to the LRU here, so we can simply remove it from the list
  803. * here regardless of whether it is referenced or not.
  804. */
  805. d_shrink_del(dentry);
  806. /*
  807. * We found an inuse dentry which was not removed from
  808. * the LRU because of laziness during lookup. Do not free it.
  809. */
  810. if (dentry->d_lockref.count) {
  811. spin_unlock(&dentry->d_lock);
  812. continue;
  813. }
  814. rcu_read_unlock();
  815. /*
  816. * If 'try_to_prune()' returns a dentry, it will
  817. * be the same one we passed in, and d_lock will
  818. * have been held the whole time, so it will not
  819. * have been added to any other lists. We failed
  820. * to get the inode lock.
  821. *
  822. * We just add it back to the shrink list.
  823. */
  824. dentry = try_prune_one_dentry(dentry);
  825. rcu_read_lock();
  826. if (dentry) {
  827. d_shrink_add(dentry, list);
  828. spin_unlock(&dentry->d_lock);
  829. }
  830. }
  831. rcu_read_unlock();
  832. }
  833. static enum lru_status
  834. dentry_lru_isolate(struct list_head *item, spinlock_t *lru_lock, void *arg)
  835. {
  836. struct list_head *freeable = arg;
  837. struct dentry *dentry = container_of(item, struct dentry, d_lru);
  838. /*
  839. * we are inverting the lru lock/dentry->d_lock here,
  840. * so use a trylock. If we fail to get the lock, just skip
  841. * it
  842. */
  843. if (!spin_trylock(&dentry->d_lock))
  844. return LRU_SKIP;
  845. /*
  846. * Referenced dentries are still in use. If they have active
  847. * counts, just remove them from the LRU. Otherwise give them
  848. * another pass through the LRU.
  849. */
  850. if (dentry->d_lockref.count) {
  851. d_lru_isolate(dentry);
  852. spin_unlock(&dentry->d_lock);
  853. return LRU_REMOVED;
  854. }
  855. if (dentry->d_flags & DCACHE_REFERENCED) {
  856. dentry->d_flags &= ~DCACHE_REFERENCED;
  857. spin_unlock(&dentry->d_lock);
  858. /*
  859. * The list move itself will be made by the common LRU code. At
  860. * this point, we've dropped the dentry->d_lock but keep the
  861. * lru lock. This is safe to do, since every list movement is
  862. * protected by the lru lock even if both locks are held.
  863. *
  864. * This is guaranteed by the fact that all LRU management
  865. * functions are intermediated by the LRU API calls like
  866. * list_lru_add and list_lru_del. List movement in this file
  867. * only ever occur through this functions or through callbacks
  868. * like this one, that are called from the LRU API.
  869. *
  870. * The only exceptions to this are functions like
  871. * shrink_dentry_list, and code that first checks for the
  872. * DCACHE_SHRINK_LIST flag. Those are guaranteed to be
  873. * operating only with stack provided lists after they are
  874. * properly isolated from the main list. It is thus, always a
  875. * local access.
  876. */
  877. return LRU_ROTATE;
  878. }
  879. d_lru_shrink_move(dentry, freeable);
  880. spin_unlock(&dentry->d_lock);
  881. return LRU_REMOVED;
  882. }
  883. /**
  884. * prune_dcache_sb - shrink the dcache
  885. * @sb: superblock
  886. * @nr_to_scan : number of entries to try to free
  887. * @nid: which node to scan for freeable entities
  888. *
  889. * Attempt to shrink the superblock dcache LRU by @nr_to_scan entries. This is
  890. * done when we need more memory an called from the superblock shrinker
  891. * function.
  892. *
  893. * This function may fail to free any resources if all the dentries are in
  894. * use.
  895. */
  896. long prune_dcache_sb(struct super_block *sb, unsigned long nr_to_scan,
  897. int nid)
  898. {
  899. LIST_HEAD(dispose);
  900. long freed;
  901. freed = list_lru_walk_node(&sb->s_dentry_lru, nid, dentry_lru_isolate,
  902. &dispose, &nr_to_scan);
  903. shrink_dentry_list(&dispose);
  904. return freed;
  905. }
  906. static enum lru_status dentry_lru_isolate_shrink(struct list_head *item,
  907. spinlock_t *lru_lock, void *arg)
  908. {
  909. struct list_head *freeable = arg;
  910. struct dentry *dentry = container_of(item, struct dentry, d_lru);
  911. /*
  912. * we are inverting the lru lock/dentry->d_lock here,
  913. * so use a trylock. If we fail to get the lock, just skip
  914. * it
  915. */
  916. if (!spin_trylock(&dentry->d_lock))
  917. return LRU_SKIP;
  918. d_lru_shrink_move(dentry, freeable);
  919. spin_unlock(&dentry->d_lock);
  920. return LRU_REMOVED;
  921. }
  922. /**
  923. * shrink_dcache_sb - shrink dcache for a superblock
  924. * @sb: superblock
  925. *
  926. * Shrink the dcache for the specified super block. This is used to free
  927. * the dcache before unmounting a file system.
  928. */
  929. void shrink_dcache_sb(struct super_block *sb)
  930. {
  931. long freed;
  932. do {
  933. LIST_HEAD(dispose);
  934. freed = list_lru_walk(&sb->s_dentry_lru,
  935. dentry_lru_isolate_shrink, &dispose, UINT_MAX);
  936. this_cpu_sub(nr_dentry_unused, freed);
  937. shrink_dentry_list(&dispose);
  938. } while (freed > 0);
  939. }
  940. EXPORT_SYMBOL(shrink_dcache_sb);
  941. /**
  942. * enum d_walk_ret - action to talke during tree walk
  943. * @D_WALK_CONTINUE: contrinue walk
  944. * @D_WALK_QUIT: quit walk
  945. * @D_WALK_NORETRY: quit when retry is needed
  946. * @D_WALK_SKIP: skip this dentry and its children
  947. */
  948. enum d_walk_ret {
  949. D_WALK_CONTINUE,
  950. D_WALK_QUIT,
  951. D_WALK_NORETRY,
  952. D_WALK_SKIP,
  953. };
  954. /**
  955. * d_walk - walk the dentry tree
  956. * @parent: start of walk
  957. * @data: data passed to @enter() and @finish()
  958. * @enter: callback when first entering the dentry
  959. * @finish: callback when successfully finished the walk
  960. *
  961. * The @enter() and @finish() callbacks are called with d_lock held.
  962. */
  963. static void d_walk(struct dentry *parent, void *data,
  964. enum d_walk_ret (*enter)(void *, struct dentry *),
  965. void (*finish)(void *))
  966. {
  967. struct dentry *this_parent;
  968. struct list_head *next;
  969. unsigned seq = 0;
  970. enum d_walk_ret ret;
  971. bool retry = true;
  972. again:
  973. read_seqbegin_or_lock(&rename_lock, &seq);
  974. this_parent = parent;
  975. spin_lock(&this_parent->d_lock);
  976. ret = enter(data, this_parent);
  977. switch (ret) {
  978. case D_WALK_CONTINUE:
  979. break;
  980. case D_WALK_QUIT:
  981. case D_WALK_SKIP:
  982. goto out_unlock;
  983. case D_WALK_NORETRY:
  984. retry = false;
  985. break;
  986. }
  987. repeat:
  988. next = this_parent->d_subdirs.next;
  989. resume:
  990. while (next != &this_parent->d_subdirs) {
  991. struct list_head *tmp = next;
  992. struct dentry *dentry = list_entry(tmp, struct dentry, d_u.d_child);
  993. next = tmp->next;
  994. spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
  995. ret = enter(data, dentry);
  996. switch (ret) {
  997. case D_WALK_CONTINUE:
  998. break;
  999. case D_WALK_QUIT:
  1000. spin_unlock(&dentry->d_lock);
  1001. goto out_unlock;
  1002. case D_WALK_NORETRY:
  1003. retry = false;
  1004. break;
  1005. case D_WALK_SKIP:
  1006. spin_unlock(&dentry->d_lock);
  1007. continue;
  1008. }
  1009. if (!list_empty(&dentry->d_subdirs)) {
  1010. spin_unlock(&this_parent->d_lock);
  1011. spin_release(&dentry->d_lock.dep_map, 1, _RET_IP_);
  1012. this_parent = dentry;
  1013. spin_acquire(&this_parent->d_lock.dep_map, 0, 1, _RET_IP_);
  1014. goto repeat;
  1015. }
  1016. spin_unlock(&dentry->d_lock);
  1017. }
  1018. /*
  1019. * All done at this level ... ascend and resume the search.
  1020. */
  1021. if (this_parent != parent) {
  1022. struct dentry *child = this_parent;
  1023. this_parent = child->d_parent;
  1024. rcu_read_lock();
  1025. spin_unlock(&child->d_lock);
  1026. spin_lock(&this_parent->d_lock);
  1027. /*
  1028. * might go back up the wrong parent if we have had a rename
  1029. * or deletion
  1030. */
  1031. if (this_parent != child->d_parent ||
  1032. (child->d_flags & DCACHE_DENTRY_KILLED) ||
  1033. need_seqretry(&rename_lock, seq)) {
  1034. spin_unlock(&this_parent->d_lock);
  1035. rcu_read_unlock();
  1036. goto rename_retry;
  1037. }
  1038. rcu_read_unlock();
  1039. next = child->d_u.d_child.next;
  1040. goto resume;
  1041. }
  1042. if (need_seqretry(&rename_lock, seq)) {
  1043. spin_unlock(&this_parent->d_lock);
  1044. goto rename_retry;
  1045. }
  1046. if (finish)
  1047. finish(data);
  1048. out_unlock:
  1049. spin_unlock(&this_parent->d_lock);
  1050. done_seqretry(&rename_lock, seq);
  1051. return;
  1052. rename_retry:
  1053. if (!retry)
  1054. return;
  1055. seq = 1;
  1056. goto again;
  1057. }
  1058. /*
  1059. * Search for at least 1 mount point in the dentry's subdirs.
  1060. * We descend to the next level whenever the d_subdirs
  1061. * list is non-empty and continue searching.
  1062. */
  1063. static enum d_walk_ret check_mount(void *data, struct dentry *dentry)
  1064. {
  1065. int *ret = data;
  1066. if (d_mountpoint(dentry)) {
  1067. *ret = 1;
  1068. return D_WALK_QUIT;
  1069. }
  1070. return D_WALK_CONTINUE;
  1071. }
  1072. /**
  1073. * have_submounts - check for mounts over a dentry
  1074. * @parent: dentry to check.
  1075. *
  1076. * Return true if the parent or its subdirectories contain
  1077. * a mount point
  1078. */
  1079. int have_submounts(struct dentry *parent)
  1080. {
  1081. int ret = 0;
  1082. d_walk(parent, &ret, check_mount, NULL);
  1083. return ret;
  1084. }
  1085. EXPORT_SYMBOL(have_submounts);
  1086. /*
  1087. * Called by mount code to set a mountpoint and check if the mountpoint is
  1088. * reachable (e.g. NFS can unhash a directory dentry and then the complete
  1089. * subtree can become unreachable).
  1090. *
  1091. * Only one of check_submounts_and_drop() and d_set_mounted() must succeed. For
  1092. * this reason take rename_lock and d_lock on dentry and ancestors.
  1093. */
  1094. int d_set_mounted(struct dentry *dentry)
  1095. {
  1096. struct dentry *p;
  1097. int ret = -ENOENT;
  1098. write_seqlock(&rename_lock);
  1099. for (p = dentry->d_parent; !IS_ROOT(p); p = p->d_parent) {
  1100. /* Need exclusion wrt. check_submounts_and_drop() */
  1101. spin_lock(&p->d_lock);
  1102. if (unlikely(d_unhashed(p))) {
  1103. spin_unlock(&p->d_lock);
  1104. goto out;
  1105. }
  1106. spin_unlock(&p->d_lock);
  1107. }
  1108. spin_lock(&dentry->d_lock);
  1109. if (!d_unlinked(dentry)) {
  1110. dentry->d_flags |= DCACHE_MOUNTED;
  1111. ret = 0;
  1112. }
  1113. spin_unlock(&dentry->d_lock);
  1114. out:
  1115. write_sequnlock(&rename_lock);
  1116. return ret;
  1117. }
  1118. /*
  1119. * Search the dentry child list of the specified parent,
  1120. * and move any unused dentries to the end of the unused
  1121. * list for prune_dcache(). We descend to the next level
  1122. * whenever the d_subdirs list is non-empty and continue
  1123. * searching.
  1124. *
  1125. * It returns zero iff there are no unused children,
  1126. * otherwise it returns the number of children moved to
  1127. * the end of the unused list. This may not be the total
  1128. * number of unused children, because select_parent can
  1129. * drop the lock and return early due to latency
  1130. * constraints.
  1131. */
  1132. struct select_data {
  1133. struct dentry *start;
  1134. struct list_head dispose;
  1135. int found;
  1136. };
  1137. static enum d_walk_ret select_collect(void *_data, struct dentry *dentry)
  1138. {
  1139. struct select_data *data = _data;
  1140. enum d_walk_ret ret = D_WALK_CONTINUE;
  1141. if (data->start == dentry)
  1142. goto out;
  1143. /*
  1144. * move only zero ref count dentries to the dispose list.
  1145. *
  1146. * Those which are presently on the shrink list, being processed
  1147. * by shrink_dentry_list(), shouldn't be moved. Otherwise the
  1148. * loop in shrink_dcache_parent() might not make any progress
  1149. * and loop forever.
  1150. */
  1151. if (dentry->d_lockref.count) {
  1152. dentry_lru_del(dentry);
  1153. } else if (!(dentry->d_flags & DCACHE_SHRINK_LIST)) {
  1154. /*
  1155. * We can't use d_lru_shrink_move() because we
  1156. * need to get the global LRU lock and do the
  1157. * LRU accounting.
  1158. */
  1159. d_lru_del(dentry);
  1160. d_shrink_add(dentry, &data->dispose);
  1161. data->found++;
  1162. ret = D_WALK_NORETRY;
  1163. }
  1164. /*
  1165. * We can return to the caller if we have found some (this
  1166. * ensures forward progress). We'll be coming back to find
  1167. * the rest.
  1168. */
  1169. if (data->found && need_resched())
  1170. ret = D_WALK_QUIT;
  1171. out:
  1172. return ret;
  1173. }
  1174. /**
  1175. * shrink_dcache_parent - prune dcache
  1176. * @parent: parent of entries to prune
  1177. *
  1178. * Prune the dcache to remove unused children of the parent dentry.
  1179. */
  1180. void shrink_dcache_parent(struct dentry *parent)
  1181. {
  1182. for (;;) {
  1183. struct select_data data;
  1184. INIT_LIST_HEAD(&data.dispose);
  1185. data.start = parent;
  1186. data.found = 0;
  1187. d_walk(parent, &data, select_collect, NULL);
  1188. if (!data.found)
  1189. break;
  1190. shrink_dentry_list(&data.dispose);
  1191. cond_resched();
  1192. }
  1193. }
  1194. EXPORT_SYMBOL(shrink_dcache_parent);
  1195. static enum d_walk_ret umount_collect(void *_data, struct dentry *dentry)
  1196. {
  1197. struct select_data *data = _data;
  1198. enum d_walk_ret ret = D_WALK_CONTINUE;
  1199. if (dentry->d_lockref.count) {
  1200. dentry_lru_del(dentry);
  1201. if (likely(!list_empty(&dentry->d_subdirs)))
  1202. goto out;
  1203. if (dentry == data->start && dentry->d_lockref.count == 1)
  1204. goto out;
  1205. printk(KERN_ERR
  1206. "BUG: Dentry %p{i=%lx,n=%s}"
  1207. " still in use (%d)"
  1208. " [unmount of %s %s]\n",
  1209. dentry,
  1210. dentry->d_inode ?
  1211. dentry->d_inode->i_ino : 0UL,
  1212. dentry->d_name.name,
  1213. dentry->d_lockref.count,
  1214. dentry->d_sb->s_type->name,
  1215. dentry->d_sb->s_id);
  1216. BUG();
  1217. } else if (!(dentry->d_flags & DCACHE_SHRINK_LIST)) {
  1218. /*
  1219. * We can't use d_lru_shrink_move() because we
  1220. * need to get the global LRU lock and do the
  1221. * LRU accounting.
  1222. */
  1223. if (dentry->d_flags & DCACHE_LRU_LIST)
  1224. d_lru_del(dentry);
  1225. d_shrink_add(dentry, &data->dispose);
  1226. data->found++;
  1227. ret = D_WALK_NORETRY;
  1228. }
  1229. out:
  1230. if (data->found && need_resched())
  1231. ret = D_WALK_QUIT;
  1232. return ret;
  1233. }
  1234. /*
  1235. * destroy the dentries attached to a superblock on unmounting
  1236. */
  1237. void shrink_dcache_for_umount(struct super_block *sb)
  1238. {
  1239. struct dentry *dentry;
  1240. if (down_read_trylock(&sb->s_umount))
  1241. BUG();
  1242. dentry = sb->s_root;
  1243. sb->s_root = NULL;
  1244. for (;;) {
  1245. struct select_data data;
  1246. INIT_LIST_HEAD(&data.dispose);
  1247. data.start = dentry;
  1248. data.found = 0;
  1249. d_walk(dentry, &data, umount_collect, NULL);
  1250. if (!data.found)
  1251. break;
  1252. shrink_dentry_list(&data.dispose);
  1253. cond_resched();
  1254. }
  1255. d_drop(dentry);
  1256. dput(dentry);
  1257. while (!hlist_bl_empty(&sb->s_anon)) {
  1258. struct select_data data;
  1259. dentry = hlist_bl_entry(hlist_bl_first(&sb->s_anon), struct dentry, d_hash);
  1260. INIT_LIST_HEAD(&data.dispose);
  1261. data.start = NULL;
  1262. data.found = 0;
  1263. d_walk(dentry, &data, umount_collect, NULL);
  1264. if (data.found)
  1265. shrink_dentry_list(&data.dispose);
  1266. cond_resched();
  1267. }
  1268. }
  1269. static enum d_walk_ret check_and_collect(void *_data, struct dentry *dentry)
  1270. {
  1271. struct select_data *data = _data;
  1272. if (d_mountpoint(dentry)) {
  1273. data->found = -EBUSY;
  1274. return D_WALK_QUIT;
  1275. }
  1276. return select_collect(_data, dentry);
  1277. }
  1278. static void check_and_drop(void *_data)
  1279. {
  1280. struct select_data *data = _data;
  1281. if (d_mountpoint(data->start))
  1282. data->found = -EBUSY;
  1283. if (!data->found)
  1284. __d_drop(data->start);
  1285. }
  1286. /**
  1287. * check_submounts_and_drop - prune dcache, check for submounts and drop
  1288. *
  1289. * All done as a single atomic operation relative to has_unlinked_ancestor().
  1290. * Returns 0 if successfully unhashed @parent. If there were submounts then
  1291. * return -EBUSY.
  1292. *
  1293. * @dentry: dentry to prune and drop
  1294. */
  1295. int check_submounts_and_drop(struct dentry *dentry)
  1296. {
  1297. int ret = 0;
  1298. /* Negative dentries can be dropped without further checks */
  1299. if (!dentry->d_inode) {
  1300. d_drop(dentry);
  1301. goto out;
  1302. }
  1303. for (;;) {
  1304. struct select_data data;
  1305. INIT_LIST_HEAD(&data.dispose);
  1306. data.start = dentry;
  1307. data.found = 0;
  1308. d_walk(dentry, &data, check_and_collect, check_and_drop);
  1309. ret = data.found;
  1310. if (!list_empty(&data.dispose))
  1311. shrink_dentry_list(&data.dispose);
  1312. if (ret <= 0)
  1313. break;
  1314. cond_resched();
  1315. }
  1316. out:
  1317. return ret;
  1318. }
  1319. EXPORT_SYMBOL(check_submounts_and_drop);
  1320. /**
  1321. * __d_alloc - allocate a dcache entry
  1322. * @sb: filesystem it will belong to
  1323. * @name: qstr of the name
  1324. *
  1325. * Allocates a dentry. It returns %NULL if there is insufficient memory
  1326. * available. On a success the dentry is returned. The name passed in is
  1327. * copied and the copy passed in may be reused after this call.
  1328. */
  1329. struct dentry *__d_alloc(struct super_block *sb, const struct qstr *name)
  1330. {
  1331. struct dentry *dentry;
  1332. char *dname;
  1333. dentry = kmem_cache_alloc(dentry_cache, GFP_KERNEL);
  1334. if (!dentry)
  1335. return NULL;
  1336. /*
  1337. * We guarantee that the inline name is always NUL-terminated.
  1338. * This way the memcpy() done by the name switching in rename
  1339. * will still always have a NUL at the end, even if we might
  1340. * be overwriting an internal NUL character
  1341. */
  1342. dentry->d_iname[DNAME_INLINE_LEN-1] = 0;
  1343. if (name->len > DNAME_INLINE_LEN-1) {
  1344. dname = kmalloc(name->len + 1, GFP_KERNEL);
  1345. if (!dname) {
  1346. kmem_cache_free(dentry_cache, dentry);
  1347. return NULL;
  1348. }
  1349. } else {
  1350. dname = dentry->d_iname;
  1351. }
  1352. dentry->d_name.len = name->len;
  1353. dentry->d_name.hash = name->hash;
  1354. memcpy(dname, name->name, name->len);
  1355. dname[name->len] = 0;
  1356. /* Make sure we always see the terminating NUL character */
  1357. smp_wmb();
  1358. dentry->d_name.name = dname;
  1359. dentry->d_lockref.count = 1;
  1360. dentry->d_flags = 0;
  1361. spin_lock_init(&dentry->d_lock);
  1362. seqcount_init(&dentry->d_seq);
  1363. dentry->d_inode = NULL;
  1364. dentry->d_parent = dentry;
  1365. dentry->d_sb = sb;
  1366. dentry->d_op = NULL;
  1367. dentry->d_fsdata = NULL;
  1368. INIT_HLIST_BL_NODE(&dentry->d_hash);
  1369. INIT_LIST_HEAD(&dentry->d_lru);
  1370. INIT_LIST_HEAD(&dentry->d_subdirs);
  1371. INIT_HLIST_NODE(&dentry->d_alias);
  1372. INIT_LIST_HEAD(&dentry->d_u.d_child);
  1373. d_set_d_op(dentry, dentry->d_sb->s_d_op);
  1374. this_cpu_inc(nr_dentry);
  1375. return dentry;
  1376. }
  1377. /**
  1378. * d_alloc - allocate a dcache entry
  1379. * @parent: parent of entry to allocate
  1380. * @name: qstr of the name
  1381. *
  1382. * Allocates a dentry. It returns %NULL if there is insufficient memory
  1383. * available. On a success the dentry is returned. The name passed in is
  1384. * copied and the copy passed in may be reused after this call.
  1385. */
  1386. struct dentry *d_alloc(struct dentry * parent, const struct qstr *name)
  1387. {
  1388. struct dentry *dentry = __d_alloc(parent->d_sb, name);
  1389. if (!dentry)
  1390. return NULL;
  1391. spin_lock(&parent->d_lock);
  1392. /*
  1393. * don't need child lock because it is not subject
  1394. * to concurrency here
  1395. */
  1396. __dget_dlock(parent);
  1397. dentry->d_parent = parent;
  1398. list_add(&dentry->d_u.d_child, &parent->d_subdirs);
  1399. spin_unlock(&parent->d_lock);
  1400. return dentry;
  1401. }
  1402. EXPORT_SYMBOL(d_alloc);
  1403. /**
  1404. * d_alloc_pseudo - allocate a dentry (for lookup-less filesystems)
  1405. * @sb: the superblock
  1406. * @name: qstr of the name
  1407. *
  1408. * For a filesystem that just pins its dentries in memory and never
  1409. * performs lookups at all, return an unhashed IS_ROOT dentry.
  1410. */
  1411. struct dentry *d_alloc_pseudo(struct super_block *sb, const struct qstr *name)
  1412. {
  1413. return __d_alloc(sb, name);
  1414. }
  1415. EXPORT_SYMBOL(d_alloc_pseudo);
  1416. struct dentry *d_alloc_name(struct dentry *parent, const char *name)
  1417. {
  1418. struct qstr q;
  1419. q.name = name;
  1420. q.len = strlen(name);
  1421. q.hash = full_name_hash(q.name, q.len);
  1422. return d_alloc(parent, &q);
  1423. }
  1424. EXPORT_SYMBOL(d_alloc_name);
  1425. void d_set_d_op(struct dentry *dentry, const struct dentry_operations *op)
  1426. {
  1427. WARN_ON_ONCE(dentry->d_op);
  1428. WARN_ON_ONCE(dentry->d_flags & (DCACHE_OP_HASH |
  1429. DCACHE_OP_COMPARE |
  1430. DCACHE_OP_REVALIDATE |
  1431. DCACHE_OP_WEAK_REVALIDATE |
  1432. DCACHE_OP_DELETE ));
  1433. dentry->d_op = op;
  1434. if (!op)
  1435. return;
  1436. if (op->d_hash)
  1437. dentry->d_flags |= DCACHE_OP_HASH;
  1438. if (op->d_compare)
  1439. dentry->d_flags |= DCACHE_OP_COMPARE;
  1440. if (op->d_revalidate)
  1441. dentry->d_flags |= DCACHE_OP_REVALIDATE;
  1442. if (op->d_weak_revalidate)
  1443. dentry->d_flags |= DCACHE_OP_WEAK_REVALIDATE;
  1444. if (op->d_delete)
  1445. dentry->d_flags |= DCACHE_OP_DELETE;
  1446. if (op->d_prune)
  1447. dentry->d_flags |= DCACHE_OP_PRUNE;
  1448. }
  1449. EXPORT_SYMBOL(d_set_d_op);
  1450. static unsigned d_flags_for_inode(struct inode *inode)
  1451. {
  1452. unsigned add_flags = DCACHE_FILE_TYPE;
  1453. if (!inode)
  1454. return DCACHE_MISS_TYPE;
  1455. if (S_ISDIR(inode->i_mode)) {
  1456. add_flags = DCACHE_DIRECTORY_TYPE;
  1457. if (unlikely(!(inode->i_opflags & IOP_LOOKUP))) {
  1458. if (unlikely(!inode->i_op->lookup))
  1459. add_flags = DCACHE_AUTODIR_TYPE;
  1460. else
  1461. inode->i_opflags |= IOP_LOOKUP;
  1462. }
  1463. } else if (unlikely(!(inode->i_opflags & IOP_NOFOLLOW))) {
  1464. if (unlikely(inode->i_op->follow_link))
  1465. add_flags = DCACHE_SYMLINK_TYPE;
  1466. else
  1467. inode->i_opflags |= IOP_NOFOLLOW;
  1468. }
  1469. if (unlikely(IS_AUTOMOUNT(inode)))
  1470. add_flags |= DCACHE_NEED_AUTOMOUNT;
  1471. return add_flags;
  1472. }
  1473. static void __d_instantiate(struct dentry *dentry, struct inode *inode)
  1474. {
  1475. unsigned add_flags = d_flags_for_inode(inode);
  1476. spin_lock(&dentry->d_lock);
  1477. dentry->d_flags &= ~DCACHE_ENTRY_TYPE;
  1478. dentry->d_flags |= add_flags;
  1479. if (inode)
  1480. hlist_add_head(&dentry->d_alias, &inode->i_dentry);
  1481. dentry->d_inode = inode;
  1482. dentry_rcuwalk_barrier(dentry);
  1483. spin_unlock(&dentry->d_lock);
  1484. fsnotify_d_instantiate(dentry, inode);
  1485. }
  1486. /**
  1487. * d_instantiate - fill in inode information for a dentry
  1488. * @entry: dentry to complete
  1489. * @inode: inode to attach to this dentry
  1490. *
  1491. * Fill in inode information in the entry.
  1492. *
  1493. * This turns negative dentries into productive full members
  1494. * of society.
  1495. *
  1496. * NOTE! This assumes that the inode count has been incremented
  1497. * (or otherwise set) by the caller to indicate that it is now
  1498. * in use by the dcache.
  1499. */
  1500. void d_instantiate(struct dentry *entry, struct inode * inode)
  1501. {
  1502. BUG_ON(!hlist_unhashed(&entry->d_alias));
  1503. if (inode)
  1504. spin_lock(&inode->i_lock);
  1505. __d_instantiate(entry, inode);
  1506. if (inode)
  1507. spin_unlock(&inode->i_lock);
  1508. security_d_instantiate(entry, inode);
  1509. }
  1510. EXPORT_SYMBOL(d_instantiate);
  1511. /**
  1512. * d_instantiate_unique - instantiate a non-aliased dentry
  1513. * @entry: dentry to instantiate
  1514. * @inode: inode to attach to this dentry
  1515. *
  1516. * Fill in inode information in the entry. On success, it returns NULL.
  1517. * If an unhashed alias of "entry" already exists, then we return the
  1518. * aliased dentry instead and drop one reference to inode.
  1519. *
  1520. * Note that in order to avoid conflicts with rename() etc, the caller
  1521. * had better be holding the parent directory semaphore.
  1522. *
  1523. * This also assumes that the inode count has been incremented
  1524. * (or otherwise set) by the caller to indicate that it is now
  1525. * in use by the dcache.
  1526. */
  1527. static struct dentry *__d_instantiate_unique(struct dentry *entry,
  1528. struct inode *inode)
  1529. {
  1530. struct dentry *alias;
  1531. int len = entry->d_name.len;
  1532. const char *name = entry->d_name.name;
  1533. unsigned int hash = entry->d_name.hash;
  1534. if (!inode) {
  1535. __d_instantiate(entry, NULL);
  1536. return NULL;
  1537. }
  1538. hlist_for_each_entry(alias, &inode->i_dentry, d_alias) {
  1539. /*
  1540. * Don't need alias->d_lock here, because aliases with
  1541. * d_parent == entry->d_parent are not subject to name or
  1542. * parent changes, because the parent inode i_mutex is held.
  1543. */
  1544. if (alias->d_name.hash != hash)
  1545. continue;
  1546. if (alias->d_parent != entry->d_parent)
  1547. continue;
  1548. if (alias->d_name.len != len)
  1549. continue;
  1550. if (dentry_cmp(alias, name, len))
  1551. continue;
  1552. __dget(alias);
  1553. return alias;
  1554. }
  1555. __d_instantiate(entry, inode);
  1556. return NULL;
  1557. }
  1558. struct dentry *d_instantiate_unique(struct dentry *entry, struct inode *inode)
  1559. {
  1560. struct dentry *result;
  1561. BUG_ON(!hlist_unhashed(&entry->d_alias));
  1562. if (inode)
  1563. spin_lock(&inode->i_lock);
  1564. result = __d_instantiate_unique(entry, inode);
  1565. if (inode)
  1566. spin_unlock(&inode->i_lock);
  1567. if (!result) {
  1568. security_d_instantiate(entry, inode);
  1569. return NULL;
  1570. }
  1571. BUG_ON(!d_unhashed(result));
  1572. iput(inode);
  1573. return result;
  1574. }
  1575. EXPORT_SYMBOL(d_instantiate_unique);
  1576. /**
  1577. * d_instantiate_no_diralias - instantiate a non-aliased dentry
  1578. * @entry: dentry to complete
  1579. * @inode: inode to attach to this dentry
  1580. *
  1581. * Fill in inode information in the entry. If a directory alias is found, then
  1582. * return an error (and drop inode). Together with d_materialise_unique() this
  1583. * guarantees that a directory inode may never have more than one alias.
  1584. */
  1585. int d_instantiate_no_diralias(struct dentry *entry, struct inode *inode)
  1586. {
  1587. BUG_ON(!hlist_unhashed(&entry->d_alias));
  1588. spin_lock(&inode->i_lock);
  1589. if (S_ISDIR(inode->i_mode) && !hlist_empty(&inode->i_dentry)) {
  1590. spin_unlock(&inode->i_lock);
  1591. iput(inode);
  1592. return -EBUSY;
  1593. }
  1594. __d_instantiate(entry, inode);
  1595. spin_unlock(&inode->i_lock);
  1596. security_d_instantiate(entry, inode);
  1597. return 0;
  1598. }
  1599. EXPORT_SYMBOL(d_instantiate_no_diralias);
  1600. struct dentry *d_make_root(struct inode *root_inode)
  1601. {
  1602. struct dentry *res = NULL;
  1603. if (root_inode) {
  1604. static const struct qstr name = QSTR_INIT("/", 1);
  1605. res = __d_alloc(root_inode->i_sb, &name);
  1606. if (res)
  1607. d_instantiate(res, root_inode);
  1608. else
  1609. iput(root_inode);
  1610. }
  1611. return res;
  1612. }
  1613. EXPORT_SYMBOL(d_make_root);
  1614. static struct dentry * __d_find_any_alias(struct inode *inode)
  1615. {
  1616. struct dentry *alias;
  1617. if (hlist_empty(&inode->i_dentry))
  1618. return NULL;
  1619. alias = hlist_entry(inode->i_dentry.first, struct dentry, d_alias);
  1620. __dget(alias);
  1621. return alias;
  1622. }
  1623. /**
  1624. * d_find_any_alias - find any alias for a given inode
  1625. * @inode: inode to find an alias for
  1626. *
  1627. * If any aliases exist for the given inode, take and return a
  1628. * reference for one of them. If no aliases exist, return %NULL.
  1629. */
  1630. struct dentry *d_find_any_alias(struct inode *inode)
  1631. {
  1632. struct dentry *de;
  1633. spin_lock(&inode->i_lock);
  1634. de = __d_find_any_alias(inode);
  1635. spin_unlock(&inode->i_lock);
  1636. return de;
  1637. }
  1638. EXPORT_SYMBOL(d_find_any_alias);
  1639. /**
  1640. * d_obtain_alias - find or allocate a dentry for a given inode
  1641. * @inode: inode to allocate the dentry for
  1642. *
  1643. * Obtain a dentry for an inode resulting from NFS filehandle conversion or
  1644. * similar open by handle operations. The returned dentry may be anonymous,
  1645. * or may have a full name (if the inode was already in the cache).
  1646. *
  1647. * When called on a directory inode, we must ensure that the inode only ever
  1648. * has one dentry. If a dentry is found, that is returned instead of
  1649. * allocating a new one.
  1650. *
  1651. * On successful return, the reference to the inode has been transferred
  1652. * to the dentry. In case of an error the reference on the inode is released.
  1653. * To make it easier to use in export operations a %NULL or IS_ERR inode may
  1654. * be passed in and will be the error will be propagate to the return value,
  1655. * with a %NULL @inode replaced by ERR_PTR(-ESTALE).
  1656. */
  1657. struct dentry *d_obtain_alias(struct inode *inode)
  1658. {
  1659. static const struct qstr anonstring = QSTR_INIT("/", 1);
  1660. struct dentry *tmp;
  1661. struct dentry *res;
  1662. unsigned add_flags;
  1663. if (!inode)
  1664. return ERR_PTR(-ESTALE);
  1665. if (IS_ERR(inode))
  1666. return ERR_CAST(inode);
  1667. res = d_find_any_alias(inode);
  1668. if (res)
  1669. goto out_iput;
  1670. tmp = __d_alloc(inode->i_sb, &anonstring);
  1671. if (!tmp) {
  1672. res = ERR_PTR(-ENOMEM);
  1673. goto out_iput;
  1674. }
  1675. spin_lock(&inode->i_lock);
  1676. res = __d_find_any_alias(inode);
  1677. if (res) {
  1678. spin_unlock(&inode->i_lock);
  1679. dput(tmp);
  1680. goto out_iput;
  1681. }
  1682. /* attach a disconnected dentry */
  1683. add_flags = d_flags_for_inode(inode) | DCACHE_DISCONNECTED;
  1684. spin_lock(&tmp->d_lock);
  1685. tmp->d_inode = inode;
  1686. tmp->d_flags |= add_flags;
  1687. hlist_add_head(&tmp->d_alias, &inode->i_dentry);
  1688. hlist_bl_lock(&tmp->d_sb->s_anon);
  1689. hlist_bl_add_head(&tmp->d_hash, &tmp->d_sb->s_anon);
  1690. hlist_bl_unlock(&tmp->d_sb->s_anon);
  1691. spin_unlock(&tmp->d_lock);
  1692. spin_unlock(&inode->i_lock);
  1693. security_d_instantiate(tmp, inode);
  1694. return tmp;
  1695. out_iput:
  1696. if (res && !IS_ERR(res))
  1697. security_d_instantiate(res, inode);
  1698. iput(inode);
  1699. return res;
  1700. }
  1701. EXPORT_SYMBOL(d_obtain_alias);
  1702. /**
  1703. * d_splice_alias - splice a disconnected dentry into the tree if one exists
  1704. * @inode: the inode which may have a disconnected dentry
  1705. * @dentry: a negative dentry which we want to point to the inode.
  1706. *
  1707. * If inode is a directory and has a 'disconnected' dentry (i.e. IS_ROOT and
  1708. * DCACHE_DISCONNECTED), then d_move that in place of the given dentry
  1709. * and return it, else simply d_add the inode to the dentry and return NULL.
  1710. *
  1711. * This is needed in the lookup routine of any filesystem that is exportable
  1712. * (via knfsd) so that we can build dcache paths to directories effectively.
  1713. *
  1714. * If a dentry was found and moved, then it is returned. Otherwise NULL
  1715. * is returned. This matches the expected return value of ->lookup.
  1716. *
  1717. * Cluster filesystems may call this function with a negative, hashed dentry.
  1718. * In that case, we know that the inode will be a regular file, and also this
  1719. * will only occur during atomic_open. So we need to check for the dentry
  1720. * being already hashed only in the final case.
  1721. */
  1722. struct dentry *d_splice_alias(struct inode *inode, struct dentry *dentry)
  1723. {
  1724. struct dentry *new = NULL;
  1725. if (IS_ERR(inode))
  1726. return ERR_CAST(inode);
  1727. if (inode && S_ISDIR(inode->i_mode)) {
  1728. spin_lock(&inode->i_lock);
  1729. new = __d_find_alias(inode, 1);
  1730. if (new) {
  1731. BUG_ON(!(new->d_flags & DCACHE_DISCONNECTED));
  1732. spin_unlock(&inode->i_lock);
  1733. security_d_instantiate(new, inode);
  1734. d_move(new, dentry);
  1735. iput(inode);
  1736. } else {
  1737. /* already taking inode->i_lock, so d_add() by hand */
  1738. __d_instantiate(dentry, inode);
  1739. spin_unlock(&inode->i_lock);
  1740. security_d_instantiate(dentry, inode);
  1741. d_rehash(dentry);
  1742. }
  1743. } else {
  1744. d_instantiate(dentry, inode);
  1745. if (d_unhashed(dentry))
  1746. d_rehash(dentry);
  1747. }
  1748. return new;
  1749. }
  1750. EXPORT_SYMBOL(d_splice_alias);
  1751. /**
  1752. * d_add_ci - lookup or allocate new dentry with case-exact name
  1753. * @inode: the inode case-insensitive lookup has found
  1754. * @dentry: the negative dentry that was passed to the parent's lookup func
  1755. * @name: the case-exact name to be associated with the returned dentry
  1756. *
  1757. * This is to avoid filling the dcache with case-insensitive names to the
  1758. * same inode, only the actual correct case is stored in the dcache for
  1759. * case-insensitive filesystems.
  1760. *
  1761. * For a case-insensitive lookup match and if the the case-exact dentry
  1762. * already exists in in the dcache, use it and return it.
  1763. *
  1764. * If no entry exists with the exact case name, allocate new dentry with
  1765. * the exact case, and return the spliced entry.
  1766. */
  1767. struct dentry *d_add_ci(struct dentry *dentry, struct inode *inode,
  1768. struct qstr *name)
  1769. {
  1770. struct dentry *found;
  1771. struct dentry *new;
  1772. /*
  1773. * First check if a dentry matching the name already exists,
  1774. * if not go ahead and create it now.
  1775. */
  1776. found = d_hash_and_lookup(dentry->d_parent, name);
  1777. if (unlikely(IS_ERR(found)))
  1778. goto err_out;
  1779. if (!found) {
  1780. new = d_alloc(dentry->d_parent, name);
  1781. if (!new) {
  1782. found = ERR_PTR(-ENOMEM);
  1783. goto err_out;
  1784. }
  1785. found = d_splice_alias(inode, new);
  1786. if (found) {
  1787. dput(new);
  1788. return found;
  1789. }
  1790. return new;
  1791. }
  1792. /*
  1793. * If a matching dentry exists, and it's not negative use it.
  1794. *
  1795. * Decrement the reference count to balance the iget() done
  1796. * earlier on.
  1797. */
  1798. if (found->d_inode) {
  1799. if (unlikely(found->d_inode != inode)) {
  1800. /* This can't happen because bad inodes are unhashed. */
  1801. BUG_ON(!is_bad_inode(inode));
  1802. BUG_ON(!is_bad_inode(found->d_inode));
  1803. }
  1804. iput(inode);
  1805. return found;
  1806. }
  1807. /*
  1808. * Negative dentry: instantiate it unless the inode is a directory and
  1809. * already has a dentry.
  1810. */
  1811. new = d_splice_alias(inode, found);
  1812. if (new) {
  1813. dput(found);
  1814. found = new;
  1815. }
  1816. return found;
  1817. err_out:
  1818. iput(inode);
  1819. return found;
  1820. }
  1821. EXPORT_SYMBOL(d_add_ci);
  1822. /*
  1823. * Do the slow-case of the dentry name compare.
  1824. *
  1825. * Unlike the dentry_cmp() function, we need to atomically
  1826. * load the name and length information, so that the
  1827. * filesystem can rely on them, and can use the 'name' and
  1828. * 'len' information without worrying about walking off the
  1829. * end of memory etc.
  1830. *
  1831. * Thus the read_seqcount_retry() and the "duplicate" info
  1832. * in arguments (the low-level filesystem should not look
  1833. * at the dentry inode or name contents directly, since
  1834. * rename can change them while we're in RCU mode).
  1835. */
  1836. enum slow_d_compare {
  1837. D_COMP_OK,
  1838. D_COMP_NOMATCH,
  1839. D_COMP_SEQRETRY,
  1840. };
  1841. static noinline enum slow_d_compare slow_dentry_cmp(
  1842. const struct dentry *parent,
  1843. struct dentry *dentry,
  1844. unsigned int seq,
  1845. const struct qstr *name)
  1846. {
  1847. int tlen = dentry->d_name.len;
  1848. const char *tname = dentry->d_name.name;
  1849. if (read_seqcount_retry(&dentry->d_seq, seq)) {
  1850. cpu_relax();
  1851. return D_COMP_SEQRETRY;
  1852. }
  1853. if (parent->d_op->d_compare(parent, dentry, tlen, tname, name))
  1854. return D_COMP_NOMATCH;
  1855. return D_COMP_OK;
  1856. }
  1857. /**
  1858. * __d_lookup_rcu - search for a dentry (racy, store-free)
  1859. * @parent: parent dentry
  1860. * @name: qstr of name we wish to find
  1861. * @seqp: returns d_seq value at the point where the dentry was found
  1862. * Returns: dentry, or NULL
  1863. *
  1864. * __d_lookup_rcu is the dcache lookup function for rcu-walk name
  1865. * resolution (store-free path walking) design described in
  1866. * Documentation/filesystems/path-lookup.txt.
  1867. *
  1868. * This is not to be used outside core vfs.
  1869. *
  1870. * __d_lookup_rcu must only be used in rcu-walk mode, ie. with vfsmount lock
  1871. * held, and rcu_read_lock held. The returned dentry must not be stored into
  1872. * without taking d_lock and checking d_seq sequence count against @seq
  1873. * returned here.
  1874. *
  1875. * A refcount may be taken on the found dentry with the d_rcu_to_refcount
  1876. * function.
  1877. *
  1878. * Alternatively, __d_lookup_rcu may be called again to look up the child of
  1879. * the returned dentry, so long as its parent's seqlock is checked after the
  1880. * child is looked up. Thus, an interlocking stepping of sequence lock checks
  1881. * is formed, giving integrity down the path walk.
  1882. *
  1883. * NOTE! The caller *has* to check the resulting dentry against the sequence
  1884. * number we've returned before using any of the resulting dentry state!
  1885. */
  1886. struct dentry *__d_lookup_rcu(const struct dentry *parent,
  1887. const struct qstr *name,
  1888. unsigned *seqp)
  1889. {
  1890. u64 hashlen = name->hash_len;
  1891. const unsigned char *str = name->name;
  1892. struct hlist_bl_head *b = d_hash(parent, hashlen_hash(hashlen));
  1893. struct hlist_bl_node *node;
  1894. struct dentry *dentry;
  1895. /*
  1896. * Note: There is significant duplication with __d_lookup_rcu which is
  1897. * required to prevent single threaded performance regressions
  1898. * especially on architectures where smp_rmb (in seqcounts) are costly.
  1899. * Keep the two functions in sync.
  1900. */
  1901. /*
  1902. * The hash list is protected using RCU.
  1903. *
  1904. * Carefully use d_seq when comparing a candidate dentry, to avoid
  1905. * races with d_move().
  1906. *
  1907. * It is possible that concurrent renames can mess up our list
  1908. * walk here and result in missing our dentry, resulting in the
  1909. * false-negative result. d_lookup() protects against concurrent
  1910. * renames using rename_lock seqlock.
  1911. *
  1912. * See Documentation/filesystems/path-lookup.txt for more details.
  1913. */
  1914. hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) {
  1915. unsigned seq;
  1916. seqretry:
  1917. /*
  1918. * The dentry sequence count protects us from concurrent
  1919. * renames, and thus protects parent and name fields.
  1920. *
  1921. * The caller must perform a seqcount check in order
  1922. * to do anything useful with the returned dentry.
  1923. *
  1924. * NOTE! We do a "raw" seqcount_begin here. That means that
  1925. * we don't wait for the sequence count to stabilize if it
  1926. * is in the middle of a sequence change. If we do the slow
  1927. * dentry compare, we will do seqretries until it is stable,
  1928. * and if we end up with a successful lookup, we actually
  1929. * want to exit RCU lookup anyway.
  1930. */
  1931. seq = raw_seqcount_begin(&dentry->d_seq);
  1932. if (dentry->d_parent != parent)
  1933. continue;
  1934. if (d_unhashed(dentry))
  1935. continue;
  1936. if (unlikely(parent->d_flags & DCACHE_OP_COMPARE)) {
  1937. if (dentry->d_name.hash != hashlen_hash(hashlen))
  1938. continue;
  1939. *seqp = seq;
  1940. switch (slow_dentry_cmp(parent, dentry, seq, name)) {
  1941. case D_COMP_OK:
  1942. return dentry;
  1943. case D_COMP_NOMATCH:
  1944. continue;
  1945. default:
  1946. goto seqretry;
  1947. }
  1948. }
  1949. if (dentry->d_name.hash_len != hashlen)
  1950. continue;
  1951. *seqp = seq;
  1952. if (!dentry_cmp(dentry, str, hashlen_len(hashlen)))
  1953. return dentry;
  1954. }
  1955. return NULL;
  1956. }
  1957. /**
  1958. * d_lookup - search for a dentry
  1959. * @parent: parent dentry
  1960. * @name: qstr of name we wish to find
  1961. * Returns: dentry, or NULL
  1962. *
  1963. * d_lookup searches the children of the parent dentry for the name in
  1964. * question. If the dentry is found its reference count is incremented and the
  1965. * dentry is returned. The caller must use dput to free the entry when it has
  1966. * finished using it. %NULL is returned if the dentry does not exist.
  1967. */
  1968. struct dentry *d_lookup(const struct dentry *parent, const struct qstr *name)
  1969. {
  1970. struct dentry *dentry;
  1971. unsigned seq;
  1972. do {
  1973. seq = read_seqbegin(&rename_lock);
  1974. dentry = __d_lookup(parent, name);
  1975. if (dentry)
  1976. break;
  1977. } while (read_seqretry(&rename_lock, seq));
  1978. return dentry;
  1979. }
  1980. EXPORT_SYMBOL(d_lookup);
  1981. /**
  1982. * __d_lookup - search for a dentry (racy)
  1983. * @parent: parent dentry
  1984. * @name: qstr of name we wish to find
  1985. * Returns: dentry, or NULL
  1986. *
  1987. * __d_lookup is like d_lookup, however it may (rarely) return a
  1988. * false-negative result due to unrelated rename activity.
  1989. *
  1990. * __d_lookup is slightly faster by avoiding rename_lock read seqlock,
  1991. * however it must be used carefully, eg. with a following d_lookup in
  1992. * the case of failure.
  1993. *
  1994. * __d_lookup callers must be commented.
  1995. */
  1996. struct dentry *__d_lookup(const struct dentry *parent, const struct qstr *name)
  1997. {
  1998. unsigned int len = name->len;
  1999. unsigned int hash = name->hash;
  2000. const unsigned char *str = name->name;
  2001. struct hlist_bl_head *b = d_hash(parent, hash);
  2002. struct hlist_bl_node *node;
  2003. struct dentry *found = NULL;
  2004. struct dentry *dentry;
  2005. /*
  2006. * Note: There is significant duplication with __d_lookup_rcu which is
  2007. * required to prevent single threaded performance regressions
  2008. * especially on architectures where smp_rmb (in seqcounts) are costly.
  2009. * Keep the two functions in sync.
  2010. */
  2011. /*
  2012. * The hash list is protected using RCU.
  2013. *
  2014. * Take d_lock when comparing a candidate dentry, to avoid races
  2015. * with d_move().
  2016. *
  2017. * It is possible that concurrent renames can mess up our list
  2018. * walk here and result in missing our dentry, resulting in the
  2019. * false-negative result. d_lookup() protects against concurrent
  2020. * renames using rename_lock seqlock.
  2021. *
  2022. * See Documentation/filesystems/path-lookup.txt for more details.
  2023. */
  2024. rcu_read_lock();
  2025. hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) {
  2026. if (dentry->d_name.hash != hash)
  2027. continue;
  2028. spin_lock(&dentry->d_lock);
  2029. if (dentry->d_parent != parent)
  2030. goto next;
  2031. if (d_unhashed(dentry))
  2032. goto next;
  2033. /*
  2034. * It is safe to compare names since d_move() cannot
  2035. * change the qstr (protected by d_lock).
  2036. */
  2037. if (parent->d_flags & DCACHE_OP_COMPARE) {
  2038. int tlen = dentry->d_name.len;
  2039. const char *tname = dentry->d_name.name;
  2040. if (parent->d_op->d_compare(parent, dentry, tlen, tname, name))
  2041. goto next;
  2042. } else {
  2043. if (dentry->d_name.len != len)
  2044. goto next;
  2045. if (dentry_cmp(dentry, str, len))
  2046. goto next;
  2047. }
  2048. dentry->d_lockref.count++;
  2049. found = dentry;
  2050. spin_unlock(&dentry->d_lock);
  2051. break;
  2052. next:
  2053. spin_unlock(&dentry->d_lock);
  2054. }
  2055. rcu_read_unlock();
  2056. return found;
  2057. }
  2058. /**
  2059. * d_hash_and_lookup - hash the qstr then search for a dentry
  2060. * @dir: Directory to search in
  2061. * @name: qstr of name we wish to find
  2062. *
  2063. * On lookup failure NULL is returned; on bad name - ERR_PTR(-error)
  2064. */
  2065. struct dentry *d_hash_and_lookup(struct dentry *dir, struct qstr *name)
  2066. {
  2067. /*
  2068. * Check for a fs-specific hash function. Note that we must
  2069. * calculate the standard hash first, as the d_op->d_hash()
  2070. * routine may choose to leave the hash value unchanged.
  2071. */
  2072. name->hash = full_name_hash(name->name, name->len);
  2073. if (dir->d_flags & DCACHE_OP_HASH) {
  2074. int err = dir->d_op->d_hash(dir, name);
  2075. if (unlikely(err < 0))
  2076. return ERR_PTR(err);
  2077. }
  2078. return d_lookup(dir, name);
  2079. }
  2080. EXPORT_SYMBOL(d_hash_and_lookup);
  2081. /**
  2082. * d_validate - verify dentry provided from insecure source (deprecated)
  2083. * @dentry: The dentry alleged to be valid child of @dparent
  2084. * @dparent: The parent dentry (known to be valid)
  2085. *
  2086. * An insecure source has sent us a dentry, here we verify it and dget() it.
  2087. * This is used by ncpfs in its readdir implementation.
  2088. * Zero is returned in the dentry is invalid.
  2089. *
  2090. * This function is slow for big directories, and deprecated, do not use it.
  2091. */
  2092. int d_validate(struct dentry *dentry, struct dentry *dparent)
  2093. {
  2094. struct dentry *child;
  2095. spin_lock(&dparent->d_lock);
  2096. list_for_each_entry(child, &dparent->d_subdirs, d_u.d_child) {
  2097. if (dentry == child) {
  2098. spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
  2099. __dget_dlock(dentry);
  2100. spin_unlock(&dentry->d_lock);
  2101. spin_unlock(&dparent->d_lock);
  2102. return 1;
  2103. }
  2104. }
  2105. spin_unlock(&dparent->d_lock);
  2106. return 0;
  2107. }
  2108. EXPORT_SYMBOL(d_validate);
  2109. /*
  2110. * When a file is deleted, we have two options:
  2111. * - turn this dentry into a negative dentry
  2112. * - unhash this dentry and free it.
  2113. *
  2114. * Usually, we want to just turn this into
  2115. * a negative dentry, but if anybody else is
  2116. * currently using the dentry or the inode
  2117. * we can't do that and we fall back on removing
  2118. * it from the hash queues and waiting for
  2119. * it to be deleted later when it has no users
  2120. */
  2121. /**
  2122. * d_delete - delete a dentry
  2123. * @dentry: The dentry to delete
  2124. *
  2125. * Turn the dentry into a negative dentry if possible, otherwise
  2126. * remove it from the hash queues so it can be deleted later
  2127. */
  2128. void d_delete(struct dentry * dentry)
  2129. {
  2130. struct inode *inode;
  2131. int isdir = 0;
  2132. /*
  2133. * Are we the only user?
  2134. */
  2135. again:
  2136. spin_lock(&dentry->d_lock);
  2137. inode = dentry->d_inode;
  2138. isdir = S_ISDIR(inode->i_mode);
  2139. if (dentry->d_lockref.count == 1) {
  2140. if (!spin_trylock(&inode->i_lock)) {
  2141. spin_unlock(&dentry->d_lock);
  2142. cpu_relax();
  2143. goto again;
  2144. }
  2145. dentry->d_flags &= ~DCACHE_CANT_MOUNT;
  2146. dentry_unlink_inode(dentry);
  2147. fsnotify_nameremove(dentry, isdir);
  2148. return;
  2149. }
  2150. if (!d_unhashed(dentry))
  2151. __d_drop(dentry);
  2152. spin_unlock(&dentry->d_lock);
  2153. fsnotify_nameremove(dentry, isdir);
  2154. }
  2155. EXPORT_SYMBOL(d_delete);
  2156. static void __d_rehash(struct dentry * entry, struct hlist_bl_head *b)
  2157. {
  2158. BUG_ON(!d_unhashed(entry));
  2159. hlist_bl_lock(b);
  2160. entry->d_flags |= DCACHE_RCUACCESS;
  2161. hlist_bl_add_head_rcu(&entry->d_hash, b);
  2162. hlist_bl_unlock(b);
  2163. }
  2164. static void _d_rehash(struct dentry * entry)
  2165. {
  2166. __d_rehash(entry, d_hash(entry->d_parent, entry->d_name.hash));
  2167. }
  2168. /**
  2169. * d_rehash - add an entry back to the hash
  2170. * @entry: dentry to add to the hash
  2171. *
  2172. * Adds a dentry to the hash according to its name.
  2173. */
  2174. void d_rehash(struct dentry * entry)
  2175. {
  2176. spin_lock(&entry->d_lock);
  2177. _d_rehash(entry);
  2178. spin_unlock(&entry->d_lock);
  2179. }
  2180. EXPORT_SYMBOL(d_rehash);
  2181. /**
  2182. * dentry_update_name_case - update case insensitive dentry with a new name
  2183. * @dentry: dentry to be updated
  2184. * @name: new name
  2185. *
  2186. * Update a case insensitive dentry with new case of name.
  2187. *
  2188. * dentry must have been returned by d_lookup with name @name. Old and new
  2189. * name lengths must match (ie. no d_compare which allows mismatched name
  2190. * lengths).
  2191. *
  2192. * Parent inode i_mutex must be held over d_lookup and into this call (to
  2193. * keep renames and concurrent inserts, and readdir(2) away).
  2194. */
  2195. void dentry_update_name_case(struct dentry *dentry, struct qstr *name)
  2196. {
  2197. BUG_ON(!mutex_is_locked(&dentry->d_parent->d_inode->i_mutex));
  2198. BUG_ON(dentry->d_name.len != name->len); /* d_lookup gives this */
  2199. spin_lock(&dentry->d_lock);
  2200. write_seqcount_begin(&dentry->d_seq);
  2201. memcpy((unsigned char *)dentry->d_name.name, name->name, name->len);
  2202. write_seqcount_end(&dentry->d_seq);
  2203. spin_unlock(&dentry->d_lock);
  2204. }
  2205. EXPORT_SYMBOL(dentry_update_name_case);
  2206. static void switch_names(struct dentry *dentry, struct dentry *target)
  2207. {
  2208. if (dname_external(target)) {
  2209. if (dname_external(dentry)) {
  2210. /*
  2211. * Both external: swap the pointers
  2212. */
  2213. swap(target->d_name.name, dentry->d_name.name);
  2214. } else {
  2215. /*
  2216. * dentry:internal, target:external. Steal target's
  2217. * storage and make target internal.
  2218. */
  2219. memcpy(target->d_iname, dentry->d_name.name,
  2220. dentry->d_name.len + 1);
  2221. dentry->d_name.name = target->d_name.name;
  2222. target->d_name.name = target->d_iname;
  2223. }
  2224. } else {
  2225. if (dname_external(dentry)) {
  2226. /*
  2227. * dentry:external, target:internal. Give dentry's
  2228. * storage to target and make dentry internal
  2229. */
  2230. memcpy(dentry->d_iname, target->d_name.name,
  2231. target->d_name.len + 1);
  2232. target->d_name.name = dentry->d_name.name;
  2233. dentry->d_name.name = dentry->d_iname;
  2234. } else {
  2235. /*
  2236. * Both are internal. Just copy target to dentry
  2237. */
  2238. memcpy(dentry->d_iname, target->d_name.name,
  2239. target->d_name.len + 1);
  2240. dentry->d_name.len = target->d_name.len;
  2241. return;
  2242. }
  2243. }
  2244. swap(dentry->d_name.len, target->d_name.len);
  2245. }
  2246. static void dentry_lock_for_move(struct dentry *dentry, struct dentry *target)
  2247. {
  2248. /*
  2249. * XXXX: do we really need to take target->d_lock?
  2250. */
  2251. if (IS_ROOT(dentry) || dentry->d_parent == target->d_parent)
  2252. spin_lock(&target->d_parent->d_lock);
  2253. else {
  2254. if (d_ancestor(dentry->d_parent, target->d_parent)) {
  2255. spin_lock(&dentry->d_parent->d_lock);
  2256. spin_lock_nested(&target->d_parent->d_lock,
  2257. DENTRY_D_LOCK_NESTED);
  2258. } else {
  2259. spin_lock(&target->d_parent->d_lock);
  2260. spin_lock_nested(&dentry->d_parent->d_lock,
  2261. DENTRY_D_LOCK_NESTED);
  2262. }
  2263. }
  2264. if (target < dentry) {
  2265. spin_lock_nested(&target->d_lock, 2);
  2266. spin_lock_nested(&dentry->d_lock, 3);
  2267. } else {
  2268. spin_lock_nested(&dentry->d_lock, 2);
  2269. spin_lock_nested(&target->d_lock, 3);
  2270. }
  2271. }
  2272. static void dentry_unlock_parents_for_move(struct dentry *dentry,
  2273. struct dentry *target)
  2274. {
  2275. if (target->d_parent != dentry->d_parent)
  2276. spin_unlock(&dentry->d_parent->d_lock);
  2277. if (target->d_parent != target)
  2278. spin_unlock(&target->d_parent->d_lock);
  2279. }
  2280. /*
  2281. * When switching names, the actual string doesn't strictly have to
  2282. * be preserved in the target - because we're dropping the target
  2283. * anyway. As such, we can just do a simple memcpy() to copy over
  2284. * the new name before we switch.
  2285. *
  2286. * Note that we have to be a lot more careful about getting the hash
  2287. * switched - we have to switch the hash value properly even if it
  2288. * then no longer matches the actual (corrupted) string of the target.
  2289. * The hash value has to match the hash queue that the dentry is on..
  2290. */
  2291. /*
  2292. * __d_move - move a dentry
  2293. * @dentry: entry to move
  2294. * @target: new dentry
  2295. *
  2296. * Update the dcache to reflect the move of a file name. Negative
  2297. * dcache entries should not be moved in this way. Caller must hold
  2298. * rename_lock, the i_mutex of the source and target directories,
  2299. * and the sb->s_vfs_rename_mutex if they differ. See lock_rename().
  2300. */
  2301. static void __d_move(struct dentry * dentry, struct dentry * target)
  2302. {
  2303. if (!dentry->d_inode)
  2304. printk(KERN_WARNING "VFS: moving negative dcache entry\n");
  2305. BUG_ON(d_ancestor(dentry, target));
  2306. BUG_ON(d_ancestor(target, dentry));
  2307. dentry_lock_for_move(dentry, target);
  2308. write_seqcount_begin(&dentry->d_seq);
  2309. write_seqcount_begin_nested(&target->d_seq, DENTRY_D_LOCK_NESTED);
  2310. /* __d_drop does write_seqcount_barrier, but they're OK to nest. */
  2311. /*
  2312. * Move the dentry to the target hash queue. Don't bother checking
  2313. * for the same hash queue because of how unlikely it is.
  2314. */
  2315. __d_drop(dentry);
  2316. __d_rehash(dentry, d_hash(target->d_parent, target->d_name.hash));
  2317. /* Unhash the target: dput() will then get rid of it */
  2318. __d_drop(target);
  2319. list_del(&dentry->d_u.d_child);
  2320. list_del(&target->d_u.d_child);
  2321. /* Switch the names.. */
  2322. switch_names(dentry, target);
  2323. swap(dentry->d_name.hash, target->d_name.hash);
  2324. /* ... and switch the parents */
  2325. if (IS_ROOT(dentry)) {
  2326. dentry->d_parent = target->d_parent;
  2327. target->d_parent = target;
  2328. INIT_LIST_HEAD(&target->d_u.d_child);
  2329. } else {
  2330. swap(dentry->d_parent, target->d_parent);
  2331. /* And add them back to the (new) parent lists */
  2332. list_add(&target->d_u.d_child, &target->d_parent->d_subdirs);
  2333. }
  2334. list_add(&dentry->d_u.d_child, &dentry->d_parent->d_subdirs);
  2335. write_seqcount_end(&target->d_seq);
  2336. write_seqcount_end(&dentry->d_seq);
  2337. dentry_unlock_parents_for_move(dentry, target);
  2338. spin_unlock(&target->d_lock);
  2339. fsnotify_d_move(dentry);
  2340. spin_unlock(&dentry->d_lock);
  2341. }
  2342. /*
  2343. * d_move - move a dentry
  2344. * @dentry: entry to move
  2345. * @target: new dentry
  2346. *
  2347. * Update the dcache to reflect the move of a file name. Negative
  2348. * dcache entries should not be moved in this way. See the locking
  2349. * requirements for __d_move.
  2350. */
  2351. void d_move(struct dentry *dentry, struct dentry *target)
  2352. {
  2353. write_seqlock(&rename_lock);
  2354. __d_move(dentry, target);
  2355. write_sequnlock(&rename_lock);
  2356. }
  2357. EXPORT_SYMBOL(d_move);
  2358. /**
  2359. * d_ancestor - search for an ancestor
  2360. * @p1: ancestor dentry
  2361. * @p2: child dentry
  2362. *
  2363. * Returns the ancestor dentry of p2 which is a child of p1, if p1 is
  2364. * an ancestor of p2, else NULL.
  2365. */
  2366. struct dentry *d_ancestor(struct dentry *p1, struct dentry *p2)
  2367. {
  2368. struct dentry *p;
  2369. for (p = p2; !IS_ROOT(p); p = p->d_parent) {
  2370. if (p->d_parent == p1)
  2371. return p;
  2372. }
  2373. return NULL;
  2374. }
  2375. /*
  2376. * This helper attempts to cope with remotely renamed directories
  2377. *
  2378. * It assumes that the caller is already holding
  2379. * dentry->d_parent->d_inode->i_mutex, inode->i_lock and rename_lock
  2380. *
  2381. * Note: If ever the locking in lock_rename() changes, then please
  2382. * remember to update this too...
  2383. */
  2384. static struct dentry *__d_unalias(struct inode *inode,
  2385. struct dentry *dentry, struct dentry *alias)
  2386. {
  2387. struct mutex *m1 = NULL, *m2 = NULL;
  2388. struct dentry *ret = ERR_PTR(-EBUSY);
  2389. /* If alias and dentry share a parent, then no extra locks required */
  2390. if (alias->d_parent == dentry->d_parent)
  2391. goto out_unalias;
  2392. /* See lock_rename() */
  2393. if (!mutex_trylock(&dentry->d_sb->s_vfs_rename_mutex))
  2394. goto out_err;
  2395. m1 = &dentry->d_sb->s_vfs_rename_mutex;
  2396. if (!mutex_trylock(&alias->d_parent->d_inode->i_mutex))
  2397. goto out_err;
  2398. m2 = &alias->d_parent->d_inode->i_mutex;
  2399. out_unalias:
  2400. if (likely(!d_mountpoint(alias))) {
  2401. __d_move(alias, dentry);
  2402. ret = alias;
  2403. }
  2404. out_err:
  2405. spin_unlock(&inode->i_lock);
  2406. if (m2)
  2407. mutex_unlock(m2);
  2408. if (m1)
  2409. mutex_unlock(m1);
  2410. return ret;
  2411. }
  2412. /*
  2413. * Prepare an anonymous dentry for life in the superblock's dentry tree as a
  2414. * named dentry in place of the dentry to be replaced.
  2415. * returns with anon->d_lock held!
  2416. */
  2417. static void __d_materialise_dentry(struct dentry *dentry, struct dentry *anon)
  2418. {
  2419. struct dentry *dparent;
  2420. dentry_lock_for_move(anon, dentry);
  2421. write_seqcount_begin(&dentry->d_seq);
  2422. write_seqcount_begin_nested(&anon->d_seq, DENTRY_D_LOCK_NESTED);
  2423. dparent = dentry->d_parent;
  2424. switch_names(dentry, anon);
  2425. swap(dentry->d_name.hash, anon->d_name.hash);
  2426. dentry->d_parent = dentry;
  2427. list_del_init(&dentry->d_u.d_child);
  2428. anon->d_parent = dparent;
  2429. list_move(&anon->d_u.d_child, &dparent->d_subdirs);
  2430. write_seqcount_end(&dentry->d_seq);
  2431. write_seqcount_end(&anon->d_seq);
  2432. dentry_unlock_parents_for_move(anon, dentry);
  2433. spin_unlock(&dentry->d_lock);
  2434. /* anon->d_lock still locked, returns locked */
  2435. }
  2436. /**
  2437. * d_materialise_unique - introduce an inode into the tree
  2438. * @dentry: candidate dentry
  2439. * @inode: inode to bind to the dentry, to which aliases may be attached
  2440. *
  2441. * Introduces an dentry into the tree, substituting an extant disconnected
  2442. * root directory alias in its place if there is one. Caller must hold the
  2443. * i_mutex of the parent directory.
  2444. */
  2445. struct dentry *d_materialise_unique(struct dentry *dentry, struct inode *inode)
  2446. {
  2447. struct dentry *actual;
  2448. BUG_ON(!d_unhashed(dentry));
  2449. if (!inode) {
  2450. actual = dentry;
  2451. __d_instantiate(dentry, NULL);
  2452. d_rehash(actual);
  2453. goto out_nolock;
  2454. }
  2455. spin_lock(&inode->i_lock);
  2456. if (S_ISDIR(inode->i_mode)) {
  2457. struct dentry *alias;
  2458. /* Does an aliased dentry already exist? */
  2459. alias = __d_find_alias(inode, 0);
  2460. if (alias) {
  2461. actual = alias;
  2462. write_seqlock(&rename_lock);
  2463. if (d_ancestor(alias, dentry)) {
  2464. /* Check for loops */
  2465. actual = ERR_PTR(-ELOOP);
  2466. spin_unlock(&inode->i_lock);
  2467. } else if (IS_ROOT(alias)) {
  2468. /* Is this an anonymous mountpoint that we
  2469. * could splice into our tree? */
  2470. __d_materialise_dentry(dentry, alias);
  2471. write_sequnlock(&rename_lock);
  2472. __d_drop(alias);
  2473. goto found;
  2474. } else {
  2475. /* Nope, but we must(!) avoid directory
  2476. * aliasing. This drops inode->i_lock */
  2477. actual = __d_unalias(inode, dentry, alias);
  2478. }
  2479. write_sequnlock(&rename_lock);
  2480. if (IS_ERR(actual)) {
  2481. if (PTR_ERR(actual) == -ELOOP)
  2482. pr_warn_ratelimited(
  2483. "VFS: Lookup of '%s' in %s %s"
  2484. " would have caused loop\n",
  2485. dentry->d_name.name,
  2486. inode->i_sb->s_type->name,
  2487. inode->i_sb->s_id);
  2488. dput(alias);
  2489. }
  2490. goto out_nolock;
  2491. }
  2492. }
  2493. /* Add a unique reference */
  2494. actual = __d_instantiate_unique(dentry, inode);
  2495. if (!actual)
  2496. actual = dentry;
  2497. else
  2498. BUG_ON(!d_unhashed(actual));
  2499. spin_lock(&actual->d_lock);
  2500. found:
  2501. _d_rehash(actual);
  2502. spin_unlock(&actual->d_lock);
  2503. spin_unlock(&inode->i_lock);
  2504. out_nolock:
  2505. if (actual == dentry) {
  2506. security_d_instantiate(dentry, inode);
  2507. return NULL;
  2508. }
  2509. iput(inode);
  2510. return actual;
  2511. }
  2512. EXPORT_SYMBOL_GPL(d_materialise_unique);
  2513. static int prepend(char **buffer, int *buflen, const char *str, int namelen)
  2514. {
  2515. *buflen -= namelen;
  2516. if (*buflen < 0)
  2517. return -ENAMETOOLONG;
  2518. *buffer -= namelen;
  2519. memcpy(*buffer, str, namelen);
  2520. return 0;
  2521. }
  2522. /**
  2523. * prepend_name - prepend a pathname in front of current buffer pointer
  2524. * @buffer: buffer pointer
  2525. * @buflen: allocated length of the buffer
  2526. * @name: name string and length qstr structure
  2527. *
  2528. * With RCU path tracing, it may race with d_move(). Use ACCESS_ONCE() to
  2529. * make sure that either the old or the new name pointer and length are
  2530. * fetched. However, there may be mismatch between length and pointer.
  2531. * The length cannot be trusted, we need to copy it byte-by-byte until
  2532. * the length is reached or a null byte is found. It also prepends "/" at
  2533. * the beginning of the name. The sequence number check at the caller will
  2534. * retry it again when a d_move() does happen. So any garbage in the buffer
  2535. * due to mismatched pointer and length will be discarded.
  2536. */
  2537. static int prepend_name(char **buffer, int *buflen, struct qstr *name)
  2538. {
  2539. const char *dname = ACCESS_ONCE(name->name);
  2540. u32 dlen = ACCESS_ONCE(name->len);
  2541. char *p;
  2542. if (*buflen < dlen + 1)
  2543. return -ENAMETOOLONG;
  2544. *buflen -= dlen + 1;
  2545. p = *buffer -= dlen + 1;
  2546. *p++ = '/';
  2547. while (dlen--) {
  2548. char c = *dname++;
  2549. if (!c)
  2550. break;
  2551. *p++ = c;
  2552. }
  2553. return 0;
  2554. }
  2555. /**
  2556. * prepend_path - Prepend path string to a buffer
  2557. * @path: the dentry/vfsmount to report
  2558. * @root: root vfsmnt/dentry
  2559. * @buffer: pointer to the end of the buffer
  2560. * @buflen: pointer to buffer length
  2561. *
  2562. * The function will first try to write out the pathname without taking any
  2563. * lock other than the RCU read lock to make sure that dentries won't go away.
  2564. * It only checks the sequence number of the global rename_lock as any change
  2565. * in the dentry's d_seq will be preceded by changes in the rename_lock
  2566. * sequence number. If the sequence number had been changed, it will restart
  2567. * the whole pathname back-tracing sequence again by taking the rename_lock.
  2568. * In this case, there is no need to take the RCU read lock as the recursive
  2569. * parent pointer references will keep the dentry chain alive as long as no
  2570. * rename operation is performed.
  2571. */
  2572. static int prepend_path(const struct path *path,
  2573. const struct path *root,
  2574. char **buffer, int *buflen)
  2575. {
  2576. struct dentry *dentry;
  2577. struct vfsmount *vfsmnt;
  2578. struct mount *mnt;
  2579. int error = 0;
  2580. unsigned seq, m_seq = 0;
  2581. char *bptr;
  2582. int blen;
  2583. rcu_read_lock();
  2584. restart_mnt:
  2585. read_seqbegin_or_lock(&mount_lock, &m_seq);
  2586. seq = 0;
  2587. rcu_read_lock();
  2588. restart:
  2589. bptr = *buffer;
  2590. blen = *buflen;
  2591. error = 0;
  2592. dentry = path->dentry;
  2593. vfsmnt = path->mnt;
  2594. mnt = real_mount(vfsmnt);
  2595. read_seqbegin_or_lock(&rename_lock, &seq);
  2596. while (dentry != root->dentry || vfsmnt != root->mnt) {
  2597. struct dentry * parent;
  2598. if (dentry == vfsmnt->mnt_root || IS_ROOT(dentry)) {
  2599. struct mount *parent = ACCESS_ONCE(mnt->mnt_parent);
  2600. /* Global root? */
  2601. if (mnt != parent) {
  2602. dentry = ACCESS_ONCE(mnt->mnt_mountpoint);
  2603. mnt = parent;
  2604. vfsmnt = &mnt->mnt;
  2605. continue;
  2606. }
  2607. /*
  2608. * Filesystems needing to implement special "root names"
  2609. * should do so with ->d_dname()
  2610. */
  2611. if (IS_ROOT(dentry) &&
  2612. (dentry->d_name.len != 1 ||
  2613. dentry->d_name.name[0] != '/')) {
  2614. WARN(1, "Root dentry has weird name <%.*s>\n",
  2615. (int) dentry->d_name.len,
  2616. dentry->d_name.name);
  2617. }
  2618. if (!error)
  2619. error = is_mounted(vfsmnt) ? 1 : 2;
  2620. break;
  2621. }
  2622. parent = dentry->d_parent;
  2623. prefetch(parent);
  2624. error = prepend_name(&bptr, &blen, &dentry->d_name);
  2625. if (error)
  2626. break;
  2627. dentry = parent;
  2628. }
  2629. if (!(seq & 1))
  2630. rcu_read_unlock();
  2631. if (need_seqretry(&rename_lock, seq)) {
  2632. seq = 1;
  2633. goto restart;
  2634. }
  2635. done_seqretry(&rename_lock, seq);
  2636. if (!(m_seq & 1))
  2637. rcu_read_unlock();
  2638. if (need_seqretry(&mount_lock, m_seq)) {
  2639. m_seq = 1;
  2640. goto restart_mnt;
  2641. }
  2642. done_seqretry(&mount_lock, m_seq);
  2643. if (error >= 0 && bptr == *buffer) {
  2644. if (--blen < 0)
  2645. error = -ENAMETOOLONG;
  2646. else
  2647. *--bptr = '/';
  2648. }
  2649. *buffer = bptr;
  2650. *buflen = blen;
  2651. return error;
  2652. }
  2653. /**
  2654. * __d_path - return the path of a dentry
  2655. * @path: the dentry/vfsmount to report
  2656. * @root: root vfsmnt/dentry
  2657. * @buf: buffer to return value in
  2658. * @buflen: buffer length
  2659. *
  2660. * Convert a dentry into an ASCII path name.
  2661. *
  2662. * Returns a pointer into the buffer or an error code if the
  2663. * path was too long.
  2664. *
  2665. * "buflen" should be positive.
  2666. *
  2667. * If the path is not reachable from the supplied root, return %NULL.
  2668. */
  2669. char *__d_path(const struct path *path,
  2670. const struct path *root,
  2671. char *buf, int buflen)
  2672. {
  2673. char *res = buf + buflen;
  2674. int error;
  2675. prepend(&res, &buflen, "\0", 1);
  2676. error = prepend_path(path, root, &res, &buflen);
  2677. if (error < 0)
  2678. return ERR_PTR(error);
  2679. if (error > 0)
  2680. return NULL;
  2681. return res;
  2682. }
  2683. char *d_absolute_path(const struct path *path,
  2684. char *buf, int buflen)
  2685. {
  2686. struct path root = {};
  2687. char *res = buf + buflen;
  2688. int error;
  2689. prepend(&res, &buflen, "\0", 1);
  2690. error = prepend_path(path, &root, &res, &buflen);
  2691. if (error > 1)
  2692. error = -EINVAL;
  2693. if (error < 0)
  2694. return ERR_PTR(error);
  2695. return res;
  2696. }
  2697. /*
  2698. * same as __d_path but appends "(deleted)" for unlinked files.
  2699. */
  2700. static int path_with_deleted(const struct path *path,
  2701. const struct path *root,
  2702. char **buf, int *buflen)
  2703. {
  2704. prepend(buf, buflen, "\0", 1);
  2705. if (d_unlinked(path->dentry)) {
  2706. int error = prepend(buf, buflen, " (deleted)", 10);
  2707. if (error)
  2708. return error;
  2709. }
  2710. return prepend_path(path, root, buf, buflen);
  2711. }
  2712. static int prepend_unreachable(char **buffer, int *buflen)
  2713. {
  2714. return prepend(buffer, buflen, "(unreachable)", 13);
  2715. }
  2716. static void get_fs_root_rcu(struct fs_struct *fs, struct path *root)
  2717. {
  2718. unsigned seq;
  2719. do {
  2720. seq = read_seqcount_begin(&fs->seq);
  2721. *root = fs->root;
  2722. } while (read_seqcount_retry(&fs->seq, seq));
  2723. }
  2724. /**
  2725. * d_path - return the path of a dentry
  2726. * @path: path to report
  2727. * @buf: buffer to return value in
  2728. * @buflen: buffer length
  2729. *
  2730. * Convert a dentry into an ASCII path name. If the entry has been deleted
  2731. * the string " (deleted)" is appended. Note that this is ambiguous.
  2732. *
  2733. * Returns a pointer into the buffer or an error code if the path was
  2734. * too long. Note: Callers should use the returned pointer, not the passed
  2735. * in buffer, to use the name! The implementation often starts at an offset
  2736. * into the buffer, and may leave 0 bytes at the start.
  2737. *
  2738. * "buflen" should be positive.
  2739. */
  2740. char *d_path(const struct path *path, char *buf, int buflen)
  2741. {
  2742. char *res = buf + buflen;
  2743. struct path root;
  2744. int error;
  2745. /*
  2746. * We have various synthetic filesystems that never get mounted. On
  2747. * these filesystems dentries are never used for lookup purposes, and
  2748. * thus don't need to be hashed. They also don't need a name until a
  2749. * user wants to identify the object in /proc/pid/fd/. The little hack
  2750. * below allows us to generate a name for these objects on demand:
  2751. */
  2752. if (path->dentry->d_op && path->dentry->d_op->d_dname)
  2753. return path->dentry->d_op->d_dname(path->dentry, buf, buflen);
  2754. rcu_read_lock();
  2755. get_fs_root_rcu(current->fs, &root);
  2756. error = path_with_deleted(path, &root, &res, &buflen);
  2757. rcu_read_unlock();
  2758. if (error < 0)
  2759. res = ERR_PTR(error);
  2760. return res;
  2761. }
  2762. EXPORT_SYMBOL(d_path);
  2763. /*
  2764. * Helper function for dentry_operations.d_dname() members
  2765. */
  2766. char *dynamic_dname(struct dentry *dentry, char *buffer, int buflen,
  2767. const char *fmt, ...)
  2768. {
  2769. va_list args;
  2770. char temp[64];
  2771. int sz;
  2772. va_start(args, fmt);
  2773. sz = vsnprintf(temp, sizeof(temp), fmt, args) + 1;
  2774. va_end(args);
  2775. if (sz > sizeof(temp) || sz > buflen)
  2776. return ERR_PTR(-ENAMETOOLONG);
  2777. buffer += buflen - sz;
  2778. return memcpy(buffer, temp, sz);
  2779. }
  2780. char *simple_dname(struct dentry *dentry, char *buffer, int buflen)
  2781. {
  2782. char *end = buffer + buflen;
  2783. /* these dentries are never renamed, so d_lock is not needed */
  2784. if (prepend(&end, &buflen, " (deleted)", 11) ||
  2785. prepend(&end, &buflen, dentry->d_name.name, dentry->d_name.len) ||
  2786. prepend(&end, &buflen, "/", 1))
  2787. end = ERR_PTR(-ENAMETOOLONG);
  2788. return end;
  2789. }
  2790. /*
  2791. * Write full pathname from the root of the filesystem into the buffer.
  2792. */
  2793. static char *__dentry_path(struct dentry *dentry, char *buf, int buflen)
  2794. {
  2795. char *end, *retval;
  2796. int len, seq = 0;
  2797. int error = 0;
  2798. rcu_read_lock();
  2799. restart:
  2800. end = buf + buflen;
  2801. len = buflen;
  2802. prepend(&end, &len, "\0", 1);
  2803. if (buflen < 1)
  2804. goto Elong;
  2805. /* Get '/' right */
  2806. retval = end-1;
  2807. *retval = '/';
  2808. read_seqbegin_or_lock(&rename_lock, &seq);
  2809. while (!IS_ROOT(dentry)) {
  2810. struct dentry *parent = dentry->d_parent;
  2811. int error;
  2812. prefetch(parent);
  2813. error = prepend_name(&end, &len, &dentry->d_name);
  2814. if (error)
  2815. break;
  2816. retval = end;
  2817. dentry = parent;
  2818. }
  2819. if (!(seq & 1))
  2820. rcu_read_unlock();
  2821. if (need_seqretry(&rename_lock, seq)) {
  2822. seq = 1;
  2823. goto restart;
  2824. }
  2825. done_seqretry(&rename_lock, seq);
  2826. if (error)
  2827. goto Elong;
  2828. return retval;
  2829. Elong:
  2830. return ERR_PTR(-ENAMETOOLONG);
  2831. }
  2832. char *dentry_path_raw(struct dentry *dentry, char *buf, int buflen)
  2833. {
  2834. return __dentry_path(dentry, buf, buflen);
  2835. }
  2836. EXPORT_SYMBOL(dentry_path_raw);
  2837. char *dentry_path(struct dentry *dentry, char *buf, int buflen)
  2838. {
  2839. char *p = NULL;
  2840. char *retval;
  2841. if (d_unlinked(dentry)) {
  2842. p = buf + buflen;
  2843. if (prepend(&p, &buflen, "//deleted", 10) != 0)
  2844. goto Elong;
  2845. buflen++;
  2846. }
  2847. retval = __dentry_path(dentry, buf, buflen);
  2848. if (!IS_ERR(retval) && p)
  2849. *p = '/'; /* restore '/' overriden with '\0' */
  2850. return retval;
  2851. Elong:
  2852. return ERR_PTR(-ENAMETOOLONG);
  2853. }
  2854. static void get_fs_root_and_pwd_rcu(struct fs_struct *fs, struct path *root,
  2855. struct path *pwd)
  2856. {
  2857. unsigned seq;
  2858. do {
  2859. seq = read_seqcount_begin(&fs->seq);
  2860. *root = fs->root;
  2861. *pwd = fs->pwd;
  2862. } while (read_seqcount_retry(&fs->seq, seq));
  2863. }
  2864. /*
  2865. * NOTE! The user-level library version returns a
  2866. * character pointer. The kernel system call just
  2867. * returns the length of the buffer filled (which
  2868. * includes the ending '\0' character), or a negative
  2869. * error value. So libc would do something like
  2870. *
  2871. * char *getcwd(char * buf, size_t size)
  2872. * {
  2873. * int retval;
  2874. *
  2875. * retval = sys_getcwd(buf, size);
  2876. * if (retval >= 0)
  2877. * return buf;
  2878. * errno = -retval;
  2879. * return NULL;
  2880. * }
  2881. */
  2882. SYSCALL_DEFINE2(getcwd, char __user *, buf, unsigned long, size)
  2883. {
  2884. int error;
  2885. struct path pwd, root;
  2886. char *page = __getname();
  2887. if (!page)
  2888. return -ENOMEM;
  2889. rcu_read_lock();
  2890. get_fs_root_and_pwd_rcu(current->fs, &root, &pwd);
  2891. error = -ENOENT;
  2892. if (!d_unlinked(pwd.dentry)) {
  2893. unsigned long len;
  2894. char *cwd = page + PATH_MAX;
  2895. int buflen = PATH_MAX;
  2896. prepend(&cwd, &buflen, "\0", 1);
  2897. error = prepend_path(&pwd, &root, &cwd, &buflen);
  2898. rcu_read_unlock();
  2899. if (error < 0)
  2900. goto out;
  2901. /* Unreachable from current root */
  2902. if (error > 0) {
  2903. error = prepend_unreachable(&cwd, &buflen);
  2904. if (error)
  2905. goto out;
  2906. }
  2907. error = -ERANGE;
  2908. len = PATH_MAX + page - cwd;
  2909. if (len <= size) {
  2910. error = len;
  2911. if (copy_to_user(buf, cwd, len))
  2912. error = -EFAULT;
  2913. }
  2914. } else {
  2915. rcu_read_unlock();
  2916. }
  2917. out:
  2918. __putname(page);
  2919. return error;
  2920. }
  2921. /*
  2922. * Test whether new_dentry is a subdirectory of old_dentry.
  2923. *
  2924. * Trivially implemented using the dcache structure
  2925. */
  2926. /**
  2927. * is_subdir - is new dentry a subdirectory of old_dentry
  2928. * @new_dentry: new dentry
  2929. * @old_dentry: old dentry
  2930. *
  2931. * Returns 1 if new_dentry is a subdirectory of the parent (at any depth).
  2932. * Returns 0 otherwise.
  2933. * Caller must ensure that "new_dentry" is pinned before calling is_subdir()
  2934. */
  2935. int is_subdir(struct dentry *new_dentry, struct dentry *old_dentry)
  2936. {
  2937. int result;
  2938. unsigned seq;
  2939. if (new_dentry == old_dentry)
  2940. return 1;
  2941. do {
  2942. /* for restarting inner loop in case of seq retry */
  2943. seq = read_seqbegin(&rename_lock);
  2944. /*
  2945. * Need rcu_readlock to protect against the d_parent trashing
  2946. * due to d_move
  2947. */
  2948. rcu_read_lock();
  2949. if (d_ancestor(old_dentry, new_dentry))
  2950. result = 1;
  2951. else
  2952. result = 0;
  2953. rcu_read_unlock();
  2954. } while (read_seqretry(&rename_lock, seq));
  2955. return result;
  2956. }
  2957. static enum d_walk_ret d_genocide_kill(void *data, struct dentry *dentry)
  2958. {
  2959. struct dentry *root = data;
  2960. if (dentry != root) {
  2961. if (d_unhashed(dentry) || !dentry->d_inode)
  2962. return D_WALK_SKIP;
  2963. if (!(dentry->d_flags & DCACHE_GENOCIDE)) {
  2964. dentry->d_flags |= DCACHE_GENOCIDE;
  2965. dentry->d_lockref.count--;
  2966. }
  2967. }
  2968. return D_WALK_CONTINUE;
  2969. }
  2970. void d_genocide(struct dentry *parent)
  2971. {
  2972. d_walk(parent, parent, d_genocide_kill, NULL);
  2973. }
  2974. void d_tmpfile(struct dentry *dentry, struct inode *inode)
  2975. {
  2976. inode_dec_link_count(inode);
  2977. BUG_ON(dentry->d_name.name != dentry->d_iname ||
  2978. !hlist_unhashed(&dentry->d_alias) ||
  2979. !d_unlinked(dentry));
  2980. spin_lock(&dentry->d_parent->d_lock);
  2981. spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
  2982. dentry->d_name.len = sprintf(dentry->d_iname, "#%llu",
  2983. (unsigned long long)inode->i_ino);
  2984. spin_unlock(&dentry->d_lock);
  2985. spin_unlock(&dentry->d_parent->d_lock);
  2986. d_instantiate(dentry, inode);
  2987. }
  2988. EXPORT_SYMBOL(d_tmpfile);
  2989. static __initdata unsigned long dhash_entries;
  2990. static int __init set_dhash_entries(char *str)
  2991. {
  2992. if (!str)
  2993. return 0;
  2994. dhash_entries = simple_strtoul(str, &str, 0);
  2995. return 1;
  2996. }
  2997. __setup("dhash_entries=", set_dhash_entries);
  2998. static void __init dcache_init_early(void)
  2999. {
  3000. unsigned int loop;
  3001. /* If hashes are distributed across NUMA nodes, defer
  3002. * hash allocation until vmalloc space is available.
  3003. */
  3004. if (hashdist)
  3005. return;
  3006. dentry_hashtable =
  3007. alloc_large_system_hash("Dentry cache",
  3008. sizeof(struct hlist_bl_head),
  3009. dhash_entries,
  3010. 13,
  3011. HASH_EARLY,
  3012. &d_hash_shift,
  3013. &d_hash_mask,
  3014. 0,
  3015. 0);
  3016. for (loop = 0; loop < (1U << d_hash_shift); loop++)
  3017. INIT_HLIST_BL_HEAD(dentry_hashtable + loop);
  3018. }
  3019. static void __init dcache_init(void)
  3020. {
  3021. unsigned int loop;
  3022. /*
  3023. * A constructor could be added for stable state like the lists,
  3024. * but it is probably not worth it because of the cache nature
  3025. * of the dcache.
  3026. */
  3027. dentry_cache = KMEM_CACHE(dentry,
  3028. SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|SLAB_MEM_SPREAD);
  3029. /* Hash may have been set up in dcache_init_early */
  3030. if (!hashdist)
  3031. return;
  3032. dentry_hashtable =
  3033. alloc_large_system_hash("Dentry cache",
  3034. sizeof(struct hlist_bl_head),
  3035. dhash_entries,
  3036. 13,
  3037. 0,
  3038. &d_hash_shift,
  3039. &d_hash_mask,
  3040. 0,
  3041. 0);
  3042. for (loop = 0; loop < (1U << d_hash_shift); loop++)
  3043. INIT_HLIST_BL_HEAD(dentry_hashtable + loop);
  3044. }
  3045. /* SLAB cache for __getname() consumers */
  3046. struct kmem_cache *names_cachep __read_mostly;
  3047. EXPORT_SYMBOL(names_cachep);
  3048. EXPORT_SYMBOL(d_genocide);
  3049. void __init vfs_caches_init_early(void)
  3050. {
  3051. dcache_init_early();
  3052. inode_init_early();
  3053. }
  3054. void __init vfs_caches_init(unsigned long mempages)
  3055. {
  3056. unsigned long reserve;
  3057. /* Base hash sizes on available memory, with a reserve equal to
  3058. 150% of current kernel size */
  3059. reserve = min((mempages - nr_free_pages()) * 3/2, mempages - 1);
  3060. mempages -= reserve;
  3061. names_cachep = kmem_cache_create("names_cache", PATH_MAX, 0,
  3062. SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
  3063. dcache_init();
  3064. inode_init();
  3065. files_init(mempages);
  3066. mnt_init();
  3067. bdev_cache_init();
  3068. chrdev_init();
  3069. }