dcache.c 88 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440
  1. /*
  2. * fs/dcache.c
  3. *
  4. * Complete reimplementation
  5. * (C) 1997 Thomas Schoebel-Theuer,
  6. * with heavy changes by Linus Torvalds
  7. */
  8. /*
  9. * Notes on the allocation strategy:
  10. *
  11. * The dcache is a master of the icache - whenever a dcache entry
  12. * exists, the inode will always exist. "iput()" is done either when
  13. * the dcache entry is deleted or garbage collected.
  14. */
  15. #include <linux/syscalls.h>
  16. #include <linux/string.h>
  17. #include <linux/mm.h>
  18. #include <linux/fs.h>
  19. #include <linux/fsnotify.h>
  20. #include <linux/slab.h>
  21. #include <linux/init.h>
  22. #include <linux/hash.h>
  23. #include <linux/cache.h>
  24. #include <linux/export.h>
  25. #include <linux/mount.h>
  26. #include <linux/file.h>
  27. #include <asm/uaccess.h>
  28. #include <linux/security.h>
  29. #include <linux/seqlock.h>
  30. #include <linux/swap.h>
  31. #include <linux/bootmem.h>
  32. #include <linux/fs_struct.h>
  33. #include <linux/hardirq.h>
  34. #include <linux/bit_spinlock.h>
  35. #include <linux/rculist_bl.h>
  36. #include <linux/prefetch.h>
  37. #include <linux/ratelimit.h>
  38. #include <linux/list_lru.h>
  39. #include "internal.h"
  40. #include "mount.h"
  41. /*
  42. * Usage:
  43. * dcache->d_inode->i_lock protects:
  44. * - i_dentry, d_alias, d_inode of aliases
  45. * dcache_hash_bucket lock protects:
  46. * - the dcache hash table
  47. * s_anon bl list spinlock protects:
  48. * - the s_anon list (see __d_drop)
  49. * dentry->d_sb->s_dentry_lru_lock protects:
  50. * - the dcache lru lists and counters
  51. * d_lock protects:
  52. * - d_flags
  53. * - d_name
  54. * - d_lru
  55. * - d_count
  56. * - d_unhashed()
  57. * - d_parent and d_subdirs
  58. * - childrens' d_child and d_parent
  59. * - d_alias, d_inode
  60. *
  61. * Ordering:
  62. * dentry->d_inode->i_lock
  63. * dentry->d_lock
  64. * dentry->d_sb->s_dentry_lru_lock
  65. * dcache_hash_bucket lock
  66. * s_anon lock
  67. *
  68. * If there is an ancestor relationship:
  69. * dentry->d_parent->...->d_parent->d_lock
  70. * ...
  71. * dentry->d_parent->d_lock
  72. * dentry->d_lock
  73. *
  74. * If no ancestor relationship:
  75. * if (dentry1 < dentry2)
  76. * dentry1->d_lock
  77. * dentry2->d_lock
  78. */
  79. int sysctl_vfs_cache_pressure __read_mostly = 100;
  80. EXPORT_SYMBOL_GPL(sysctl_vfs_cache_pressure);
  81. __cacheline_aligned_in_smp DEFINE_SEQLOCK(rename_lock);
  82. EXPORT_SYMBOL(rename_lock);
  83. static struct kmem_cache *dentry_cache __read_mostly;
  84. /**
  85. * read_seqbegin_or_lock - begin a sequence number check or locking block
  86. * @lock: sequence lock
  87. * @seq : sequence number to be checked
  88. *
  89. * First try it once optimistically without taking the lock. If that fails,
  90. * take the lock. The sequence number is also used as a marker for deciding
  91. * whether to be a reader (even) or writer (odd).
  92. * N.B. seq must be initialized to an even number to begin with.
  93. */
  94. static inline void read_seqbegin_or_lock(seqlock_t *lock, int *seq)
  95. {
  96. if (!(*seq & 1)) /* Even */
  97. *seq = read_seqbegin(lock);
  98. else /* Odd */
  99. read_seqlock_excl(lock);
  100. }
  101. static inline int need_seqretry(seqlock_t *lock, int seq)
  102. {
  103. return !(seq & 1) && read_seqretry(lock, seq);
  104. }
  105. static inline void done_seqretry(seqlock_t *lock, int seq)
  106. {
  107. if (seq & 1)
  108. read_sequnlock_excl(lock);
  109. }
  110. /*
  111. * This is the single most critical data structure when it comes
  112. * to the dcache: the hashtable for lookups. Somebody should try
  113. * to make this good - I've just made it work.
  114. *
  115. * This hash-function tries to avoid losing too many bits of hash
  116. * information, yet avoid using a prime hash-size or similar.
  117. */
  118. #define D_HASHBITS d_hash_shift
  119. #define D_HASHMASK d_hash_mask
  120. static unsigned int d_hash_mask __read_mostly;
  121. static unsigned int d_hash_shift __read_mostly;
  122. static struct hlist_bl_head *dentry_hashtable __read_mostly;
  123. static inline struct hlist_bl_head *d_hash(const struct dentry *parent,
  124. unsigned int hash)
  125. {
  126. hash += (unsigned long) parent / L1_CACHE_BYTES;
  127. hash = hash + (hash >> D_HASHBITS);
  128. return dentry_hashtable + (hash & D_HASHMASK);
  129. }
  130. /* Statistics gathering. */
  131. struct dentry_stat_t dentry_stat = {
  132. .age_limit = 45,
  133. };
  134. static DEFINE_PER_CPU(long, nr_dentry);
  135. static DEFINE_PER_CPU(long, nr_dentry_unused);
  136. #if defined(CONFIG_SYSCTL) && defined(CONFIG_PROC_FS)
  137. /*
  138. * Here we resort to our own counters instead of using generic per-cpu counters
  139. * for consistency with what the vfs inode code does. We are expected to harvest
  140. * better code and performance by having our own specialized counters.
  141. *
  142. * Please note that the loop is done over all possible CPUs, not over all online
  143. * CPUs. The reason for this is that we don't want to play games with CPUs going
  144. * on and off. If one of them goes off, we will just keep their counters.
  145. *
  146. * glommer: See cffbc8a for details, and if you ever intend to change this,
  147. * please update all vfs counters to match.
  148. */
  149. static long get_nr_dentry(void)
  150. {
  151. int i;
  152. long sum = 0;
  153. for_each_possible_cpu(i)
  154. sum += per_cpu(nr_dentry, i);
  155. return sum < 0 ? 0 : sum;
  156. }
  157. static long get_nr_dentry_unused(void)
  158. {
  159. int i;
  160. long sum = 0;
  161. for_each_possible_cpu(i)
  162. sum += per_cpu(nr_dentry_unused, i);
  163. return sum < 0 ? 0 : sum;
  164. }
  165. int proc_nr_dentry(ctl_table *table, int write, void __user *buffer,
  166. size_t *lenp, loff_t *ppos)
  167. {
  168. dentry_stat.nr_dentry = get_nr_dentry();
  169. dentry_stat.nr_unused = get_nr_dentry_unused();
  170. return proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
  171. }
  172. #endif
  173. /*
  174. * Compare 2 name strings, return 0 if they match, otherwise non-zero.
  175. * The strings are both count bytes long, and count is non-zero.
  176. */
  177. #ifdef CONFIG_DCACHE_WORD_ACCESS
  178. #include <asm/word-at-a-time.h>
  179. /*
  180. * NOTE! 'cs' and 'scount' come from a dentry, so it has a
  181. * aligned allocation for this particular component. We don't
  182. * strictly need the load_unaligned_zeropad() safety, but it
  183. * doesn't hurt either.
  184. *
  185. * In contrast, 'ct' and 'tcount' can be from a pathname, and do
  186. * need the careful unaligned handling.
  187. */
  188. static inline int dentry_string_cmp(const unsigned char *cs, const unsigned char *ct, unsigned tcount)
  189. {
  190. unsigned long a,b,mask;
  191. for (;;) {
  192. a = *(unsigned long *)cs;
  193. b = load_unaligned_zeropad(ct);
  194. if (tcount < sizeof(unsigned long))
  195. break;
  196. if (unlikely(a != b))
  197. return 1;
  198. cs += sizeof(unsigned long);
  199. ct += sizeof(unsigned long);
  200. tcount -= sizeof(unsigned long);
  201. if (!tcount)
  202. return 0;
  203. }
  204. mask = ~(~0ul << tcount*8);
  205. return unlikely(!!((a ^ b) & mask));
  206. }
  207. #else
  208. static inline int dentry_string_cmp(const unsigned char *cs, const unsigned char *ct, unsigned tcount)
  209. {
  210. do {
  211. if (*cs != *ct)
  212. return 1;
  213. cs++;
  214. ct++;
  215. tcount--;
  216. } while (tcount);
  217. return 0;
  218. }
  219. #endif
  220. static inline int dentry_cmp(const struct dentry *dentry, const unsigned char *ct, unsigned tcount)
  221. {
  222. const unsigned char *cs;
  223. /*
  224. * Be careful about RCU walk racing with rename:
  225. * use ACCESS_ONCE to fetch the name pointer.
  226. *
  227. * NOTE! Even if a rename will mean that the length
  228. * was not loaded atomically, we don't care. The
  229. * RCU walk will check the sequence count eventually,
  230. * and catch it. And we won't overrun the buffer,
  231. * because we're reading the name pointer atomically,
  232. * and a dentry name is guaranteed to be properly
  233. * terminated with a NUL byte.
  234. *
  235. * End result: even if 'len' is wrong, we'll exit
  236. * early because the data cannot match (there can
  237. * be no NUL in the ct/tcount data)
  238. */
  239. cs = ACCESS_ONCE(dentry->d_name.name);
  240. smp_read_barrier_depends();
  241. return dentry_string_cmp(cs, ct, tcount);
  242. }
  243. static void __d_free(struct rcu_head *head)
  244. {
  245. struct dentry *dentry = container_of(head, struct dentry, d_u.d_rcu);
  246. WARN_ON(!hlist_unhashed(&dentry->d_alias));
  247. if (dname_external(dentry))
  248. kfree(dentry->d_name.name);
  249. kmem_cache_free(dentry_cache, dentry);
  250. }
  251. /*
  252. * no locks, please.
  253. */
  254. static void d_free(struct dentry *dentry)
  255. {
  256. BUG_ON((int)dentry->d_lockref.count > 0);
  257. this_cpu_dec(nr_dentry);
  258. if (dentry->d_op && dentry->d_op->d_release)
  259. dentry->d_op->d_release(dentry);
  260. /* if dentry was never visible to RCU, immediate free is OK */
  261. if (!(dentry->d_flags & DCACHE_RCUACCESS))
  262. __d_free(&dentry->d_u.d_rcu);
  263. else
  264. call_rcu(&dentry->d_u.d_rcu, __d_free);
  265. }
  266. /**
  267. * dentry_rcuwalk_barrier - invalidate in-progress rcu-walk lookups
  268. * @dentry: the target dentry
  269. * After this call, in-progress rcu-walk path lookup will fail. This
  270. * should be called after unhashing, and after changing d_inode (if
  271. * the dentry has not already been unhashed).
  272. */
  273. static inline void dentry_rcuwalk_barrier(struct dentry *dentry)
  274. {
  275. assert_spin_locked(&dentry->d_lock);
  276. /* Go through a barrier */
  277. write_seqcount_barrier(&dentry->d_seq);
  278. }
  279. /*
  280. * Release the dentry's inode, using the filesystem
  281. * d_iput() operation if defined. Dentry has no refcount
  282. * and is unhashed.
  283. */
  284. static void dentry_iput(struct dentry * dentry)
  285. __releases(dentry->d_lock)
  286. __releases(dentry->d_inode->i_lock)
  287. {
  288. struct inode *inode = dentry->d_inode;
  289. if (inode) {
  290. dentry->d_inode = NULL;
  291. hlist_del_init(&dentry->d_alias);
  292. spin_unlock(&dentry->d_lock);
  293. spin_unlock(&inode->i_lock);
  294. if (!inode->i_nlink)
  295. fsnotify_inoderemove(inode);
  296. if (dentry->d_op && dentry->d_op->d_iput)
  297. dentry->d_op->d_iput(dentry, inode);
  298. else
  299. iput(inode);
  300. } else {
  301. spin_unlock(&dentry->d_lock);
  302. }
  303. }
  304. /*
  305. * Release the dentry's inode, using the filesystem
  306. * d_iput() operation if defined. dentry remains in-use.
  307. */
  308. static void dentry_unlink_inode(struct dentry * dentry)
  309. __releases(dentry->d_lock)
  310. __releases(dentry->d_inode->i_lock)
  311. {
  312. struct inode *inode = dentry->d_inode;
  313. dentry->d_inode = NULL;
  314. hlist_del_init(&dentry->d_alias);
  315. dentry_rcuwalk_barrier(dentry);
  316. spin_unlock(&dentry->d_lock);
  317. spin_unlock(&inode->i_lock);
  318. if (!inode->i_nlink)
  319. fsnotify_inoderemove(inode);
  320. if (dentry->d_op && dentry->d_op->d_iput)
  321. dentry->d_op->d_iput(dentry, inode);
  322. else
  323. iput(inode);
  324. }
  325. /*
  326. * The DCACHE_LRU_LIST bit is set whenever the 'd_lru' entry
  327. * is in use - which includes both the "real" per-superblock
  328. * LRU list _and_ the DCACHE_SHRINK_LIST use.
  329. *
  330. * The DCACHE_SHRINK_LIST bit is set whenever the dentry is
  331. * on the shrink list (ie not on the superblock LRU list).
  332. *
  333. * The per-cpu "nr_dentry_unused" counters are updated with
  334. * the DCACHE_LRU_LIST bit.
  335. *
  336. * These helper functions make sure we always follow the
  337. * rules. d_lock must be held by the caller.
  338. */
  339. #define D_FLAG_VERIFY(dentry,x) WARN_ON_ONCE(((dentry)->d_flags & (DCACHE_LRU_LIST | DCACHE_SHRINK_LIST)) != (x))
  340. static void d_lru_add(struct dentry *dentry)
  341. {
  342. D_FLAG_VERIFY(dentry, 0);
  343. dentry->d_flags |= DCACHE_LRU_LIST;
  344. this_cpu_inc(nr_dentry_unused);
  345. WARN_ON_ONCE(!list_lru_add(&dentry->d_sb->s_dentry_lru, &dentry->d_lru));
  346. }
  347. static void d_lru_del(struct dentry *dentry)
  348. {
  349. D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);
  350. dentry->d_flags &= ~DCACHE_LRU_LIST;
  351. this_cpu_dec(nr_dentry_unused);
  352. WARN_ON_ONCE(!list_lru_del(&dentry->d_sb->s_dentry_lru, &dentry->d_lru));
  353. }
  354. static void d_shrink_del(struct dentry *dentry)
  355. {
  356. D_FLAG_VERIFY(dentry, DCACHE_SHRINK_LIST | DCACHE_LRU_LIST);
  357. list_del_init(&dentry->d_lru);
  358. dentry->d_flags &= ~(DCACHE_SHRINK_LIST | DCACHE_LRU_LIST);
  359. this_cpu_dec(nr_dentry_unused);
  360. }
  361. static void d_shrink_add(struct dentry *dentry, struct list_head *list)
  362. {
  363. D_FLAG_VERIFY(dentry, 0);
  364. list_add(&dentry->d_lru, list);
  365. dentry->d_flags |= DCACHE_SHRINK_LIST | DCACHE_LRU_LIST;
  366. this_cpu_inc(nr_dentry_unused);
  367. }
  368. /*
  369. * These can only be called under the global LRU lock, ie during the
  370. * callback for freeing the LRU list. "isolate" removes it from the
  371. * LRU lists entirely, while shrink_move moves it to the indicated
  372. * private list.
  373. */
  374. static void d_lru_isolate(struct dentry *dentry)
  375. {
  376. D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);
  377. dentry->d_flags &= ~DCACHE_LRU_LIST;
  378. this_cpu_dec(nr_dentry_unused);
  379. list_del_init(&dentry->d_lru);
  380. }
  381. static void d_lru_shrink_move(struct dentry *dentry, struct list_head *list)
  382. {
  383. D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);
  384. dentry->d_flags |= DCACHE_SHRINK_LIST;
  385. list_move_tail(&dentry->d_lru, list);
  386. }
  387. /*
  388. * dentry_lru_(add|del)_list) must be called with d_lock held.
  389. */
  390. static void dentry_lru_add(struct dentry *dentry)
  391. {
  392. if (unlikely(!(dentry->d_flags & DCACHE_LRU_LIST)))
  393. d_lru_add(dentry);
  394. }
  395. /*
  396. * Remove a dentry with references from the LRU.
  397. *
  398. * If we are on the shrink list, then we can get to try_prune_one_dentry() and
  399. * lose our last reference through the parent walk. In this case, we need to
  400. * remove ourselves from the shrink list, not the LRU.
  401. */
  402. static void dentry_lru_del(struct dentry *dentry)
  403. {
  404. if (dentry->d_flags & DCACHE_LRU_LIST) {
  405. if (dentry->d_flags & DCACHE_SHRINK_LIST)
  406. return d_shrink_del(dentry);
  407. d_lru_del(dentry);
  408. }
  409. }
  410. /**
  411. * d_kill - kill dentry and return parent
  412. * @dentry: dentry to kill
  413. * @parent: parent dentry
  414. *
  415. * The dentry must already be unhashed and removed from the LRU.
  416. *
  417. * If this is the root of the dentry tree, return NULL.
  418. *
  419. * dentry->d_lock and parent->d_lock must be held by caller, and are dropped by
  420. * d_kill.
  421. */
  422. static struct dentry *d_kill(struct dentry *dentry, struct dentry *parent)
  423. __releases(dentry->d_lock)
  424. __releases(parent->d_lock)
  425. __releases(dentry->d_inode->i_lock)
  426. {
  427. list_del(&dentry->d_u.d_child);
  428. /*
  429. * Inform try_to_ascend() that we are no longer attached to the
  430. * dentry tree
  431. */
  432. dentry->d_flags |= DCACHE_DENTRY_KILLED;
  433. if (parent)
  434. spin_unlock(&parent->d_lock);
  435. dentry_iput(dentry);
  436. /*
  437. * dentry_iput drops the locks, at which point nobody (except
  438. * transient RCU lookups) can reach this dentry.
  439. */
  440. d_free(dentry);
  441. return parent;
  442. }
  443. /*
  444. * Unhash a dentry without inserting an RCU walk barrier or checking that
  445. * dentry->d_lock is locked. The caller must take care of that, if
  446. * appropriate.
  447. */
  448. static void __d_shrink(struct dentry *dentry)
  449. {
  450. if (!d_unhashed(dentry)) {
  451. struct hlist_bl_head *b;
  452. if (unlikely(dentry->d_flags & DCACHE_DISCONNECTED))
  453. b = &dentry->d_sb->s_anon;
  454. else
  455. b = d_hash(dentry->d_parent, dentry->d_name.hash);
  456. hlist_bl_lock(b);
  457. __hlist_bl_del(&dentry->d_hash);
  458. dentry->d_hash.pprev = NULL;
  459. hlist_bl_unlock(b);
  460. }
  461. }
  462. /**
  463. * d_drop - drop a dentry
  464. * @dentry: dentry to drop
  465. *
  466. * d_drop() unhashes the entry from the parent dentry hashes, so that it won't
  467. * be found through a VFS lookup any more. Note that this is different from
  468. * deleting the dentry - d_delete will try to mark the dentry negative if
  469. * possible, giving a successful _negative_ lookup, while d_drop will
  470. * just make the cache lookup fail.
  471. *
  472. * d_drop() is used mainly for stuff that wants to invalidate a dentry for some
  473. * reason (NFS timeouts or autofs deletes).
  474. *
  475. * __d_drop requires dentry->d_lock.
  476. */
  477. void __d_drop(struct dentry *dentry)
  478. {
  479. if (!d_unhashed(dentry)) {
  480. __d_shrink(dentry);
  481. dentry_rcuwalk_barrier(dentry);
  482. }
  483. }
  484. EXPORT_SYMBOL(__d_drop);
  485. void d_drop(struct dentry *dentry)
  486. {
  487. spin_lock(&dentry->d_lock);
  488. __d_drop(dentry);
  489. spin_unlock(&dentry->d_lock);
  490. }
  491. EXPORT_SYMBOL(d_drop);
  492. /*
  493. * Finish off a dentry we've decided to kill.
  494. * dentry->d_lock must be held, returns with it unlocked.
  495. * If ref is non-zero, then decrement the refcount too.
  496. * Returns dentry requiring refcount drop, or NULL if we're done.
  497. */
  498. static struct dentry *
  499. dentry_kill(struct dentry *dentry, int unlock_on_failure)
  500. __releases(dentry->d_lock)
  501. {
  502. struct inode *inode;
  503. struct dentry *parent;
  504. inode = dentry->d_inode;
  505. if (inode && !spin_trylock(&inode->i_lock)) {
  506. relock:
  507. if (unlock_on_failure) {
  508. spin_unlock(&dentry->d_lock);
  509. cpu_relax();
  510. }
  511. return dentry; /* try again with same dentry */
  512. }
  513. if (IS_ROOT(dentry))
  514. parent = NULL;
  515. else
  516. parent = dentry->d_parent;
  517. if (parent && !spin_trylock(&parent->d_lock)) {
  518. if (inode)
  519. spin_unlock(&inode->i_lock);
  520. goto relock;
  521. }
  522. /*
  523. * The dentry is now unrecoverably dead to the world.
  524. */
  525. lockref_mark_dead(&dentry->d_lockref);
  526. /*
  527. * inform the fs via d_prune that this dentry is about to be
  528. * unhashed and destroyed.
  529. */
  530. if ((dentry->d_flags & DCACHE_OP_PRUNE) && !d_unhashed(dentry))
  531. dentry->d_op->d_prune(dentry);
  532. dentry_lru_del(dentry);
  533. /* if it was on the hash then remove it */
  534. __d_drop(dentry);
  535. return d_kill(dentry, parent);
  536. }
  537. /*
  538. * This is dput
  539. *
  540. * This is complicated by the fact that we do not want to put
  541. * dentries that are no longer on any hash chain on the unused
  542. * list: we'd much rather just get rid of them immediately.
  543. *
  544. * However, that implies that we have to traverse the dentry
  545. * tree upwards to the parents which might _also_ now be
  546. * scheduled for deletion (it may have been only waiting for
  547. * its last child to go away).
  548. *
  549. * This tail recursion is done by hand as we don't want to depend
  550. * on the compiler to always get this right (gcc generally doesn't).
  551. * Real recursion would eat up our stack space.
  552. */
  553. /*
  554. * dput - release a dentry
  555. * @dentry: dentry to release
  556. *
  557. * Release a dentry. This will drop the usage count and if appropriate
  558. * call the dentry unlink method as well as removing it from the queues and
  559. * releasing its resources. If the parent dentries were scheduled for release
  560. * they too may now get deleted.
  561. */
  562. void dput(struct dentry *dentry)
  563. {
  564. if (unlikely(!dentry))
  565. return;
  566. repeat:
  567. if (lockref_put_or_lock(&dentry->d_lockref))
  568. return;
  569. /* Unreachable? Get rid of it */
  570. if (unlikely(d_unhashed(dentry)))
  571. goto kill_it;
  572. if (unlikely(dentry->d_flags & DCACHE_OP_DELETE)) {
  573. if (dentry->d_op->d_delete(dentry))
  574. goto kill_it;
  575. }
  576. if (!(dentry->d_flags & DCACHE_REFERENCED))
  577. dentry->d_flags |= DCACHE_REFERENCED;
  578. dentry_lru_add(dentry);
  579. dentry->d_lockref.count--;
  580. spin_unlock(&dentry->d_lock);
  581. return;
  582. kill_it:
  583. dentry = dentry_kill(dentry, 1);
  584. if (dentry)
  585. goto repeat;
  586. }
  587. EXPORT_SYMBOL(dput);
  588. /**
  589. * d_invalidate - invalidate a dentry
  590. * @dentry: dentry to invalidate
  591. *
  592. * Try to invalidate the dentry if it turns out to be
  593. * possible. If there are other dentries that can be
  594. * reached through this one we can't delete it and we
  595. * return -EBUSY. On success we return 0.
  596. *
  597. * no dcache lock.
  598. */
  599. int d_invalidate(struct dentry * dentry)
  600. {
  601. /*
  602. * If it's already been dropped, return OK.
  603. */
  604. spin_lock(&dentry->d_lock);
  605. if (d_unhashed(dentry)) {
  606. spin_unlock(&dentry->d_lock);
  607. return 0;
  608. }
  609. /*
  610. * Check whether to do a partial shrink_dcache
  611. * to get rid of unused child entries.
  612. */
  613. if (!list_empty(&dentry->d_subdirs)) {
  614. spin_unlock(&dentry->d_lock);
  615. shrink_dcache_parent(dentry);
  616. spin_lock(&dentry->d_lock);
  617. }
  618. /*
  619. * Somebody else still using it?
  620. *
  621. * If it's a directory, we can't drop it
  622. * for fear of somebody re-populating it
  623. * with children (even though dropping it
  624. * would make it unreachable from the root,
  625. * we might still populate it if it was a
  626. * working directory or similar).
  627. * We also need to leave mountpoints alone,
  628. * directory or not.
  629. */
  630. if (dentry->d_lockref.count > 1 && dentry->d_inode) {
  631. if (S_ISDIR(dentry->d_inode->i_mode) || d_mountpoint(dentry)) {
  632. spin_unlock(&dentry->d_lock);
  633. return -EBUSY;
  634. }
  635. }
  636. __d_drop(dentry);
  637. spin_unlock(&dentry->d_lock);
  638. return 0;
  639. }
  640. EXPORT_SYMBOL(d_invalidate);
  641. /* This must be called with d_lock held */
  642. static inline void __dget_dlock(struct dentry *dentry)
  643. {
  644. dentry->d_lockref.count++;
  645. }
  646. static inline void __dget(struct dentry *dentry)
  647. {
  648. lockref_get(&dentry->d_lockref);
  649. }
  650. struct dentry *dget_parent(struct dentry *dentry)
  651. {
  652. int gotref;
  653. struct dentry *ret;
  654. /*
  655. * Do optimistic parent lookup without any
  656. * locking.
  657. */
  658. rcu_read_lock();
  659. ret = ACCESS_ONCE(dentry->d_parent);
  660. gotref = lockref_get_not_zero(&ret->d_lockref);
  661. rcu_read_unlock();
  662. if (likely(gotref)) {
  663. if (likely(ret == ACCESS_ONCE(dentry->d_parent)))
  664. return ret;
  665. dput(ret);
  666. }
  667. repeat:
  668. /*
  669. * Don't need rcu_dereference because we re-check it was correct under
  670. * the lock.
  671. */
  672. rcu_read_lock();
  673. ret = dentry->d_parent;
  674. spin_lock(&ret->d_lock);
  675. if (unlikely(ret != dentry->d_parent)) {
  676. spin_unlock(&ret->d_lock);
  677. rcu_read_unlock();
  678. goto repeat;
  679. }
  680. rcu_read_unlock();
  681. BUG_ON(!ret->d_lockref.count);
  682. ret->d_lockref.count++;
  683. spin_unlock(&ret->d_lock);
  684. return ret;
  685. }
  686. EXPORT_SYMBOL(dget_parent);
  687. /**
  688. * d_find_alias - grab a hashed alias of inode
  689. * @inode: inode in question
  690. * @want_discon: flag, used by d_splice_alias, to request
  691. * that only a DISCONNECTED alias be returned.
  692. *
  693. * If inode has a hashed alias, or is a directory and has any alias,
  694. * acquire the reference to alias and return it. Otherwise return NULL.
  695. * Notice that if inode is a directory there can be only one alias and
  696. * it can be unhashed only if it has no children, or if it is the root
  697. * of a filesystem.
  698. *
  699. * If the inode has an IS_ROOT, DCACHE_DISCONNECTED alias, then prefer
  700. * any other hashed alias over that one unless @want_discon is set,
  701. * in which case only return an IS_ROOT, DCACHE_DISCONNECTED alias.
  702. */
  703. static struct dentry *__d_find_alias(struct inode *inode, int want_discon)
  704. {
  705. struct dentry *alias, *discon_alias;
  706. again:
  707. discon_alias = NULL;
  708. hlist_for_each_entry(alias, &inode->i_dentry, d_alias) {
  709. spin_lock(&alias->d_lock);
  710. if (S_ISDIR(inode->i_mode) || !d_unhashed(alias)) {
  711. if (IS_ROOT(alias) &&
  712. (alias->d_flags & DCACHE_DISCONNECTED)) {
  713. discon_alias = alias;
  714. } else if (!want_discon) {
  715. __dget_dlock(alias);
  716. spin_unlock(&alias->d_lock);
  717. return alias;
  718. }
  719. }
  720. spin_unlock(&alias->d_lock);
  721. }
  722. if (discon_alias) {
  723. alias = discon_alias;
  724. spin_lock(&alias->d_lock);
  725. if (S_ISDIR(inode->i_mode) || !d_unhashed(alias)) {
  726. if (IS_ROOT(alias) &&
  727. (alias->d_flags & DCACHE_DISCONNECTED)) {
  728. __dget_dlock(alias);
  729. spin_unlock(&alias->d_lock);
  730. return alias;
  731. }
  732. }
  733. spin_unlock(&alias->d_lock);
  734. goto again;
  735. }
  736. return NULL;
  737. }
  738. struct dentry *d_find_alias(struct inode *inode)
  739. {
  740. struct dentry *de = NULL;
  741. if (!hlist_empty(&inode->i_dentry)) {
  742. spin_lock(&inode->i_lock);
  743. de = __d_find_alias(inode, 0);
  744. spin_unlock(&inode->i_lock);
  745. }
  746. return de;
  747. }
  748. EXPORT_SYMBOL(d_find_alias);
  749. /*
  750. * Try to kill dentries associated with this inode.
  751. * WARNING: you must own a reference to inode.
  752. */
  753. void d_prune_aliases(struct inode *inode)
  754. {
  755. struct dentry *dentry;
  756. restart:
  757. spin_lock(&inode->i_lock);
  758. hlist_for_each_entry(dentry, &inode->i_dentry, d_alias) {
  759. spin_lock(&dentry->d_lock);
  760. if (!dentry->d_lockref.count) {
  761. /*
  762. * inform the fs via d_prune that this dentry
  763. * is about to be unhashed and destroyed.
  764. */
  765. if ((dentry->d_flags & DCACHE_OP_PRUNE) &&
  766. !d_unhashed(dentry))
  767. dentry->d_op->d_prune(dentry);
  768. __dget_dlock(dentry);
  769. __d_drop(dentry);
  770. spin_unlock(&dentry->d_lock);
  771. spin_unlock(&inode->i_lock);
  772. dput(dentry);
  773. goto restart;
  774. }
  775. spin_unlock(&dentry->d_lock);
  776. }
  777. spin_unlock(&inode->i_lock);
  778. }
  779. EXPORT_SYMBOL(d_prune_aliases);
  780. /*
  781. * Try to throw away a dentry - free the inode, dput the parent.
  782. * Requires dentry->d_lock is held, and dentry->d_count == 0.
  783. * Releases dentry->d_lock.
  784. *
  785. * This may fail if locks cannot be acquired no problem, just try again.
  786. */
  787. static struct dentry * try_prune_one_dentry(struct dentry *dentry)
  788. __releases(dentry->d_lock)
  789. {
  790. struct dentry *parent;
  791. parent = dentry_kill(dentry, 0);
  792. /*
  793. * If dentry_kill returns NULL, we have nothing more to do.
  794. * if it returns the same dentry, trylocks failed. In either
  795. * case, just loop again.
  796. *
  797. * Otherwise, we need to prune ancestors too. This is necessary
  798. * to prevent quadratic behavior of shrink_dcache_parent(), but
  799. * is also expected to be beneficial in reducing dentry cache
  800. * fragmentation.
  801. */
  802. if (!parent)
  803. return NULL;
  804. if (parent == dentry)
  805. return dentry;
  806. /* Prune ancestors. */
  807. dentry = parent;
  808. while (dentry) {
  809. if (lockref_put_or_lock(&dentry->d_lockref))
  810. return NULL;
  811. dentry = dentry_kill(dentry, 1);
  812. }
  813. return NULL;
  814. }
  815. static void shrink_dentry_list(struct list_head *list)
  816. {
  817. struct dentry *dentry;
  818. rcu_read_lock();
  819. for (;;) {
  820. dentry = list_entry_rcu(list->prev, struct dentry, d_lru);
  821. if (&dentry->d_lru == list)
  822. break; /* empty */
  823. /*
  824. * Get the dentry lock, and re-verify that the dentry is
  825. * this on the shrinking list. If it is, we know that
  826. * DCACHE_SHRINK_LIST and DCACHE_LRU_LIST are set.
  827. */
  828. spin_lock(&dentry->d_lock);
  829. if (dentry != list_entry(list->prev, struct dentry, d_lru)) {
  830. spin_unlock(&dentry->d_lock);
  831. continue;
  832. }
  833. /*
  834. * The dispose list is isolated and dentries are not accounted
  835. * to the LRU here, so we can simply remove it from the list
  836. * here regardless of whether it is referenced or not.
  837. */
  838. d_shrink_del(dentry);
  839. /*
  840. * We found an inuse dentry which was not removed from
  841. * the LRU because of laziness during lookup. Do not free it.
  842. */
  843. if (dentry->d_lockref.count) {
  844. spin_unlock(&dentry->d_lock);
  845. continue;
  846. }
  847. rcu_read_unlock();
  848. /*
  849. * If 'try_to_prune()' returns a dentry, it will
  850. * be the same one we passed in, and d_lock will
  851. * have been held the whole time, so it will not
  852. * have been added to any other lists. We failed
  853. * to get the inode lock.
  854. *
  855. * We just add it back to the shrink list.
  856. */
  857. dentry = try_prune_one_dentry(dentry);
  858. rcu_read_lock();
  859. if (dentry) {
  860. d_shrink_add(dentry, list);
  861. spin_unlock(&dentry->d_lock);
  862. }
  863. }
  864. rcu_read_unlock();
  865. }
  866. static enum lru_status
  867. dentry_lru_isolate(struct list_head *item, spinlock_t *lru_lock, void *arg)
  868. {
  869. struct list_head *freeable = arg;
  870. struct dentry *dentry = container_of(item, struct dentry, d_lru);
  871. /*
  872. * we are inverting the lru lock/dentry->d_lock here,
  873. * so use a trylock. If we fail to get the lock, just skip
  874. * it
  875. */
  876. if (!spin_trylock(&dentry->d_lock))
  877. return LRU_SKIP;
  878. /*
  879. * Referenced dentries are still in use. If they have active
  880. * counts, just remove them from the LRU. Otherwise give them
  881. * another pass through the LRU.
  882. */
  883. if (dentry->d_lockref.count) {
  884. d_lru_isolate(dentry);
  885. spin_unlock(&dentry->d_lock);
  886. return LRU_REMOVED;
  887. }
  888. if (dentry->d_flags & DCACHE_REFERENCED) {
  889. dentry->d_flags &= ~DCACHE_REFERENCED;
  890. spin_unlock(&dentry->d_lock);
  891. /*
  892. * The list move itself will be made by the common LRU code. At
  893. * this point, we've dropped the dentry->d_lock but keep the
  894. * lru lock. This is safe to do, since every list movement is
  895. * protected by the lru lock even if both locks are held.
  896. *
  897. * This is guaranteed by the fact that all LRU management
  898. * functions are intermediated by the LRU API calls like
  899. * list_lru_add and list_lru_del. List movement in this file
  900. * only ever occur through this functions or through callbacks
  901. * like this one, that are called from the LRU API.
  902. *
  903. * The only exceptions to this are functions like
  904. * shrink_dentry_list, and code that first checks for the
  905. * DCACHE_SHRINK_LIST flag. Those are guaranteed to be
  906. * operating only with stack provided lists after they are
  907. * properly isolated from the main list. It is thus, always a
  908. * local access.
  909. */
  910. return LRU_ROTATE;
  911. }
  912. d_lru_shrink_move(dentry, freeable);
  913. spin_unlock(&dentry->d_lock);
  914. return LRU_REMOVED;
  915. }
  916. /**
  917. * prune_dcache_sb - shrink the dcache
  918. * @sb: superblock
  919. * @nr_to_scan : number of entries to try to free
  920. * @nid: which node to scan for freeable entities
  921. *
  922. * Attempt to shrink the superblock dcache LRU by @nr_to_scan entries. This is
  923. * done when we need more memory an called from the superblock shrinker
  924. * function.
  925. *
  926. * This function may fail to free any resources if all the dentries are in
  927. * use.
  928. */
  929. long prune_dcache_sb(struct super_block *sb, unsigned long nr_to_scan,
  930. int nid)
  931. {
  932. LIST_HEAD(dispose);
  933. long freed;
  934. freed = list_lru_walk_node(&sb->s_dentry_lru, nid, dentry_lru_isolate,
  935. &dispose, &nr_to_scan);
  936. shrink_dentry_list(&dispose);
  937. return freed;
  938. }
  939. static enum lru_status dentry_lru_isolate_shrink(struct list_head *item,
  940. spinlock_t *lru_lock, void *arg)
  941. {
  942. struct list_head *freeable = arg;
  943. struct dentry *dentry = container_of(item, struct dentry, d_lru);
  944. /*
  945. * we are inverting the lru lock/dentry->d_lock here,
  946. * so use a trylock. If we fail to get the lock, just skip
  947. * it
  948. */
  949. if (!spin_trylock(&dentry->d_lock))
  950. return LRU_SKIP;
  951. d_lru_shrink_move(dentry, freeable);
  952. spin_unlock(&dentry->d_lock);
  953. return LRU_REMOVED;
  954. }
  955. /**
  956. * shrink_dcache_sb - shrink dcache for a superblock
  957. * @sb: superblock
  958. *
  959. * Shrink the dcache for the specified super block. This is used to free
  960. * the dcache before unmounting a file system.
  961. */
  962. void shrink_dcache_sb(struct super_block *sb)
  963. {
  964. long freed;
  965. do {
  966. LIST_HEAD(dispose);
  967. freed = list_lru_walk(&sb->s_dentry_lru,
  968. dentry_lru_isolate_shrink, &dispose, UINT_MAX);
  969. this_cpu_sub(nr_dentry_unused, freed);
  970. shrink_dentry_list(&dispose);
  971. } while (freed > 0);
  972. }
  973. EXPORT_SYMBOL(shrink_dcache_sb);
  974. /*
  975. * destroy a single subtree of dentries for unmount
  976. * - see the comments on shrink_dcache_for_umount() for a description of the
  977. * locking
  978. */
  979. static void shrink_dcache_for_umount_subtree(struct dentry *dentry)
  980. {
  981. struct dentry *parent;
  982. BUG_ON(!IS_ROOT(dentry));
  983. for (;;) {
  984. /* descend to the first leaf in the current subtree */
  985. while (!list_empty(&dentry->d_subdirs))
  986. dentry = list_entry(dentry->d_subdirs.next,
  987. struct dentry, d_u.d_child);
  988. /* consume the dentries from this leaf up through its parents
  989. * until we find one with children or run out altogether */
  990. do {
  991. struct inode *inode;
  992. /*
  993. * inform the fs that this dentry is about to be
  994. * unhashed and destroyed.
  995. */
  996. if ((dentry->d_flags & DCACHE_OP_PRUNE) &&
  997. !d_unhashed(dentry))
  998. dentry->d_op->d_prune(dentry);
  999. dentry_lru_del(dentry);
  1000. __d_shrink(dentry);
  1001. if (dentry->d_lockref.count != 0) {
  1002. printk(KERN_ERR
  1003. "BUG: Dentry %p{i=%lx,n=%s}"
  1004. " still in use (%d)"
  1005. " [unmount of %s %s]\n",
  1006. dentry,
  1007. dentry->d_inode ?
  1008. dentry->d_inode->i_ino : 0UL,
  1009. dentry->d_name.name,
  1010. dentry->d_lockref.count,
  1011. dentry->d_sb->s_type->name,
  1012. dentry->d_sb->s_id);
  1013. BUG();
  1014. }
  1015. if (IS_ROOT(dentry)) {
  1016. parent = NULL;
  1017. list_del(&dentry->d_u.d_child);
  1018. } else {
  1019. parent = dentry->d_parent;
  1020. parent->d_lockref.count--;
  1021. list_del(&dentry->d_u.d_child);
  1022. }
  1023. inode = dentry->d_inode;
  1024. if (inode) {
  1025. dentry->d_inode = NULL;
  1026. hlist_del_init(&dentry->d_alias);
  1027. if (dentry->d_op && dentry->d_op->d_iput)
  1028. dentry->d_op->d_iput(dentry, inode);
  1029. else
  1030. iput(inode);
  1031. }
  1032. d_free(dentry);
  1033. /* finished when we fall off the top of the tree,
  1034. * otherwise we ascend to the parent and move to the
  1035. * next sibling if there is one */
  1036. if (!parent)
  1037. return;
  1038. dentry = parent;
  1039. } while (list_empty(&dentry->d_subdirs));
  1040. dentry = list_entry(dentry->d_subdirs.next,
  1041. struct dentry, d_u.d_child);
  1042. }
  1043. }
  1044. /*
  1045. * destroy the dentries attached to a superblock on unmounting
  1046. * - we don't need to use dentry->d_lock because:
  1047. * - the superblock is detached from all mountings and open files, so the
  1048. * dentry trees will not be rearranged by the VFS
  1049. * - s_umount is write-locked, so the memory pressure shrinker will ignore
  1050. * any dentries belonging to this superblock that it comes across
  1051. * - the filesystem itself is no longer permitted to rearrange the dentries
  1052. * in this superblock
  1053. */
  1054. void shrink_dcache_for_umount(struct super_block *sb)
  1055. {
  1056. struct dentry *dentry;
  1057. if (down_read_trylock(&sb->s_umount))
  1058. BUG();
  1059. dentry = sb->s_root;
  1060. sb->s_root = NULL;
  1061. dentry->d_lockref.count--;
  1062. shrink_dcache_for_umount_subtree(dentry);
  1063. while (!hlist_bl_empty(&sb->s_anon)) {
  1064. dentry = hlist_bl_entry(hlist_bl_first(&sb->s_anon), struct dentry, d_hash);
  1065. shrink_dcache_for_umount_subtree(dentry);
  1066. }
  1067. }
  1068. /*
  1069. * This tries to ascend one level of parenthood, but
  1070. * we can race with renaming, so we need to re-check
  1071. * the parenthood after dropping the lock and check
  1072. * that the sequence number still matches.
  1073. */
  1074. static struct dentry *try_to_ascend(struct dentry *old, unsigned seq)
  1075. {
  1076. struct dentry *new = old->d_parent;
  1077. rcu_read_lock();
  1078. spin_unlock(&old->d_lock);
  1079. spin_lock(&new->d_lock);
  1080. /*
  1081. * might go back up the wrong parent if we have had a rename
  1082. * or deletion
  1083. */
  1084. if (new != old->d_parent ||
  1085. (old->d_flags & DCACHE_DENTRY_KILLED) ||
  1086. need_seqretry(&rename_lock, seq)) {
  1087. spin_unlock(&new->d_lock);
  1088. new = NULL;
  1089. }
  1090. rcu_read_unlock();
  1091. return new;
  1092. }
  1093. /**
  1094. * enum d_walk_ret - action to talke during tree walk
  1095. * @D_WALK_CONTINUE: contrinue walk
  1096. * @D_WALK_QUIT: quit walk
  1097. * @D_WALK_NORETRY: quit when retry is needed
  1098. * @D_WALK_SKIP: skip this dentry and its children
  1099. */
  1100. enum d_walk_ret {
  1101. D_WALK_CONTINUE,
  1102. D_WALK_QUIT,
  1103. D_WALK_NORETRY,
  1104. D_WALK_SKIP,
  1105. };
  1106. /**
  1107. * d_walk - walk the dentry tree
  1108. * @parent: start of walk
  1109. * @data: data passed to @enter() and @finish()
  1110. * @enter: callback when first entering the dentry
  1111. * @finish: callback when successfully finished the walk
  1112. *
  1113. * The @enter() and @finish() callbacks are called with d_lock held.
  1114. */
  1115. static void d_walk(struct dentry *parent, void *data,
  1116. enum d_walk_ret (*enter)(void *, struct dentry *),
  1117. void (*finish)(void *))
  1118. {
  1119. struct dentry *this_parent;
  1120. struct list_head *next;
  1121. unsigned seq = 0;
  1122. enum d_walk_ret ret;
  1123. bool retry = true;
  1124. again:
  1125. read_seqbegin_or_lock(&rename_lock, &seq);
  1126. this_parent = parent;
  1127. spin_lock(&this_parent->d_lock);
  1128. ret = enter(data, this_parent);
  1129. switch (ret) {
  1130. case D_WALK_CONTINUE:
  1131. break;
  1132. case D_WALK_QUIT:
  1133. case D_WALK_SKIP:
  1134. goto out_unlock;
  1135. case D_WALK_NORETRY:
  1136. retry = false;
  1137. break;
  1138. }
  1139. repeat:
  1140. next = this_parent->d_subdirs.next;
  1141. resume:
  1142. while (next != &this_parent->d_subdirs) {
  1143. struct list_head *tmp = next;
  1144. struct dentry *dentry = list_entry(tmp, struct dentry, d_u.d_child);
  1145. next = tmp->next;
  1146. spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
  1147. ret = enter(data, dentry);
  1148. switch (ret) {
  1149. case D_WALK_CONTINUE:
  1150. break;
  1151. case D_WALK_QUIT:
  1152. spin_unlock(&dentry->d_lock);
  1153. goto out_unlock;
  1154. case D_WALK_NORETRY:
  1155. retry = false;
  1156. break;
  1157. case D_WALK_SKIP:
  1158. spin_unlock(&dentry->d_lock);
  1159. continue;
  1160. }
  1161. if (!list_empty(&dentry->d_subdirs)) {
  1162. spin_unlock(&this_parent->d_lock);
  1163. spin_release(&dentry->d_lock.dep_map, 1, _RET_IP_);
  1164. this_parent = dentry;
  1165. spin_acquire(&this_parent->d_lock.dep_map, 0, 1, _RET_IP_);
  1166. goto repeat;
  1167. }
  1168. spin_unlock(&dentry->d_lock);
  1169. }
  1170. /*
  1171. * All done at this level ... ascend and resume the search.
  1172. */
  1173. if (this_parent != parent) {
  1174. struct dentry *child = this_parent;
  1175. this_parent = try_to_ascend(this_parent, seq);
  1176. if (!this_parent)
  1177. goto rename_retry;
  1178. next = child->d_u.d_child.next;
  1179. goto resume;
  1180. }
  1181. if (need_seqretry(&rename_lock, seq)) {
  1182. spin_unlock(&this_parent->d_lock);
  1183. goto rename_retry;
  1184. }
  1185. if (finish)
  1186. finish(data);
  1187. out_unlock:
  1188. spin_unlock(&this_parent->d_lock);
  1189. done_seqretry(&rename_lock, seq);
  1190. return;
  1191. rename_retry:
  1192. if (!retry)
  1193. return;
  1194. seq = 1;
  1195. goto again;
  1196. }
  1197. /*
  1198. * Search for at least 1 mount point in the dentry's subdirs.
  1199. * We descend to the next level whenever the d_subdirs
  1200. * list is non-empty and continue searching.
  1201. */
  1202. static enum d_walk_ret check_mount(void *data, struct dentry *dentry)
  1203. {
  1204. int *ret = data;
  1205. if (d_mountpoint(dentry)) {
  1206. *ret = 1;
  1207. return D_WALK_QUIT;
  1208. }
  1209. return D_WALK_CONTINUE;
  1210. }
  1211. /**
  1212. * have_submounts - check for mounts over a dentry
  1213. * @parent: dentry to check.
  1214. *
  1215. * Return true if the parent or its subdirectories contain
  1216. * a mount point
  1217. */
  1218. int have_submounts(struct dentry *parent)
  1219. {
  1220. int ret = 0;
  1221. d_walk(parent, &ret, check_mount, NULL);
  1222. return ret;
  1223. }
  1224. EXPORT_SYMBOL(have_submounts);
  1225. /*
  1226. * Called by mount code to set a mountpoint and check if the mountpoint is
  1227. * reachable (e.g. NFS can unhash a directory dentry and then the complete
  1228. * subtree can become unreachable).
  1229. *
  1230. * Only one of check_submounts_and_drop() and d_set_mounted() must succeed. For
  1231. * this reason take rename_lock and d_lock on dentry and ancestors.
  1232. */
  1233. int d_set_mounted(struct dentry *dentry)
  1234. {
  1235. struct dentry *p;
  1236. int ret = -ENOENT;
  1237. write_seqlock(&rename_lock);
  1238. for (p = dentry->d_parent; !IS_ROOT(p); p = p->d_parent) {
  1239. /* Need exclusion wrt. check_submounts_and_drop() */
  1240. spin_lock(&p->d_lock);
  1241. if (unlikely(d_unhashed(p))) {
  1242. spin_unlock(&p->d_lock);
  1243. goto out;
  1244. }
  1245. spin_unlock(&p->d_lock);
  1246. }
  1247. spin_lock(&dentry->d_lock);
  1248. if (!d_unlinked(dentry)) {
  1249. dentry->d_flags |= DCACHE_MOUNTED;
  1250. ret = 0;
  1251. }
  1252. spin_unlock(&dentry->d_lock);
  1253. out:
  1254. write_sequnlock(&rename_lock);
  1255. return ret;
  1256. }
  1257. /*
  1258. * Search the dentry child list of the specified parent,
  1259. * and move any unused dentries to the end of the unused
  1260. * list for prune_dcache(). We descend to the next level
  1261. * whenever the d_subdirs list is non-empty and continue
  1262. * searching.
  1263. *
  1264. * It returns zero iff there are no unused children,
  1265. * otherwise it returns the number of children moved to
  1266. * the end of the unused list. This may not be the total
  1267. * number of unused children, because select_parent can
  1268. * drop the lock and return early due to latency
  1269. * constraints.
  1270. */
  1271. struct select_data {
  1272. struct dentry *start;
  1273. struct list_head dispose;
  1274. int found;
  1275. };
  1276. static enum d_walk_ret select_collect(void *_data, struct dentry *dentry)
  1277. {
  1278. struct select_data *data = _data;
  1279. enum d_walk_ret ret = D_WALK_CONTINUE;
  1280. if (data->start == dentry)
  1281. goto out;
  1282. /*
  1283. * move only zero ref count dentries to the dispose list.
  1284. *
  1285. * Those which are presently on the shrink list, being processed
  1286. * by shrink_dentry_list(), shouldn't be moved. Otherwise the
  1287. * loop in shrink_dcache_parent() might not make any progress
  1288. * and loop forever.
  1289. */
  1290. if (dentry->d_lockref.count) {
  1291. dentry_lru_del(dentry);
  1292. } else if (!(dentry->d_flags & DCACHE_SHRINK_LIST)) {
  1293. /*
  1294. * We can't use d_lru_shrink_move() because we
  1295. * need to get the global LRU lock and do the
  1296. * LRU accounting.
  1297. */
  1298. d_lru_del(dentry);
  1299. d_shrink_add(dentry, &data->dispose);
  1300. data->found++;
  1301. ret = D_WALK_NORETRY;
  1302. }
  1303. /*
  1304. * We can return to the caller if we have found some (this
  1305. * ensures forward progress). We'll be coming back to find
  1306. * the rest.
  1307. */
  1308. if (data->found && need_resched())
  1309. ret = D_WALK_QUIT;
  1310. out:
  1311. return ret;
  1312. }
  1313. /**
  1314. * shrink_dcache_parent - prune dcache
  1315. * @parent: parent of entries to prune
  1316. *
  1317. * Prune the dcache to remove unused children of the parent dentry.
  1318. */
  1319. void shrink_dcache_parent(struct dentry *parent)
  1320. {
  1321. for (;;) {
  1322. struct select_data data;
  1323. INIT_LIST_HEAD(&data.dispose);
  1324. data.start = parent;
  1325. data.found = 0;
  1326. d_walk(parent, &data, select_collect, NULL);
  1327. if (!data.found)
  1328. break;
  1329. shrink_dentry_list(&data.dispose);
  1330. cond_resched();
  1331. }
  1332. }
  1333. EXPORT_SYMBOL(shrink_dcache_parent);
  1334. static enum d_walk_ret check_and_collect(void *_data, struct dentry *dentry)
  1335. {
  1336. struct select_data *data = _data;
  1337. if (d_mountpoint(dentry)) {
  1338. data->found = -EBUSY;
  1339. return D_WALK_QUIT;
  1340. }
  1341. return select_collect(_data, dentry);
  1342. }
  1343. static void check_and_drop(void *_data)
  1344. {
  1345. struct select_data *data = _data;
  1346. if (d_mountpoint(data->start))
  1347. data->found = -EBUSY;
  1348. if (!data->found)
  1349. __d_drop(data->start);
  1350. }
  1351. /**
  1352. * check_submounts_and_drop - prune dcache, check for submounts and drop
  1353. *
  1354. * All done as a single atomic operation relative to has_unlinked_ancestor().
  1355. * Returns 0 if successfully unhashed @parent. If there were submounts then
  1356. * return -EBUSY.
  1357. *
  1358. * @dentry: dentry to prune and drop
  1359. */
  1360. int check_submounts_and_drop(struct dentry *dentry)
  1361. {
  1362. int ret = 0;
  1363. /* Negative dentries can be dropped without further checks */
  1364. if (!dentry->d_inode) {
  1365. d_drop(dentry);
  1366. goto out;
  1367. }
  1368. for (;;) {
  1369. struct select_data data;
  1370. INIT_LIST_HEAD(&data.dispose);
  1371. data.start = dentry;
  1372. data.found = 0;
  1373. d_walk(dentry, &data, check_and_collect, check_and_drop);
  1374. ret = data.found;
  1375. if (!list_empty(&data.dispose))
  1376. shrink_dentry_list(&data.dispose);
  1377. if (ret <= 0)
  1378. break;
  1379. cond_resched();
  1380. }
  1381. out:
  1382. return ret;
  1383. }
  1384. EXPORT_SYMBOL(check_submounts_and_drop);
  1385. /**
  1386. * __d_alloc - allocate a dcache entry
  1387. * @sb: filesystem it will belong to
  1388. * @name: qstr of the name
  1389. *
  1390. * Allocates a dentry. It returns %NULL if there is insufficient memory
  1391. * available. On a success the dentry is returned. The name passed in is
  1392. * copied and the copy passed in may be reused after this call.
  1393. */
  1394. struct dentry *__d_alloc(struct super_block *sb, const struct qstr *name)
  1395. {
  1396. struct dentry *dentry;
  1397. char *dname;
  1398. dentry = kmem_cache_alloc(dentry_cache, GFP_KERNEL);
  1399. if (!dentry)
  1400. return NULL;
  1401. /*
  1402. * We guarantee that the inline name is always NUL-terminated.
  1403. * This way the memcpy() done by the name switching in rename
  1404. * will still always have a NUL at the end, even if we might
  1405. * be overwriting an internal NUL character
  1406. */
  1407. dentry->d_iname[DNAME_INLINE_LEN-1] = 0;
  1408. if (name->len > DNAME_INLINE_LEN-1) {
  1409. dname = kmalloc(name->len + 1, GFP_KERNEL);
  1410. if (!dname) {
  1411. kmem_cache_free(dentry_cache, dentry);
  1412. return NULL;
  1413. }
  1414. } else {
  1415. dname = dentry->d_iname;
  1416. }
  1417. dentry->d_name.len = name->len;
  1418. dentry->d_name.hash = name->hash;
  1419. memcpy(dname, name->name, name->len);
  1420. dname[name->len] = 0;
  1421. /* Make sure we always see the terminating NUL character */
  1422. smp_wmb();
  1423. dentry->d_name.name = dname;
  1424. dentry->d_lockref.count = 1;
  1425. dentry->d_flags = 0;
  1426. spin_lock_init(&dentry->d_lock);
  1427. seqcount_init(&dentry->d_seq);
  1428. dentry->d_inode = NULL;
  1429. dentry->d_parent = dentry;
  1430. dentry->d_sb = sb;
  1431. dentry->d_op = NULL;
  1432. dentry->d_fsdata = NULL;
  1433. INIT_HLIST_BL_NODE(&dentry->d_hash);
  1434. INIT_LIST_HEAD(&dentry->d_lru);
  1435. INIT_LIST_HEAD(&dentry->d_subdirs);
  1436. INIT_HLIST_NODE(&dentry->d_alias);
  1437. INIT_LIST_HEAD(&dentry->d_u.d_child);
  1438. d_set_d_op(dentry, dentry->d_sb->s_d_op);
  1439. this_cpu_inc(nr_dentry);
  1440. return dentry;
  1441. }
  1442. /**
  1443. * d_alloc - allocate a dcache entry
  1444. * @parent: parent of entry to allocate
  1445. * @name: qstr of the name
  1446. *
  1447. * Allocates a dentry. It returns %NULL if there is insufficient memory
  1448. * available. On a success the dentry is returned. The name passed in is
  1449. * copied and the copy passed in may be reused after this call.
  1450. */
  1451. struct dentry *d_alloc(struct dentry * parent, const struct qstr *name)
  1452. {
  1453. struct dentry *dentry = __d_alloc(parent->d_sb, name);
  1454. if (!dentry)
  1455. return NULL;
  1456. spin_lock(&parent->d_lock);
  1457. /*
  1458. * don't need child lock because it is not subject
  1459. * to concurrency here
  1460. */
  1461. __dget_dlock(parent);
  1462. dentry->d_parent = parent;
  1463. list_add(&dentry->d_u.d_child, &parent->d_subdirs);
  1464. spin_unlock(&parent->d_lock);
  1465. return dentry;
  1466. }
  1467. EXPORT_SYMBOL(d_alloc);
  1468. struct dentry *d_alloc_pseudo(struct super_block *sb, const struct qstr *name)
  1469. {
  1470. struct dentry *dentry = __d_alloc(sb, name);
  1471. if (dentry)
  1472. dentry->d_flags |= DCACHE_DISCONNECTED;
  1473. return dentry;
  1474. }
  1475. EXPORT_SYMBOL(d_alloc_pseudo);
  1476. struct dentry *d_alloc_name(struct dentry *parent, const char *name)
  1477. {
  1478. struct qstr q;
  1479. q.name = name;
  1480. q.len = strlen(name);
  1481. q.hash = full_name_hash(q.name, q.len);
  1482. return d_alloc(parent, &q);
  1483. }
  1484. EXPORT_SYMBOL(d_alloc_name);
  1485. void d_set_d_op(struct dentry *dentry, const struct dentry_operations *op)
  1486. {
  1487. WARN_ON_ONCE(dentry->d_op);
  1488. WARN_ON_ONCE(dentry->d_flags & (DCACHE_OP_HASH |
  1489. DCACHE_OP_COMPARE |
  1490. DCACHE_OP_REVALIDATE |
  1491. DCACHE_OP_WEAK_REVALIDATE |
  1492. DCACHE_OP_DELETE ));
  1493. dentry->d_op = op;
  1494. if (!op)
  1495. return;
  1496. if (op->d_hash)
  1497. dentry->d_flags |= DCACHE_OP_HASH;
  1498. if (op->d_compare)
  1499. dentry->d_flags |= DCACHE_OP_COMPARE;
  1500. if (op->d_revalidate)
  1501. dentry->d_flags |= DCACHE_OP_REVALIDATE;
  1502. if (op->d_weak_revalidate)
  1503. dentry->d_flags |= DCACHE_OP_WEAK_REVALIDATE;
  1504. if (op->d_delete)
  1505. dentry->d_flags |= DCACHE_OP_DELETE;
  1506. if (op->d_prune)
  1507. dentry->d_flags |= DCACHE_OP_PRUNE;
  1508. }
  1509. EXPORT_SYMBOL(d_set_d_op);
  1510. static void __d_instantiate(struct dentry *dentry, struct inode *inode)
  1511. {
  1512. spin_lock(&dentry->d_lock);
  1513. if (inode) {
  1514. if (unlikely(IS_AUTOMOUNT(inode)))
  1515. dentry->d_flags |= DCACHE_NEED_AUTOMOUNT;
  1516. hlist_add_head(&dentry->d_alias, &inode->i_dentry);
  1517. }
  1518. dentry->d_inode = inode;
  1519. dentry_rcuwalk_barrier(dentry);
  1520. spin_unlock(&dentry->d_lock);
  1521. fsnotify_d_instantiate(dentry, inode);
  1522. }
  1523. /**
  1524. * d_instantiate - fill in inode information for a dentry
  1525. * @entry: dentry to complete
  1526. * @inode: inode to attach to this dentry
  1527. *
  1528. * Fill in inode information in the entry.
  1529. *
  1530. * This turns negative dentries into productive full members
  1531. * of society.
  1532. *
  1533. * NOTE! This assumes that the inode count has been incremented
  1534. * (or otherwise set) by the caller to indicate that it is now
  1535. * in use by the dcache.
  1536. */
  1537. void d_instantiate(struct dentry *entry, struct inode * inode)
  1538. {
  1539. BUG_ON(!hlist_unhashed(&entry->d_alias));
  1540. if (inode)
  1541. spin_lock(&inode->i_lock);
  1542. __d_instantiate(entry, inode);
  1543. if (inode)
  1544. spin_unlock(&inode->i_lock);
  1545. security_d_instantiate(entry, inode);
  1546. }
  1547. EXPORT_SYMBOL(d_instantiate);
  1548. /**
  1549. * d_instantiate_unique - instantiate a non-aliased dentry
  1550. * @entry: dentry to instantiate
  1551. * @inode: inode to attach to this dentry
  1552. *
  1553. * Fill in inode information in the entry. On success, it returns NULL.
  1554. * If an unhashed alias of "entry" already exists, then we return the
  1555. * aliased dentry instead and drop one reference to inode.
  1556. *
  1557. * Note that in order to avoid conflicts with rename() etc, the caller
  1558. * had better be holding the parent directory semaphore.
  1559. *
  1560. * This also assumes that the inode count has been incremented
  1561. * (or otherwise set) by the caller to indicate that it is now
  1562. * in use by the dcache.
  1563. */
  1564. static struct dentry *__d_instantiate_unique(struct dentry *entry,
  1565. struct inode *inode)
  1566. {
  1567. struct dentry *alias;
  1568. int len = entry->d_name.len;
  1569. const char *name = entry->d_name.name;
  1570. unsigned int hash = entry->d_name.hash;
  1571. if (!inode) {
  1572. __d_instantiate(entry, NULL);
  1573. return NULL;
  1574. }
  1575. hlist_for_each_entry(alias, &inode->i_dentry, d_alias) {
  1576. /*
  1577. * Don't need alias->d_lock here, because aliases with
  1578. * d_parent == entry->d_parent are not subject to name or
  1579. * parent changes, because the parent inode i_mutex is held.
  1580. */
  1581. if (alias->d_name.hash != hash)
  1582. continue;
  1583. if (alias->d_parent != entry->d_parent)
  1584. continue;
  1585. if (alias->d_name.len != len)
  1586. continue;
  1587. if (dentry_cmp(alias, name, len))
  1588. continue;
  1589. __dget(alias);
  1590. return alias;
  1591. }
  1592. __d_instantiate(entry, inode);
  1593. return NULL;
  1594. }
  1595. struct dentry *d_instantiate_unique(struct dentry *entry, struct inode *inode)
  1596. {
  1597. struct dentry *result;
  1598. BUG_ON(!hlist_unhashed(&entry->d_alias));
  1599. if (inode)
  1600. spin_lock(&inode->i_lock);
  1601. result = __d_instantiate_unique(entry, inode);
  1602. if (inode)
  1603. spin_unlock(&inode->i_lock);
  1604. if (!result) {
  1605. security_d_instantiate(entry, inode);
  1606. return NULL;
  1607. }
  1608. BUG_ON(!d_unhashed(result));
  1609. iput(inode);
  1610. return result;
  1611. }
  1612. EXPORT_SYMBOL(d_instantiate_unique);
  1613. struct dentry *d_make_root(struct inode *root_inode)
  1614. {
  1615. struct dentry *res = NULL;
  1616. if (root_inode) {
  1617. static const struct qstr name = QSTR_INIT("/", 1);
  1618. res = __d_alloc(root_inode->i_sb, &name);
  1619. if (res)
  1620. d_instantiate(res, root_inode);
  1621. else
  1622. iput(root_inode);
  1623. }
  1624. return res;
  1625. }
  1626. EXPORT_SYMBOL(d_make_root);
  1627. static struct dentry * __d_find_any_alias(struct inode *inode)
  1628. {
  1629. struct dentry *alias;
  1630. if (hlist_empty(&inode->i_dentry))
  1631. return NULL;
  1632. alias = hlist_entry(inode->i_dentry.first, struct dentry, d_alias);
  1633. __dget(alias);
  1634. return alias;
  1635. }
  1636. /**
  1637. * d_find_any_alias - find any alias for a given inode
  1638. * @inode: inode to find an alias for
  1639. *
  1640. * If any aliases exist for the given inode, take and return a
  1641. * reference for one of them. If no aliases exist, return %NULL.
  1642. */
  1643. struct dentry *d_find_any_alias(struct inode *inode)
  1644. {
  1645. struct dentry *de;
  1646. spin_lock(&inode->i_lock);
  1647. de = __d_find_any_alias(inode);
  1648. spin_unlock(&inode->i_lock);
  1649. return de;
  1650. }
  1651. EXPORT_SYMBOL(d_find_any_alias);
  1652. /**
  1653. * d_obtain_alias - find or allocate a dentry for a given inode
  1654. * @inode: inode to allocate the dentry for
  1655. *
  1656. * Obtain a dentry for an inode resulting from NFS filehandle conversion or
  1657. * similar open by handle operations. The returned dentry may be anonymous,
  1658. * or may have a full name (if the inode was already in the cache).
  1659. *
  1660. * When called on a directory inode, we must ensure that the inode only ever
  1661. * has one dentry. If a dentry is found, that is returned instead of
  1662. * allocating a new one.
  1663. *
  1664. * On successful return, the reference to the inode has been transferred
  1665. * to the dentry. In case of an error the reference on the inode is released.
  1666. * To make it easier to use in export operations a %NULL or IS_ERR inode may
  1667. * be passed in and will be the error will be propagate to the return value,
  1668. * with a %NULL @inode replaced by ERR_PTR(-ESTALE).
  1669. */
  1670. struct dentry *d_obtain_alias(struct inode *inode)
  1671. {
  1672. static const struct qstr anonstring = QSTR_INIT("/", 1);
  1673. struct dentry *tmp;
  1674. struct dentry *res;
  1675. if (!inode)
  1676. return ERR_PTR(-ESTALE);
  1677. if (IS_ERR(inode))
  1678. return ERR_CAST(inode);
  1679. res = d_find_any_alias(inode);
  1680. if (res)
  1681. goto out_iput;
  1682. tmp = __d_alloc(inode->i_sb, &anonstring);
  1683. if (!tmp) {
  1684. res = ERR_PTR(-ENOMEM);
  1685. goto out_iput;
  1686. }
  1687. spin_lock(&inode->i_lock);
  1688. res = __d_find_any_alias(inode);
  1689. if (res) {
  1690. spin_unlock(&inode->i_lock);
  1691. dput(tmp);
  1692. goto out_iput;
  1693. }
  1694. /* attach a disconnected dentry */
  1695. spin_lock(&tmp->d_lock);
  1696. tmp->d_inode = inode;
  1697. tmp->d_flags |= DCACHE_DISCONNECTED;
  1698. hlist_add_head(&tmp->d_alias, &inode->i_dentry);
  1699. hlist_bl_lock(&tmp->d_sb->s_anon);
  1700. hlist_bl_add_head(&tmp->d_hash, &tmp->d_sb->s_anon);
  1701. hlist_bl_unlock(&tmp->d_sb->s_anon);
  1702. spin_unlock(&tmp->d_lock);
  1703. spin_unlock(&inode->i_lock);
  1704. security_d_instantiate(tmp, inode);
  1705. return tmp;
  1706. out_iput:
  1707. if (res && !IS_ERR(res))
  1708. security_d_instantiate(res, inode);
  1709. iput(inode);
  1710. return res;
  1711. }
  1712. EXPORT_SYMBOL(d_obtain_alias);
  1713. /**
  1714. * d_splice_alias - splice a disconnected dentry into the tree if one exists
  1715. * @inode: the inode which may have a disconnected dentry
  1716. * @dentry: a negative dentry which we want to point to the inode.
  1717. *
  1718. * If inode is a directory and has a 'disconnected' dentry (i.e. IS_ROOT and
  1719. * DCACHE_DISCONNECTED), then d_move that in place of the given dentry
  1720. * and return it, else simply d_add the inode to the dentry and return NULL.
  1721. *
  1722. * This is needed in the lookup routine of any filesystem that is exportable
  1723. * (via knfsd) so that we can build dcache paths to directories effectively.
  1724. *
  1725. * If a dentry was found and moved, then it is returned. Otherwise NULL
  1726. * is returned. This matches the expected return value of ->lookup.
  1727. *
  1728. * Cluster filesystems may call this function with a negative, hashed dentry.
  1729. * In that case, we know that the inode will be a regular file, and also this
  1730. * will only occur during atomic_open. So we need to check for the dentry
  1731. * being already hashed only in the final case.
  1732. */
  1733. struct dentry *d_splice_alias(struct inode *inode, struct dentry *dentry)
  1734. {
  1735. struct dentry *new = NULL;
  1736. if (IS_ERR(inode))
  1737. return ERR_CAST(inode);
  1738. if (inode && S_ISDIR(inode->i_mode)) {
  1739. spin_lock(&inode->i_lock);
  1740. new = __d_find_alias(inode, 1);
  1741. if (new) {
  1742. BUG_ON(!(new->d_flags & DCACHE_DISCONNECTED));
  1743. spin_unlock(&inode->i_lock);
  1744. security_d_instantiate(new, inode);
  1745. d_move(new, dentry);
  1746. iput(inode);
  1747. } else {
  1748. /* already taking inode->i_lock, so d_add() by hand */
  1749. __d_instantiate(dentry, inode);
  1750. spin_unlock(&inode->i_lock);
  1751. security_d_instantiate(dentry, inode);
  1752. d_rehash(dentry);
  1753. }
  1754. } else {
  1755. d_instantiate(dentry, inode);
  1756. if (d_unhashed(dentry))
  1757. d_rehash(dentry);
  1758. }
  1759. return new;
  1760. }
  1761. EXPORT_SYMBOL(d_splice_alias);
  1762. /**
  1763. * d_add_ci - lookup or allocate new dentry with case-exact name
  1764. * @inode: the inode case-insensitive lookup has found
  1765. * @dentry: the negative dentry that was passed to the parent's lookup func
  1766. * @name: the case-exact name to be associated with the returned dentry
  1767. *
  1768. * This is to avoid filling the dcache with case-insensitive names to the
  1769. * same inode, only the actual correct case is stored in the dcache for
  1770. * case-insensitive filesystems.
  1771. *
  1772. * For a case-insensitive lookup match and if the the case-exact dentry
  1773. * already exists in in the dcache, use it and return it.
  1774. *
  1775. * If no entry exists with the exact case name, allocate new dentry with
  1776. * the exact case, and return the spliced entry.
  1777. */
  1778. struct dentry *d_add_ci(struct dentry *dentry, struct inode *inode,
  1779. struct qstr *name)
  1780. {
  1781. struct dentry *found;
  1782. struct dentry *new;
  1783. /*
  1784. * First check if a dentry matching the name already exists,
  1785. * if not go ahead and create it now.
  1786. */
  1787. found = d_hash_and_lookup(dentry->d_parent, name);
  1788. if (unlikely(IS_ERR(found)))
  1789. goto err_out;
  1790. if (!found) {
  1791. new = d_alloc(dentry->d_parent, name);
  1792. if (!new) {
  1793. found = ERR_PTR(-ENOMEM);
  1794. goto err_out;
  1795. }
  1796. found = d_splice_alias(inode, new);
  1797. if (found) {
  1798. dput(new);
  1799. return found;
  1800. }
  1801. return new;
  1802. }
  1803. /*
  1804. * If a matching dentry exists, and it's not negative use it.
  1805. *
  1806. * Decrement the reference count to balance the iget() done
  1807. * earlier on.
  1808. */
  1809. if (found->d_inode) {
  1810. if (unlikely(found->d_inode != inode)) {
  1811. /* This can't happen because bad inodes are unhashed. */
  1812. BUG_ON(!is_bad_inode(inode));
  1813. BUG_ON(!is_bad_inode(found->d_inode));
  1814. }
  1815. iput(inode);
  1816. return found;
  1817. }
  1818. /*
  1819. * Negative dentry: instantiate it unless the inode is a directory and
  1820. * already has a dentry.
  1821. */
  1822. new = d_splice_alias(inode, found);
  1823. if (new) {
  1824. dput(found);
  1825. found = new;
  1826. }
  1827. return found;
  1828. err_out:
  1829. iput(inode);
  1830. return found;
  1831. }
  1832. EXPORT_SYMBOL(d_add_ci);
  1833. /*
  1834. * Do the slow-case of the dentry name compare.
  1835. *
  1836. * Unlike the dentry_cmp() function, we need to atomically
  1837. * load the name and length information, so that the
  1838. * filesystem can rely on them, and can use the 'name' and
  1839. * 'len' information without worrying about walking off the
  1840. * end of memory etc.
  1841. *
  1842. * Thus the read_seqcount_retry() and the "duplicate" info
  1843. * in arguments (the low-level filesystem should not look
  1844. * at the dentry inode or name contents directly, since
  1845. * rename can change them while we're in RCU mode).
  1846. */
  1847. enum slow_d_compare {
  1848. D_COMP_OK,
  1849. D_COMP_NOMATCH,
  1850. D_COMP_SEQRETRY,
  1851. };
  1852. static noinline enum slow_d_compare slow_dentry_cmp(
  1853. const struct dentry *parent,
  1854. struct dentry *dentry,
  1855. unsigned int seq,
  1856. const struct qstr *name)
  1857. {
  1858. int tlen = dentry->d_name.len;
  1859. const char *tname = dentry->d_name.name;
  1860. if (read_seqcount_retry(&dentry->d_seq, seq)) {
  1861. cpu_relax();
  1862. return D_COMP_SEQRETRY;
  1863. }
  1864. if (parent->d_op->d_compare(parent, dentry, tlen, tname, name))
  1865. return D_COMP_NOMATCH;
  1866. return D_COMP_OK;
  1867. }
  1868. /**
  1869. * __d_lookup_rcu - search for a dentry (racy, store-free)
  1870. * @parent: parent dentry
  1871. * @name: qstr of name we wish to find
  1872. * @seqp: returns d_seq value at the point where the dentry was found
  1873. * Returns: dentry, or NULL
  1874. *
  1875. * __d_lookup_rcu is the dcache lookup function for rcu-walk name
  1876. * resolution (store-free path walking) design described in
  1877. * Documentation/filesystems/path-lookup.txt.
  1878. *
  1879. * This is not to be used outside core vfs.
  1880. *
  1881. * __d_lookup_rcu must only be used in rcu-walk mode, ie. with vfsmount lock
  1882. * held, and rcu_read_lock held. The returned dentry must not be stored into
  1883. * without taking d_lock and checking d_seq sequence count against @seq
  1884. * returned here.
  1885. *
  1886. * A refcount may be taken on the found dentry with the d_rcu_to_refcount
  1887. * function.
  1888. *
  1889. * Alternatively, __d_lookup_rcu may be called again to look up the child of
  1890. * the returned dentry, so long as its parent's seqlock is checked after the
  1891. * child is looked up. Thus, an interlocking stepping of sequence lock checks
  1892. * is formed, giving integrity down the path walk.
  1893. *
  1894. * NOTE! The caller *has* to check the resulting dentry against the sequence
  1895. * number we've returned before using any of the resulting dentry state!
  1896. */
  1897. struct dentry *__d_lookup_rcu(const struct dentry *parent,
  1898. const struct qstr *name,
  1899. unsigned *seqp)
  1900. {
  1901. u64 hashlen = name->hash_len;
  1902. const unsigned char *str = name->name;
  1903. struct hlist_bl_head *b = d_hash(parent, hashlen_hash(hashlen));
  1904. struct hlist_bl_node *node;
  1905. struct dentry *dentry;
  1906. /*
  1907. * Note: There is significant duplication with __d_lookup_rcu which is
  1908. * required to prevent single threaded performance regressions
  1909. * especially on architectures where smp_rmb (in seqcounts) are costly.
  1910. * Keep the two functions in sync.
  1911. */
  1912. /*
  1913. * The hash list is protected using RCU.
  1914. *
  1915. * Carefully use d_seq when comparing a candidate dentry, to avoid
  1916. * races with d_move().
  1917. *
  1918. * It is possible that concurrent renames can mess up our list
  1919. * walk here and result in missing our dentry, resulting in the
  1920. * false-negative result. d_lookup() protects against concurrent
  1921. * renames using rename_lock seqlock.
  1922. *
  1923. * See Documentation/filesystems/path-lookup.txt for more details.
  1924. */
  1925. hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) {
  1926. unsigned seq;
  1927. seqretry:
  1928. /*
  1929. * The dentry sequence count protects us from concurrent
  1930. * renames, and thus protects parent and name fields.
  1931. *
  1932. * The caller must perform a seqcount check in order
  1933. * to do anything useful with the returned dentry.
  1934. *
  1935. * NOTE! We do a "raw" seqcount_begin here. That means that
  1936. * we don't wait for the sequence count to stabilize if it
  1937. * is in the middle of a sequence change. If we do the slow
  1938. * dentry compare, we will do seqretries until it is stable,
  1939. * and if we end up with a successful lookup, we actually
  1940. * want to exit RCU lookup anyway.
  1941. */
  1942. seq = raw_seqcount_begin(&dentry->d_seq);
  1943. if (dentry->d_parent != parent)
  1944. continue;
  1945. if (d_unhashed(dentry))
  1946. continue;
  1947. if (unlikely(parent->d_flags & DCACHE_OP_COMPARE)) {
  1948. if (dentry->d_name.hash != hashlen_hash(hashlen))
  1949. continue;
  1950. *seqp = seq;
  1951. switch (slow_dentry_cmp(parent, dentry, seq, name)) {
  1952. case D_COMP_OK:
  1953. return dentry;
  1954. case D_COMP_NOMATCH:
  1955. continue;
  1956. default:
  1957. goto seqretry;
  1958. }
  1959. }
  1960. if (dentry->d_name.hash_len != hashlen)
  1961. continue;
  1962. *seqp = seq;
  1963. if (!dentry_cmp(dentry, str, hashlen_len(hashlen)))
  1964. return dentry;
  1965. }
  1966. return NULL;
  1967. }
  1968. /**
  1969. * d_lookup - search for a dentry
  1970. * @parent: parent dentry
  1971. * @name: qstr of name we wish to find
  1972. * Returns: dentry, or NULL
  1973. *
  1974. * d_lookup searches the children of the parent dentry for the name in
  1975. * question. If the dentry is found its reference count is incremented and the
  1976. * dentry is returned. The caller must use dput to free the entry when it has
  1977. * finished using it. %NULL is returned if the dentry does not exist.
  1978. */
  1979. struct dentry *d_lookup(const struct dentry *parent, const struct qstr *name)
  1980. {
  1981. struct dentry *dentry;
  1982. unsigned seq;
  1983. do {
  1984. seq = read_seqbegin(&rename_lock);
  1985. dentry = __d_lookup(parent, name);
  1986. if (dentry)
  1987. break;
  1988. } while (read_seqretry(&rename_lock, seq));
  1989. return dentry;
  1990. }
  1991. EXPORT_SYMBOL(d_lookup);
  1992. /**
  1993. * __d_lookup - search for a dentry (racy)
  1994. * @parent: parent dentry
  1995. * @name: qstr of name we wish to find
  1996. * Returns: dentry, or NULL
  1997. *
  1998. * __d_lookup is like d_lookup, however it may (rarely) return a
  1999. * false-negative result due to unrelated rename activity.
  2000. *
  2001. * __d_lookup is slightly faster by avoiding rename_lock read seqlock,
  2002. * however it must be used carefully, eg. with a following d_lookup in
  2003. * the case of failure.
  2004. *
  2005. * __d_lookup callers must be commented.
  2006. */
  2007. struct dentry *__d_lookup(const struct dentry *parent, const struct qstr *name)
  2008. {
  2009. unsigned int len = name->len;
  2010. unsigned int hash = name->hash;
  2011. const unsigned char *str = name->name;
  2012. struct hlist_bl_head *b = d_hash(parent, hash);
  2013. struct hlist_bl_node *node;
  2014. struct dentry *found = NULL;
  2015. struct dentry *dentry;
  2016. /*
  2017. * Note: There is significant duplication with __d_lookup_rcu which is
  2018. * required to prevent single threaded performance regressions
  2019. * especially on architectures where smp_rmb (in seqcounts) are costly.
  2020. * Keep the two functions in sync.
  2021. */
  2022. /*
  2023. * The hash list is protected using RCU.
  2024. *
  2025. * Take d_lock when comparing a candidate dentry, to avoid races
  2026. * with d_move().
  2027. *
  2028. * It is possible that concurrent renames can mess up our list
  2029. * walk here and result in missing our dentry, resulting in the
  2030. * false-negative result. d_lookup() protects against concurrent
  2031. * renames using rename_lock seqlock.
  2032. *
  2033. * See Documentation/filesystems/path-lookup.txt for more details.
  2034. */
  2035. rcu_read_lock();
  2036. hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) {
  2037. if (dentry->d_name.hash != hash)
  2038. continue;
  2039. spin_lock(&dentry->d_lock);
  2040. if (dentry->d_parent != parent)
  2041. goto next;
  2042. if (d_unhashed(dentry))
  2043. goto next;
  2044. /*
  2045. * It is safe to compare names since d_move() cannot
  2046. * change the qstr (protected by d_lock).
  2047. */
  2048. if (parent->d_flags & DCACHE_OP_COMPARE) {
  2049. int tlen = dentry->d_name.len;
  2050. const char *tname = dentry->d_name.name;
  2051. if (parent->d_op->d_compare(parent, dentry, tlen, tname, name))
  2052. goto next;
  2053. } else {
  2054. if (dentry->d_name.len != len)
  2055. goto next;
  2056. if (dentry_cmp(dentry, str, len))
  2057. goto next;
  2058. }
  2059. dentry->d_lockref.count++;
  2060. found = dentry;
  2061. spin_unlock(&dentry->d_lock);
  2062. break;
  2063. next:
  2064. spin_unlock(&dentry->d_lock);
  2065. }
  2066. rcu_read_unlock();
  2067. return found;
  2068. }
  2069. /**
  2070. * d_hash_and_lookup - hash the qstr then search for a dentry
  2071. * @dir: Directory to search in
  2072. * @name: qstr of name we wish to find
  2073. *
  2074. * On lookup failure NULL is returned; on bad name - ERR_PTR(-error)
  2075. */
  2076. struct dentry *d_hash_and_lookup(struct dentry *dir, struct qstr *name)
  2077. {
  2078. /*
  2079. * Check for a fs-specific hash function. Note that we must
  2080. * calculate the standard hash first, as the d_op->d_hash()
  2081. * routine may choose to leave the hash value unchanged.
  2082. */
  2083. name->hash = full_name_hash(name->name, name->len);
  2084. if (dir->d_flags & DCACHE_OP_HASH) {
  2085. int err = dir->d_op->d_hash(dir, name);
  2086. if (unlikely(err < 0))
  2087. return ERR_PTR(err);
  2088. }
  2089. return d_lookup(dir, name);
  2090. }
  2091. EXPORT_SYMBOL(d_hash_and_lookup);
  2092. /**
  2093. * d_validate - verify dentry provided from insecure source (deprecated)
  2094. * @dentry: The dentry alleged to be valid child of @dparent
  2095. * @dparent: The parent dentry (known to be valid)
  2096. *
  2097. * An insecure source has sent us a dentry, here we verify it and dget() it.
  2098. * This is used by ncpfs in its readdir implementation.
  2099. * Zero is returned in the dentry is invalid.
  2100. *
  2101. * This function is slow for big directories, and deprecated, do not use it.
  2102. */
  2103. int d_validate(struct dentry *dentry, struct dentry *dparent)
  2104. {
  2105. struct dentry *child;
  2106. spin_lock(&dparent->d_lock);
  2107. list_for_each_entry(child, &dparent->d_subdirs, d_u.d_child) {
  2108. if (dentry == child) {
  2109. spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
  2110. __dget_dlock(dentry);
  2111. spin_unlock(&dentry->d_lock);
  2112. spin_unlock(&dparent->d_lock);
  2113. return 1;
  2114. }
  2115. }
  2116. spin_unlock(&dparent->d_lock);
  2117. return 0;
  2118. }
  2119. EXPORT_SYMBOL(d_validate);
  2120. /*
  2121. * When a file is deleted, we have two options:
  2122. * - turn this dentry into a negative dentry
  2123. * - unhash this dentry and free it.
  2124. *
  2125. * Usually, we want to just turn this into
  2126. * a negative dentry, but if anybody else is
  2127. * currently using the dentry or the inode
  2128. * we can't do that and we fall back on removing
  2129. * it from the hash queues and waiting for
  2130. * it to be deleted later when it has no users
  2131. */
  2132. /**
  2133. * d_delete - delete a dentry
  2134. * @dentry: The dentry to delete
  2135. *
  2136. * Turn the dentry into a negative dentry if possible, otherwise
  2137. * remove it from the hash queues so it can be deleted later
  2138. */
  2139. void d_delete(struct dentry * dentry)
  2140. {
  2141. struct inode *inode;
  2142. int isdir = 0;
  2143. /*
  2144. * Are we the only user?
  2145. */
  2146. again:
  2147. spin_lock(&dentry->d_lock);
  2148. inode = dentry->d_inode;
  2149. isdir = S_ISDIR(inode->i_mode);
  2150. if (dentry->d_lockref.count == 1) {
  2151. if (!spin_trylock(&inode->i_lock)) {
  2152. spin_unlock(&dentry->d_lock);
  2153. cpu_relax();
  2154. goto again;
  2155. }
  2156. dentry->d_flags &= ~DCACHE_CANT_MOUNT;
  2157. dentry_unlink_inode(dentry);
  2158. fsnotify_nameremove(dentry, isdir);
  2159. return;
  2160. }
  2161. if (!d_unhashed(dentry))
  2162. __d_drop(dentry);
  2163. spin_unlock(&dentry->d_lock);
  2164. fsnotify_nameremove(dentry, isdir);
  2165. }
  2166. EXPORT_SYMBOL(d_delete);
  2167. static void __d_rehash(struct dentry * entry, struct hlist_bl_head *b)
  2168. {
  2169. BUG_ON(!d_unhashed(entry));
  2170. hlist_bl_lock(b);
  2171. entry->d_flags |= DCACHE_RCUACCESS;
  2172. hlist_bl_add_head_rcu(&entry->d_hash, b);
  2173. hlist_bl_unlock(b);
  2174. }
  2175. static void _d_rehash(struct dentry * entry)
  2176. {
  2177. __d_rehash(entry, d_hash(entry->d_parent, entry->d_name.hash));
  2178. }
  2179. /**
  2180. * d_rehash - add an entry back to the hash
  2181. * @entry: dentry to add to the hash
  2182. *
  2183. * Adds a dentry to the hash according to its name.
  2184. */
  2185. void d_rehash(struct dentry * entry)
  2186. {
  2187. spin_lock(&entry->d_lock);
  2188. _d_rehash(entry);
  2189. spin_unlock(&entry->d_lock);
  2190. }
  2191. EXPORT_SYMBOL(d_rehash);
  2192. /**
  2193. * dentry_update_name_case - update case insensitive dentry with a new name
  2194. * @dentry: dentry to be updated
  2195. * @name: new name
  2196. *
  2197. * Update a case insensitive dentry with new case of name.
  2198. *
  2199. * dentry must have been returned by d_lookup with name @name. Old and new
  2200. * name lengths must match (ie. no d_compare which allows mismatched name
  2201. * lengths).
  2202. *
  2203. * Parent inode i_mutex must be held over d_lookup and into this call (to
  2204. * keep renames and concurrent inserts, and readdir(2) away).
  2205. */
  2206. void dentry_update_name_case(struct dentry *dentry, struct qstr *name)
  2207. {
  2208. BUG_ON(!mutex_is_locked(&dentry->d_parent->d_inode->i_mutex));
  2209. BUG_ON(dentry->d_name.len != name->len); /* d_lookup gives this */
  2210. spin_lock(&dentry->d_lock);
  2211. write_seqcount_begin(&dentry->d_seq);
  2212. memcpy((unsigned char *)dentry->d_name.name, name->name, name->len);
  2213. write_seqcount_end(&dentry->d_seq);
  2214. spin_unlock(&dentry->d_lock);
  2215. }
  2216. EXPORT_SYMBOL(dentry_update_name_case);
  2217. static void switch_names(struct dentry *dentry, struct dentry *target)
  2218. {
  2219. if (dname_external(target)) {
  2220. if (dname_external(dentry)) {
  2221. /*
  2222. * Both external: swap the pointers
  2223. */
  2224. swap(target->d_name.name, dentry->d_name.name);
  2225. } else {
  2226. /*
  2227. * dentry:internal, target:external. Steal target's
  2228. * storage and make target internal.
  2229. */
  2230. memcpy(target->d_iname, dentry->d_name.name,
  2231. dentry->d_name.len + 1);
  2232. dentry->d_name.name = target->d_name.name;
  2233. target->d_name.name = target->d_iname;
  2234. }
  2235. } else {
  2236. if (dname_external(dentry)) {
  2237. /*
  2238. * dentry:external, target:internal. Give dentry's
  2239. * storage to target and make dentry internal
  2240. */
  2241. memcpy(dentry->d_iname, target->d_name.name,
  2242. target->d_name.len + 1);
  2243. target->d_name.name = dentry->d_name.name;
  2244. dentry->d_name.name = dentry->d_iname;
  2245. } else {
  2246. /*
  2247. * Both are internal. Just copy target to dentry
  2248. */
  2249. memcpy(dentry->d_iname, target->d_name.name,
  2250. target->d_name.len + 1);
  2251. dentry->d_name.len = target->d_name.len;
  2252. return;
  2253. }
  2254. }
  2255. swap(dentry->d_name.len, target->d_name.len);
  2256. }
  2257. static void dentry_lock_for_move(struct dentry *dentry, struct dentry *target)
  2258. {
  2259. /*
  2260. * XXXX: do we really need to take target->d_lock?
  2261. */
  2262. if (IS_ROOT(dentry) || dentry->d_parent == target->d_parent)
  2263. spin_lock(&target->d_parent->d_lock);
  2264. else {
  2265. if (d_ancestor(dentry->d_parent, target->d_parent)) {
  2266. spin_lock(&dentry->d_parent->d_lock);
  2267. spin_lock_nested(&target->d_parent->d_lock,
  2268. DENTRY_D_LOCK_NESTED);
  2269. } else {
  2270. spin_lock(&target->d_parent->d_lock);
  2271. spin_lock_nested(&dentry->d_parent->d_lock,
  2272. DENTRY_D_LOCK_NESTED);
  2273. }
  2274. }
  2275. if (target < dentry) {
  2276. spin_lock_nested(&target->d_lock, 2);
  2277. spin_lock_nested(&dentry->d_lock, 3);
  2278. } else {
  2279. spin_lock_nested(&dentry->d_lock, 2);
  2280. spin_lock_nested(&target->d_lock, 3);
  2281. }
  2282. }
  2283. static void dentry_unlock_parents_for_move(struct dentry *dentry,
  2284. struct dentry *target)
  2285. {
  2286. if (target->d_parent != dentry->d_parent)
  2287. spin_unlock(&dentry->d_parent->d_lock);
  2288. if (target->d_parent != target)
  2289. spin_unlock(&target->d_parent->d_lock);
  2290. }
  2291. /*
  2292. * When switching names, the actual string doesn't strictly have to
  2293. * be preserved in the target - because we're dropping the target
  2294. * anyway. As such, we can just do a simple memcpy() to copy over
  2295. * the new name before we switch.
  2296. *
  2297. * Note that we have to be a lot more careful about getting the hash
  2298. * switched - we have to switch the hash value properly even if it
  2299. * then no longer matches the actual (corrupted) string of the target.
  2300. * The hash value has to match the hash queue that the dentry is on..
  2301. */
  2302. /*
  2303. * __d_move - move a dentry
  2304. * @dentry: entry to move
  2305. * @target: new dentry
  2306. *
  2307. * Update the dcache to reflect the move of a file name. Negative
  2308. * dcache entries should not be moved in this way. Caller must hold
  2309. * rename_lock, the i_mutex of the source and target directories,
  2310. * and the sb->s_vfs_rename_mutex if they differ. See lock_rename().
  2311. */
  2312. static void __d_move(struct dentry * dentry, struct dentry * target)
  2313. {
  2314. if (!dentry->d_inode)
  2315. printk(KERN_WARNING "VFS: moving negative dcache entry\n");
  2316. BUG_ON(d_ancestor(dentry, target));
  2317. BUG_ON(d_ancestor(target, dentry));
  2318. dentry_lock_for_move(dentry, target);
  2319. write_seqcount_begin(&dentry->d_seq);
  2320. write_seqcount_begin(&target->d_seq);
  2321. /* __d_drop does write_seqcount_barrier, but they're OK to nest. */
  2322. /*
  2323. * Move the dentry to the target hash queue. Don't bother checking
  2324. * for the same hash queue because of how unlikely it is.
  2325. */
  2326. __d_drop(dentry);
  2327. __d_rehash(dentry, d_hash(target->d_parent, target->d_name.hash));
  2328. /* Unhash the target: dput() will then get rid of it */
  2329. __d_drop(target);
  2330. list_del(&dentry->d_u.d_child);
  2331. list_del(&target->d_u.d_child);
  2332. /* Switch the names.. */
  2333. switch_names(dentry, target);
  2334. swap(dentry->d_name.hash, target->d_name.hash);
  2335. /* ... and switch the parents */
  2336. if (IS_ROOT(dentry)) {
  2337. dentry->d_parent = target->d_parent;
  2338. target->d_parent = target;
  2339. INIT_LIST_HEAD(&target->d_u.d_child);
  2340. } else {
  2341. swap(dentry->d_parent, target->d_parent);
  2342. /* And add them back to the (new) parent lists */
  2343. list_add(&target->d_u.d_child, &target->d_parent->d_subdirs);
  2344. }
  2345. list_add(&dentry->d_u.d_child, &dentry->d_parent->d_subdirs);
  2346. write_seqcount_end(&target->d_seq);
  2347. write_seqcount_end(&dentry->d_seq);
  2348. dentry_unlock_parents_for_move(dentry, target);
  2349. spin_unlock(&target->d_lock);
  2350. fsnotify_d_move(dentry);
  2351. spin_unlock(&dentry->d_lock);
  2352. }
  2353. /*
  2354. * d_move - move a dentry
  2355. * @dentry: entry to move
  2356. * @target: new dentry
  2357. *
  2358. * Update the dcache to reflect the move of a file name. Negative
  2359. * dcache entries should not be moved in this way. See the locking
  2360. * requirements for __d_move.
  2361. */
  2362. void d_move(struct dentry *dentry, struct dentry *target)
  2363. {
  2364. write_seqlock(&rename_lock);
  2365. __d_move(dentry, target);
  2366. write_sequnlock(&rename_lock);
  2367. }
  2368. EXPORT_SYMBOL(d_move);
  2369. /**
  2370. * d_ancestor - search for an ancestor
  2371. * @p1: ancestor dentry
  2372. * @p2: child dentry
  2373. *
  2374. * Returns the ancestor dentry of p2 which is a child of p1, if p1 is
  2375. * an ancestor of p2, else NULL.
  2376. */
  2377. struct dentry *d_ancestor(struct dentry *p1, struct dentry *p2)
  2378. {
  2379. struct dentry *p;
  2380. for (p = p2; !IS_ROOT(p); p = p->d_parent) {
  2381. if (p->d_parent == p1)
  2382. return p;
  2383. }
  2384. return NULL;
  2385. }
  2386. /*
  2387. * This helper attempts to cope with remotely renamed directories
  2388. *
  2389. * It assumes that the caller is already holding
  2390. * dentry->d_parent->d_inode->i_mutex, inode->i_lock and rename_lock
  2391. *
  2392. * Note: If ever the locking in lock_rename() changes, then please
  2393. * remember to update this too...
  2394. */
  2395. static struct dentry *__d_unalias(struct inode *inode,
  2396. struct dentry *dentry, struct dentry *alias)
  2397. {
  2398. struct mutex *m1 = NULL, *m2 = NULL;
  2399. struct dentry *ret = ERR_PTR(-EBUSY);
  2400. /* If alias and dentry share a parent, then no extra locks required */
  2401. if (alias->d_parent == dentry->d_parent)
  2402. goto out_unalias;
  2403. /* See lock_rename() */
  2404. if (!mutex_trylock(&dentry->d_sb->s_vfs_rename_mutex))
  2405. goto out_err;
  2406. m1 = &dentry->d_sb->s_vfs_rename_mutex;
  2407. if (!mutex_trylock(&alias->d_parent->d_inode->i_mutex))
  2408. goto out_err;
  2409. m2 = &alias->d_parent->d_inode->i_mutex;
  2410. out_unalias:
  2411. if (likely(!d_mountpoint(alias))) {
  2412. __d_move(alias, dentry);
  2413. ret = alias;
  2414. }
  2415. out_err:
  2416. spin_unlock(&inode->i_lock);
  2417. if (m2)
  2418. mutex_unlock(m2);
  2419. if (m1)
  2420. mutex_unlock(m1);
  2421. return ret;
  2422. }
  2423. /*
  2424. * Prepare an anonymous dentry for life in the superblock's dentry tree as a
  2425. * named dentry in place of the dentry to be replaced.
  2426. * returns with anon->d_lock held!
  2427. */
  2428. static void __d_materialise_dentry(struct dentry *dentry, struct dentry *anon)
  2429. {
  2430. struct dentry *dparent;
  2431. dentry_lock_for_move(anon, dentry);
  2432. write_seqcount_begin(&dentry->d_seq);
  2433. write_seqcount_begin(&anon->d_seq);
  2434. dparent = dentry->d_parent;
  2435. switch_names(dentry, anon);
  2436. swap(dentry->d_name.hash, anon->d_name.hash);
  2437. dentry->d_parent = dentry;
  2438. list_del_init(&dentry->d_u.d_child);
  2439. anon->d_parent = dparent;
  2440. list_move(&anon->d_u.d_child, &dparent->d_subdirs);
  2441. write_seqcount_end(&dentry->d_seq);
  2442. write_seqcount_end(&anon->d_seq);
  2443. dentry_unlock_parents_for_move(anon, dentry);
  2444. spin_unlock(&dentry->d_lock);
  2445. /* anon->d_lock still locked, returns locked */
  2446. anon->d_flags &= ~DCACHE_DISCONNECTED;
  2447. }
  2448. /**
  2449. * d_materialise_unique - introduce an inode into the tree
  2450. * @dentry: candidate dentry
  2451. * @inode: inode to bind to the dentry, to which aliases may be attached
  2452. *
  2453. * Introduces an dentry into the tree, substituting an extant disconnected
  2454. * root directory alias in its place if there is one. Caller must hold the
  2455. * i_mutex of the parent directory.
  2456. */
  2457. struct dentry *d_materialise_unique(struct dentry *dentry, struct inode *inode)
  2458. {
  2459. struct dentry *actual;
  2460. BUG_ON(!d_unhashed(dentry));
  2461. if (!inode) {
  2462. actual = dentry;
  2463. __d_instantiate(dentry, NULL);
  2464. d_rehash(actual);
  2465. goto out_nolock;
  2466. }
  2467. spin_lock(&inode->i_lock);
  2468. if (S_ISDIR(inode->i_mode)) {
  2469. struct dentry *alias;
  2470. /* Does an aliased dentry already exist? */
  2471. alias = __d_find_alias(inode, 0);
  2472. if (alias) {
  2473. actual = alias;
  2474. write_seqlock(&rename_lock);
  2475. if (d_ancestor(alias, dentry)) {
  2476. /* Check for loops */
  2477. actual = ERR_PTR(-ELOOP);
  2478. spin_unlock(&inode->i_lock);
  2479. } else if (IS_ROOT(alias)) {
  2480. /* Is this an anonymous mountpoint that we
  2481. * could splice into our tree? */
  2482. __d_materialise_dentry(dentry, alias);
  2483. write_sequnlock(&rename_lock);
  2484. __d_drop(alias);
  2485. goto found;
  2486. } else {
  2487. /* Nope, but we must(!) avoid directory
  2488. * aliasing. This drops inode->i_lock */
  2489. actual = __d_unalias(inode, dentry, alias);
  2490. }
  2491. write_sequnlock(&rename_lock);
  2492. if (IS_ERR(actual)) {
  2493. if (PTR_ERR(actual) == -ELOOP)
  2494. pr_warn_ratelimited(
  2495. "VFS: Lookup of '%s' in %s %s"
  2496. " would have caused loop\n",
  2497. dentry->d_name.name,
  2498. inode->i_sb->s_type->name,
  2499. inode->i_sb->s_id);
  2500. dput(alias);
  2501. }
  2502. goto out_nolock;
  2503. }
  2504. }
  2505. /* Add a unique reference */
  2506. actual = __d_instantiate_unique(dentry, inode);
  2507. if (!actual)
  2508. actual = dentry;
  2509. else
  2510. BUG_ON(!d_unhashed(actual));
  2511. spin_lock(&actual->d_lock);
  2512. found:
  2513. _d_rehash(actual);
  2514. spin_unlock(&actual->d_lock);
  2515. spin_unlock(&inode->i_lock);
  2516. out_nolock:
  2517. if (actual == dentry) {
  2518. security_d_instantiate(dentry, inode);
  2519. return NULL;
  2520. }
  2521. iput(inode);
  2522. return actual;
  2523. }
  2524. EXPORT_SYMBOL_GPL(d_materialise_unique);
  2525. static int prepend(char **buffer, int *buflen, const char *str, int namelen)
  2526. {
  2527. *buflen -= namelen;
  2528. if (*buflen < 0)
  2529. return -ENAMETOOLONG;
  2530. *buffer -= namelen;
  2531. memcpy(*buffer, str, namelen);
  2532. return 0;
  2533. }
  2534. /**
  2535. * prepend_name - prepend a pathname in front of current buffer pointer
  2536. * @buffer: buffer pointer
  2537. * @buflen: allocated length of the buffer
  2538. * @name: name string and length qstr structure
  2539. *
  2540. * With RCU path tracing, it may race with d_move(). Use ACCESS_ONCE() to
  2541. * make sure that either the old or the new name pointer and length are
  2542. * fetched. However, there may be mismatch between length and pointer.
  2543. * The length cannot be trusted, we need to copy it byte-by-byte until
  2544. * the length is reached or a null byte is found. It also prepends "/" at
  2545. * the beginning of the name. The sequence number check at the caller will
  2546. * retry it again when a d_move() does happen. So any garbage in the buffer
  2547. * due to mismatched pointer and length will be discarded.
  2548. */
  2549. static int prepend_name(char **buffer, int *buflen, struct qstr *name)
  2550. {
  2551. const char *dname = ACCESS_ONCE(name->name);
  2552. u32 dlen = ACCESS_ONCE(name->len);
  2553. char *p;
  2554. if (*buflen < dlen + 1)
  2555. return -ENAMETOOLONG;
  2556. *buflen -= dlen + 1;
  2557. p = *buffer -= dlen + 1;
  2558. *p++ = '/';
  2559. while (dlen--) {
  2560. char c = *dname++;
  2561. if (!c)
  2562. break;
  2563. *p++ = c;
  2564. }
  2565. return 0;
  2566. }
  2567. /**
  2568. * prepend_path - Prepend path string to a buffer
  2569. * @path: the dentry/vfsmount to report
  2570. * @root: root vfsmnt/dentry
  2571. * @buffer: pointer to the end of the buffer
  2572. * @buflen: pointer to buffer length
  2573. *
  2574. * The function will first try to write out the pathname without taking any
  2575. * lock other than the RCU read lock to make sure that dentries won't go away.
  2576. * It only checks the sequence number of the global rename_lock as any change
  2577. * in the dentry's d_seq will be preceded by changes in the rename_lock
  2578. * sequence number. If the sequence number had been changed, it will restart
  2579. * the whole pathname back-tracing sequence again by taking the rename_lock.
  2580. * In this case, there is no need to take the RCU read lock as the recursive
  2581. * parent pointer references will keep the dentry chain alive as long as no
  2582. * rename operation is performed.
  2583. */
  2584. static int prepend_path(const struct path *path,
  2585. const struct path *root,
  2586. char **buffer, int *buflen)
  2587. {
  2588. struct dentry *dentry = path->dentry;
  2589. struct vfsmount *vfsmnt = path->mnt;
  2590. struct mount *mnt = real_mount(vfsmnt);
  2591. int error = 0;
  2592. unsigned seq = 0;
  2593. char *bptr;
  2594. int blen;
  2595. rcu_read_lock();
  2596. restart:
  2597. bptr = *buffer;
  2598. blen = *buflen;
  2599. read_seqbegin_or_lock(&rename_lock, &seq);
  2600. while (dentry != root->dentry || vfsmnt != root->mnt) {
  2601. struct dentry * parent;
  2602. if (dentry == vfsmnt->mnt_root || IS_ROOT(dentry)) {
  2603. /* Global root? */
  2604. if (mnt_has_parent(mnt)) {
  2605. dentry = mnt->mnt_mountpoint;
  2606. mnt = mnt->mnt_parent;
  2607. vfsmnt = &mnt->mnt;
  2608. continue;
  2609. }
  2610. /*
  2611. * Filesystems needing to implement special "root names"
  2612. * should do so with ->d_dname()
  2613. */
  2614. if (IS_ROOT(dentry) &&
  2615. (dentry->d_name.len != 1 ||
  2616. dentry->d_name.name[0] != '/')) {
  2617. WARN(1, "Root dentry has weird name <%.*s>\n",
  2618. (int) dentry->d_name.len,
  2619. dentry->d_name.name);
  2620. }
  2621. if (!error)
  2622. error = is_mounted(vfsmnt) ? 1 : 2;
  2623. break;
  2624. }
  2625. parent = dentry->d_parent;
  2626. prefetch(parent);
  2627. error = prepend_name(&bptr, &blen, &dentry->d_name);
  2628. if (error)
  2629. break;
  2630. dentry = parent;
  2631. }
  2632. if (!(seq & 1))
  2633. rcu_read_unlock();
  2634. if (need_seqretry(&rename_lock, seq)) {
  2635. seq = 1;
  2636. goto restart;
  2637. }
  2638. done_seqretry(&rename_lock, seq);
  2639. if (error >= 0 && bptr == *buffer) {
  2640. if (--blen < 0)
  2641. error = -ENAMETOOLONG;
  2642. else
  2643. *--bptr = '/';
  2644. }
  2645. *buffer = bptr;
  2646. *buflen = blen;
  2647. return error;
  2648. }
  2649. /**
  2650. * __d_path - return the path of a dentry
  2651. * @path: the dentry/vfsmount to report
  2652. * @root: root vfsmnt/dentry
  2653. * @buf: buffer to return value in
  2654. * @buflen: buffer length
  2655. *
  2656. * Convert a dentry into an ASCII path name.
  2657. *
  2658. * Returns a pointer into the buffer or an error code if the
  2659. * path was too long.
  2660. *
  2661. * "buflen" should be positive.
  2662. *
  2663. * If the path is not reachable from the supplied root, return %NULL.
  2664. */
  2665. char *__d_path(const struct path *path,
  2666. const struct path *root,
  2667. char *buf, int buflen)
  2668. {
  2669. char *res = buf + buflen;
  2670. int error;
  2671. prepend(&res, &buflen, "\0", 1);
  2672. br_read_lock(&vfsmount_lock);
  2673. error = prepend_path(path, root, &res, &buflen);
  2674. br_read_unlock(&vfsmount_lock);
  2675. if (error < 0)
  2676. return ERR_PTR(error);
  2677. if (error > 0)
  2678. return NULL;
  2679. return res;
  2680. }
  2681. char *d_absolute_path(const struct path *path,
  2682. char *buf, int buflen)
  2683. {
  2684. struct path root = {};
  2685. char *res = buf + buflen;
  2686. int error;
  2687. prepend(&res, &buflen, "\0", 1);
  2688. br_read_lock(&vfsmount_lock);
  2689. error = prepend_path(path, &root, &res, &buflen);
  2690. br_read_unlock(&vfsmount_lock);
  2691. if (error > 1)
  2692. error = -EINVAL;
  2693. if (error < 0)
  2694. return ERR_PTR(error);
  2695. return res;
  2696. }
  2697. /*
  2698. * same as __d_path but appends "(deleted)" for unlinked files.
  2699. */
  2700. static int path_with_deleted(const struct path *path,
  2701. const struct path *root,
  2702. char **buf, int *buflen)
  2703. {
  2704. prepend(buf, buflen, "\0", 1);
  2705. if (d_unlinked(path->dentry)) {
  2706. int error = prepend(buf, buflen, " (deleted)", 10);
  2707. if (error)
  2708. return error;
  2709. }
  2710. return prepend_path(path, root, buf, buflen);
  2711. }
  2712. static int prepend_unreachable(char **buffer, int *buflen)
  2713. {
  2714. return prepend(buffer, buflen, "(unreachable)", 13);
  2715. }
  2716. static void get_fs_root_rcu(struct fs_struct *fs, struct path *root)
  2717. {
  2718. unsigned seq;
  2719. do {
  2720. seq = read_seqcount_begin(&fs->seq);
  2721. *root = fs->root;
  2722. } while (read_seqcount_retry(&fs->seq, seq));
  2723. }
  2724. /**
  2725. * d_path - return the path of a dentry
  2726. * @path: path to report
  2727. * @buf: buffer to return value in
  2728. * @buflen: buffer length
  2729. *
  2730. * Convert a dentry into an ASCII path name. If the entry has been deleted
  2731. * the string " (deleted)" is appended. Note that this is ambiguous.
  2732. *
  2733. * Returns a pointer into the buffer or an error code if the path was
  2734. * too long. Note: Callers should use the returned pointer, not the passed
  2735. * in buffer, to use the name! The implementation often starts at an offset
  2736. * into the buffer, and may leave 0 bytes at the start.
  2737. *
  2738. * "buflen" should be positive.
  2739. */
  2740. char *d_path(const struct path *path, char *buf, int buflen)
  2741. {
  2742. char *res = buf + buflen;
  2743. struct path root;
  2744. int error;
  2745. /*
  2746. * We have various synthetic filesystems that never get mounted. On
  2747. * these filesystems dentries are never used for lookup purposes, and
  2748. * thus don't need to be hashed. They also don't need a name until a
  2749. * user wants to identify the object in /proc/pid/fd/. The little hack
  2750. * below allows us to generate a name for these objects on demand:
  2751. */
  2752. if (path->dentry->d_op && path->dentry->d_op->d_dname)
  2753. return path->dentry->d_op->d_dname(path->dentry, buf, buflen);
  2754. rcu_read_lock();
  2755. get_fs_root_rcu(current->fs, &root);
  2756. br_read_lock(&vfsmount_lock);
  2757. error = path_with_deleted(path, &root, &res, &buflen);
  2758. br_read_unlock(&vfsmount_lock);
  2759. rcu_read_unlock();
  2760. if (error < 0)
  2761. res = ERR_PTR(error);
  2762. return res;
  2763. }
  2764. EXPORT_SYMBOL(d_path);
  2765. /*
  2766. * Helper function for dentry_operations.d_dname() members
  2767. */
  2768. char *dynamic_dname(struct dentry *dentry, char *buffer, int buflen,
  2769. const char *fmt, ...)
  2770. {
  2771. va_list args;
  2772. char temp[64];
  2773. int sz;
  2774. va_start(args, fmt);
  2775. sz = vsnprintf(temp, sizeof(temp), fmt, args) + 1;
  2776. va_end(args);
  2777. if (sz > sizeof(temp) || sz > buflen)
  2778. return ERR_PTR(-ENAMETOOLONG);
  2779. buffer += buflen - sz;
  2780. return memcpy(buffer, temp, sz);
  2781. }
  2782. char *simple_dname(struct dentry *dentry, char *buffer, int buflen)
  2783. {
  2784. char *end = buffer + buflen;
  2785. /* these dentries are never renamed, so d_lock is not needed */
  2786. if (prepend(&end, &buflen, " (deleted)", 11) ||
  2787. prepend(&end, &buflen, dentry->d_name.name, dentry->d_name.len) ||
  2788. prepend(&end, &buflen, "/", 1))
  2789. end = ERR_PTR(-ENAMETOOLONG);
  2790. return end;
  2791. }
  2792. /*
  2793. * Write full pathname from the root of the filesystem into the buffer.
  2794. */
  2795. static char *__dentry_path(struct dentry *dentry, char *buf, int buflen)
  2796. {
  2797. char *end, *retval;
  2798. int len, seq = 0;
  2799. int error = 0;
  2800. rcu_read_lock();
  2801. restart:
  2802. end = buf + buflen;
  2803. len = buflen;
  2804. prepend(&end, &len, "\0", 1);
  2805. if (buflen < 1)
  2806. goto Elong;
  2807. /* Get '/' right */
  2808. retval = end-1;
  2809. *retval = '/';
  2810. read_seqbegin_or_lock(&rename_lock, &seq);
  2811. while (!IS_ROOT(dentry)) {
  2812. struct dentry *parent = dentry->d_parent;
  2813. int error;
  2814. prefetch(parent);
  2815. error = prepend_name(&end, &len, &dentry->d_name);
  2816. if (error)
  2817. break;
  2818. retval = end;
  2819. dentry = parent;
  2820. }
  2821. if (!(seq & 1))
  2822. rcu_read_unlock();
  2823. if (need_seqretry(&rename_lock, seq)) {
  2824. seq = 1;
  2825. goto restart;
  2826. }
  2827. done_seqretry(&rename_lock, seq);
  2828. if (error)
  2829. goto Elong;
  2830. return retval;
  2831. Elong:
  2832. return ERR_PTR(-ENAMETOOLONG);
  2833. }
  2834. char *dentry_path_raw(struct dentry *dentry, char *buf, int buflen)
  2835. {
  2836. return __dentry_path(dentry, buf, buflen);
  2837. }
  2838. EXPORT_SYMBOL(dentry_path_raw);
  2839. char *dentry_path(struct dentry *dentry, char *buf, int buflen)
  2840. {
  2841. char *p = NULL;
  2842. char *retval;
  2843. if (d_unlinked(dentry)) {
  2844. p = buf + buflen;
  2845. if (prepend(&p, &buflen, "//deleted", 10) != 0)
  2846. goto Elong;
  2847. buflen++;
  2848. }
  2849. retval = __dentry_path(dentry, buf, buflen);
  2850. if (!IS_ERR(retval) && p)
  2851. *p = '/'; /* restore '/' overriden with '\0' */
  2852. return retval;
  2853. Elong:
  2854. return ERR_PTR(-ENAMETOOLONG);
  2855. }
  2856. static void get_fs_root_and_pwd_rcu(struct fs_struct *fs, struct path *root,
  2857. struct path *pwd)
  2858. {
  2859. unsigned seq;
  2860. do {
  2861. seq = read_seqcount_begin(&fs->seq);
  2862. *root = fs->root;
  2863. *pwd = fs->pwd;
  2864. } while (read_seqcount_retry(&fs->seq, seq));
  2865. }
  2866. /*
  2867. * NOTE! The user-level library version returns a
  2868. * character pointer. The kernel system call just
  2869. * returns the length of the buffer filled (which
  2870. * includes the ending '\0' character), or a negative
  2871. * error value. So libc would do something like
  2872. *
  2873. * char *getcwd(char * buf, size_t size)
  2874. * {
  2875. * int retval;
  2876. *
  2877. * retval = sys_getcwd(buf, size);
  2878. * if (retval >= 0)
  2879. * return buf;
  2880. * errno = -retval;
  2881. * return NULL;
  2882. * }
  2883. */
  2884. SYSCALL_DEFINE2(getcwd, char __user *, buf, unsigned long, size)
  2885. {
  2886. int error;
  2887. struct path pwd, root;
  2888. char *page = __getname();
  2889. if (!page)
  2890. return -ENOMEM;
  2891. rcu_read_lock();
  2892. get_fs_root_and_pwd_rcu(current->fs, &root, &pwd);
  2893. error = -ENOENT;
  2894. br_read_lock(&vfsmount_lock);
  2895. if (!d_unlinked(pwd.dentry)) {
  2896. unsigned long len;
  2897. char *cwd = page + PATH_MAX;
  2898. int buflen = PATH_MAX;
  2899. prepend(&cwd, &buflen, "\0", 1);
  2900. error = prepend_path(&pwd, &root, &cwd, &buflen);
  2901. br_read_unlock(&vfsmount_lock);
  2902. rcu_read_unlock();
  2903. if (error < 0)
  2904. goto out;
  2905. /* Unreachable from current root */
  2906. if (error > 0) {
  2907. error = prepend_unreachable(&cwd, &buflen);
  2908. if (error)
  2909. goto out;
  2910. }
  2911. error = -ERANGE;
  2912. len = PATH_MAX + page - cwd;
  2913. if (len <= size) {
  2914. error = len;
  2915. if (copy_to_user(buf, cwd, len))
  2916. error = -EFAULT;
  2917. }
  2918. } else {
  2919. br_read_unlock(&vfsmount_lock);
  2920. rcu_read_unlock();
  2921. }
  2922. out:
  2923. __putname(page);
  2924. return error;
  2925. }
  2926. /*
  2927. * Test whether new_dentry is a subdirectory of old_dentry.
  2928. *
  2929. * Trivially implemented using the dcache structure
  2930. */
  2931. /**
  2932. * is_subdir - is new dentry a subdirectory of old_dentry
  2933. * @new_dentry: new dentry
  2934. * @old_dentry: old dentry
  2935. *
  2936. * Returns 1 if new_dentry is a subdirectory of the parent (at any depth).
  2937. * Returns 0 otherwise.
  2938. * Caller must ensure that "new_dentry" is pinned before calling is_subdir()
  2939. */
  2940. int is_subdir(struct dentry *new_dentry, struct dentry *old_dentry)
  2941. {
  2942. int result;
  2943. unsigned seq;
  2944. if (new_dentry == old_dentry)
  2945. return 1;
  2946. do {
  2947. /* for restarting inner loop in case of seq retry */
  2948. seq = read_seqbegin(&rename_lock);
  2949. /*
  2950. * Need rcu_readlock to protect against the d_parent trashing
  2951. * due to d_move
  2952. */
  2953. rcu_read_lock();
  2954. if (d_ancestor(old_dentry, new_dentry))
  2955. result = 1;
  2956. else
  2957. result = 0;
  2958. rcu_read_unlock();
  2959. } while (read_seqretry(&rename_lock, seq));
  2960. return result;
  2961. }
  2962. static enum d_walk_ret d_genocide_kill(void *data, struct dentry *dentry)
  2963. {
  2964. struct dentry *root = data;
  2965. if (dentry != root) {
  2966. if (d_unhashed(dentry) || !dentry->d_inode)
  2967. return D_WALK_SKIP;
  2968. if (!(dentry->d_flags & DCACHE_GENOCIDE)) {
  2969. dentry->d_flags |= DCACHE_GENOCIDE;
  2970. dentry->d_lockref.count--;
  2971. }
  2972. }
  2973. return D_WALK_CONTINUE;
  2974. }
  2975. void d_genocide(struct dentry *parent)
  2976. {
  2977. d_walk(parent, parent, d_genocide_kill, NULL);
  2978. }
  2979. void d_tmpfile(struct dentry *dentry, struct inode *inode)
  2980. {
  2981. inode_dec_link_count(inode);
  2982. BUG_ON(dentry->d_name.name != dentry->d_iname ||
  2983. !hlist_unhashed(&dentry->d_alias) ||
  2984. !d_unlinked(dentry));
  2985. spin_lock(&dentry->d_parent->d_lock);
  2986. spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
  2987. dentry->d_name.len = sprintf(dentry->d_iname, "#%llu",
  2988. (unsigned long long)inode->i_ino);
  2989. spin_unlock(&dentry->d_lock);
  2990. spin_unlock(&dentry->d_parent->d_lock);
  2991. d_instantiate(dentry, inode);
  2992. }
  2993. EXPORT_SYMBOL(d_tmpfile);
  2994. static __initdata unsigned long dhash_entries;
  2995. static int __init set_dhash_entries(char *str)
  2996. {
  2997. if (!str)
  2998. return 0;
  2999. dhash_entries = simple_strtoul(str, &str, 0);
  3000. return 1;
  3001. }
  3002. __setup("dhash_entries=", set_dhash_entries);
  3003. static void __init dcache_init_early(void)
  3004. {
  3005. unsigned int loop;
  3006. /* If hashes are distributed across NUMA nodes, defer
  3007. * hash allocation until vmalloc space is available.
  3008. */
  3009. if (hashdist)
  3010. return;
  3011. dentry_hashtable =
  3012. alloc_large_system_hash("Dentry cache",
  3013. sizeof(struct hlist_bl_head),
  3014. dhash_entries,
  3015. 13,
  3016. HASH_EARLY,
  3017. &d_hash_shift,
  3018. &d_hash_mask,
  3019. 0,
  3020. 0);
  3021. for (loop = 0; loop < (1U << d_hash_shift); loop++)
  3022. INIT_HLIST_BL_HEAD(dentry_hashtable + loop);
  3023. }
  3024. static void __init dcache_init(void)
  3025. {
  3026. unsigned int loop;
  3027. /*
  3028. * A constructor could be added for stable state like the lists,
  3029. * but it is probably not worth it because of the cache nature
  3030. * of the dcache.
  3031. */
  3032. dentry_cache = KMEM_CACHE(dentry,
  3033. SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|SLAB_MEM_SPREAD);
  3034. /* Hash may have been set up in dcache_init_early */
  3035. if (!hashdist)
  3036. return;
  3037. dentry_hashtable =
  3038. alloc_large_system_hash("Dentry cache",
  3039. sizeof(struct hlist_bl_head),
  3040. dhash_entries,
  3041. 13,
  3042. 0,
  3043. &d_hash_shift,
  3044. &d_hash_mask,
  3045. 0,
  3046. 0);
  3047. for (loop = 0; loop < (1U << d_hash_shift); loop++)
  3048. INIT_HLIST_BL_HEAD(dentry_hashtable + loop);
  3049. }
  3050. /* SLAB cache for __getname() consumers */
  3051. struct kmem_cache *names_cachep __read_mostly;
  3052. EXPORT_SYMBOL(names_cachep);
  3053. EXPORT_SYMBOL(d_genocide);
  3054. void __init vfs_caches_init_early(void)
  3055. {
  3056. dcache_init_early();
  3057. inode_init_early();
  3058. }
  3059. void __init vfs_caches_init(unsigned long mempages)
  3060. {
  3061. unsigned long reserve;
  3062. /* Base hash sizes on available memory, with a reserve equal to
  3063. 150% of current kernel size */
  3064. reserve = min((mempages - nr_free_pages()) * 3/2, mempages - 1);
  3065. mempages -= reserve;
  3066. names_cachep = kmem_cache_create("names_cache", PATH_MAX, 0,
  3067. SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
  3068. dcache_init();
  3069. inode_init();
  3070. files_init(mempages);
  3071. mnt_init();
  3072. bdev_cache_init();
  3073. chrdev_init();
  3074. }