dcache.c 82 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260
  1. /*
  2. * fs/dcache.c
  3. *
  4. * Complete reimplementation
  5. * (C) 1997 Thomas Schoebel-Theuer,
  6. * with heavy changes by Linus Torvalds
  7. */
  8. /*
  9. * Notes on the allocation strategy:
  10. *
  11. * The dcache is a master of the icache - whenever a dcache entry
  12. * exists, the inode will always exist. "iput()" is done either when
  13. * the dcache entry is deleted or garbage collected.
  14. */
  15. #include <linux/syscalls.h>
  16. #include <linux/string.h>
  17. #include <linux/mm.h>
  18. #include <linux/fs.h>
  19. #include <linux/fsnotify.h>
  20. #include <linux/slab.h>
  21. #include <linux/init.h>
  22. #include <linux/hash.h>
  23. #include <linux/cache.h>
  24. #include <linux/export.h>
  25. #include <linux/mount.h>
  26. #include <linux/file.h>
  27. #include <asm/uaccess.h>
  28. #include <linux/security.h>
  29. #include <linux/seqlock.h>
  30. #include <linux/swap.h>
  31. #include <linux/bootmem.h>
  32. #include <linux/fs_struct.h>
  33. #include <linux/hardirq.h>
  34. #include <linux/bit_spinlock.h>
  35. #include <linux/rculist_bl.h>
  36. #include <linux/prefetch.h>
  37. #include <linux/ratelimit.h>
  38. #include "internal.h"
  39. #include "mount.h"
  40. /*
  41. * Usage:
  42. * dcache->d_inode->i_lock protects:
  43. * - i_dentry, d_alias, d_inode of aliases
  44. * dcache_hash_bucket lock protects:
  45. * - the dcache hash table
  46. * s_anon bl list spinlock protects:
  47. * - the s_anon list (see __d_drop)
  48. * dcache_lru_lock protects:
  49. * - the dcache lru lists and counters
  50. * d_lock protects:
  51. * - d_flags
  52. * - d_name
  53. * - d_lru
  54. * - d_count
  55. * - d_unhashed()
  56. * - d_parent and d_subdirs
  57. * - childrens' d_child and d_parent
  58. * - d_alias, d_inode
  59. *
  60. * Ordering:
  61. * dentry->d_inode->i_lock
  62. * dentry->d_lock
  63. * dcache_lru_lock
  64. * dcache_hash_bucket lock
  65. * s_anon lock
  66. *
  67. * If there is an ancestor relationship:
  68. * dentry->d_parent->...->d_parent->d_lock
  69. * ...
  70. * dentry->d_parent->d_lock
  71. * dentry->d_lock
  72. *
  73. * If no ancestor relationship:
  74. * if (dentry1 < dentry2)
  75. * dentry1->d_lock
  76. * dentry2->d_lock
  77. */
  78. int sysctl_vfs_cache_pressure __read_mostly = 100;
  79. EXPORT_SYMBOL_GPL(sysctl_vfs_cache_pressure);
  80. static __cacheline_aligned_in_smp DEFINE_SPINLOCK(dcache_lru_lock);
  81. __cacheline_aligned_in_smp DEFINE_SEQLOCK(rename_lock);
  82. EXPORT_SYMBOL(rename_lock);
  83. static struct kmem_cache *dentry_cache __read_mostly;
  84. /**
  85. * read_seqbegin_or_lock - begin a sequence number check or locking block
  86. * lock: sequence lock
  87. * seq : sequence number to be checked
  88. *
  89. * First try it once optimistically without taking the lock. If that fails,
  90. * take the lock. The sequence number is also used as a marker for deciding
  91. * whether to be a reader (even) or writer (odd).
  92. * N.B. seq must be initialized to an even number to begin with.
  93. */
  94. static inline void read_seqbegin_or_lock(seqlock_t *lock, int *seq)
  95. {
  96. if (!(*seq & 1)) /* Even */
  97. *seq = read_seqbegin(lock);
  98. else /* Odd */
  99. write_seqlock(lock);
  100. }
  101. static inline int need_seqretry(seqlock_t *lock, int seq)
  102. {
  103. return !(seq & 1) && read_seqretry(lock, seq);
  104. }
  105. static inline void done_seqretry(seqlock_t *lock, int seq)
  106. {
  107. if (seq & 1)
  108. write_sequnlock(lock);
  109. }
  110. /*
  111. * This is the single most critical data structure when it comes
  112. * to the dcache: the hashtable for lookups. Somebody should try
  113. * to make this good - I've just made it work.
  114. *
  115. * This hash-function tries to avoid losing too many bits of hash
  116. * information, yet avoid using a prime hash-size or similar.
  117. */
  118. #define D_HASHBITS d_hash_shift
  119. #define D_HASHMASK d_hash_mask
  120. static unsigned int d_hash_mask __read_mostly;
  121. static unsigned int d_hash_shift __read_mostly;
  122. static struct hlist_bl_head *dentry_hashtable __read_mostly;
  123. static inline struct hlist_bl_head *d_hash(const struct dentry *parent,
  124. unsigned int hash)
  125. {
  126. hash += (unsigned long) parent / L1_CACHE_BYTES;
  127. hash = hash + (hash >> D_HASHBITS);
  128. return dentry_hashtable + (hash & D_HASHMASK);
  129. }
  130. /* Statistics gathering. */
  131. struct dentry_stat_t dentry_stat = {
  132. .age_limit = 45,
  133. };
  134. static DEFINE_PER_CPU(unsigned int, nr_dentry);
  135. #if defined(CONFIG_SYSCTL) && defined(CONFIG_PROC_FS)
  136. static int get_nr_dentry(void)
  137. {
  138. int i;
  139. int sum = 0;
  140. for_each_possible_cpu(i)
  141. sum += per_cpu(nr_dentry, i);
  142. return sum < 0 ? 0 : sum;
  143. }
  144. int proc_nr_dentry(ctl_table *table, int write, void __user *buffer,
  145. size_t *lenp, loff_t *ppos)
  146. {
  147. dentry_stat.nr_dentry = get_nr_dentry();
  148. return proc_dointvec(table, write, buffer, lenp, ppos);
  149. }
  150. #endif
  151. /*
  152. * Compare 2 name strings, return 0 if they match, otherwise non-zero.
  153. * The strings are both count bytes long, and count is non-zero.
  154. */
  155. #ifdef CONFIG_DCACHE_WORD_ACCESS
  156. #include <asm/word-at-a-time.h>
  157. /*
  158. * NOTE! 'cs' and 'scount' come from a dentry, so it has a
  159. * aligned allocation for this particular component. We don't
  160. * strictly need the load_unaligned_zeropad() safety, but it
  161. * doesn't hurt either.
  162. *
  163. * In contrast, 'ct' and 'tcount' can be from a pathname, and do
  164. * need the careful unaligned handling.
  165. */
  166. static inline int dentry_string_cmp(const unsigned char *cs, const unsigned char *ct, unsigned tcount)
  167. {
  168. unsigned long a,b,mask;
  169. for (;;) {
  170. a = *(unsigned long *)cs;
  171. b = load_unaligned_zeropad(ct);
  172. if (tcount < sizeof(unsigned long))
  173. break;
  174. if (unlikely(a != b))
  175. return 1;
  176. cs += sizeof(unsigned long);
  177. ct += sizeof(unsigned long);
  178. tcount -= sizeof(unsigned long);
  179. if (!tcount)
  180. return 0;
  181. }
  182. mask = ~(~0ul << tcount*8);
  183. return unlikely(!!((a ^ b) & mask));
  184. }
  185. #else
  186. static inline int dentry_string_cmp(const unsigned char *cs, const unsigned char *ct, unsigned tcount)
  187. {
  188. do {
  189. if (*cs != *ct)
  190. return 1;
  191. cs++;
  192. ct++;
  193. tcount--;
  194. } while (tcount);
  195. return 0;
  196. }
  197. #endif
  198. static inline int dentry_cmp(const struct dentry *dentry, const unsigned char *ct, unsigned tcount)
  199. {
  200. const unsigned char *cs;
  201. /*
  202. * Be careful about RCU walk racing with rename:
  203. * use ACCESS_ONCE to fetch the name pointer.
  204. *
  205. * NOTE! Even if a rename will mean that the length
  206. * was not loaded atomically, we don't care. The
  207. * RCU walk will check the sequence count eventually,
  208. * and catch it. And we won't overrun the buffer,
  209. * because we're reading the name pointer atomically,
  210. * and a dentry name is guaranteed to be properly
  211. * terminated with a NUL byte.
  212. *
  213. * End result: even if 'len' is wrong, we'll exit
  214. * early because the data cannot match (there can
  215. * be no NUL in the ct/tcount data)
  216. */
  217. cs = ACCESS_ONCE(dentry->d_name.name);
  218. smp_read_barrier_depends();
  219. return dentry_string_cmp(cs, ct, tcount);
  220. }
  221. static void __d_free(struct rcu_head *head)
  222. {
  223. struct dentry *dentry = container_of(head, struct dentry, d_u.d_rcu);
  224. WARN_ON(!hlist_unhashed(&dentry->d_alias));
  225. if (dname_external(dentry))
  226. kfree(dentry->d_name.name);
  227. kmem_cache_free(dentry_cache, dentry);
  228. }
  229. /*
  230. * no locks, please.
  231. */
  232. static void d_free(struct dentry *dentry)
  233. {
  234. BUG_ON((int)dentry->d_lockref.count > 0);
  235. this_cpu_dec(nr_dentry);
  236. if (dentry->d_op && dentry->d_op->d_release)
  237. dentry->d_op->d_release(dentry);
  238. /* if dentry was never visible to RCU, immediate free is OK */
  239. if (!(dentry->d_flags & DCACHE_RCUACCESS))
  240. __d_free(&dentry->d_u.d_rcu);
  241. else
  242. call_rcu(&dentry->d_u.d_rcu, __d_free);
  243. }
  244. /**
  245. * dentry_rcuwalk_barrier - invalidate in-progress rcu-walk lookups
  246. * @dentry: the target dentry
  247. * After this call, in-progress rcu-walk path lookup will fail. This
  248. * should be called after unhashing, and after changing d_inode (if
  249. * the dentry has not already been unhashed).
  250. */
  251. static inline void dentry_rcuwalk_barrier(struct dentry *dentry)
  252. {
  253. assert_spin_locked(&dentry->d_lock);
  254. /* Go through a barrier */
  255. write_seqcount_barrier(&dentry->d_seq);
  256. }
  257. /*
  258. * Release the dentry's inode, using the filesystem
  259. * d_iput() operation if defined. Dentry has no refcount
  260. * and is unhashed.
  261. */
  262. static void dentry_iput(struct dentry * dentry)
  263. __releases(dentry->d_lock)
  264. __releases(dentry->d_inode->i_lock)
  265. {
  266. struct inode *inode = dentry->d_inode;
  267. if (inode) {
  268. dentry->d_inode = NULL;
  269. hlist_del_init(&dentry->d_alias);
  270. spin_unlock(&dentry->d_lock);
  271. spin_unlock(&inode->i_lock);
  272. if (!inode->i_nlink)
  273. fsnotify_inoderemove(inode);
  274. if (dentry->d_op && dentry->d_op->d_iput)
  275. dentry->d_op->d_iput(dentry, inode);
  276. else
  277. iput(inode);
  278. } else {
  279. spin_unlock(&dentry->d_lock);
  280. }
  281. }
  282. /*
  283. * Release the dentry's inode, using the filesystem
  284. * d_iput() operation if defined. dentry remains in-use.
  285. */
  286. static void dentry_unlink_inode(struct dentry * dentry)
  287. __releases(dentry->d_lock)
  288. __releases(dentry->d_inode->i_lock)
  289. {
  290. struct inode *inode = dentry->d_inode;
  291. dentry->d_inode = NULL;
  292. hlist_del_init(&dentry->d_alias);
  293. dentry_rcuwalk_barrier(dentry);
  294. spin_unlock(&dentry->d_lock);
  295. spin_unlock(&inode->i_lock);
  296. if (!inode->i_nlink)
  297. fsnotify_inoderemove(inode);
  298. if (dentry->d_op && dentry->d_op->d_iput)
  299. dentry->d_op->d_iput(dentry, inode);
  300. else
  301. iput(inode);
  302. }
  303. /*
  304. * dentry_lru_(add|del|prune|move_tail) must be called with d_lock held.
  305. */
  306. static void dentry_lru_add(struct dentry *dentry)
  307. {
  308. if (unlikely(!(dentry->d_flags & DCACHE_LRU_LIST))) {
  309. spin_lock(&dcache_lru_lock);
  310. dentry->d_flags |= DCACHE_LRU_LIST;
  311. list_add(&dentry->d_lru, &dentry->d_sb->s_dentry_lru);
  312. dentry->d_sb->s_nr_dentry_unused++;
  313. dentry_stat.nr_unused++;
  314. spin_unlock(&dcache_lru_lock);
  315. }
  316. }
  317. static void __dentry_lru_del(struct dentry *dentry)
  318. {
  319. list_del_init(&dentry->d_lru);
  320. dentry->d_flags &= ~(DCACHE_SHRINK_LIST | DCACHE_LRU_LIST);
  321. dentry->d_sb->s_nr_dentry_unused--;
  322. dentry_stat.nr_unused--;
  323. }
  324. /*
  325. * Remove a dentry with references from the LRU.
  326. */
  327. static void dentry_lru_del(struct dentry *dentry)
  328. {
  329. if (!list_empty(&dentry->d_lru)) {
  330. spin_lock(&dcache_lru_lock);
  331. __dentry_lru_del(dentry);
  332. spin_unlock(&dcache_lru_lock);
  333. }
  334. }
  335. static void dentry_lru_move_list(struct dentry *dentry, struct list_head *list)
  336. {
  337. spin_lock(&dcache_lru_lock);
  338. if (list_empty(&dentry->d_lru)) {
  339. dentry->d_flags |= DCACHE_LRU_LIST;
  340. list_add_tail(&dentry->d_lru, list);
  341. dentry->d_sb->s_nr_dentry_unused++;
  342. dentry_stat.nr_unused++;
  343. } else {
  344. list_move_tail(&dentry->d_lru, list);
  345. }
  346. spin_unlock(&dcache_lru_lock);
  347. }
  348. /**
  349. * d_kill - kill dentry and return parent
  350. * @dentry: dentry to kill
  351. * @parent: parent dentry
  352. *
  353. * The dentry must already be unhashed and removed from the LRU.
  354. *
  355. * If this is the root of the dentry tree, return NULL.
  356. *
  357. * dentry->d_lock and parent->d_lock must be held by caller, and are dropped by
  358. * d_kill.
  359. */
  360. static struct dentry *d_kill(struct dentry *dentry, struct dentry *parent)
  361. __releases(dentry->d_lock)
  362. __releases(parent->d_lock)
  363. __releases(dentry->d_inode->i_lock)
  364. {
  365. list_del(&dentry->d_u.d_child);
  366. /*
  367. * Inform try_to_ascend() that we are no longer attached to the
  368. * dentry tree
  369. */
  370. dentry->d_flags |= DCACHE_DENTRY_KILLED;
  371. if (parent)
  372. spin_unlock(&parent->d_lock);
  373. dentry_iput(dentry);
  374. /*
  375. * dentry_iput drops the locks, at which point nobody (except
  376. * transient RCU lookups) can reach this dentry.
  377. */
  378. d_free(dentry);
  379. return parent;
  380. }
  381. /*
  382. * Unhash a dentry without inserting an RCU walk barrier or checking that
  383. * dentry->d_lock is locked. The caller must take care of that, if
  384. * appropriate.
  385. */
  386. static void __d_shrink(struct dentry *dentry)
  387. {
  388. if (!d_unhashed(dentry)) {
  389. struct hlist_bl_head *b;
  390. if (unlikely(dentry->d_flags & DCACHE_DISCONNECTED))
  391. b = &dentry->d_sb->s_anon;
  392. else
  393. b = d_hash(dentry->d_parent, dentry->d_name.hash);
  394. hlist_bl_lock(b);
  395. __hlist_bl_del(&dentry->d_hash);
  396. dentry->d_hash.pprev = NULL;
  397. hlist_bl_unlock(b);
  398. }
  399. }
  400. /**
  401. * d_drop - drop a dentry
  402. * @dentry: dentry to drop
  403. *
  404. * d_drop() unhashes the entry from the parent dentry hashes, so that it won't
  405. * be found through a VFS lookup any more. Note that this is different from
  406. * deleting the dentry - d_delete will try to mark the dentry negative if
  407. * possible, giving a successful _negative_ lookup, while d_drop will
  408. * just make the cache lookup fail.
  409. *
  410. * d_drop() is used mainly for stuff that wants to invalidate a dentry for some
  411. * reason (NFS timeouts or autofs deletes).
  412. *
  413. * __d_drop requires dentry->d_lock.
  414. */
  415. void __d_drop(struct dentry *dentry)
  416. {
  417. if (!d_unhashed(dentry)) {
  418. __d_shrink(dentry);
  419. dentry_rcuwalk_barrier(dentry);
  420. }
  421. }
  422. EXPORT_SYMBOL(__d_drop);
  423. void d_drop(struct dentry *dentry)
  424. {
  425. spin_lock(&dentry->d_lock);
  426. __d_drop(dentry);
  427. spin_unlock(&dentry->d_lock);
  428. }
  429. EXPORT_SYMBOL(d_drop);
  430. /*
  431. * Finish off a dentry we've decided to kill.
  432. * dentry->d_lock must be held, returns with it unlocked.
  433. * If ref is non-zero, then decrement the refcount too.
  434. * Returns dentry requiring refcount drop, or NULL if we're done.
  435. */
  436. static inline struct dentry *dentry_kill(struct dentry *dentry)
  437. __releases(dentry->d_lock)
  438. {
  439. struct inode *inode;
  440. struct dentry *parent;
  441. inode = dentry->d_inode;
  442. if (inode && !spin_trylock(&inode->i_lock)) {
  443. relock:
  444. spin_unlock(&dentry->d_lock);
  445. cpu_relax();
  446. return dentry; /* try again with same dentry */
  447. }
  448. if (IS_ROOT(dentry))
  449. parent = NULL;
  450. else
  451. parent = dentry->d_parent;
  452. if (parent && !spin_trylock(&parent->d_lock)) {
  453. if (inode)
  454. spin_unlock(&inode->i_lock);
  455. goto relock;
  456. }
  457. /*
  458. * The dentry is now unrecoverably dead to the world.
  459. */
  460. lockref_mark_dead(&dentry->d_lockref);
  461. /*
  462. * inform the fs via d_prune that this dentry is about to be
  463. * unhashed and destroyed.
  464. */
  465. if ((dentry->d_flags & DCACHE_OP_PRUNE) && !d_unhashed(dentry))
  466. dentry->d_op->d_prune(dentry);
  467. dentry_lru_del(dentry);
  468. /* if it was on the hash then remove it */
  469. __d_drop(dentry);
  470. return d_kill(dentry, parent);
  471. }
  472. /*
  473. * This is dput
  474. *
  475. * This is complicated by the fact that we do not want to put
  476. * dentries that are no longer on any hash chain on the unused
  477. * list: we'd much rather just get rid of them immediately.
  478. *
  479. * However, that implies that we have to traverse the dentry
  480. * tree upwards to the parents which might _also_ now be
  481. * scheduled for deletion (it may have been only waiting for
  482. * its last child to go away).
  483. *
  484. * This tail recursion is done by hand as we don't want to depend
  485. * on the compiler to always get this right (gcc generally doesn't).
  486. * Real recursion would eat up our stack space.
  487. */
  488. /*
  489. * dput - release a dentry
  490. * @dentry: dentry to release
  491. *
  492. * Release a dentry. This will drop the usage count and if appropriate
  493. * call the dentry unlink method as well as removing it from the queues and
  494. * releasing its resources. If the parent dentries were scheduled for release
  495. * they too may now get deleted.
  496. */
  497. void dput(struct dentry *dentry)
  498. {
  499. if (unlikely(!dentry))
  500. return;
  501. repeat:
  502. if (lockref_put_or_lock(&dentry->d_lockref))
  503. return;
  504. /* Unreachable? Get rid of it */
  505. if (unlikely(d_unhashed(dentry)))
  506. goto kill_it;
  507. if (unlikely(dentry->d_flags & DCACHE_OP_DELETE)) {
  508. if (dentry->d_op->d_delete(dentry))
  509. goto kill_it;
  510. }
  511. dentry->d_flags |= DCACHE_REFERENCED;
  512. dentry_lru_add(dentry);
  513. dentry->d_lockref.count--;
  514. spin_unlock(&dentry->d_lock);
  515. return;
  516. kill_it:
  517. dentry = dentry_kill(dentry);
  518. if (dentry)
  519. goto repeat;
  520. }
  521. EXPORT_SYMBOL(dput);
  522. /**
  523. * d_invalidate - invalidate a dentry
  524. * @dentry: dentry to invalidate
  525. *
  526. * Try to invalidate the dentry if it turns out to be
  527. * possible. If there are other dentries that can be
  528. * reached through this one we can't delete it and we
  529. * return -EBUSY. On success we return 0.
  530. *
  531. * no dcache lock.
  532. */
  533. int d_invalidate(struct dentry * dentry)
  534. {
  535. /*
  536. * If it's already been dropped, return OK.
  537. */
  538. spin_lock(&dentry->d_lock);
  539. if (d_unhashed(dentry)) {
  540. spin_unlock(&dentry->d_lock);
  541. return 0;
  542. }
  543. /*
  544. * Check whether to do a partial shrink_dcache
  545. * to get rid of unused child entries.
  546. */
  547. if (!list_empty(&dentry->d_subdirs)) {
  548. spin_unlock(&dentry->d_lock);
  549. shrink_dcache_parent(dentry);
  550. spin_lock(&dentry->d_lock);
  551. }
  552. /*
  553. * Somebody else still using it?
  554. *
  555. * If it's a directory, we can't drop it
  556. * for fear of somebody re-populating it
  557. * with children (even though dropping it
  558. * would make it unreachable from the root,
  559. * we might still populate it if it was a
  560. * working directory or similar).
  561. * We also need to leave mountpoints alone,
  562. * directory or not.
  563. */
  564. if (dentry->d_lockref.count > 1 && dentry->d_inode) {
  565. if (S_ISDIR(dentry->d_inode->i_mode) || d_mountpoint(dentry)) {
  566. spin_unlock(&dentry->d_lock);
  567. return -EBUSY;
  568. }
  569. }
  570. __d_drop(dentry);
  571. spin_unlock(&dentry->d_lock);
  572. return 0;
  573. }
  574. EXPORT_SYMBOL(d_invalidate);
  575. /* This must be called with d_lock held */
  576. static inline void __dget_dlock(struct dentry *dentry)
  577. {
  578. dentry->d_lockref.count++;
  579. }
  580. static inline void __dget(struct dentry *dentry)
  581. {
  582. lockref_get(&dentry->d_lockref);
  583. }
  584. struct dentry *dget_parent(struct dentry *dentry)
  585. {
  586. int gotref;
  587. struct dentry *ret;
  588. /*
  589. * Do optimistic parent lookup without any
  590. * locking.
  591. */
  592. rcu_read_lock();
  593. ret = ACCESS_ONCE(dentry->d_parent);
  594. gotref = lockref_get_not_zero(&ret->d_lockref);
  595. rcu_read_unlock();
  596. if (likely(gotref)) {
  597. if (likely(ret == ACCESS_ONCE(dentry->d_parent)))
  598. return ret;
  599. dput(ret);
  600. }
  601. repeat:
  602. /*
  603. * Don't need rcu_dereference because we re-check it was correct under
  604. * the lock.
  605. */
  606. rcu_read_lock();
  607. ret = dentry->d_parent;
  608. spin_lock(&ret->d_lock);
  609. if (unlikely(ret != dentry->d_parent)) {
  610. spin_unlock(&ret->d_lock);
  611. rcu_read_unlock();
  612. goto repeat;
  613. }
  614. rcu_read_unlock();
  615. BUG_ON(!ret->d_lockref.count);
  616. ret->d_lockref.count++;
  617. spin_unlock(&ret->d_lock);
  618. return ret;
  619. }
  620. EXPORT_SYMBOL(dget_parent);
  621. /**
  622. * d_find_alias - grab a hashed alias of inode
  623. * @inode: inode in question
  624. * @want_discon: flag, used by d_splice_alias, to request
  625. * that only a DISCONNECTED alias be returned.
  626. *
  627. * If inode has a hashed alias, or is a directory and has any alias,
  628. * acquire the reference to alias and return it. Otherwise return NULL.
  629. * Notice that if inode is a directory there can be only one alias and
  630. * it can be unhashed only if it has no children, or if it is the root
  631. * of a filesystem.
  632. *
  633. * If the inode has an IS_ROOT, DCACHE_DISCONNECTED alias, then prefer
  634. * any other hashed alias over that one unless @want_discon is set,
  635. * in which case only return an IS_ROOT, DCACHE_DISCONNECTED alias.
  636. */
  637. static struct dentry *__d_find_alias(struct inode *inode, int want_discon)
  638. {
  639. struct dentry *alias, *discon_alias;
  640. again:
  641. discon_alias = NULL;
  642. hlist_for_each_entry(alias, &inode->i_dentry, d_alias) {
  643. spin_lock(&alias->d_lock);
  644. if (S_ISDIR(inode->i_mode) || !d_unhashed(alias)) {
  645. if (IS_ROOT(alias) &&
  646. (alias->d_flags & DCACHE_DISCONNECTED)) {
  647. discon_alias = alias;
  648. } else if (!want_discon) {
  649. __dget_dlock(alias);
  650. spin_unlock(&alias->d_lock);
  651. return alias;
  652. }
  653. }
  654. spin_unlock(&alias->d_lock);
  655. }
  656. if (discon_alias) {
  657. alias = discon_alias;
  658. spin_lock(&alias->d_lock);
  659. if (S_ISDIR(inode->i_mode) || !d_unhashed(alias)) {
  660. if (IS_ROOT(alias) &&
  661. (alias->d_flags & DCACHE_DISCONNECTED)) {
  662. __dget_dlock(alias);
  663. spin_unlock(&alias->d_lock);
  664. return alias;
  665. }
  666. }
  667. spin_unlock(&alias->d_lock);
  668. goto again;
  669. }
  670. return NULL;
  671. }
  672. struct dentry *d_find_alias(struct inode *inode)
  673. {
  674. struct dentry *de = NULL;
  675. if (!hlist_empty(&inode->i_dentry)) {
  676. spin_lock(&inode->i_lock);
  677. de = __d_find_alias(inode, 0);
  678. spin_unlock(&inode->i_lock);
  679. }
  680. return de;
  681. }
  682. EXPORT_SYMBOL(d_find_alias);
  683. /*
  684. * Try to kill dentries associated with this inode.
  685. * WARNING: you must own a reference to inode.
  686. */
  687. void d_prune_aliases(struct inode *inode)
  688. {
  689. struct dentry *dentry;
  690. restart:
  691. spin_lock(&inode->i_lock);
  692. hlist_for_each_entry(dentry, &inode->i_dentry, d_alias) {
  693. spin_lock(&dentry->d_lock);
  694. if (!dentry->d_lockref.count) {
  695. /*
  696. * inform the fs via d_prune that this dentry
  697. * is about to be unhashed and destroyed.
  698. */
  699. if ((dentry->d_flags & DCACHE_OP_PRUNE) &&
  700. !d_unhashed(dentry))
  701. dentry->d_op->d_prune(dentry);
  702. __dget_dlock(dentry);
  703. __d_drop(dentry);
  704. spin_unlock(&dentry->d_lock);
  705. spin_unlock(&inode->i_lock);
  706. dput(dentry);
  707. goto restart;
  708. }
  709. spin_unlock(&dentry->d_lock);
  710. }
  711. spin_unlock(&inode->i_lock);
  712. }
  713. EXPORT_SYMBOL(d_prune_aliases);
  714. /*
  715. * Try to throw away a dentry - free the inode, dput the parent.
  716. * Requires dentry->d_lock is held, and dentry->d_count == 0.
  717. * Releases dentry->d_lock.
  718. *
  719. * This may fail if locks cannot be acquired no problem, just try again.
  720. */
  721. static void try_prune_one_dentry(struct dentry *dentry)
  722. __releases(dentry->d_lock)
  723. {
  724. struct dentry *parent;
  725. parent = dentry_kill(dentry);
  726. /*
  727. * If dentry_kill returns NULL, we have nothing more to do.
  728. * if it returns the same dentry, trylocks failed. In either
  729. * case, just loop again.
  730. *
  731. * Otherwise, we need to prune ancestors too. This is necessary
  732. * to prevent quadratic behavior of shrink_dcache_parent(), but
  733. * is also expected to be beneficial in reducing dentry cache
  734. * fragmentation.
  735. */
  736. if (!parent)
  737. return;
  738. if (parent == dentry)
  739. return;
  740. /* Prune ancestors. */
  741. dentry = parent;
  742. while (dentry) {
  743. if (lockref_put_or_lock(&dentry->d_lockref))
  744. return;
  745. dentry = dentry_kill(dentry);
  746. }
  747. }
  748. static void shrink_dentry_list(struct list_head *list)
  749. {
  750. struct dentry *dentry;
  751. rcu_read_lock();
  752. for (;;) {
  753. dentry = list_entry_rcu(list->prev, struct dentry, d_lru);
  754. if (&dentry->d_lru == list)
  755. break; /* empty */
  756. spin_lock(&dentry->d_lock);
  757. if (dentry != list_entry(list->prev, struct dentry, d_lru)) {
  758. spin_unlock(&dentry->d_lock);
  759. continue;
  760. }
  761. /*
  762. * We found an inuse dentry which was not removed from
  763. * the LRU because of laziness during lookup. Do not free
  764. * it - just keep it off the LRU list.
  765. */
  766. if (dentry->d_lockref.count) {
  767. dentry_lru_del(dentry);
  768. spin_unlock(&dentry->d_lock);
  769. continue;
  770. }
  771. rcu_read_unlock();
  772. try_prune_one_dentry(dentry);
  773. rcu_read_lock();
  774. }
  775. rcu_read_unlock();
  776. }
  777. /**
  778. * prune_dcache_sb - shrink the dcache
  779. * @sb: superblock
  780. * @count: number of entries to try to free
  781. *
  782. * Attempt to shrink the superblock dcache LRU by @count entries. This is
  783. * done when we need more memory an called from the superblock shrinker
  784. * function.
  785. *
  786. * This function may fail to free any resources if all the dentries are in
  787. * use.
  788. */
  789. void prune_dcache_sb(struct super_block *sb, int count)
  790. {
  791. struct dentry *dentry;
  792. LIST_HEAD(referenced);
  793. LIST_HEAD(tmp);
  794. relock:
  795. spin_lock(&dcache_lru_lock);
  796. while (!list_empty(&sb->s_dentry_lru)) {
  797. dentry = list_entry(sb->s_dentry_lru.prev,
  798. struct dentry, d_lru);
  799. BUG_ON(dentry->d_sb != sb);
  800. if (!spin_trylock(&dentry->d_lock)) {
  801. spin_unlock(&dcache_lru_lock);
  802. cpu_relax();
  803. goto relock;
  804. }
  805. if (dentry->d_flags & DCACHE_REFERENCED) {
  806. dentry->d_flags &= ~DCACHE_REFERENCED;
  807. list_move(&dentry->d_lru, &referenced);
  808. spin_unlock(&dentry->d_lock);
  809. } else {
  810. list_move_tail(&dentry->d_lru, &tmp);
  811. dentry->d_flags |= DCACHE_SHRINK_LIST;
  812. spin_unlock(&dentry->d_lock);
  813. if (!--count)
  814. break;
  815. }
  816. cond_resched_lock(&dcache_lru_lock);
  817. }
  818. if (!list_empty(&referenced))
  819. list_splice(&referenced, &sb->s_dentry_lru);
  820. spin_unlock(&dcache_lru_lock);
  821. shrink_dentry_list(&tmp);
  822. }
  823. /**
  824. * shrink_dcache_sb - shrink dcache for a superblock
  825. * @sb: superblock
  826. *
  827. * Shrink the dcache for the specified super block. This is used to free
  828. * the dcache before unmounting a file system.
  829. */
  830. void shrink_dcache_sb(struct super_block *sb)
  831. {
  832. LIST_HEAD(tmp);
  833. spin_lock(&dcache_lru_lock);
  834. while (!list_empty(&sb->s_dentry_lru)) {
  835. list_splice_init(&sb->s_dentry_lru, &tmp);
  836. spin_unlock(&dcache_lru_lock);
  837. shrink_dentry_list(&tmp);
  838. spin_lock(&dcache_lru_lock);
  839. }
  840. spin_unlock(&dcache_lru_lock);
  841. }
  842. EXPORT_SYMBOL(shrink_dcache_sb);
  843. /*
  844. * destroy a single subtree of dentries for unmount
  845. * - see the comments on shrink_dcache_for_umount() for a description of the
  846. * locking
  847. */
  848. static void shrink_dcache_for_umount_subtree(struct dentry *dentry)
  849. {
  850. struct dentry *parent;
  851. BUG_ON(!IS_ROOT(dentry));
  852. for (;;) {
  853. /* descend to the first leaf in the current subtree */
  854. while (!list_empty(&dentry->d_subdirs))
  855. dentry = list_entry(dentry->d_subdirs.next,
  856. struct dentry, d_u.d_child);
  857. /* consume the dentries from this leaf up through its parents
  858. * until we find one with children or run out altogether */
  859. do {
  860. struct inode *inode;
  861. /*
  862. * inform the fs that this dentry is about to be
  863. * unhashed and destroyed.
  864. */
  865. if ((dentry->d_flags & DCACHE_OP_PRUNE) &&
  866. !d_unhashed(dentry))
  867. dentry->d_op->d_prune(dentry);
  868. dentry_lru_del(dentry);
  869. __d_shrink(dentry);
  870. if (dentry->d_lockref.count != 0) {
  871. printk(KERN_ERR
  872. "BUG: Dentry %p{i=%lx,n=%s}"
  873. " still in use (%d)"
  874. " [unmount of %s %s]\n",
  875. dentry,
  876. dentry->d_inode ?
  877. dentry->d_inode->i_ino : 0UL,
  878. dentry->d_name.name,
  879. dentry->d_lockref.count,
  880. dentry->d_sb->s_type->name,
  881. dentry->d_sb->s_id);
  882. BUG();
  883. }
  884. if (IS_ROOT(dentry)) {
  885. parent = NULL;
  886. list_del(&dentry->d_u.d_child);
  887. } else {
  888. parent = dentry->d_parent;
  889. parent->d_lockref.count--;
  890. list_del(&dentry->d_u.d_child);
  891. }
  892. inode = dentry->d_inode;
  893. if (inode) {
  894. dentry->d_inode = NULL;
  895. hlist_del_init(&dentry->d_alias);
  896. if (dentry->d_op && dentry->d_op->d_iput)
  897. dentry->d_op->d_iput(dentry, inode);
  898. else
  899. iput(inode);
  900. }
  901. d_free(dentry);
  902. /* finished when we fall off the top of the tree,
  903. * otherwise we ascend to the parent and move to the
  904. * next sibling if there is one */
  905. if (!parent)
  906. return;
  907. dentry = parent;
  908. } while (list_empty(&dentry->d_subdirs));
  909. dentry = list_entry(dentry->d_subdirs.next,
  910. struct dentry, d_u.d_child);
  911. }
  912. }
  913. /*
  914. * destroy the dentries attached to a superblock on unmounting
  915. * - we don't need to use dentry->d_lock because:
  916. * - the superblock is detached from all mountings and open files, so the
  917. * dentry trees will not be rearranged by the VFS
  918. * - s_umount is write-locked, so the memory pressure shrinker will ignore
  919. * any dentries belonging to this superblock that it comes across
  920. * - the filesystem itself is no longer permitted to rearrange the dentries
  921. * in this superblock
  922. */
  923. void shrink_dcache_for_umount(struct super_block *sb)
  924. {
  925. struct dentry *dentry;
  926. if (down_read_trylock(&sb->s_umount))
  927. BUG();
  928. dentry = sb->s_root;
  929. sb->s_root = NULL;
  930. dentry->d_lockref.count--;
  931. shrink_dcache_for_umount_subtree(dentry);
  932. while (!hlist_bl_empty(&sb->s_anon)) {
  933. dentry = hlist_bl_entry(hlist_bl_first(&sb->s_anon), struct dentry, d_hash);
  934. shrink_dcache_for_umount_subtree(dentry);
  935. }
  936. }
  937. /*
  938. * This tries to ascend one level of parenthood, but
  939. * we can race with renaming, so we need to re-check
  940. * the parenthood after dropping the lock and check
  941. * that the sequence number still matches.
  942. */
  943. static struct dentry *try_to_ascend(struct dentry *old, unsigned seq)
  944. {
  945. struct dentry *new = old->d_parent;
  946. rcu_read_lock();
  947. spin_unlock(&old->d_lock);
  948. spin_lock(&new->d_lock);
  949. /*
  950. * might go back up the wrong parent if we have had a rename
  951. * or deletion
  952. */
  953. if (new != old->d_parent ||
  954. (old->d_flags & DCACHE_DENTRY_KILLED) ||
  955. need_seqretry(&rename_lock, seq)) {
  956. spin_unlock(&new->d_lock);
  957. new = NULL;
  958. }
  959. rcu_read_unlock();
  960. return new;
  961. }
  962. /**
  963. * enum d_walk_ret - action to talke during tree walk
  964. * @D_WALK_CONTINUE: contrinue walk
  965. * @D_WALK_QUIT: quit walk
  966. * @D_WALK_NORETRY: quit when retry is needed
  967. * @D_WALK_SKIP: skip this dentry and its children
  968. */
  969. enum d_walk_ret {
  970. D_WALK_CONTINUE,
  971. D_WALK_QUIT,
  972. D_WALK_NORETRY,
  973. D_WALK_SKIP,
  974. };
  975. /**
  976. * d_walk - walk the dentry tree
  977. * @parent: start of walk
  978. * @data: data passed to @enter() and @finish()
  979. * @enter: callback when first entering the dentry
  980. * @finish: callback when successfully finished the walk
  981. *
  982. * The @enter() and @finish() callbacks are called with d_lock held.
  983. */
  984. static void d_walk(struct dentry *parent, void *data,
  985. enum d_walk_ret (*enter)(void *, struct dentry *),
  986. void (*finish)(void *))
  987. {
  988. struct dentry *this_parent;
  989. struct list_head *next;
  990. unsigned seq = 0;
  991. enum d_walk_ret ret;
  992. bool retry = true;
  993. again:
  994. read_seqbegin_or_lock(&rename_lock, &seq);
  995. this_parent = parent;
  996. spin_lock(&this_parent->d_lock);
  997. ret = enter(data, this_parent);
  998. switch (ret) {
  999. case D_WALK_CONTINUE:
  1000. break;
  1001. case D_WALK_QUIT:
  1002. case D_WALK_SKIP:
  1003. goto out_unlock;
  1004. case D_WALK_NORETRY:
  1005. retry = false;
  1006. break;
  1007. }
  1008. repeat:
  1009. next = this_parent->d_subdirs.next;
  1010. resume:
  1011. while (next != &this_parent->d_subdirs) {
  1012. struct list_head *tmp = next;
  1013. struct dentry *dentry = list_entry(tmp, struct dentry, d_u.d_child);
  1014. next = tmp->next;
  1015. spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
  1016. ret = enter(data, dentry);
  1017. switch (ret) {
  1018. case D_WALK_CONTINUE:
  1019. break;
  1020. case D_WALK_QUIT:
  1021. spin_unlock(&dentry->d_lock);
  1022. goto out_unlock;
  1023. case D_WALK_NORETRY:
  1024. retry = false;
  1025. break;
  1026. case D_WALK_SKIP:
  1027. spin_unlock(&dentry->d_lock);
  1028. continue;
  1029. }
  1030. if (!list_empty(&dentry->d_subdirs)) {
  1031. spin_unlock(&this_parent->d_lock);
  1032. spin_release(&dentry->d_lock.dep_map, 1, _RET_IP_);
  1033. this_parent = dentry;
  1034. spin_acquire(&this_parent->d_lock.dep_map, 0, 1, _RET_IP_);
  1035. goto repeat;
  1036. }
  1037. spin_unlock(&dentry->d_lock);
  1038. }
  1039. /*
  1040. * All done at this level ... ascend and resume the search.
  1041. */
  1042. if (this_parent != parent) {
  1043. struct dentry *child = this_parent;
  1044. this_parent = try_to_ascend(this_parent, seq);
  1045. if (!this_parent)
  1046. goto rename_retry;
  1047. next = child->d_u.d_child.next;
  1048. goto resume;
  1049. }
  1050. if (need_seqretry(&rename_lock, seq)) {
  1051. spin_unlock(&this_parent->d_lock);
  1052. goto rename_retry;
  1053. }
  1054. if (finish)
  1055. finish(data);
  1056. out_unlock:
  1057. spin_unlock(&this_parent->d_lock);
  1058. done_seqretry(&rename_lock, seq);
  1059. return;
  1060. rename_retry:
  1061. if (!retry)
  1062. return;
  1063. seq = 1;
  1064. goto again;
  1065. }
  1066. /*
  1067. * Search for at least 1 mount point in the dentry's subdirs.
  1068. * We descend to the next level whenever the d_subdirs
  1069. * list is non-empty and continue searching.
  1070. */
  1071. /**
  1072. * have_submounts - check for mounts over a dentry
  1073. * @parent: dentry to check.
  1074. *
  1075. * Return true if the parent or its subdirectories contain
  1076. * a mount point
  1077. */
  1078. static enum d_walk_ret check_mount(void *data, struct dentry *dentry)
  1079. {
  1080. int *ret = data;
  1081. if (d_mountpoint(dentry)) {
  1082. *ret = 1;
  1083. return D_WALK_QUIT;
  1084. }
  1085. return D_WALK_CONTINUE;
  1086. }
  1087. int have_submounts(struct dentry *parent)
  1088. {
  1089. int ret = 0;
  1090. d_walk(parent, &ret, check_mount, NULL);
  1091. return ret;
  1092. }
  1093. EXPORT_SYMBOL(have_submounts);
  1094. /*
  1095. * Called by mount code to set a mountpoint and check if the mountpoint is
  1096. * reachable (e.g. NFS can unhash a directory dentry and then the complete
  1097. * subtree can become unreachable).
  1098. *
  1099. * Only one of check_submounts_and_drop() and d_set_mounted() must succeed. For
  1100. * this reason take rename_lock and d_lock on dentry and ancestors.
  1101. */
  1102. int d_set_mounted(struct dentry *dentry)
  1103. {
  1104. struct dentry *p;
  1105. int ret = -ENOENT;
  1106. write_seqlock(&rename_lock);
  1107. for (p = dentry->d_parent; !IS_ROOT(p); p = p->d_parent) {
  1108. /* Need exclusion wrt. check_submounts_and_drop() */
  1109. spin_lock(&p->d_lock);
  1110. if (unlikely(d_unhashed(p))) {
  1111. spin_unlock(&p->d_lock);
  1112. goto out;
  1113. }
  1114. spin_unlock(&p->d_lock);
  1115. }
  1116. spin_lock(&dentry->d_lock);
  1117. if (!d_unlinked(dentry)) {
  1118. dentry->d_flags |= DCACHE_MOUNTED;
  1119. ret = 0;
  1120. }
  1121. spin_unlock(&dentry->d_lock);
  1122. out:
  1123. write_sequnlock(&rename_lock);
  1124. return ret;
  1125. }
  1126. /*
  1127. * Search the dentry child list of the specified parent,
  1128. * and move any unused dentries to the end of the unused
  1129. * list for prune_dcache(). We descend to the next level
  1130. * whenever the d_subdirs list is non-empty and continue
  1131. * searching.
  1132. *
  1133. * It returns zero iff there are no unused children,
  1134. * otherwise it returns the number of children moved to
  1135. * the end of the unused list. This may not be the total
  1136. * number of unused children, because select_parent can
  1137. * drop the lock and return early due to latency
  1138. * constraints.
  1139. */
  1140. struct select_data {
  1141. struct dentry *start;
  1142. struct list_head dispose;
  1143. int found;
  1144. };
  1145. static enum d_walk_ret select_collect(void *_data, struct dentry *dentry)
  1146. {
  1147. struct select_data *data = _data;
  1148. enum d_walk_ret ret = D_WALK_CONTINUE;
  1149. if (data->start == dentry)
  1150. goto out;
  1151. /*
  1152. * move only zero ref count dentries to the dispose list.
  1153. *
  1154. * Those which are presently on the shrink list, being processed
  1155. * by shrink_dentry_list(), shouldn't be moved. Otherwise the
  1156. * loop in shrink_dcache_parent() might not make any progress
  1157. * and loop forever.
  1158. */
  1159. if (dentry->d_lockref.count) {
  1160. dentry_lru_del(dentry);
  1161. } else if (!(dentry->d_flags & DCACHE_SHRINK_LIST)) {
  1162. dentry_lru_move_list(dentry, &data->dispose);
  1163. dentry->d_flags |= DCACHE_SHRINK_LIST;
  1164. data->found++;
  1165. ret = D_WALK_NORETRY;
  1166. }
  1167. /*
  1168. * We can return to the caller if we have found some (this
  1169. * ensures forward progress). We'll be coming back to find
  1170. * the rest.
  1171. */
  1172. if (data->found && need_resched())
  1173. ret = D_WALK_QUIT;
  1174. out:
  1175. return ret;
  1176. }
  1177. /**
  1178. * shrink_dcache_parent - prune dcache
  1179. * @parent: parent of entries to prune
  1180. *
  1181. * Prune the dcache to remove unused children of the parent dentry.
  1182. */
  1183. void shrink_dcache_parent(struct dentry *parent)
  1184. {
  1185. for (;;) {
  1186. struct select_data data;
  1187. INIT_LIST_HEAD(&data.dispose);
  1188. data.start = parent;
  1189. data.found = 0;
  1190. d_walk(parent, &data, select_collect, NULL);
  1191. if (!data.found)
  1192. break;
  1193. shrink_dentry_list(&data.dispose);
  1194. cond_resched();
  1195. }
  1196. }
  1197. EXPORT_SYMBOL(shrink_dcache_parent);
  1198. static enum d_walk_ret check_and_collect(void *_data, struct dentry *dentry)
  1199. {
  1200. struct select_data *data = _data;
  1201. if (d_mountpoint(dentry)) {
  1202. data->found = -EBUSY;
  1203. return D_WALK_QUIT;
  1204. }
  1205. return select_collect(_data, dentry);
  1206. }
  1207. static void check_and_drop(void *_data)
  1208. {
  1209. struct select_data *data = _data;
  1210. if (d_mountpoint(data->start))
  1211. data->found = -EBUSY;
  1212. if (!data->found)
  1213. __d_drop(data->start);
  1214. }
  1215. /**
  1216. * check_submounts_and_drop - prune dcache, check for submounts and drop
  1217. *
  1218. * All done as a single atomic operation relative to has_unlinked_ancestor().
  1219. * Returns 0 if successfully unhashed @parent. If there were submounts then
  1220. * return -EBUSY.
  1221. *
  1222. * @dentry: dentry to prune and drop
  1223. */
  1224. int check_submounts_and_drop(struct dentry *dentry)
  1225. {
  1226. int ret = 0;
  1227. /* Negative dentries can be dropped without further checks */
  1228. if (!dentry->d_inode) {
  1229. d_drop(dentry);
  1230. goto out;
  1231. }
  1232. for (;;) {
  1233. struct select_data data;
  1234. INIT_LIST_HEAD(&data.dispose);
  1235. data.start = dentry;
  1236. data.found = 0;
  1237. d_walk(dentry, &data, check_and_collect, check_and_drop);
  1238. ret = data.found;
  1239. if (!list_empty(&data.dispose))
  1240. shrink_dentry_list(&data.dispose);
  1241. if (ret <= 0)
  1242. break;
  1243. cond_resched();
  1244. }
  1245. out:
  1246. return ret;
  1247. }
  1248. EXPORT_SYMBOL(check_submounts_and_drop);
  1249. /**
  1250. * __d_alloc - allocate a dcache entry
  1251. * @sb: filesystem it will belong to
  1252. * @name: qstr of the name
  1253. *
  1254. * Allocates a dentry. It returns %NULL if there is insufficient memory
  1255. * available. On a success the dentry is returned. The name passed in is
  1256. * copied and the copy passed in may be reused after this call.
  1257. */
  1258. struct dentry *__d_alloc(struct super_block *sb, const struct qstr *name)
  1259. {
  1260. struct dentry *dentry;
  1261. char *dname;
  1262. dentry = kmem_cache_alloc(dentry_cache, GFP_KERNEL);
  1263. if (!dentry)
  1264. return NULL;
  1265. /*
  1266. * We guarantee that the inline name is always NUL-terminated.
  1267. * This way the memcpy() done by the name switching in rename
  1268. * will still always have a NUL at the end, even if we might
  1269. * be overwriting an internal NUL character
  1270. */
  1271. dentry->d_iname[DNAME_INLINE_LEN-1] = 0;
  1272. if (name->len > DNAME_INLINE_LEN-1) {
  1273. dname = kmalloc(name->len + 1, GFP_KERNEL);
  1274. if (!dname) {
  1275. kmem_cache_free(dentry_cache, dentry);
  1276. return NULL;
  1277. }
  1278. } else {
  1279. dname = dentry->d_iname;
  1280. }
  1281. dentry->d_name.len = name->len;
  1282. dentry->d_name.hash = name->hash;
  1283. memcpy(dname, name->name, name->len);
  1284. dname[name->len] = 0;
  1285. /* Make sure we always see the terminating NUL character */
  1286. smp_wmb();
  1287. dentry->d_name.name = dname;
  1288. dentry->d_lockref.count = 1;
  1289. dentry->d_flags = 0;
  1290. spin_lock_init(&dentry->d_lock);
  1291. seqcount_init(&dentry->d_seq);
  1292. dentry->d_inode = NULL;
  1293. dentry->d_parent = dentry;
  1294. dentry->d_sb = sb;
  1295. dentry->d_op = NULL;
  1296. dentry->d_fsdata = NULL;
  1297. INIT_HLIST_BL_NODE(&dentry->d_hash);
  1298. INIT_LIST_HEAD(&dentry->d_lru);
  1299. INIT_LIST_HEAD(&dentry->d_subdirs);
  1300. INIT_HLIST_NODE(&dentry->d_alias);
  1301. INIT_LIST_HEAD(&dentry->d_u.d_child);
  1302. d_set_d_op(dentry, dentry->d_sb->s_d_op);
  1303. this_cpu_inc(nr_dentry);
  1304. return dentry;
  1305. }
  1306. /**
  1307. * d_alloc - allocate a dcache entry
  1308. * @parent: parent of entry to allocate
  1309. * @name: qstr of the name
  1310. *
  1311. * Allocates a dentry. It returns %NULL if there is insufficient memory
  1312. * available. On a success the dentry is returned. The name passed in is
  1313. * copied and the copy passed in may be reused after this call.
  1314. */
  1315. struct dentry *d_alloc(struct dentry * parent, const struct qstr *name)
  1316. {
  1317. struct dentry *dentry = __d_alloc(parent->d_sb, name);
  1318. if (!dentry)
  1319. return NULL;
  1320. spin_lock(&parent->d_lock);
  1321. /*
  1322. * don't need child lock because it is not subject
  1323. * to concurrency here
  1324. */
  1325. __dget_dlock(parent);
  1326. dentry->d_parent = parent;
  1327. list_add(&dentry->d_u.d_child, &parent->d_subdirs);
  1328. spin_unlock(&parent->d_lock);
  1329. return dentry;
  1330. }
  1331. EXPORT_SYMBOL(d_alloc);
  1332. struct dentry *d_alloc_pseudo(struct super_block *sb, const struct qstr *name)
  1333. {
  1334. struct dentry *dentry = __d_alloc(sb, name);
  1335. if (dentry)
  1336. dentry->d_flags |= DCACHE_DISCONNECTED;
  1337. return dentry;
  1338. }
  1339. EXPORT_SYMBOL(d_alloc_pseudo);
  1340. struct dentry *d_alloc_name(struct dentry *parent, const char *name)
  1341. {
  1342. struct qstr q;
  1343. q.name = name;
  1344. q.len = strlen(name);
  1345. q.hash = full_name_hash(q.name, q.len);
  1346. return d_alloc(parent, &q);
  1347. }
  1348. EXPORT_SYMBOL(d_alloc_name);
  1349. void d_set_d_op(struct dentry *dentry, const struct dentry_operations *op)
  1350. {
  1351. WARN_ON_ONCE(dentry->d_op);
  1352. WARN_ON_ONCE(dentry->d_flags & (DCACHE_OP_HASH |
  1353. DCACHE_OP_COMPARE |
  1354. DCACHE_OP_REVALIDATE |
  1355. DCACHE_OP_WEAK_REVALIDATE |
  1356. DCACHE_OP_DELETE ));
  1357. dentry->d_op = op;
  1358. if (!op)
  1359. return;
  1360. if (op->d_hash)
  1361. dentry->d_flags |= DCACHE_OP_HASH;
  1362. if (op->d_compare)
  1363. dentry->d_flags |= DCACHE_OP_COMPARE;
  1364. if (op->d_revalidate)
  1365. dentry->d_flags |= DCACHE_OP_REVALIDATE;
  1366. if (op->d_weak_revalidate)
  1367. dentry->d_flags |= DCACHE_OP_WEAK_REVALIDATE;
  1368. if (op->d_delete)
  1369. dentry->d_flags |= DCACHE_OP_DELETE;
  1370. if (op->d_prune)
  1371. dentry->d_flags |= DCACHE_OP_PRUNE;
  1372. }
  1373. EXPORT_SYMBOL(d_set_d_op);
  1374. static void __d_instantiate(struct dentry *dentry, struct inode *inode)
  1375. {
  1376. spin_lock(&dentry->d_lock);
  1377. if (inode) {
  1378. if (unlikely(IS_AUTOMOUNT(inode)))
  1379. dentry->d_flags |= DCACHE_NEED_AUTOMOUNT;
  1380. hlist_add_head(&dentry->d_alias, &inode->i_dentry);
  1381. }
  1382. dentry->d_inode = inode;
  1383. dentry_rcuwalk_barrier(dentry);
  1384. spin_unlock(&dentry->d_lock);
  1385. fsnotify_d_instantiate(dentry, inode);
  1386. }
  1387. /**
  1388. * d_instantiate - fill in inode information for a dentry
  1389. * @entry: dentry to complete
  1390. * @inode: inode to attach to this dentry
  1391. *
  1392. * Fill in inode information in the entry.
  1393. *
  1394. * This turns negative dentries into productive full members
  1395. * of society.
  1396. *
  1397. * NOTE! This assumes that the inode count has been incremented
  1398. * (or otherwise set) by the caller to indicate that it is now
  1399. * in use by the dcache.
  1400. */
  1401. void d_instantiate(struct dentry *entry, struct inode * inode)
  1402. {
  1403. BUG_ON(!hlist_unhashed(&entry->d_alias));
  1404. if (inode)
  1405. spin_lock(&inode->i_lock);
  1406. __d_instantiate(entry, inode);
  1407. if (inode)
  1408. spin_unlock(&inode->i_lock);
  1409. security_d_instantiate(entry, inode);
  1410. }
  1411. EXPORT_SYMBOL(d_instantiate);
  1412. /**
  1413. * d_instantiate_unique - instantiate a non-aliased dentry
  1414. * @entry: dentry to instantiate
  1415. * @inode: inode to attach to this dentry
  1416. *
  1417. * Fill in inode information in the entry. On success, it returns NULL.
  1418. * If an unhashed alias of "entry" already exists, then we return the
  1419. * aliased dentry instead and drop one reference to inode.
  1420. *
  1421. * Note that in order to avoid conflicts with rename() etc, the caller
  1422. * had better be holding the parent directory semaphore.
  1423. *
  1424. * This also assumes that the inode count has been incremented
  1425. * (or otherwise set) by the caller to indicate that it is now
  1426. * in use by the dcache.
  1427. */
  1428. static struct dentry *__d_instantiate_unique(struct dentry *entry,
  1429. struct inode *inode)
  1430. {
  1431. struct dentry *alias;
  1432. int len = entry->d_name.len;
  1433. const char *name = entry->d_name.name;
  1434. unsigned int hash = entry->d_name.hash;
  1435. if (!inode) {
  1436. __d_instantiate(entry, NULL);
  1437. return NULL;
  1438. }
  1439. hlist_for_each_entry(alias, &inode->i_dentry, d_alias) {
  1440. /*
  1441. * Don't need alias->d_lock here, because aliases with
  1442. * d_parent == entry->d_parent are not subject to name or
  1443. * parent changes, because the parent inode i_mutex is held.
  1444. */
  1445. if (alias->d_name.hash != hash)
  1446. continue;
  1447. if (alias->d_parent != entry->d_parent)
  1448. continue;
  1449. if (alias->d_name.len != len)
  1450. continue;
  1451. if (dentry_cmp(alias, name, len))
  1452. continue;
  1453. __dget(alias);
  1454. return alias;
  1455. }
  1456. __d_instantiate(entry, inode);
  1457. return NULL;
  1458. }
  1459. struct dentry *d_instantiate_unique(struct dentry *entry, struct inode *inode)
  1460. {
  1461. struct dentry *result;
  1462. BUG_ON(!hlist_unhashed(&entry->d_alias));
  1463. if (inode)
  1464. spin_lock(&inode->i_lock);
  1465. result = __d_instantiate_unique(entry, inode);
  1466. if (inode)
  1467. spin_unlock(&inode->i_lock);
  1468. if (!result) {
  1469. security_d_instantiate(entry, inode);
  1470. return NULL;
  1471. }
  1472. BUG_ON(!d_unhashed(result));
  1473. iput(inode);
  1474. return result;
  1475. }
  1476. EXPORT_SYMBOL(d_instantiate_unique);
  1477. struct dentry *d_make_root(struct inode *root_inode)
  1478. {
  1479. struct dentry *res = NULL;
  1480. if (root_inode) {
  1481. static const struct qstr name = QSTR_INIT("/", 1);
  1482. res = __d_alloc(root_inode->i_sb, &name);
  1483. if (res)
  1484. d_instantiate(res, root_inode);
  1485. else
  1486. iput(root_inode);
  1487. }
  1488. return res;
  1489. }
  1490. EXPORT_SYMBOL(d_make_root);
  1491. static struct dentry * __d_find_any_alias(struct inode *inode)
  1492. {
  1493. struct dentry *alias;
  1494. if (hlist_empty(&inode->i_dentry))
  1495. return NULL;
  1496. alias = hlist_entry(inode->i_dentry.first, struct dentry, d_alias);
  1497. __dget(alias);
  1498. return alias;
  1499. }
  1500. /**
  1501. * d_find_any_alias - find any alias for a given inode
  1502. * @inode: inode to find an alias for
  1503. *
  1504. * If any aliases exist for the given inode, take and return a
  1505. * reference for one of them. If no aliases exist, return %NULL.
  1506. */
  1507. struct dentry *d_find_any_alias(struct inode *inode)
  1508. {
  1509. struct dentry *de;
  1510. spin_lock(&inode->i_lock);
  1511. de = __d_find_any_alias(inode);
  1512. spin_unlock(&inode->i_lock);
  1513. return de;
  1514. }
  1515. EXPORT_SYMBOL(d_find_any_alias);
  1516. /**
  1517. * d_obtain_alias - find or allocate a dentry for a given inode
  1518. * @inode: inode to allocate the dentry for
  1519. *
  1520. * Obtain a dentry for an inode resulting from NFS filehandle conversion or
  1521. * similar open by handle operations. The returned dentry may be anonymous,
  1522. * or may have a full name (if the inode was already in the cache).
  1523. *
  1524. * When called on a directory inode, we must ensure that the inode only ever
  1525. * has one dentry. If a dentry is found, that is returned instead of
  1526. * allocating a new one.
  1527. *
  1528. * On successful return, the reference to the inode has been transferred
  1529. * to the dentry. In case of an error the reference on the inode is released.
  1530. * To make it easier to use in export operations a %NULL or IS_ERR inode may
  1531. * be passed in and will be the error will be propagate to the return value,
  1532. * with a %NULL @inode replaced by ERR_PTR(-ESTALE).
  1533. */
  1534. struct dentry *d_obtain_alias(struct inode *inode)
  1535. {
  1536. static const struct qstr anonstring = QSTR_INIT("/", 1);
  1537. struct dentry *tmp;
  1538. struct dentry *res;
  1539. if (!inode)
  1540. return ERR_PTR(-ESTALE);
  1541. if (IS_ERR(inode))
  1542. return ERR_CAST(inode);
  1543. res = d_find_any_alias(inode);
  1544. if (res)
  1545. goto out_iput;
  1546. tmp = __d_alloc(inode->i_sb, &anonstring);
  1547. if (!tmp) {
  1548. res = ERR_PTR(-ENOMEM);
  1549. goto out_iput;
  1550. }
  1551. spin_lock(&inode->i_lock);
  1552. res = __d_find_any_alias(inode);
  1553. if (res) {
  1554. spin_unlock(&inode->i_lock);
  1555. dput(tmp);
  1556. goto out_iput;
  1557. }
  1558. /* attach a disconnected dentry */
  1559. spin_lock(&tmp->d_lock);
  1560. tmp->d_inode = inode;
  1561. tmp->d_flags |= DCACHE_DISCONNECTED;
  1562. hlist_add_head(&tmp->d_alias, &inode->i_dentry);
  1563. hlist_bl_lock(&tmp->d_sb->s_anon);
  1564. hlist_bl_add_head(&tmp->d_hash, &tmp->d_sb->s_anon);
  1565. hlist_bl_unlock(&tmp->d_sb->s_anon);
  1566. spin_unlock(&tmp->d_lock);
  1567. spin_unlock(&inode->i_lock);
  1568. security_d_instantiate(tmp, inode);
  1569. return tmp;
  1570. out_iput:
  1571. if (res && !IS_ERR(res))
  1572. security_d_instantiate(res, inode);
  1573. iput(inode);
  1574. return res;
  1575. }
  1576. EXPORT_SYMBOL(d_obtain_alias);
  1577. /**
  1578. * d_splice_alias - splice a disconnected dentry into the tree if one exists
  1579. * @inode: the inode which may have a disconnected dentry
  1580. * @dentry: a negative dentry which we want to point to the inode.
  1581. *
  1582. * If inode is a directory and has a 'disconnected' dentry (i.e. IS_ROOT and
  1583. * DCACHE_DISCONNECTED), then d_move that in place of the given dentry
  1584. * and return it, else simply d_add the inode to the dentry and return NULL.
  1585. *
  1586. * This is needed in the lookup routine of any filesystem that is exportable
  1587. * (via knfsd) so that we can build dcache paths to directories effectively.
  1588. *
  1589. * If a dentry was found and moved, then it is returned. Otherwise NULL
  1590. * is returned. This matches the expected return value of ->lookup.
  1591. *
  1592. * Cluster filesystems may call this function with a negative, hashed dentry.
  1593. * In that case, we know that the inode will be a regular file, and also this
  1594. * will only occur during atomic_open. So we need to check for the dentry
  1595. * being already hashed only in the final case.
  1596. */
  1597. struct dentry *d_splice_alias(struct inode *inode, struct dentry *dentry)
  1598. {
  1599. struct dentry *new = NULL;
  1600. if (IS_ERR(inode))
  1601. return ERR_CAST(inode);
  1602. if (inode && S_ISDIR(inode->i_mode)) {
  1603. spin_lock(&inode->i_lock);
  1604. new = __d_find_alias(inode, 1);
  1605. if (new) {
  1606. BUG_ON(!(new->d_flags & DCACHE_DISCONNECTED));
  1607. spin_unlock(&inode->i_lock);
  1608. security_d_instantiate(new, inode);
  1609. d_move(new, dentry);
  1610. iput(inode);
  1611. } else {
  1612. /* already taking inode->i_lock, so d_add() by hand */
  1613. __d_instantiate(dentry, inode);
  1614. spin_unlock(&inode->i_lock);
  1615. security_d_instantiate(dentry, inode);
  1616. d_rehash(dentry);
  1617. }
  1618. } else {
  1619. d_instantiate(dentry, inode);
  1620. if (d_unhashed(dentry))
  1621. d_rehash(dentry);
  1622. }
  1623. return new;
  1624. }
  1625. EXPORT_SYMBOL(d_splice_alias);
  1626. /**
  1627. * d_add_ci - lookup or allocate new dentry with case-exact name
  1628. * @inode: the inode case-insensitive lookup has found
  1629. * @dentry: the negative dentry that was passed to the parent's lookup func
  1630. * @name: the case-exact name to be associated with the returned dentry
  1631. *
  1632. * This is to avoid filling the dcache with case-insensitive names to the
  1633. * same inode, only the actual correct case is stored in the dcache for
  1634. * case-insensitive filesystems.
  1635. *
  1636. * For a case-insensitive lookup match and if the the case-exact dentry
  1637. * already exists in in the dcache, use it and return it.
  1638. *
  1639. * If no entry exists with the exact case name, allocate new dentry with
  1640. * the exact case, and return the spliced entry.
  1641. */
  1642. struct dentry *d_add_ci(struct dentry *dentry, struct inode *inode,
  1643. struct qstr *name)
  1644. {
  1645. struct dentry *found;
  1646. struct dentry *new;
  1647. /*
  1648. * First check if a dentry matching the name already exists,
  1649. * if not go ahead and create it now.
  1650. */
  1651. found = d_hash_and_lookup(dentry->d_parent, name);
  1652. if (unlikely(IS_ERR(found)))
  1653. goto err_out;
  1654. if (!found) {
  1655. new = d_alloc(dentry->d_parent, name);
  1656. if (!new) {
  1657. found = ERR_PTR(-ENOMEM);
  1658. goto err_out;
  1659. }
  1660. found = d_splice_alias(inode, new);
  1661. if (found) {
  1662. dput(new);
  1663. return found;
  1664. }
  1665. return new;
  1666. }
  1667. /*
  1668. * If a matching dentry exists, and it's not negative use it.
  1669. *
  1670. * Decrement the reference count to balance the iget() done
  1671. * earlier on.
  1672. */
  1673. if (found->d_inode) {
  1674. if (unlikely(found->d_inode != inode)) {
  1675. /* This can't happen because bad inodes are unhashed. */
  1676. BUG_ON(!is_bad_inode(inode));
  1677. BUG_ON(!is_bad_inode(found->d_inode));
  1678. }
  1679. iput(inode);
  1680. return found;
  1681. }
  1682. /*
  1683. * Negative dentry: instantiate it unless the inode is a directory and
  1684. * already has a dentry.
  1685. */
  1686. new = d_splice_alias(inode, found);
  1687. if (new) {
  1688. dput(found);
  1689. found = new;
  1690. }
  1691. return found;
  1692. err_out:
  1693. iput(inode);
  1694. return found;
  1695. }
  1696. EXPORT_SYMBOL(d_add_ci);
  1697. /*
  1698. * Do the slow-case of the dentry name compare.
  1699. *
  1700. * Unlike the dentry_cmp() function, we need to atomically
  1701. * load the name and length information, so that the
  1702. * filesystem can rely on them, and can use the 'name' and
  1703. * 'len' information without worrying about walking off the
  1704. * end of memory etc.
  1705. *
  1706. * Thus the read_seqcount_retry() and the "duplicate" info
  1707. * in arguments (the low-level filesystem should not look
  1708. * at the dentry inode or name contents directly, since
  1709. * rename can change them while we're in RCU mode).
  1710. */
  1711. enum slow_d_compare {
  1712. D_COMP_OK,
  1713. D_COMP_NOMATCH,
  1714. D_COMP_SEQRETRY,
  1715. };
  1716. static noinline enum slow_d_compare slow_dentry_cmp(
  1717. const struct dentry *parent,
  1718. struct dentry *dentry,
  1719. unsigned int seq,
  1720. const struct qstr *name)
  1721. {
  1722. int tlen = dentry->d_name.len;
  1723. const char *tname = dentry->d_name.name;
  1724. if (read_seqcount_retry(&dentry->d_seq, seq)) {
  1725. cpu_relax();
  1726. return D_COMP_SEQRETRY;
  1727. }
  1728. if (parent->d_op->d_compare(parent, dentry, tlen, tname, name))
  1729. return D_COMP_NOMATCH;
  1730. return D_COMP_OK;
  1731. }
  1732. /**
  1733. * __d_lookup_rcu - search for a dentry (racy, store-free)
  1734. * @parent: parent dentry
  1735. * @name: qstr of name we wish to find
  1736. * @seqp: returns d_seq value at the point where the dentry was found
  1737. * Returns: dentry, or NULL
  1738. *
  1739. * __d_lookup_rcu is the dcache lookup function for rcu-walk name
  1740. * resolution (store-free path walking) design described in
  1741. * Documentation/filesystems/path-lookup.txt.
  1742. *
  1743. * This is not to be used outside core vfs.
  1744. *
  1745. * __d_lookup_rcu must only be used in rcu-walk mode, ie. with vfsmount lock
  1746. * held, and rcu_read_lock held. The returned dentry must not be stored into
  1747. * without taking d_lock and checking d_seq sequence count against @seq
  1748. * returned here.
  1749. *
  1750. * A refcount may be taken on the found dentry with the d_rcu_to_refcount
  1751. * function.
  1752. *
  1753. * Alternatively, __d_lookup_rcu may be called again to look up the child of
  1754. * the returned dentry, so long as its parent's seqlock is checked after the
  1755. * child is looked up. Thus, an interlocking stepping of sequence lock checks
  1756. * is formed, giving integrity down the path walk.
  1757. *
  1758. * NOTE! The caller *has* to check the resulting dentry against the sequence
  1759. * number we've returned before using any of the resulting dentry state!
  1760. */
  1761. struct dentry *__d_lookup_rcu(const struct dentry *parent,
  1762. const struct qstr *name,
  1763. unsigned *seqp)
  1764. {
  1765. u64 hashlen = name->hash_len;
  1766. const unsigned char *str = name->name;
  1767. struct hlist_bl_head *b = d_hash(parent, hashlen_hash(hashlen));
  1768. struct hlist_bl_node *node;
  1769. struct dentry *dentry;
  1770. /*
  1771. * Note: There is significant duplication with __d_lookup_rcu which is
  1772. * required to prevent single threaded performance regressions
  1773. * especially on architectures where smp_rmb (in seqcounts) are costly.
  1774. * Keep the two functions in sync.
  1775. */
  1776. /*
  1777. * The hash list is protected using RCU.
  1778. *
  1779. * Carefully use d_seq when comparing a candidate dentry, to avoid
  1780. * races with d_move().
  1781. *
  1782. * It is possible that concurrent renames can mess up our list
  1783. * walk here and result in missing our dentry, resulting in the
  1784. * false-negative result. d_lookup() protects against concurrent
  1785. * renames using rename_lock seqlock.
  1786. *
  1787. * See Documentation/filesystems/path-lookup.txt for more details.
  1788. */
  1789. hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) {
  1790. unsigned seq;
  1791. seqretry:
  1792. /*
  1793. * The dentry sequence count protects us from concurrent
  1794. * renames, and thus protects parent and name fields.
  1795. *
  1796. * The caller must perform a seqcount check in order
  1797. * to do anything useful with the returned dentry.
  1798. *
  1799. * NOTE! We do a "raw" seqcount_begin here. That means that
  1800. * we don't wait for the sequence count to stabilize if it
  1801. * is in the middle of a sequence change. If we do the slow
  1802. * dentry compare, we will do seqretries until it is stable,
  1803. * and if we end up with a successful lookup, we actually
  1804. * want to exit RCU lookup anyway.
  1805. */
  1806. seq = raw_seqcount_begin(&dentry->d_seq);
  1807. if (dentry->d_parent != parent)
  1808. continue;
  1809. if (d_unhashed(dentry))
  1810. continue;
  1811. if (unlikely(parent->d_flags & DCACHE_OP_COMPARE)) {
  1812. if (dentry->d_name.hash != hashlen_hash(hashlen))
  1813. continue;
  1814. *seqp = seq;
  1815. switch (slow_dentry_cmp(parent, dentry, seq, name)) {
  1816. case D_COMP_OK:
  1817. return dentry;
  1818. case D_COMP_NOMATCH:
  1819. continue;
  1820. default:
  1821. goto seqretry;
  1822. }
  1823. }
  1824. if (dentry->d_name.hash_len != hashlen)
  1825. continue;
  1826. *seqp = seq;
  1827. if (!dentry_cmp(dentry, str, hashlen_len(hashlen)))
  1828. return dentry;
  1829. }
  1830. return NULL;
  1831. }
  1832. /**
  1833. * d_lookup - search for a dentry
  1834. * @parent: parent dentry
  1835. * @name: qstr of name we wish to find
  1836. * Returns: dentry, or NULL
  1837. *
  1838. * d_lookup searches the children of the parent dentry for the name in
  1839. * question. If the dentry is found its reference count is incremented and the
  1840. * dentry is returned. The caller must use dput to free the entry when it has
  1841. * finished using it. %NULL is returned if the dentry does not exist.
  1842. */
  1843. struct dentry *d_lookup(const struct dentry *parent, const struct qstr *name)
  1844. {
  1845. struct dentry *dentry;
  1846. unsigned seq;
  1847. do {
  1848. seq = read_seqbegin(&rename_lock);
  1849. dentry = __d_lookup(parent, name);
  1850. if (dentry)
  1851. break;
  1852. } while (read_seqretry(&rename_lock, seq));
  1853. return dentry;
  1854. }
  1855. EXPORT_SYMBOL(d_lookup);
  1856. /**
  1857. * __d_lookup - search for a dentry (racy)
  1858. * @parent: parent dentry
  1859. * @name: qstr of name we wish to find
  1860. * Returns: dentry, or NULL
  1861. *
  1862. * __d_lookup is like d_lookup, however it may (rarely) return a
  1863. * false-negative result due to unrelated rename activity.
  1864. *
  1865. * __d_lookup is slightly faster by avoiding rename_lock read seqlock,
  1866. * however it must be used carefully, eg. with a following d_lookup in
  1867. * the case of failure.
  1868. *
  1869. * __d_lookup callers must be commented.
  1870. */
  1871. struct dentry *__d_lookup(const struct dentry *parent, const struct qstr *name)
  1872. {
  1873. unsigned int len = name->len;
  1874. unsigned int hash = name->hash;
  1875. const unsigned char *str = name->name;
  1876. struct hlist_bl_head *b = d_hash(parent, hash);
  1877. struct hlist_bl_node *node;
  1878. struct dentry *found = NULL;
  1879. struct dentry *dentry;
  1880. /*
  1881. * Note: There is significant duplication with __d_lookup_rcu which is
  1882. * required to prevent single threaded performance regressions
  1883. * especially on architectures where smp_rmb (in seqcounts) are costly.
  1884. * Keep the two functions in sync.
  1885. */
  1886. /*
  1887. * The hash list is protected using RCU.
  1888. *
  1889. * Take d_lock when comparing a candidate dentry, to avoid races
  1890. * with d_move().
  1891. *
  1892. * It is possible that concurrent renames can mess up our list
  1893. * walk here and result in missing our dentry, resulting in the
  1894. * false-negative result. d_lookup() protects against concurrent
  1895. * renames using rename_lock seqlock.
  1896. *
  1897. * See Documentation/filesystems/path-lookup.txt for more details.
  1898. */
  1899. rcu_read_lock();
  1900. hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) {
  1901. if (dentry->d_name.hash != hash)
  1902. continue;
  1903. spin_lock(&dentry->d_lock);
  1904. if (dentry->d_parent != parent)
  1905. goto next;
  1906. if (d_unhashed(dentry))
  1907. goto next;
  1908. /*
  1909. * It is safe to compare names since d_move() cannot
  1910. * change the qstr (protected by d_lock).
  1911. */
  1912. if (parent->d_flags & DCACHE_OP_COMPARE) {
  1913. int tlen = dentry->d_name.len;
  1914. const char *tname = dentry->d_name.name;
  1915. if (parent->d_op->d_compare(parent, dentry, tlen, tname, name))
  1916. goto next;
  1917. } else {
  1918. if (dentry->d_name.len != len)
  1919. goto next;
  1920. if (dentry_cmp(dentry, str, len))
  1921. goto next;
  1922. }
  1923. dentry->d_lockref.count++;
  1924. found = dentry;
  1925. spin_unlock(&dentry->d_lock);
  1926. break;
  1927. next:
  1928. spin_unlock(&dentry->d_lock);
  1929. }
  1930. rcu_read_unlock();
  1931. return found;
  1932. }
  1933. /**
  1934. * d_hash_and_lookup - hash the qstr then search for a dentry
  1935. * @dir: Directory to search in
  1936. * @name: qstr of name we wish to find
  1937. *
  1938. * On lookup failure NULL is returned; on bad name - ERR_PTR(-error)
  1939. */
  1940. struct dentry *d_hash_and_lookup(struct dentry *dir, struct qstr *name)
  1941. {
  1942. /*
  1943. * Check for a fs-specific hash function. Note that we must
  1944. * calculate the standard hash first, as the d_op->d_hash()
  1945. * routine may choose to leave the hash value unchanged.
  1946. */
  1947. name->hash = full_name_hash(name->name, name->len);
  1948. if (dir->d_flags & DCACHE_OP_HASH) {
  1949. int err = dir->d_op->d_hash(dir, name);
  1950. if (unlikely(err < 0))
  1951. return ERR_PTR(err);
  1952. }
  1953. return d_lookup(dir, name);
  1954. }
  1955. EXPORT_SYMBOL(d_hash_and_lookup);
  1956. /**
  1957. * d_validate - verify dentry provided from insecure source (deprecated)
  1958. * @dentry: The dentry alleged to be valid child of @dparent
  1959. * @dparent: The parent dentry (known to be valid)
  1960. *
  1961. * An insecure source has sent us a dentry, here we verify it and dget() it.
  1962. * This is used by ncpfs in its readdir implementation.
  1963. * Zero is returned in the dentry is invalid.
  1964. *
  1965. * This function is slow for big directories, and deprecated, do not use it.
  1966. */
  1967. int d_validate(struct dentry *dentry, struct dentry *dparent)
  1968. {
  1969. struct dentry *child;
  1970. spin_lock(&dparent->d_lock);
  1971. list_for_each_entry(child, &dparent->d_subdirs, d_u.d_child) {
  1972. if (dentry == child) {
  1973. spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
  1974. __dget_dlock(dentry);
  1975. spin_unlock(&dentry->d_lock);
  1976. spin_unlock(&dparent->d_lock);
  1977. return 1;
  1978. }
  1979. }
  1980. spin_unlock(&dparent->d_lock);
  1981. return 0;
  1982. }
  1983. EXPORT_SYMBOL(d_validate);
  1984. /*
  1985. * When a file is deleted, we have two options:
  1986. * - turn this dentry into a negative dentry
  1987. * - unhash this dentry and free it.
  1988. *
  1989. * Usually, we want to just turn this into
  1990. * a negative dentry, but if anybody else is
  1991. * currently using the dentry or the inode
  1992. * we can't do that and we fall back on removing
  1993. * it from the hash queues and waiting for
  1994. * it to be deleted later when it has no users
  1995. */
  1996. /**
  1997. * d_delete - delete a dentry
  1998. * @dentry: The dentry to delete
  1999. *
  2000. * Turn the dentry into a negative dentry if possible, otherwise
  2001. * remove it from the hash queues so it can be deleted later
  2002. */
  2003. void d_delete(struct dentry * dentry)
  2004. {
  2005. struct inode *inode;
  2006. int isdir = 0;
  2007. /*
  2008. * Are we the only user?
  2009. */
  2010. again:
  2011. spin_lock(&dentry->d_lock);
  2012. inode = dentry->d_inode;
  2013. isdir = S_ISDIR(inode->i_mode);
  2014. if (dentry->d_lockref.count == 1) {
  2015. if (!spin_trylock(&inode->i_lock)) {
  2016. spin_unlock(&dentry->d_lock);
  2017. cpu_relax();
  2018. goto again;
  2019. }
  2020. dentry->d_flags &= ~DCACHE_CANT_MOUNT;
  2021. dentry_unlink_inode(dentry);
  2022. fsnotify_nameremove(dentry, isdir);
  2023. return;
  2024. }
  2025. if (!d_unhashed(dentry))
  2026. __d_drop(dentry);
  2027. spin_unlock(&dentry->d_lock);
  2028. fsnotify_nameremove(dentry, isdir);
  2029. }
  2030. EXPORT_SYMBOL(d_delete);
  2031. static void __d_rehash(struct dentry * entry, struct hlist_bl_head *b)
  2032. {
  2033. BUG_ON(!d_unhashed(entry));
  2034. hlist_bl_lock(b);
  2035. entry->d_flags |= DCACHE_RCUACCESS;
  2036. hlist_bl_add_head_rcu(&entry->d_hash, b);
  2037. hlist_bl_unlock(b);
  2038. }
  2039. static void _d_rehash(struct dentry * entry)
  2040. {
  2041. __d_rehash(entry, d_hash(entry->d_parent, entry->d_name.hash));
  2042. }
  2043. /**
  2044. * d_rehash - add an entry back to the hash
  2045. * @entry: dentry to add to the hash
  2046. *
  2047. * Adds a dentry to the hash according to its name.
  2048. */
  2049. void d_rehash(struct dentry * entry)
  2050. {
  2051. spin_lock(&entry->d_lock);
  2052. _d_rehash(entry);
  2053. spin_unlock(&entry->d_lock);
  2054. }
  2055. EXPORT_SYMBOL(d_rehash);
  2056. /**
  2057. * dentry_update_name_case - update case insensitive dentry with a new name
  2058. * @dentry: dentry to be updated
  2059. * @name: new name
  2060. *
  2061. * Update a case insensitive dentry with new case of name.
  2062. *
  2063. * dentry must have been returned by d_lookup with name @name. Old and new
  2064. * name lengths must match (ie. no d_compare which allows mismatched name
  2065. * lengths).
  2066. *
  2067. * Parent inode i_mutex must be held over d_lookup and into this call (to
  2068. * keep renames and concurrent inserts, and readdir(2) away).
  2069. */
  2070. void dentry_update_name_case(struct dentry *dentry, struct qstr *name)
  2071. {
  2072. BUG_ON(!mutex_is_locked(&dentry->d_parent->d_inode->i_mutex));
  2073. BUG_ON(dentry->d_name.len != name->len); /* d_lookup gives this */
  2074. spin_lock(&dentry->d_lock);
  2075. write_seqcount_begin(&dentry->d_seq);
  2076. memcpy((unsigned char *)dentry->d_name.name, name->name, name->len);
  2077. write_seqcount_end(&dentry->d_seq);
  2078. spin_unlock(&dentry->d_lock);
  2079. }
  2080. EXPORT_SYMBOL(dentry_update_name_case);
  2081. static void switch_names(struct dentry *dentry, struct dentry *target)
  2082. {
  2083. if (dname_external(target)) {
  2084. if (dname_external(dentry)) {
  2085. /*
  2086. * Both external: swap the pointers
  2087. */
  2088. swap(target->d_name.name, dentry->d_name.name);
  2089. } else {
  2090. /*
  2091. * dentry:internal, target:external. Steal target's
  2092. * storage and make target internal.
  2093. */
  2094. memcpy(target->d_iname, dentry->d_name.name,
  2095. dentry->d_name.len + 1);
  2096. dentry->d_name.name = target->d_name.name;
  2097. target->d_name.name = target->d_iname;
  2098. }
  2099. } else {
  2100. if (dname_external(dentry)) {
  2101. /*
  2102. * dentry:external, target:internal. Give dentry's
  2103. * storage to target and make dentry internal
  2104. */
  2105. memcpy(dentry->d_iname, target->d_name.name,
  2106. target->d_name.len + 1);
  2107. target->d_name.name = dentry->d_name.name;
  2108. dentry->d_name.name = dentry->d_iname;
  2109. } else {
  2110. /*
  2111. * Both are internal. Just copy target to dentry
  2112. */
  2113. memcpy(dentry->d_iname, target->d_name.name,
  2114. target->d_name.len + 1);
  2115. dentry->d_name.len = target->d_name.len;
  2116. return;
  2117. }
  2118. }
  2119. swap(dentry->d_name.len, target->d_name.len);
  2120. }
  2121. static void dentry_lock_for_move(struct dentry *dentry, struct dentry *target)
  2122. {
  2123. /*
  2124. * XXXX: do we really need to take target->d_lock?
  2125. */
  2126. if (IS_ROOT(dentry) || dentry->d_parent == target->d_parent)
  2127. spin_lock(&target->d_parent->d_lock);
  2128. else {
  2129. if (d_ancestor(dentry->d_parent, target->d_parent)) {
  2130. spin_lock(&dentry->d_parent->d_lock);
  2131. spin_lock_nested(&target->d_parent->d_lock,
  2132. DENTRY_D_LOCK_NESTED);
  2133. } else {
  2134. spin_lock(&target->d_parent->d_lock);
  2135. spin_lock_nested(&dentry->d_parent->d_lock,
  2136. DENTRY_D_LOCK_NESTED);
  2137. }
  2138. }
  2139. if (target < dentry) {
  2140. spin_lock_nested(&target->d_lock, 2);
  2141. spin_lock_nested(&dentry->d_lock, 3);
  2142. } else {
  2143. spin_lock_nested(&dentry->d_lock, 2);
  2144. spin_lock_nested(&target->d_lock, 3);
  2145. }
  2146. }
  2147. static void dentry_unlock_parents_for_move(struct dentry *dentry,
  2148. struct dentry *target)
  2149. {
  2150. if (target->d_parent != dentry->d_parent)
  2151. spin_unlock(&dentry->d_parent->d_lock);
  2152. if (target->d_parent != target)
  2153. spin_unlock(&target->d_parent->d_lock);
  2154. }
  2155. /*
  2156. * When switching names, the actual string doesn't strictly have to
  2157. * be preserved in the target - because we're dropping the target
  2158. * anyway. As such, we can just do a simple memcpy() to copy over
  2159. * the new name before we switch.
  2160. *
  2161. * Note that we have to be a lot more careful about getting the hash
  2162. * switched - we have to switch the hash value properly even if it
  2163. * then no longer matches the actual (corrupted) string of the target.
  2164. * The hash value has to match the hash queue that the dentry is on..
  2165. */
  2166. /*
  2167. * __d_move - move a dentry
  2168. * @dentry: entry to move
  2169. * @target: new dentry
  2170. *
  2171. * Update the dcache to reflect the move of a file name. Negative
  2172. * dcache entries should not be moved in this way. Caller must hold
  2173. * rename_lock, the i_mutex of the source and target directories,
  2174. * and the sb->s_vfs_rename_mutex if they differ. See lock_rename().
  2175. */
  2176. static void __d_move(struct dentry * dentry, struct dentry * target)
  2177. {
  2178. if (!dentry->d_inode)
  2179. printk(KERN_WARNING "VFS: moving negative dcache entry\n");
  2180. BUG_ON(d_ancestor(dentry, target));
  2181. BUG_ON(d_ancestor(target, dentry));
  2182. dentry_lock_for_move(dentry, target);
  2183. write_seqcount_begin(&dentry->d_seq);
  2184. write_seqcount_begin(&target->d_seq);
  2185. /* __d_drop does write_seqcount_barrier, but they're OK to nest. */
  2186. /*
  2187. * Move the dentry to the target hash queue. Don't bother checking
  2188. * for the same hash queue because of how unlikely it is.
  2189. */
  2190. __d_drop(dentry);
  2191. __d_rehash(dentry, d_hash(target->d_parent, target->d_name.hash));
  2192. /* Unhash the target: dput() will then get rid of it */
  2193. __d_drop(target);
  2194. list_del(&dentry->d_u.d_child);
  2195. list_del(&target->d_u.d_child);
  2196. /* Switch the names.. */
  2197. switch_names(dentry, target);
  2198. swap(dentry->d_name.hash, target->d_name.hash);
  2199. /* ... and switch the parents */
  2200. if (IS_ROOT(dentry)) {
  2201. dentry->d_parent = target->d_parent;
  2202. target->d_parent = target;
  2203. INIT_LIST_HEAD(&target->d_u.d_child);
  2204. } else {
  2205. swap(dentry->d_parent, target->d_parent);
  2206. /* And add them back to the (new) parent lists */
  2207. list_add(&target->d_u.d_child, &target->d_parent->d_subdirs);
  2208. }
  2209. list_add(&dentry->d_u.d_child, &dentry->d_parent->d_subdirs);
  2210. write_seqcount_end(&target->d_seq);
  2211. write_seqcount_end(&dentry->d_seq);
  2212. dentry_unlock_parents_for_move(dentry, target);
  2213. spin_unlock(&target->d_lock);
  2214. fsnotify_d_move(dentry);
  2215. spin_unlock(&dentry->d_lock);
  2216. }
  2217. /*
  2218. * d_move - move a dentry
  2219. * @dentry: entry to move
  2220. * @target: new dentry
  2221. *
  2222. * Update the dcache to reflect the move of a file name. Negative
  2223. * dcache entries should not be moved in this way. See the locking
  2224. * requirements for __d_move.
  2225. */
  2226. void d_move(struct dentry *dentry, struct dentry *target)
  2227. {
  2228. write_seqlock(&rename_lock);
  2229. __d_move(dentry, target);
  2230. write_sequnlock(&rename_lock);
  2231. }
  2232. EXPORT_SYMBOL(d_move);
  2233. /**
  2234. * d_ancestor - search for an ancestor
  2235. * @p1: ancestor dentry
  2236. * @p2: child dentry
  2237. *
  2238. * Returns the ancestor dentry of p2 which is a child of p1, if p1 is
  2239. * an ancestor of p2, else NULL.
  2240. */
  2241. struct dentry *d_ancestor(struct dentry *p1, struct dentry *p2)
  2242. {
  2243. struct dentry *p;
  2244. for (p = p2; !IS_ROOT(p); p = p->d_parent) {
  2245. if (p->d_parent == p1)
  2246. return p;
  2247. }
  2248. return NULL;
  2249. }
  2250. /*
  2251. * This helper attempts to cope with remotely renamed directories
  2252. *
  2253. * It assumes that the caller is already holding
  2254. * dentry->d_parent->d_inode->i_mutex, inode->i_lock and rename_lock
  2255. *
  2256. * Note: If ever the locking in lock_rename() changes, then please
  2257. * remember to update this too...
  2258. */
  2259. static struct dentry *__d_unalias(struct inode *inode,
  2260. struct dentry *dentry, struct dentry *alias)
  2261. {
  2262. struct mutex *m1 = NULL, *m2 = NULL;
  2263. struct dentry *ret = ERR_PTR(-EBUSY);
  2264. /* If alias and dentry share a parent, then no extra locks required */
  2265. if (alias->d_parent == dentry->d_parent)
  2266. goto out_unalias;
  2267. /* See lock_rename() */
  2268. if (!mutex_trylock(&dentry->d_sb->s_vfs_rename_mutex))
  2269. goto out_err;
  2270. m1 = &dentry->d_sb->s_vfs_rename_mutex;
  2271. if (!mutex_trylock(&alias->d_parent->d_inode->i_mutex))
  2272. goto out_err;
  2273. m2 = &alias->d_parent->d_inode->i_mutex;
  2274. out_unalias:
  2275. if (likely(!d_mountpoint(alias))) {
  2276. __d_move(alias, dentry);
  2277. ret = alias;
  2278. }
  2279. out_err:
  2280. spin_unlock(&inode->i_lock);
  2281. if (m2)
  2282. mutex_unlock(m2);
  2283. if (m1)
  2284. mutex_unlock(m1);
  2285. return ret;
  2286. }
  2287. /*
  2288. * Prepare an anonymous dentry for life in the superblock's dentry tree as a
  2289. * named dentry in place of the dentry to be replaced.
  2290. * returns with anon->d_lock held!
  2291. */
  2292. static void __d_materialise_dentry(struct dentry *dentry, struct dentry *anon)
  2293. {
  2294. struct dentry *dparent;
  2295. dentry_lock_for_move(anon, dentry);
  2296. write_seqcount_begin(&dentry->d_seq);
  2297. write_seqcount_begin(&anon->d_seq);
  2298. dparent = dentry->d_parent;
  2299. switch_names(dentry, anon);
  2300. swap(dentry->d_name.hash, anon->d_name.hash);
  2301. dentry->d_parent = dentry;
  2302. list_del_init(&dentry->d_u.d_child);
  2303. anon->d_parent = dparent;
  2304. list_move(&anon->d_u.d_child, &dparent->d_subdirs);
  2305. write_seqcount_end(&dentry->d_seq);
  2306. write_seqcount_end(&anon->d_seq);
  2307. dentry_unlock_parents_for_move(anon, dentry);
  2308. spin_unlock(&dentry->d_lock);
  2309. /* anon->d_lock still locked, returns locked */
  2310. anon->d_flags &= ~DCACHE_DISCONNECTED;
  2311. }
  2312. /**
  2313. * d_materialise_unique - introduce an inode into the tree
  2314. * @dentry: candidate dentry
  2315. * @inode: inode to bind to the dentry, to which aliases may be attached
  2316. *
  2317. * Introduces an dentry into the tree, substituting an extant disconnected
  2318. * root directory alias in its place if there is one. Caller must hold the
  2319. * i_mutex of the parent directory.
  2320. */
  2321. struct dentry *d_materialise_unique(struct dentry *dentry, struct inode *inode)
  2322. {
  2323. struct dentry *actual;
  2324. BUG_ON(!d_unhashed(dentry));
  2325. if (!inode) {
  2326. actual = dentry;
  2327. __d_instantiate(dentry, NULL);
  2328. d_rehash(actual);
  2329. goto out_nolock;
  2330. }
  2331. spin_lock(&inode->i_lock);
  2332. if (S_ISDIR(inode->i_mode)) {
  2333. struct dentry *alias;
  2334. /* Does an aliased dentry already exist? */
  2335. alias = __d_find_alias(inode, 0);
  2336. if (alias) {
  2337. actual = alias;
  2338. write_seqlock(&rename_lock);
  2339. if (d_ancestor(alias, dentry)) {
  2340. /* Check for loops */
  2341. actual = ERR_PTR(-ELOOP);
  2342. spin_unlock(&inode->i_lock);
  2343. } else if (IS_ROOT(alias)) {
  2344. /* Is this an anonymous mountpoint that we
  2345. * could splice into our tree? */
  2346. __d_materialise_dentry(dentry, alias);
  2347. write_sequnlock(&rename_lock);
  2348. __d_drop(alias);
  2349. goto found;
  2350. } else {
  2351. /* Nope, but we must(!) avoid directory
  2352. * aliasing. This drops inode->i_lock */
  2353. actual = __d_unalias(inode, dentry, alias);
  2354. }
  2355. write_sequnlock(&rename_lock);
  2356. if (IS_ERR(actual)) {
  2357. if (PTR_ERR(actual) == -ELOOP)
  2358. pr_warn_ratelimited(
  2359. "VFS: Lookup of '%s' in %s %s"
  2360. " would have caused loop\n",
  2361. dentry->d_name.name,
  2362. inode->i_sb->s_type->name,
  2363. inode->i_sb->s_id);
  2364. dput(alias);
  2365. }
  2366. goto out_nolock;
  2367. }
  2368. }
  2369. /* Add a unique reference */
  2370. actual = __d_instantiate_unique(dentry, inode);
  2371. if (!actual)
  2372. actual = dentry;
  2373. else
  2374. BUG_ON(!d_unhashed(actual));
  2375. spin_lock(&actual->d_lock);
  2376. found:
  2377. _d_rehash(actual);
  2378. spin_unlock(&actual->d_lock);
  2379. spin_unlock(&inode->i_lock);
  2380. out_nolock:
  2381. if (actual == dentry) {
  2382. security_d_instantiate(dentry, inode);
  2383. return NULL;
  2384. }
  2385. iput(inode);
  2386. return actual;
  2387. }
  2388. EXPORT_SYMBOL_GPL(d_materialise_unique);
  2389. static int prepend(char **buffer, int *buflen, const char *str, int namelen)
  2390. {
  2391. *buflen -= namelen;
  2392. if (*buflen < 0)
  2393. return -ENAMETOOLONG;
  2394. *buffer -= namelen;
  2395. memcpy(*buffer, str, namelen);
  2396. return 0;
  2397. }
  2398. /**
  2399. * prepend_name - prepend a pathname in front of current buffer pointer
  2400. * buffer: buffer pointer
  2401. * buflen: allocated length of the buffer
  2402. * name: name string and length qstr structure
  2403. *
  2404. * With RCU path tracing, it may race with d_move(). Use ACCESS_ONCE() to
  2405. * make sure that either the old or the new name pointer and length are
  2406. * fetched. However, there may be mismatch between length and pointer.
  2407. * The length cannot be trusted, we need to copy it byte-by-byte until
  2408. * the length is reached or a null byte is found. It also prepends "/" at
  2409. * the beginning of the name. The sequence number check at the caller will
  2410. * retry it again when a d_move() does happen. So any garbage in the buffer
  2411. * due to mismatched pointer and length will be discarded.
  2412. */
  2413. static int prepend_name(char **buffer, int *buflen, struct qstr *name)
  2414. {
  2415. const char *dname = ACCESS_ONCE(name->name);
  2416. u32 dlen = ACCESS_ONCE(name->len);
  2417. char *p;
  2418. if (*buflen < dlen + 1)
  2419. return -ENAMETOOLONG;
  2420. *buflen -= dlen + 1;
  2421. p = *buffer -= dlen + 1;
  2422. *p++ = '/';
  2423. while (dlen--) {
  2424. char c = *dname++;
  2425. if (!c)
  2426. break;
  2427. *p++ = c;
  2428. }
  2429. return 0;
  2430. }
  2431. /**
  2432. * prepend_path - Prepend path string to a buffer
  2433. * @path: the dentry/vfsmount to report
  2434. * @root: root vfsmnt/dentry
  2435. * @buffer: pointer to the end of the buffer
  2436. * @buflen: pointer to buffer length
  2437. *
  2438. * The function tries to write out the pathname without taking any lock other
  2439. * than the RCU read lock to make sure that dentries won't go away. It only
  2440. * checks the sequence number of the global rename_lock as any change in the
  2441. * dentry's d_seq will be preceded by changes in the rename_lock sequence
  2442. * number. If the sequence number had been change, it will restart the whole
  2443. * pathname back-tracing sequence again. It performs a total of 3 trials of
  2444. * lockless back-tracing sequences before falling back to take the
  2445. * rename_lock.
  2446. */
  2447. static int prepend_path(const struct path *path,
  2448. const struct path *root,
  2449. char **buffer, int *buflen)
  2450. {
  2451. struct dentry *dentry = path->dentry;
  2452. struct vfsmount *vfsmnt = path->mnt;
  2453. struct mount *mnt = real_mount(vfsmnt);
  2454. int error = 0;
  2455. unsigned seq = 0;
  2456. char *bptr;
  2457. int blen;
  2458. rcu_read_lock();
  2459. restart:
  2460. bptr = *buffer;
  2461. blen = *buflen;
  2462. read_seqbegin_or_lock(&rename_lock, &seq);
  2463. while (dentry != root->dentry || vfsmnt != root->mnt) {
  2464. struct dentry * parent;
  2465. if (dentry == vfsmnt->mnt_root || IS_ROOT(dentry)) {
  2466. /* Global root? */
  2467. if (mnt_has_parent(mnt)) {
  2468. dentry = mnt->mnt_mountpoint;
  2469. mnt = mnt->mnt_parent;
  2470. vfsmnt = &mnt->mnt;
  2471. continue;
  2472. }
  2473. /*
  2474. * Filesystems needing to implement special "root names"
  2475. * should do so with ->d_dname()
  2476. */
  2477. if (IS_ROOT(dentry) &&
  2478. (dentry->d_name.len != 1 ||
  2479. dentry->d_name.name[0] != '/')) {
  2480. WARN(1, "Root dentry has weird name <%.*s>\n",
  2481. (int) dentry->d_name.len,
  2482. dentry->d_name.name);
  2483. }
  2484. if (!error)
  2485. error = is_mounted(vfsmnt) ? 1 : 2;
  2486. break;
  2487. }
  2488. parent = dentry->d_parent;
  2489. prefetch(parent);
  2490. error = prepend_name(&bptr, &blen, &dentry->d_name);
  2491. if (error)
  2492. break;
  2493. dentry = parent;
  2494. }
  2495. if (!(seq & 1))
  2496. rcu_read_unlock();
  2497. if (need_seqretry(&rename_lock, seq)) {
  2498. seq = 1;
  2499. goto restart;
  2500. }
  2501. done_seqretry(&rename_lock, seq);
  2502. if (error >= 0 && bptr == *buffer) {
  2503. if (--blen < 0)
  2504. error = -ENAMETOOLONG;
  2505. else
  2506. *--bptr = '/';
  2507. }
  2508. *buffer = bptr;
  2509. *buflen = blen;
  2510. return error;
  2511. }
  2512. /**
  2513. * __d_path - return the path of a dentry
  2514. * @path: the dentry/vfsmount to report
  2515. * @root: root vfsmnt/dentry
  2516. * @buf: buffer to return value in
  2517. * @buflen: buffer length
  2518. *
  2519. * Convert a dentry into an ASCII path name.
  2520. *
  2521. * Returns a pointer into the buffer or an error code if the
  2522. * path was too long.
  2523. *
  2524. * "buflen" should be positive.
  2525. *
  2526. * If the path is not reachable from the supplied root, return %NULL.
  2527. */
  2528. char *__d_path(const struct path *path,
  2529. const struct path *root,
  2530. char *buf, int buflen)
  2531. {
  2532. char *res = buf + buflen;
  2533. int error;
  2534. prepend(&res, &buflen, "\0", 1);
  2535. br_read_lock(&vfsmount_lock);
  2536. error = prepend_path(path, root, &res, &buflen);
  2537. br_read_unlock(&vfsmount_lock);
  2538. if (error < 0)
  2539. return ERR_PTR(error);
  2540. if (error > 0)
  2541. return NULL;
  2542. return res;
  2543. }
  2544. char *d_absolute_path(const struct path *path,
  2545. char *buf, int buflen)
  2546. {
  2547. struct path root = {};
  2548. char *res = buf + buflen;
  2549. int error;
  2550. prepend(&res, &buflen, "\0", 1);
  2551. br_read_lock(&vfsmount_lock);
  2552. error = prepend_path(path, &root, &res, &buflen);
  2553. br_read_unlock(&vfsmount_lock);
  2554. if (error > 1)
  2555. error = -EINVAL;
  2556. if (error < 0)
  2557. return ERR_PTR(error);
  2558. return res;
  2559. }
  2560. /*
  2561. * same as __d_path but appends "(deleted)" for unlinked files.
  2562. */
  2563. static int path_with_deleted(const struct path *path,
  2564. const struct path *root,
  2565. char **buf, int *buflen)
  2566. {
  2567. prepend(buf, buflen, "\0", 1);
  2568. if (d_unlinked(path->dentry)) {
  2569. int error = prepend(buf, buflen, " (deleted)", 10);
  2570. if (error)
  2571. return error;
  2572. }
  2573. return prepend_path(path, root, buf, buflen);
  2574. }
  2575. static int prepend_unreachable(char **buffer, int *buflen)
  2576. {
  2577. return prepend(buffer, buflen, "(unreachable)", 13);
  2578. }
  2579. /**
  2580. * d_path - return the path of a dentry
  2581. * @path: path to report
  2582. * @buf: buffer to return value in
  2583. * @buflen: buffer length
  2584. *
  2585. * Convert a dentry into an ASCII path name. If the entry has been deleted
  2586. * the string " (deleted)" is appended. Note that this is ambiguous.
  2587. *
  2588. * Returns a pointer into the buffer or an error code if the path was
  2589. * too long. Note: Callers should use the returned pointer, not the passed
  2590. * in buffer, to use the name! The implementation often starts at an offset
  2591. * into the buffer, and may leave 0 bytes at the start.
  2592. *
  2593. * "buflen" should be positive.
  2594. */
  2595. char *d_path(const struct path *path, char *buf, int buflen)
  2596. {
  2597. char *res = buf + buflen;
  2598. struct path root;
  2599. int error;
  2600. /*
  2601. * We have various synthetic filesystems that never get mounted. On
  2602. * these filesystems dentries are never used for lookup purposes, and
  2603. * thus don't need to be hashed. They also don't need a name until a
  2604. * user wants to identify the object in /proc/pid/fd/. The little hack
  2605. * below allows us to generate a name for these objects on demand:
  2606. */
  2607. if (path->dentry->d_op && path->dentry->d_op->d_dname)
  2608. return path->dentry->d_op->d_dname(path->dentry, buf, buflen);
  2609. get_fs_root(current->fs, &root);
  2610. br_read_lock(&vfsmount_lock);
  2611. error = path_with_deleted(path, &root, &res, &buflen);
  2612. br_read_unlock(&vfsmount_lock);
  2613. if (error < 0)
  2614. res = ERR_PTR(error);
  2615. path_put(&root);
  2616. return res;
  2617. }
  2618. EXPORT_SYMBOL(d_path);
  2619. /*
  2620. * Helper function for dentry_operations.d_dname() members
  2621. */
  2622. char *dynamic_dname(struct dentry *dentry, char *buffer, int buflen,
  2623. const char *fmt, ...)
  2624. {
  2625. va_list args;
  2626. char temp[64];
  2627. int sz;
  2628. va_start(args, fmt);
  2629. sz = vsnprintf(temp, sizeof(temp), fmt, args) + 1;
  2630. va_end(args);
  2631. if (sz > sizeof(temp) || sz > buflen)
  2632. return ERR_PTR(-ENAMETOOLONG);
  2633. buffer += buflen - sz;
  2634. return memcpy(buffer, temp, sz);
  2635. }
  2636. char *simple_dname(struct dentry *dentry, char *buffer, int buflen)
  2637. {
  2638. char *end = buffer + buflen;
  2639. /* these dentries are never renamed, so d_lock is not needed */
  2640. if (prepend(&end, &buflen, " (deleted)", 11) ||
  2641. prepend(&end, &buflen, dentry->d_name.name, dentry->d_name.len) ||
  2642. prepend(&end, &buflen, "/", 1))
  2643. end = ERR_PTR(-ENAMETOOLONG);
  2644. return end;
  2645. }
  2646. /*
  2647. * Write full pathname from the root of the filesystem into the buffer.
  2648. */
  2649. static char *__dentry_path(struct dentry *dentry, char *buf, int buflen)
  2650. {
  2651. char *end, *retval;
  2652. int len, seq = 0;
  2653. int error = 0;
  2654. rcu_read_lock();
  2655. restart:
  2656. end = buf + buflen;
  2657. len = buflen;
  2658. prepend(&end, &len, "\0", 1);
  2659. if (buflen < 1)
  2660. goto Elong;
  2661. /* Get '/' right */
  2662. retval = end-1;
  2663. *retval = '/';
  2664. read_seqbegin_or_lock(&rename_lock, &seq);
  2665. while (!IS_ROOT(dentry)) {
  2666. struct dentry *parent = dentry->d_parent;
  2667. int error;
  2668. prefetch(parent);
  2669. error = prepend_name(&end, &len, &dentry->d_name);
  2670. if (error)
  2671. break;
  2672. retval = end;
  2673. dentry = parent;
  2674. }
  2675. if (!(seq & 1))
  2676. rcu_read_unlock();
  2677. if (need_seqretry(&rename_lock, seq)) {
  2678. seq = 1;
  2679. goto restart;
  2680. }
  2681. done_seqretry(&rename_lock, seq);
  2682. if (error)
  2683. goto Elong;
  2684. return retval;
  2685. Elong:
  2686. return ERR_PTR(-ENAMETOOLONG);
  2687. }
  2688. char *dentry_path_raw(struct dentry *dentry, char *buf, int buflen)
  2689. {
  2690. return __dentry_path(dentry, buf, buflen);
  2691. }
  2692. EXPORT_SYMBOL(dentry_path_raw);
  2693. char *dentry_path(struct dentry *dentry, char *buf, int buflen)
  2694. {
  2695. char *p = NULL;
  2696. char *retval;
  2697. if (d_unlinked(dentry)) {
  2698. p = buf + buflen;
  2699. if (prepend(&p, &buflen, "//deleted", 10) != 0)
  2700. goto Elong;
  2701. buflen++;
  2702. }
  2703. retval = __dentry_path(dentry, buf, buflen);
  2704. if (!IS_ERR(retval) && p)
  2705. *p = '/'; /* restore '/' overriden with '\0' */
  2706. return retval;
  2707. Elong:
  2708. return ERR_PTR(-ENAMETOOLONG);
  2709. }
  2710. /*
  2711. * NOTE! The user-level library version returns a
  2712. * character pointer. The kernel system call just
  2713. * returns the length of the buffer filled (which
  2714. * includes the ending '\0' character), or a negative
  2715. * error value. So libc would do something like
  2716. *
  2717. * char *getcwd(char * buf, size_t size)
  2718. * {
  2719. * int retval;
  2720. *
  2721. * retval = sys_getcwd(buf, size);
  2722. * if (retval >= 0)
  2723. * return buf;
  2724. * errno = -retval;
  2725. * return NULL;
  2726. * }
  2727. */
  2728. SYSCALL_DEFINE2(getcwd, char __user *, buf, unsigned long, size)
  2729. {
  2730. int error;
  2731. struct path pwd, root;
  2732. char *page = (char *) __get_free_page(GFP_USER);
  2733. if (!page)
  2734. return -ENOMEM;
  2735. get_fs_root_and_pwd(current->fs, &root, &pwd);
  2736. error = -ENOENT;
  2737. br_read_lock(&vfsmount_lock);
  2738. if (!d_unlinked(pwd.dentry)) {
  2739. unsigned long len;
  2740. char *cwd = page + PAGE_SIZE;
  2741. int buflen = PAGE_SIZE;
  2742. prepend(&cwd, &buflen, "\0", 1);
  2743. error = prepend_path(&pwd, &root, &cwd, &buflen);
  2744. br_read_unlock(&vfsmount_lock);
  2745. if (error < 0)
  2746. goto out;
  2747. /* Unreachable from current root */
  2748. if (error > 0) {
  2749. error = prepend_unreachable(&cwd, &buflen);
  2750. if (error)
  2751. goto out;
  2752. }
  2753. error = -ERANGE;
  2754. len = PAGE_SIZE + page - cwd;
  2755. if (len <= size) {
  2756. error = len;
  2757. if (copy_to_user(buf, cwd, len))
  2758. error = -EFAULT;
  2759. }
  2760. } else {
  2761. br_read_unlock(&vfsmount_lock);
  2762. }
  2763. out:
  2764. path_put(&pwd);
  2765. path_put(&root);
  2766. free_page((unsigned long) page);
  2767. return error;
  2768. }
  2769. /*
  2770. * Test whether new_dentry is a subdirectory of old_dentry.
  2771. *
  2772. * Trivially implemented using the dcache structure
  2773. */
  2774. /**
  2775. * is_subdir - is new dentry a subdirectory of old_dentry
  2776. * @new_dentry: new dentry
  2777. * @old_dentry: old dentry
  2778. *
  2779. * Returns 1 if new_dentry is a subdirectory of the parent (at any depth).
  2780. * Returns 0 otherwise.
  2781. * Caller must ensure that "new_dentry" is pinned before calling is_subdir()
  2782. */
  2783. int is_subdir(struct dentry *new_dentry, struct dentry *old_dentry)
  2784. {
  2785. int result;
  2786. unsigned seq;
  2787. if (new_dentry == old_dentry)
  2788. return 1;
  2789. do {
  2790. /* for restarting inner loop in case of seq retry */
  2791. seq = read_seqbegin(&rename_lock);
  2792. /*
  2793. * Need rcu_readlock to protect against the d_parent trashing
  2794. * due to d_move
  2795. */
  2796. rcu_read_lock();
  2797. if (d_ancestor(old_dentry, new_dentry))
  2798. result = 1;
  2799. else
  2800. result = 0;
  2801. rcu_read_unlock();
  2802. } while (read_seqretry(&rename_lock, seq));
  2803. return result;
  2804. }
  2805. static enum d_walk_ret d_genocide_kill(void *data, struct dentry *dentry)
  2806. {
  2807. struct dentry *root = data;
  2808. if (dentry != root) {
  2809. if (d_unhashed(dentry) || !dentry->d_inode)
  2810. return D_WALK_SKIP;
  2811. if (!(dentry->d_flags & DCACHE_GENOCIDE)) {
  2812. dentry->d_flags |= DCACHE_GENOCIDE;
  2813. dentry->d_lockref.count--;
  2814. }
  2815. }
  2816. return D_WALK_CONTINUE;
  2817. }
  2818. void d_genocide(struct dentry *parent)
  2819. {
  2820. d_walk(parent, parent, d_genocide_kill, NULL);
  2821. }
  2822. void d_tmpfile(struct dentry *dentry, struct inode *inode)
  2823. {
  2824. inode_dec_link_count(inode);
  2825. BUG_ON(dentry->d_name.name != dentry->d_iname ||
  2826. !hlist_unhashed(&dentry->d_alias) ||
  2827. !d_unlinked(dentry));
  2828. spin_lock(&dentry->d_parent->d_lock);
  2829. spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
  2830. dentry->d_name.len = sprintf(dentry->d_iname, "#%llu",
  2831. (unsigned long long)inode->i_ino);
  2832. spin_unlock(&dentry->d_lock);
  2833. spin_unlock(&dentry->d_parent->d_lock);
  2834. d_instantiate(dentry, inode);
  2835. }
  2836. EXPORT_SYMBOL(d_tmpfile);
  2837. static __initdata unsigned long dhash_entries;
  2838. static int __init set_dhash_entries(char *str)
  2839. {
  2840. if (!str)
  2841. return 0;
  2842. dhash_entries = simple_strtoul(str, &str, 0);
  2843. return 1;
  2844. }
  2845. __setup("dhash_entries=", set_dhash_entries);
  2846. static void __init dcache_init_early(void)
  2847. {
  2848. unsigned int loop;
  2849. /* If hashes are distributed across NUMA nodes, defer
  2850. * hash allocation until vmalloc space is available.
  2851. */
  2852. if (hashdist)
  2853. return;
  2854. dentry_hashtable =
  2855. alloc_large_system_hash("Dentry cache",
  2856. sizeof(struct hlist_bl_head),
  2857. dhash_entries,
  2858. 13,
  2859. HASH_EARLY,
  2860. &d_hash_shift,
  2861. &d_hash_mask,
  2862. 0,
  2863. 0);
  2864. for (loop = 0; loop < (1U << d_hash_shift); loop++)
  2865. INIT_HLIST_BL_HEAD(dentry_hashtable + loop);
  2866. }
  2867. static void __init dcache_init(void)
  2868. {
  2869. unsigned int loop;
  2870. /*
  2871. * A constructor could be added for stable state like the lists,
  2872. * but it is probably not worth it because of the cache nature
  2873. * of the dcache.
  2874. */
  2875. dentry_cache = KMEM_CACHE(dentry,
  2876. SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|SLAB_MEM_SPREAD);
  2877. /* Hash may have been set up in dcache_init_early */
  2878. if (!hashdist)
  2879. return;
  2880. dentry_hashtable =
  2881. alloc_large_system_hash("Dentry cache",
  2882. sizeof(struct hlist_bl_head),
  2883. dhash_entries,
  2884. 13,
  2885. 0,
  2886. &d_hash_shift,
  2887. &d_hash_mask,
  2888. 0,
  2889. 0);
  2890. for (loop = 0; loop < (1U << d_hash_shift); loop++)
  2891. INIT_HLIST_BL_HEAD(dentry_hashtable + loop);
  2892. }
  2893. /* SLAB cache for __getname() consumers */
  2894. struct kmem_cache *names_cachep __read_mostly;
  2895. EXPORT_SYMBOL(names_cachep);
  2896. EXPORT_SYMBOL(d_genocide);
  2897. void __init vfs_caches_init_early(void)
  2898. {
  2899. dcache_init_early();
  2900. inode_init_early();
  2901. }
  2902. void __init vfs_caches_init(unsigned long mempages)
  2903. {
  2904. unsigned long reserve;
  2905. /* Base hash sizes on available memory, with a reserve equal to
  2906. 150% of current kernel size */
  2907. reserve = min((mempages - nr_free_pages()) * 3/2, mempages - 1);
  2908. mempages -= reserve;
  2909. names_cachep = kmem_cache_create("names_cache", PATH_MAX, 0,
  2910. SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
  2911. dcache_init();
  2912. inode_init();
  2913. files_init(mempages);
  2914. mnt_init();
  2915. bdev_cache_init();
  2916. chrdev_init();
  2917. }