dcache.c 89 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479
  1. /*
  2. * fs/dcache.c
  3. *
  4. * Complete reimplementation
  5. * (C) 1997 Thomas Schoebel-Theuer,
  6. * with heavy changes by Linus Torvalds
  7. */
  8. /*
  9. * Notes on the allocation strategy:
  10. *
  11. * The dcache is a master of the icache - whenever a dcache entry
  12. * exists, the inode will always exist. "iput()" is done either when
  13. * the dcache entry is deleted or garbage collected.
  14. */
  15. #include <linux/syscalls.h>
  16. #include <linux/string.h>
  17. #include <linux/mm.h>
  18. #include <linux/fs.h>
  19. #include <linux/fsnotify.h>
  20. #include <linux/slab.h>
  21. #include <linux/init.h>
  22. #include <linux/hash.h>
  23. #include <linux/cache.h>
  24. #include <linux/export.h>
  25. #include <linux/mount.h>
  26. #include <linux/file.h>
  27. #include <asm/uaccess.h>
  28. #include <linux/security.h>
  29. #include <linux/seqlock.h>
  30. #include <linux/swap.h>
  31. #include <linux/bootmem.h>
  32. #include <linux/fs_struct.h>
  33. #include <linux/hardirq.h>
  34. #include <linux/bit_spinlock.h>
  35. #include <linux/rculist_bl.h>
  36. #include <linux/prefetch.h>
  37. #include <linux/ratelimit.h>
  38. #include <linux/list_lru.h>
  39. #include "internal.h"
  40. #include "mount.h"
  41. /*
  42. * Usage:
  43. * dcache->d_inode->i_lock protects:
  44. * - i_dentry, d_alias, d_inode of aliases
  45. * dcache_hash_bucket lock protects:
  46. * - the dcache hash table
  47. * s_anon bl list spinlock protects:
  48. * - the s_anon list (see __d_drop)
  49. * dentry->d_sb->s_dentry_lru_lock protects:
  50. * - the dcache lru lists and counters
  51. * d_lock protects:
  52. * - d_flags
  53. * - d_name
  54. * - d_lru
  55. * - d_count
  56. * - d_unhashed()
  57. * - d_parent and d_subdirs
  58. * - childrens' d_child and d_parent
  59. * - d_alias, d_inode
  60. *
  61. * Ordering:
  62. * dentry->d_inode->i_lock
  63. * dentry->d_lock
  64. * dentry->d_sb->s_dentry_lru_lock
  65. * dcache_hash_bucket lock
  66. * s_anon lock
  67. *
  68. * If there is an ancestor relationship:
  69. * dentry->d_parent->...->d_parent->d_lock
  70. * ...
  71. * dentry->d_parent->d_lock
  72. * dentry->d_lock
  73. *
  74. * If no ancestor relationship:
  75. * if (dentry1 < dentry2)
  76. * dentry1->d_lock
  77. * dentry2->d_lock
  78. */
  79. int sysctl_vfs_cache_pressure __read_mostly = 100;
  80. EXPORT_SYMBOL_GPL(sysctl_vfs_cache_pressure);
  81. __cacheline_aligned_in_smp DEFINE_SEQLOCK(rename_lock);
  82. EXPORT_SYMBOL(rename_lock);
  83. static struct kmem_cache *dentry_cache __read_mostly;
  84. /**
  85. * read_seqbegin_or_lock - begin a sequence number check or locking block
  86. * @lock: sequence lock
  87. * @seq : sequence number to be checked
  88. *
  89. * First try it once optimistically without taking the lock. If that fails,
  90. * take the lock. The sequence number is also used as a marker for deciding
  91. * whether to be a reader (even) or writer (odd).
  92. * N.B. seq must be initialized to an even number to begin with.
  93. */
  94. static inline void read_seqbegin_or_lock(seqlock_t *lock, int *seq)
  95. {
  96. if (!(*seq & 1)) /* Even */
  97. *seq = read_seqbegin(lock);
  98. else /* Odd */
  99. read_seqlock_excl(lock);
  100. }
  101. static inline int need_seqretry(seqlock_t *lock, int seq)
  102. {
  103. return !(seq & 1) && read_seqretry(lock, seq);
  104. }
  105. static inline void done_seqretry(seqlock_t *lock, int seq)
  106. {
  107. if (seq & 1)
  108. read_sequnlock_excl(lock);
  109. }
  110. /*
  111. * This is the single most critical data structure when it comes
  112. * to the dcache: the hashtable for lookups. Somebody should try
  113. * to make this good - I've just made it work.
  114. *
  115. * This hash-function tries to avoid losing too many bits of hash
  116. * information, yet avoid using a prime hash-size or similar.
  117. */
  118. #define D_HASHBITS d_hash_shift
  119. #define D_HASHMASK d_hash_mask
  120. static unsigned int d_hash_mask __read_mostly;
  121. static unsigned int d_hash_shift __read_mostly;
  122. static struct hlist_bl_head *dentry_hashtable __read_mostly;
  123. static inline struct hlist_bl_head *d_hash(const struct dentry *parent,
  124. unsigned int hash)
  125. {
  126. hash += (unsigned long) parent / L1_CACHE_BYTES;
  127. hash = hash + (hash >> D_HASHBITS);
  128. return dentry_hashtable + (hash & D_HASHMASK);
  129. }
  130. /* Statistics gathering. */
  131. struct dentry_stat_t dentry_stat = {
  132. .age_limit = 45,
  133. };
  134. static DEFINE_PER_CPU(long, nr_dentry);
  135. static DEFINE_PER_CPU(long, nr_dentry_unused);
  136. #if defined(CONFIG_SYSCTL) && defined(CONFIG_PROC_FS)
  137. /*
  138. * Here we resort to our own counters instead of using generic per-cpu counters
  139. * for consistency with what the vfs inode code does. We are expected to harvest
  140. * better code and performance by having our own specialized counters.
  141. *
  142. * Please note that the loop is done over all possible CPUs, not over all online
  143. * CPUs. The reason for this is that we don't want to play games with CPUs going
  144. * on and off. If one of them goes off, we will just keep their counters.
  145. *
  146. * glommer: See cffbc8a for details, and if you ever intend to change this,
  147. * please update all vfs counters to match.
  148. */
  149. static long get_nr_dentry(void)
  150. {
  151. int i;
  152. long sum = 0;
  153. for_each_possible_cpu(i)
  154. sum += per_cpu(nr_dentry, i);
  155. return sum < 0 ? 0 : sum;
  156. }
  157. static long get_nr_dentry_unused(void)
  158. {
  159. int i;
  160. long sum = 0;
  161. for_each_possible_cpu(i)
  162. sum += per_cpu(nr_dentry_unused, i);
  163. return sum < 0 ? 0 : sum;
  164. }
  165. int proc_nr_dentry(ctl_table *table, int write, void __user *buffer,
  166. size_t *lenp, loff_t *ppos)
  167. {
  168. dentry_stat.nr_dentry = get_nr_dentry();
  169. dentry_stat.nr_unused = get_nr_dentry_unused();
  170. return proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
  171. }
  172. #endif
  173. /*
  174. * Compare 2 name strings, return 0 if they match, otherwise non-zero.
  175. * The strings are both count bytes long, and count is non-zero.
  176. */
  177. #ifdef CONFIG_DCACHE_WORD_ACCESS
  178. #include <asm/word-at-a-time.h>
  179. /*
  180. * NOTE! 'cs' and 'scount' come from a dentry, so it has a
  181. * aligned allocation for this particular component. We don't
  182. * strictly need the load_unaligned_zeropad() safety, but it
  183. * doesn't hurt either.
  184. *
  185. * In contrast, 'ct' and 'tcount' can be from a pathname, and do
  186. * need the careful unaligned handling.
  187. */
  188. static inline int dentry_string_cmp(const unsigned char *cs, const unsigned char *ct, unsigned tcount)
  189. {
  190. unsigned long a,b,mask;
  191. for (;;) {
  192. a = *(unsigned long *)cs;
  193. b = load_unaligned_zeropad(ct);
  194. if (tcount < sizeof(unsigned long))
  195. break;
  196. if (unlikely(a != b))
  197. return 1;
  198. cs += sizeof(unsigned long);
  199. ct += sizeof(unsigned long);
  200. tcount -= sizeof(unsigned long);
  201. if (!tcount)
  202. return 0;
  203. }
  204. mask = ~(~0ul << tcount*8);
  205. return unlikely(!!((a ^ b) & mask));
  206. }
  207. #else
  208. static inline int dentry_string_cmp(const unsigned char *cs, const unsigned char *ct, unsigned tcount)
  209. {
  210. do {
  211. if (*cs != *ct)
  212. return 1;
  213. cs++;
  214. ct++;
  215. tcount--;
  216. } while (tcount);
  217. return 0;
  218. }
  219. #endif
  220. static inline int dentry_cmp(const struct dentry *dentry, const unsigned char *ct, unsigned tcount)
  221. {
  222. const unsigned char *cs;
  223. /*
  224. * Be careful about RCU walk racing with rename:
  225. * use ACCESS_ONCE to fetch the name pointer.
  226. *
  227. * NOTE! Even if a rename will mean that the length
  228. * was not loaded atomically, we don't care. The
  229. * RCU walk will check the sequence count eventually,
  230. * and catch it. And we won't overrun the buffer,
  231. * because we're reading the name pointer atomically,
  232. * and a dentry name is guaranteed to be properly
  233. * terminated with a NUL byte.
  234. *
  235. * End result: even if 'len' is wrong, we'll exit
  236. * early because the data cannot match (there can
  237. * be no NUL in the ct/tcount data)
  238. */
  239. cs = ACCESS_ONCE(dentry->d_name.name);
  240. smp_read_barrier_depends();
  241. return dentry_string_cmp(cs, ct, tcount);
  242. }
  243. static void __d_free(struct rcu_head *head)
  244. {
  245. struct dentry *dentry = container_of(head, struct dentry, d_u.d_rcu);
  246. WARN_ON(!hlist_unhashed(&dentry->d_alias));
  247. if (dname_external(dentry))
  248. kfree(dentry->d_name.name);
  249. kmem_cache_free(dentry_cache, dentry);
  250. }
  251. /*
  252. * no locks, please.
  253. */
  254. static void d_free(struct dentry *dentry)
  255. {
  256. BUG_ON((int)dentry->d_lockref.count > 0);
  257. this_cpu_dec(nr_dentry);
  258. if (dentry->d_op && dentry->d_op->d_release)
  259. dentry->d_op->d_release(dentry);
  260. /* if dentry was never visible to RCU, immediate free is OK */
  261. if (!(dentry->d_flags & DCACHE_RCUACCESS))
  262. __d_free(&dentry->d_u.d_rcu);
  263. else
  264. call_rcu(&dentry->d_u.d_rcu, __d_free);
  265. }
  266. /**
  267. * dentry_rcuwalk_barrier - invalidate in-progress rcu-walk lookups
  268. * @dentry: the target dentry
  269. * After this call, in-progress rcu-walk path lookup will fail. This
  270. * should be called after unhashing, and after changing d_inode (if
  271. * the dentry has not already been unhashed).
  272. */
  273. static inline void dentry_rcuwalk_barrier(struct dentry *dentry)
  274. {
  275. assert_spin_locked(&dentry->d_lock);
  276. /* Go through a barrier */
  277. write_seqcount_barrier(&dentry->d_seq);
  278. }
  279. /*
  280. * Release the dentry's inode, using the filesystem
  281. * d_iput() operation if defined. Dentry has no refcount
  282. * and is unhashed.
  283. */
  284. static void dentry_iput(struct dentry * dentry)
  285. __releases(dentry->d_lock)
  286. __releases(dentry->d_inode->i_lock)
  287. {
  288. struct inode *inode = dentry->d_inode;
  289. if (inode) {
  290. dentry->d_inode = NULL;
  291. hlist_del_init(&dentry->d_alias);
  292. spin_unlock(&dentry->d_lock);
  293. spin_unlock(&inode->i_lock);
  294. if (!inode->i_nlink)
  295. fsnotify_inoderemove(inode);
  296. if (dentry->d_op && dentry->d_op->d_iput)
  297. dentry->d_op->d_iput(dentry, inode);
  298. else
  299. iput(inode);
  300. } else {
  301. spin_unlock(&dentry->d_lock);
  302. }
  303. }
  304. /*
  305. * Release the dentry's inode, using the filesystem
  306. * d_iput() operation if defined. dentry remains in-use.
  307. */
  308. static void dentry_unlink_inode(struct dentry * dentry)
  309. __releases(dentry->d_lock)
  310. __releases(dentry->d_inode->i_lock)
  311. {
  312. struct inode *inode = dentry->d_inode;
  313. __d_clear_type(dentry);
  314. dentry->d_inode = NULL;
  315. hlist_del_init(&dentry->d_alias);
  316. dentry_rcuwalk_barrier(dentry);
  317. spin_unlock(&dentry->d_lock);
  318. spin_unlock(&inode->i_lock);
  319. if (!inode->i_nlink)
  320. fsnotify_inoderemove(inode);
  321. if (dentry->d_op && dentry->d_op->d_iput)
  322. dentry->d_op->d_iput(dentry, inode);
  323. else
  324. iput(inode);
  325. }
  326. /*
  327. * The DCACHE_LRU_LIST bit is set whenever the 'd_lru' entry
  328. * is in use - which includes both the "real" per-superblock
  329. * LRU list _and_ the DCACHE_SHRINK_LIST use.
  330. *
  331. * The DCACHE_SHRINK_LIST bit is set whenever the dentry is
  332. * on the shrink list (ie not on the superblock LRU list).
  333. *
  334. * The per-cpu "nr_dentry_unused" counters are updated with
  335. * the DCACHE_LRU_LIST bit.
  336. *
  337. * These helper functions make sure we always follow the
  338. * rules. d_lock must be held by the caller.
  339. */
  340. #define D_FLAG_VERIFY(dentry,x) WARN_ON_ONCE(((dentry)->d_flags & (DCACHE_LRU_LIST | DCACHE_SHRINK_LIST)) != (x))
  341. static void d_lru_add(struct dentry *dentry)
  342. {
  343. D_FLAG_VERIFY(dentry, 0);
  344. dentry->d_flags |= DCACHE_LRU_LIST;
  345. this_cpu_inc(nr_dentry_unused);
  346. WARN_ON_ONCE(!list_lru_add(&dentry->d_sb->s_dentry_lru, &dentry->d_lru));
  347. }
  348. static void d_lru_del(struct dentry *dentry)
  349. {
  350. D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);
  351. dentry->d_flags &= ~DCACHE_LRU_LIST;
  352. this_cpu_dec(nr_dentry_unused);
  353. WARN_ON_ONCE(!list_lru_del(&dentry->d_sb->s_dentry_lru, &dentry->d_lru));
  354. }
  355. static void d_shrink_del(struct dentry *dentry)
  356. {
  357. D_FLAG_VERIFY(dentry, DCACHE_SHRINK_LIST | DCACHE_LRU_LIST);
  358. list_del_init(&dentry->d_lru);
  359. dentry->d_flags &= ~(DCACHE_SHRINK_LIST | DCACHE_LRU_LIST);
  360. this_cpu_dec(nr_dentry_unused);
  361. }
  362. static void d_shrink_add(struct dentry *dentry, struct list_head *list)
  363. {
  364. D_FLAG_VERIFY(dentry, 0);
  365. list_add(&dentry->d_lru, list);
  366. dentry->d_flags |= DCACHE_SHRINK_LIST | DCACHE_LRU_LIST;
  367. this_cpu_inc(nr_dentry_unused);
  368. }
  369. /*
  370. * These can only be called under the global LRU lock, ie during the
  371. * callback for freeing the LRU list. "isolate" removes it from the
  372. * LRU lists entirely, while shrink_move moves it to the indicated
  373. * private list.
  374. */
  375. static void d_lru_isolate(struct dentry *dentry)
  376. {
  377. D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);
  378. dentry->d_flags &= ~DCACHE_LRU_LIST;
  379. this_cpu_dec(nr_dentry_unused);
  380. list_del_init(&dentry->d_lru);
  381. }
  382. static void d_lru_shrink_move(struct dentry *dentry, struct list_head *list)
  383. {
  384. D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);
  385. dentry->d_flags |= DCACHE_SHRINK_LIST;
  386. list_move_tail(&dentry->d_lru, list);
  387. }
  388. /*
  389. * dentry_lru_(add|del)_list) must be called with d_lock held.
  390. */
  391. static void dentry_lru_add(struct dentry *dentry)
  392. {
  393. if (unlikely(!(dentry->d_flags & DCACHE_LRU_LIST)))
  394. d_lru_add(dentry);
  395. }
  396. /*
  397. * Remove a dentry with references from the LRU.
  398. *
  399. * If we are on the shrink list, then we can get to try_prune_one_dentry() and
  400. * lose our last reference through the parent walk. In this case, we need to
  401. * remove ourselves from the shrink list, not the LRU.
  402. */
  403. static void dentry_lru_del(struct dentry *dentry)
  404. {
  405. if (dentry->d_flags & DCACHE_LRU_LIST) {
  406. if (dentry->d_flags & DCACHE_SHRINK_LIST)
  407. return d_shrink_del(dentry);
  408. d_lru_del(dentry);
  409. }
  410. }
  411. /**
  412. * d_kill - kill dentry and return parent
  413. * @dentry: dentry to kill
  414. * @parent: parent dentry
  415. *
  416. * The dentry must already be unhashed and removed from the LRU.
  417. *
  418. * If this is the root of the dentry tree, return NULL.
  419. *
  420. * dentry->d_lock and parent->d_lock must be held by caller, and are dropped by
  421. * d_kill.
  422. */
  423. static struct dentry *d_kill(struct dentry *dentry, struct dentry *parent)
  424. __releases(dentry->d_lock)
  425. __releases(parent->d_lock)
  426. __releases(dentry->d_inode->i_lock)
  427. {
  428. list_del(&dentry->d_u.d_child);
  429. /*
  430. * Inform try_to_ascend() that we are no longer attached to the
  431. * dentry tree
  432. */
  433. dentry->d_flags |= DCACHE_DENTRY_KILLED;
  434. if (parent)
  435. spin_unlock(&parent->d_lock);
  436. dentry_iput(dentry);
  437. /*
  438. * dentry_iput drops the locks, at which point nobody (except
  439. * transient RCU lookups) can reach this dentry.
  440. */
  441. d_free(dentry);
  442. return parent;
  443. }
  444. /**
  445. * d_drop - drop a dentry
  446. * @dentry: dentry to drop
  447. *
  448. * d_drop() unhashes the entry from the parent dentry hashes, so that it won't
  449. * be found through a VFS lookup any more. Note that this is different from
  450. * deleting the dentry - d_delete will try to mark the dentry negative if
  451. * possible, giving a successful _negative_ lookup, while d_drop will
  452. * just make the cache lookup fail.
  453. *
  454. * d_drop() is used mainly for stuff that wants to invalidate a dentry for some
  455. * reason (NFS timeouts or autofs deletes).
  456. *
  457. * __d_drop requires dentry->d_lock.
  458. */
  459. void __d_drop(struct dentry *dentry)
  460. {
  461. if (!d_unhashed(dentry)) {
  462. struct hlist_bl_head *b;
  463. /*
  464. * Hashed dentries are normally on the dentry hashtable,
  465. * with the exception of those newly allocated by
  466. * d_obtain_alias, which are always IS_ROOT:
  467. */
  468. if (unlikely(IS_ROOT(dentry)))
  469. b = &dentry->d_sb->s_anon;
  470. else
  471. b = d_hash(dentry->d_parent, dentry->d_name.hash);
  472. hlist_bl_lock(b);
  473. __hlist_bl_del(&dentry->d_hash);
  474. dentry->d_hash.pprev = NULL;
  475. hlist_bl_unlock(b);
  476. dentry_rcuwalk_barrier(dentry);
  477. }
  478. }
  479. EXPORT_SYMBOL(__d_drop);
  480. void d_drop(struct dentry *dentry)
  481. {
  482. spin_lock(&dentry->d_lock);
  483. __d_drop(dentry);
  484. spin_unlock(&dentry->d_lock);
  485. }
  486. EXPORT_SYMBOL(d_drop);
  487. /*
  488. * Finish off a dentry we've decided to kill.
  489. * dentry->d_lock must be held, returns with it unlocked.
  490. * If ref is non-zero, then decrement the refcount too.
  491. * Returns dentry requiring refcount drop, or NULL if we're done.
  492. */
  493. static struct dentry *
  494. dentry_kill(struct dentry *dentry, int unlock_on_failure)
  495. __releases(dentry->d_lock)
  496. {
  497. struct inode *inode;
  498. struct dentry *parent;
  499. inode = dentry->d_inode;
  500. if (inode && !spin_trylock(&inode->i_lock)) {
  501. relock:
  502. if (unlock_on_failure) {
  503. spin_unlock(&dentry->d_lock);
  504. cpu_relax();
  505. }
  506. return dentry; /* try again with same dentry */
  507. }
  508. if (IS_ROOT(dentry))
  509. parent = NULL;
  510. else
  511. parent = dentry->d_parent;
  512. if (parent && !spin_trylock(&parent->d_lock)) {
  513. if (inode)
  514. spin_unlock(&inode->i_lock);
  515. goto relock;
  516. }
  517. /*
  518. * The dentry is now unrecoverably dead to the world.
  519. */
  520. lockref_mark_dead(&dentry->d_lockref);
  521. /*
  522. * inform the fs via d_prune that this dentry is about to be
  523. * unhashed and destroyed.
  524. */
  525. if ((dentry->d_flags & DCACHE_OP_PRUNE) && !d_unhashed(dentry))
  526. dentry->d_op->d_prune(dentry);
  527. dentry_lru_del(dentry);
  528. /* if it was on the hash then remove it */
  529. __d_drop(dentry);
  530. return d_kill(dentry, parent);
  531. }
  532. /*
  533. * This is dput
  534. *
  535. * This is complicated by the fact that we do not want to put
  536. * dentries that are no longer on any hash chain on the unused
  537. * list: we'd much rather just get rid of them immediately.
  538. *
  539. * However, that implies that we have to traverse the dentry
  540. * tree upwards to the parents which might _also_ now be
  541. * scheduled for deletion (it may have been only waiting for
  542. * its last child to go away).
  543. *
  544. * This tail recursion is done by hand as we don't want to depend
  545. * on the compiler to always get this right (gcc generally doesn't).
  546. * Real recursion would eat up our stack space.
  547. */
  548. /*
  549. * dput - release a dentry
  550. * @dentry: dentry to release
  551. *
  552. * Release a dentry. This will drop the usage count and if appropriate
  553. * call the dentry unlink method as well as removing it from the queues and
  554. * releasing its resources. If the parent dentries were scheduled for release
  555. * they too may now get deleted.
  556. */
  557. void dput(struct dentry *dentry)
  558. {
  559. if (unlikely(!dentry))
  560. return;
  561. repeat:
  562. if (lockref_put_or_lock(&dentry->d_lockref))
  563. return;
  564. /* Unreachable? Get rid of it */
  565. if (unlikely(d_unhashed(dentry)))
  566. goto kill_it;
  567. if (unlikely(dentry->d_flags & DCACHE_OP_DELETE)) {
  568. if (dentry->d_op->d_delete(dentry))
  569. goto kill_it;
  570. }
  571. if (!(dentry->d_flags & DCACHE_REFERENCED))
  572. dentry->d_flags |= DCACHE_REFERENCED;
  573. dentry_lru_add(dentry);
  574. dentry->d_lockref.count--;
  575. spin_unlock(&dentry->d_lock);
  576. return;
  577. kill_it:
  578. dentry = dentry_kill(dentry, 1);
  579. if (dentry)
  580. goto repeat;
  581. }
  582. EXPORT_SYMBOL(dput);
  583. /**
  584. * d_invalidate - invalidate a dentry
  585. * @dentry: dentry to invalidate
  586. *
  587. * Try to invalidate the dentry if it turns out to be
  588. * possible. If there are other dentries that can be
  589. * reached through this one we can't delete it and we
  590. * return -EBUSY. On success we return 0.
  591. *
  592. * no dcache lock.
  593. */
  594. int d_invalidate(struct dentry * dentry)
  595. {
  596. /*
  597. * If it's already been dropped, return OK.
  598. */
  599. spin_lock(&dentry->d_lock);
  600. if (d_unhashed(dentry)) {
  601. spin_unlock(&dentry->d_lock);
  602. return 0;
  603. }
  604. /*
  605. * Check whether to do a partial shrink_dcache
  606. * to get rid of unused child entries.
  607. */
  608. if (!list_empty(&dentry->d_subdirs)) {
  609. spin_unlock(&dentry->d_lock);
  610. shrink_dcache_parent(dentry);
  611. spin_lock(&dentry->d_lock);
  612. }
  613. /*
  614. * Somebody else still using it?
  615. *
  616. * If it's a directory, we can't drop it
  617. * for fear of somebody re-populating it
  618. * with children (even though dropping it
  619. * would make it unreachable from the root,
  620. * we might still populate it if it was a
  621. * working directory or similar).
  622. * We also need to leave mountpoints alone,
  623. * directory or not.
  624. */
  625. if (dentry->d_lockref.count > 1 && dentry->d_inode) {
  626. if (S_ISDIR(dentry->d_inode->i_mode) || d_mountpoint(dentry)) {
  627. spin_unlock(&dentry->d_lock);
  628. return -EBUSY;
  629. }
  630. }
  631. __d_drop(dentry);
  632. spin_unlock(&dentry->d_lock);
  633. return 0;
  634. }
  635. EXPORT_SYMBOL(d_invalidate);
  636. /* This must be called with d_lock held */
  637. static inline void __dget_dlock(struct dentry *dentry)
  638. {
  639. dentry->d_lockref.count++;
  640. }
  641. static inline void __dget(struct dentry *dentry)
  642. {
  643. lockref_get(&dentry->d_lockref);
  644. }
  645. struct dentry *dget_parent(struct dentry *dentry)
  646. {
  647. int gotref;
  648. struct dentry *ret;
  649. /*
  650. * Do optimistic parent lookup without any
  651. * locking.
  652. */
  653. rcu_read_lock();
  654. ret = ACCESS_ONCE(dentry->d_parent);
  655. gotref = lockref_get_not_zero(&ret->d_lockref);
  656. rcu_read_unlock();
  657. if (likely(gotref)) {
  658. if (likely(ret == ACCESS_ONCE(dentry->d_parent)))
  659. return ret;
  660. dput(ret);
  661. }
  662. repeat:
  663. /*
  664. * Don't need rcu_dereference because we re-check it was correct under
  665. * the lock.
  666. */
  667. rcu_read_lock();
  668. ret = dentry->d_parent;
  669. spin_lock(&ret->d_lock);
  670. if (unlikely(ret != dentry->d_parent)) {
  671. spin_unlock(&ret->d_lock);
  672. rcu_read_unlock();
  673. goto repeat;
  674. }
  675. rcu_read_unlock();
  676. BUG_ON(!ret->d_lockref.count);
  677. ret->d_lockref.count++;
  678. spin_unlock(&ret->d_lock);
  679. return ret;
  680. }
  681. EXPORT_SYMBOL(dget_parent);
  682. /**
  683. * d_find_alias - grab a hashed alias of inode
  684. * @inode: inode in question
  685. * @want_discon: flag, used by d_splice_alias, to request
  686. * that only a DISCONNECTED alias be returned.
  687. *
  688. * If inode has a hashed alias, or is a directory and has any alias,
  689. * acquire the reference to alias and return it. Otherwise return NULL.
  690. * Notice that if inode is a directory there can be only one alias and
  691. * it can be unhashed only if it has no children, or if it is the root
  692. * of a filesystem.
  693. *
  694. * If the inode has an IS_ROOT, DCACHE_DISCONNECTED alias, then prefer
  695. * any other hashed alias over that one unless @want_discon is set,
  696. * in which case only return an IS_ROOT, DCACHE_DISCONNECTED alias.
  697. */
  698. static struct dentry *__d_find_alias(struct inode *inode, int want_discon)
  699. {
  700. struct dentry *alias, *discon_alias;
  701. again:
  702. discon_alias = NULL;
  703. hlist_for_each_entry(alias, &inode->i_dentry, d_alias) {
  704. spin_lock(&alias->d_lock);
  705. if (S_ISDIR(inode->i_mode) || !d_unhashed(alias)) {
  706. if (IS_ROOT(alias) &&
  707. (alias->d_flags & DCACHE_DISCONNECTED)) {
  708. discon_alias = alias;
  709. } else if (!want_discon) {
  710. __dget_dlock(alias);
  711. spin_unlock(&alias->d_lock);
  712. return alias;
  713. }
  714. }
  715. spin_unlock(&alias->d_lock);
  716. }
  717. if (discon_alias) {
  718. alias = discon_alias;
  719. spin_lock(&alias->d_lock);
  720. if (S_ISDIR(inode->i_mode) || !d_unhashed(alias)) {
  721. if (IS_ROOT(alias) &&
  722. (alias->d_flags & DCACHE_DISCONNECTED)) {
  723. __dget_dlock(alias);
  724. spin_unlock(&alias->d_lock);
  725. return alias;
  726. }
  727. }
  728. spin_unlock(&alias->d_lock);
  729. goto again;
  730. }
  731. return NULL;
  732. }
  733. struct dentry *d_find_alias(struct inode *inode)
  734. {
  735. struct dentry *de = NULL;
  736. if (!hlist_empty(&inode->i_dentry)) {
  737. spin_lock(&inode->i_lock);
  738. de = __d_find_alias(inode, 0);
  739. spin_unlock(&inode->i_lock);
  740. }
  741. return de;
  742. }
  743. EXPORT_SYMBOL(d_find_alias);
  744. /*
  745. * Try to kill dentries associated with this inode.
  746. * WARNING: you must own a reference to inode.
  747. */
  748. void d_prune_aliases(struct inode *inode)
  749. {
  750. struct dentry *dentry;
  751. restart:
  752. spin_lock(&inode->i_lock);
  753. hlist_for_each_entry(dentry, &inode->i_dentry, d_alias) {
  754. spin_lock(&dentry->d_lock);
  755. if (!dentry->d_lockref.count) {
  756. /*
  757. * inform the fs via d_prune that this dentry
  758. * is about to be unhashed and destroyed.
  759. */
  760. if ((dentry->d_flags & DCACHE_OP_PRUNE) &&
  761. !d_unhashed(dentry))
  762. dentry->d_op->d_prune(dentry);
  763. __dget_dlock(dentry);
  764. __d_drop(dentry);
  765. spin_unlock(&dentry->d_lock);
  766. spin_unlock(&inode->i_lock);
  767. dput(dentry);
  768. goto restart;
  769. }
  770. spin_unlock(&dentry->d_lock);
  771. }
  772. spin_unlock(&inode->i_lock);
  773. }
  774. EXPORT_SYMBOL(d_prune_aliases);
  775. /*
  776. * Try to throw away a dentry - free the inode, dput the parent.
  777. * Requires dentry->d_lock is held, and dentry->d_count == 0.
  778. * Releases dentry->d_lock.
  779. *
  780. * This may fail if locks cannot be acquired no problem, just try again.
  781. */
  782. static struct dentry * try_prune_one_dentry(struct dentry *dentry)
  783. __releases(dentry->d_lock)
  784. {
  785. struct dentry *parent;
  786. parent = dentry_kill(dentry, 0);
  787. /*
  788. * If dentry_kill returns NULL, we have nothing more to do.
  789. * if it returns the same dentry, trylocks failed. In either
  790. * case, just loop again.
  791. *
  792. * Otherwise, we need to prune ancestors too. This is necessary
  793. * to prevent quadratic behavior of shrink_dcache_parent(), but
  794. * is also expected to be beneficial in reducing dentry cache
  795. * fragmentation.
  796. */
  797. if (!parent)
  798. return NULL;
  799. if (parent == dentry)
  800. return dentry;
  801. /* Prune ancestors. */
  802. dentry = parent;
  803. while (dentry) {
  804. if (lockref_put_or_lock(&dentry->d_lockref))
  805. return NULL;
  806. dentry = dentry_kill(dentry, 1);
  807. }
  808. return NULL;
  809. }
  810. static void shrink_dentry_list(struct list_head *list)
  811. {
  812. struct dentry *dentry;
  813. rcu_read_lock();
  814. for (;;) {
  815. dentry = list_entry_rcu(list->prev, struct dentry, d_lru);
  816. if (&dentry->d_lru == list)
  817. break; /* empty */
  818. /*
  819. * Get the dentry lock, and re-verify that the dentry is
  820. * this on the shrinking list. If it is, we know that
  821. * DCACHE_SHRINK_LIST and DCACHE_LRU_LIST are set.
  822. */
  823. spin_lock(&dentry->d_lock);
  824. if (dentry != list_entry(list->prev, struct dentry, d_lru)) {
  825. spin_unlock(&dentry->d_lock);
  826. continue;
  827. }
  828. /*
  829. * The dispose list is isolated and dentries are not accounted
  830. * to the LRU here, so we can simply remove it from the list
  831. * here regardless of whether it is referenced or not.
  832. */
  833. d_shrink_del(dentry);
  834. /*
  835. * We found an inuse dentry which was not removed from
  836. * the LRU because of laziness during lookup. Do not free it.
  837. */
  838. if (dentry->d_lockref.count) {
  839. spin_unlock(&dentry->d_lock);
  840. continue;
  841. }
  842. rcu_read_unlock();
  843. /*
  844. * If 'try_to_prune()' returns a dentry, it will
  845. * be the same one we passed in, and d_lock will
  846. * have been held the whole time, so it will not
  847. * have been added to any other lists. We failed
  848. * to get the inode lock.
  849. *
  850. * We just add it back to the shrink list.
  851. */
  852. dentry = try_prune_one_dentry(dentry);
  853. rcu_read_lock();
  854. if (dentry) {
  855. d_shrink_add(dentry, list);
  856. spin_unlock(&dentry->d_lock);
  857. }
  858. }
  859. rcu_read_unlock();
  860. }
  861. static enum lru_status
  862. dentry_lru_isolate(struct list_head *item, spinlock_t *lru_lock, void *arg)
  863. {
  864. struct list_head *freeable = arg;
  865. struct dentry *dentry = container_of(item, struct dentry, d_lru);
  866. /*
  867. * we are inverting the lru lock/dentry->d_lock here,
  868. * so use a trylock. If we fail to get the lock, just skip
  869. * it
  870. */
  871. if (!spin_trylock(&dentry->d_lock))
  872. return LRU_SKIP;
  873. /*
  874. * Referenced dentries are still in use. If they have active
  875. * counts, just remove them from the LRU. Otherwise give them
  876. * another pass through the LRU.
  877. */
  878. if (dentry->d_lockref.count) {
  879. d_lru_isolate(dentry);
  880. spin_unlock(&dentry->d_lock);
  881. return LRU_REMOVED;
  882. }
  883. if (dentry->d_flags & DCACHE_REFERENCED) {
  884. dentry->d_flags &= ~DCACHE_REFERENCED;
  885. spin_unlock(&dentry->d_lock);
  886. /*
  887. * The list move itself will be made by the common LRU code. At
  888. * this point, we've dropped the dentry->d_lock but keep the
  889. * lru lock. This is safe to do, since every list movement is
  890. * protected by the lru lock even if both locks are held.
  891. *
  892. * This is guaranteed by the fact that all LRU management
  893. * functions are intermediated by the LRU API calls like
  894. * list_lru_add and list_lru_del. List movement in this file
  895. * only ever occur through this functions or through callbacks
  896. * like this one, that are called from the LRU API.
  897. *
  898. * The only exceptions to this are functions like
  899. * shrink_dentry_list, and code that first checks for the
  900. * DCACHE_SHRINK_LIST flag. Those are guaranteed to be
  901. * operating only with stack provided lists after they are
  902. * properly isolated from the main list. It is thus, always a
  903. * local access.
  904. */
  905. return LRU_ROTATE;
  906. }
  907. d_lru_shrink_move(dentry, freeable);
  908. spin_unlock(&dentry->d_lock);
  909. return LRU_REMOVED;
  910. }
  911. /**
  912. * prune_dcache_sb - shrink the dcache
  913. * @sb: superblock
  914. * @nr_to_scan : number of entries to try to free
  915. * @nid: which node to scan for freeable entities
  916. *
  917. * Attempt to shrink the superblock dcache LRU by @nr_to_scan entries. This is
  918. * done when we need more memory an called from the superblock shrinker
  919. * function.
  920. *
  921. * This function may fail to free any resources if all the dentries are in
  922. * use.
  923. */
  924. long prune_dcache_sb(struct super_block *sb, unsigned long nr_to_scan,
  925. int nid)
  926. {
  927. LIST_HEAD(dispose);
  928. long freed;
  929. freed = list_lru_walk_node(&sb->s_dentry_lru, nid, dentry_lru_isolate,
  930. &dispose, &nr_to_scan);
  931. shrink_dentry_list(&dispose);
  932. return freed;
  933. }
  934. static enum lru_status dentry_lru_isolate_shrink(struct list_head *item,
  935. spinlock_t *lru_lock, void *arg)
  936. {
  937. struct list_head *freeable = arg;
  938. struct dentry *dentry = container_of(item, struct dentry, d_lru);
  939. /*
  940. * we are inverting the lru lock/dentry->d_lock here,
  941. * so use a trylock. If we fail to get the lock, just skip
  942. * it
  943. */
  944. if (!spin_trylock(&dentry->d_lock))
  945. return LRU_SKIP;
  946. d_lru_shrink_move(dentry, freeable);
  947. spin_unlock(&dentry->d_lock);
  948. return LRU_REMOVED;
  949. }
  950. /**
  951. * shrink_dcache_sb - shrink dcache for a superblock
  952. * @sb: superblock
  953. *
  954. * Shrink the dcache for the specified super block. This is used to free
  955. * the dcache before unmounting a file system.
  956. */
  957. void shrink_dcache_sb(struct super_block *sb)
  958. {
  959. long freed;
  960. do {
  961. LIST_HEAD(dispose);
  962. freed = list_lru_walk(&sb->s_dentry_lru,
  963. dentry_lru_isolate_shrink, &dispose, UINT_MAX);
  964. this_cpu_sub(nr_dentry_unused, freed);
  965. shrink_dentry_list(&dispose);
  966. } while (freed > 0);
  967. }
  968. EXPORT_SYMBOL(shrink_dcache_sb);
  969. /*
  970. * This tries to ascend one level of parenthood, but
  971. * we can race with renaming, so we need to re-check
  972. * the parenthood after dropping the lock and check
  973. * that the sequence number still matches.
  974. */
  975. static struct dentry *try_to_ascend(struct dentry *old, unsigned seq)
  976. {
  977. struct dentry *new = old->d_parent;
  978. rcu_read_lock();
  979. spin_unlock(&old->d_lock);
  980. spin_lock(&new->d_lock);
  981. /*
  982. * might go back up the wrong parent if we have had a rename
  983. * or deletion
  984. */
  985. if (new != old->d_parent ||
  986. (old->d_flags & DCACHE_DENTRY_KILLED) ||
  987. need_seqretry(&rename_lock, seq)) {
  988. spin_unlock(&new->d_lock);
  989. new = NULL;
  990. }
  991. rcu_read_unlock();
  992. return new;
  993. }
  994. /**
  995. * enum d_walk_ret - action to talke during tree walk
  996. * @D_WALK_CONTINUE: contrinue walk
  997. * @D_WALK_QUIT: quit walk
  998. * @D_WALK_NORETRY: quit when retry is needed
  999. * @D_WALK_SKIP: skip this dentry and its children
  1000. */
  1001. enum d_walk_ret {
  1002. D_WALK_CONTINUE,
  1003. D_WALK_QUIT,
  1004. D_WALK_NORETRY,
  1005. D_WALK_SKIP,
  1006. };
  1007. /**
  1008. * d_walk - walk the dentry tree
  1009. * @parent: start of walk
  1010. * @data: data passed to @enter() and @finish()
  1011. * @enter: callback when first entering the dentry
  1012. * @finish: callback when successfully finished the walk
  1013. *
  1014. * The @enter() and @finish() callbacks are called with d_lock held.
  1015. */
  1016. static void d_walk(struct dentry *parent, void *data,
  1017. enum d_walk_ret (*enter)(void *, struct dentry *),
  1018. void (*finish)(void *))
  1019. {
  1020. struct dentry *this_parent;
  1021. struct list_head *next;
  1022. unsigned seq = 0;
  1023. enum d_walk_ret ret;
  1024. bool retry = true;
  1025. again:
  1026. read_seqbegin_or_lock(&rename_lock, &seq);
  1027. this_parent = parent;
  1028. spin_lock(&this_parent->d_lock);
  1029. ret = enter(data, this_parent);
  1030. switch (ret) {
  1031. case D_WALK_CONTINUE:
  1032. break;
  1033. case D_WALK_QUIT:
  1034. case D_WALK_SKIP:
  1035. goto out_unlock;
  1036. case D_WALK_NORETRY:
  1037. retry = false;
  1038. break;
  1039. }
  1040. repeat:
  1041. next = this_parent->d_subdirs.next;
  1042. resume:
  1043. while (next != &this_parent->d_subdirs) {
  1044. struct list_head *tmp = next;
  1045. struct dentry *dentry = list_entry(tmp, struct dentry, d_u.d_child);
  1046. next = tmp->next;
  1047. spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
  1048. ret = enter(data, dentry);
  1049. switch (ret) {
  1050. case D_WALK_CONTINUE:
  1051. break;
  1052. case D_WALK_QUIT:
  1053. spin_unlock(&dentry->d_lock);
  1054. goto out_unlock;
  1055. case D_WALK_NORETRY:
  1056. retry = false;
  1057. break;
  1058. case D_WALK_SKIP:
  1059. spin_unlock(&dentry->d_lock);
  1060. continue;
  1061. }
  1062. if (!list_empty(&dentry->d_subdirs)) {
  1063. spin_unlock(&this_parent->d_lock);
  1064. spin_release(&dentry->d_lock.dep_map, 1, _RET_IP_);
  1065. this_parent = dentry;
  1066. spin_acquire(&this_parent->d_lock.dep_map, 0, 1, _RET_IP_);
  1067. goto repeat;
  1068. }
  1069. spin_unlock(&dentry->d_lock);
  1070. }
  1071. /*
  1072. * All done at this level ... ascend and resume the search.
  1073. */
  1074. if (this_parent != parent) {
  1075. struct dentry *child = this_parent;
  1076. this_parent = try_to_ascend(this_parent, seq);
  1077. if (!this_parent)
  1078. goto rename_retry;
  1079. next = child->d_u.d_child.next;
  1080. goto resume;
  1081. }
  1082. if (need_seqretry(&rename_lock, seq)) {
  1083. spin_unlock(&this_parent->d_lock);
  1084. goto rename_retry;
  1085. }
  1086. if (finish)
  1087. finish(data);
  1088. out_unlock:
  1089. spin_unlock(&this_parent->d_lock);
  1090. done_seqretry(&rename_lock, seq);
  1091. return;
  1092. rename_retry:
  1093. if (!retry)
  1094. return;
  1095. seq = 1;
  1096. goto again;
  1097. }
  1098. /*
  1099. * Search for at least 1 mount point in the dentry's subdirs.
  1100. * We descend to the next level whenever the d_subdirs
  1101. * list is non-empty and continue searching.
  1102. */
  1103. static enum d_walk_ret check_mount(void *data, struct dentry *dentry)
  1104. {
  1105. int *ret = data;
  1106. if (d_mountpoint(dentry)) {
  1107. *ret = 1;
  1108. return D_WALK_QUIT;
  1109. }
  1110. return D_WALK_CONTINUE;
  1111. }
  1112. /**
  1113. * have_submounts - check for mounts over a dentry
  1114. * @parent: dentry to check.
  1115. *
  1116. * Return true if the parent or its subdirectories contain
  1117. * a mount point
  1118. */
  1119. int have_submounts(struct dentry *parent)
  1120. {
  1121. int ret = 0;
  1122. d_walk(parent, &ret, check_mount, NULL);
  1123. return ret;
  1124. }
  1125. EXPORT_SYMBOL(have_submounts);
  1126. /*
  1127. * Called by mount code to set a mountpoint and check if the mountpoint is
  1128. * reachable (e.g. NFS can unhash a directory dentry and then the complete
  1129. * subtree can become unreachable).
  1130. *
  1131. * Only one of check_submounts_and_drop() and d_set_mounted() must succeed. For
  1132. * this reason take rename_lock and d_lock on dentry and ancestors.
  1133. */
  1134. int d_set_mounted(struct dentry *dentry)
  1135. {
  1136. struct dentry *p;
  1137. int ret = -ENOENT;
  1138. write_seqlock(&rename_lock);
  1139. for (p = dentry->d_parent; !IS_ROOT(p); p = p->d_parent) {
  1140. /* Need exclusion wrt. check_submounts_and_drop() */
  1141. spin_lock(&p->d_lock);
  1142. if (unlikely(d_unhashed(p))) {
  1143. spin_unlock(&p->d_lock);
  1144. goto out;
  1145. }
  1146. spin_unlock(&p->d_lock);
  1147. }
  1148. spin_lock(&dentry->d_lock);
  1149. if (!d_unlinked(dentry)) {
  1150. dentry->d_flags |= DCACHE_MOUNTED;
  1151. ret = 0;
  1152. }
  1153. spin_unlock(&dentry->d_lock);
  1154. out:
  1155. write_sequnlock(&rename_lock);
  1156. return ret;
  1157. }
  1158. /*
  1159. * Search the dentry child list of the specified parent,
  1160. * and move any unused dentries to the end of the unused
  1161. * list for prune_dcache(). We descend to the next level
  1162. * whenever the d_subdirs list is non-empty and continue
  1163. * searching.
  1164. *
  1165. * It returns zero iff there are no unused children,
  1166. * otherwise it returns the number of children moved to
  1167. * the end of the unused list. This may not be the total
  1168. * number of unused children, because select_parent can
  1169. * drop the lock and return early due to latency
  1170. * constraints.
  1171. */
  1172. struct select_data {
  1173. struct dentry *start;
  1174. struct list_head dispose;
  1175. int found;
  1176. };
  1177. static enum d_walk_ret select_collect(void *_data, struct dentry *dentry)
  1178. {
  1179. struct select_data *data = _data;
  1180. enum d_walk_ret ret = D_WALK_CONTINUE;
  1181. if (data->start == dentry)
  1182. goto out;
  1183. /*
  1184. * move only zero ref count dentries to the dispose list.
  1185. *
  1186. * Those which are presently on the shrink list, being processed
  1187. * by shrink_dentry_list(), shouldn't be moved. Otherwise the
  1188. * loop in shrink_dcache_parent() might not make any progress
  1189. * and loop forever.
  1190. */
  1191. if (dentry->d_lockref.count) {
  1192. dentry_lru_del(dentry);
  1193. } else if (!(dentry->d_flags & DCACHE_SHRINK_LIST)) {
  1194. /*
  1195. * We can't use d_lru_shrink_move() because we
  1196. * need to get the global LRU lock and do the
  1197. * LRU accounting.
  1198. */
  1199. d_lru_del(dentry);
  1200. d_shrink_add(dentry, &data->dispose);
  1201. data->found++;
  1202. ret = D_WALK_NORETRY;
  1203. }
  1204. /*
  1205. * We can return to the caller if we have found some (this
  1206. * ensures forward progress). We'll be coming back to find
  1207. * the rest.
  1208. */
  1209. if (data->found && need_resched())
  1210. ret = D_WALK_QUIT;
  1211. out:
  1212. return ret;
  1213. }
  1214. /**
  1215. * shrink_dcache_parent - prune dcache
  1216. * @parent: parent of entries to prune
  1217. *
  1218. * Prune the dcache to remove unused children of the parent dentry.
  1219. */
  1220. void shrink_dcache_parent(struct dentry *parent)
  1221. {
  1222. for (;;) {
  1223. struct select_data data;
  1224. INIT_LIST_HEAD(&data.dispose);
  1225. data.start = parent;
  1226. data.found = 0;
  1227. d_walk(parent, &data, select_collect, NULL);
  1228. if (!data.found)
  1229. break;
  1230. shrink_dentry_list(&data.dispose);
  1231. cond_resched();
  1232. }
  1233. }
  1234. EXPORT_SYMBOL(shrink_dcache_parent);
  1235. static enum d_walk_ret umount_collect(void *_data, struct dentry *dentry)
  1236. {
  1237. struct select_data *data = _data;
  1238. enum d_walk_ret ret = D_WALK_CONTINUE;
  1239. if (dentry->d_lockref.count) {
  1240. dentry_lru_del(dentry);
  1241. if (likely(!list_empty(&dentry->d_subdirs)))
  1242. goto out;
  1243. if (dentry == data->start && dentry->d_lockref.count == 1)
  1244. goto out;
  1245. printk(KERN_ERR
  1246. "BUG: Dentry %p{i=%lx,n=%s}"
  1247. " still in use (%d)"
  1248. " [unmount of %s %s]\n",
  1249. dentry,
  1250. dentry->d_inode ?
  1251. dentry->d_inode->i_ino : 0UL,
  1252. dentry->d_name.name,
  1253. dentry->d_lockref.count,
  1254. dentry->d_sb->s_type->name,
  1255. dentry->d_sb->s_id);
  1256. BUG();
  1257. } else if (!(dentry->d_flags & DCACHE_SHRINK_LIST)) {
  1258. /*
  1259. * We can't use d_lru_shrink_move() because we
  1260. * need to get the global LRU lock and do the
  1261. * LRU accounting.
  1262. */
  1263. if (dentry->d_flags & DCACHE_LRU_LIST)
  1264. d_lru_del(dentry);
  1265. d_shrink_add(dentry, &data->dispose);
  1266. data->found++;
  1267. ret = D_WALK_NORETRY;
  1268. }
  1269. out:
  1270. if (data->found && need_resched())
  1271. ret = D_WALK_QUIT;
  1272. return ret;
  1273. }
  1274. /*
  1275. * destroy the dentries attached to a superblock on unmounting
  1276. */
  1277. void shrink_dcache_for_umount(struct super_block *sb)
  1278. {
  1279. struct dentry *dentry;
  1280. if (down_read_trylock(&sb->s_umount))
  1281. BUG();
  1282. dentry = sb->s_root;
  1283. sb->s_root = NULL;
  1284. for (;;) {
  1285. struct select_data data;
  1286. INIT_LIST_HEAD(&data.dispose);
  1287. data.start = dentry;
  1288. data.found = 0;
  1289. d_walk(dentry, &data, umount_collect, NULL);
  1290. if (!data.found)
  1291. break;
  1292. shrink_dentry_list(&data.dispose);
  1293. cond_resched();
  1294. }
  1295. d_drop(dentry);
  1296. dput(dentry);
  1297. while (!hlist_bl_empty(&sb->s_anon)) {
  1298. struct select_data data;
  1299. dentry = hlist_bl_entry(hlist_bl_first(&sb->s_anon), struct dentry, d_hash);
  1300. INIT_LIST_HEAD(&data.dispose);
  1301. data.start = NULL;
  1302. data.found = 0;
  1303. d_walk(dentry, &data, umount_collect, NULL);
  1304. if (data.found)
  1305. shrink_dentry_list(&data.dispose);
  1306. cond_resched();
  1307. }
  1308. }
  1309. static enum d_walk_ret check_and_collect(void *_data, struct dentry *dentry)
  1310. {
  1311. struct select_data *data = _data;
  1312. if (d_mountpoint(dentry)) {
  1313. data->found = -EBUSY;
  1314. return D_WALK_QUIT;
  1315. }
  1316. return select_collect(_data, dentry);
  1317. }
  1318. static void check_and_drop(void *_data)
  1319. {
  1320. struct select_data *data = _data;
  1321. if (d_mountpoint(data->start))
  1322. data->found = -EBUSY;
  1323. if (!data->found)
  1324. __d_drop(data->start);
  1325. }
  1326. /**
  1327. * check_submounts_and_drop - prune dcache, check for submounts and drop
  1328. *
  1329. * All done as a single atomic operation relative to has_unlinked_ancestor().
  1330. * Returns 0 if successfully unhashed @parent. If there were submounts then
  1331. * return -EBUSY.
  1332. *
  1333. * @dentry: dentry to prune and drop
  1334. */
  1335. int check_submounts_and_drop(struct dentry *dentry)
  1336. {
  1337. int ret = 0;
  1338. /* Negative dentries can be dropped without further checks */
  1339. if (!dentry->d_inode) {
  1340. d_drop(dentry);
  1341. goto out;
  1342. }
  1343. for (;;) {
  1344. struct select_data data;
  1345. INIT_LIST_HEAD(&data.dispose);
  1346. data.start = dentry;
  1347. data.found = 0;
  1348. d_walk(dentry, &data, check_and_collect, check_and_drop);
  1349. ret = data.found;
  1350. if (!list_empty(&data.dispose))
  1351. shrink_dentry_list(&data.dispose);
  1352. if (ret <= 0)
  1353. break;
  1354. cond_resched();
  1355. }
  1356. out:
  1357. return ret;
  1358. }
  1359. EXPORT_SYMBOL(check_submounts_and_drop);
  1360. /**
  1361. * __d_alloc - allocate a dcache entry
  1362. * @sb: filesystem it will belong to
  1363. * @name: qstr of the name
  1364. *
  1365. * Allocates a dentry. It returns %NULL if there is insufficient memory
  1366. * available. On a success the dentry is returned. The name passed in is
  1367. * copied and the copy passed in may be reused after this call.
  1368. */
  1369. struct dentry *__d_alloc(struct super_block *sb, const struct qstr *name)
  1370. {
  1371. struct dentry *dentry;
  1372. char *dname;
  1373. dentry = kmem_cache_alloc(dentry_cache, GFP_KERNEL);
  1374. if (!dentry)
  1375. return NULL;
  1376. /*
  1377. * We guarantee that the inline name is always NUL-terminated.
  1378. * This way the memcpy() done by the name switching in rename
  1379. * will still always have a NUL at the end, even if we might
  1380. * be overwriting an internal NUL character
  1381. */
  1382. dentry->d_iname[DNAME_INLINE_LEN-1] = 0;
  1383. if (name->len > DNAME_INLINE_LEN-1) {
  1384. dname = kmalloc(name->len + 1, GFP_KERNEL);
  1385. if (!dname) {
  1386. kmem_cache_free(dentry_cache, dentry);
  1387. return NULL;
  1388. }
  1389. } else {
  1390. dname = dentry->d_iname;
  1391. }
  1392. dentry->d_name.len = name->len;
  1393. dentry->d_name.hash = name->hash;
  1394. memcpy(dname, name->name, name->len);
  1395. dname[name->len] = 0;
  1396. /* Make sure we always see the terminating NUL character */
  1397. smp_wmb();
  1398. dentry->d_name.name = dname;
  1399. dentry->d_lockref.count = 1;
  1400. dentry->d_flags = 0;
  1401. spin_lock_init(&dentry->d_lock);
  1402. seqcount_init(&dentry->d_seq);
  1403. dentry->d_inode = NULL;
  1404. dentry->d_parent = dentry;
  1405. dentry->d_sb = sb;
  1406. dentry->d_op = NULL;
  1407. dentry->d_fsdata = NULL;
  1408. INIT_HLIST_BL_NODE(&dentry->d_hash);
  1409. INIT_LIST_HEAD(&dentry->d_lru);
  1410. INIT_LIST_HEAD(&dentry->d_subdirs);
  1411. INIT_HLIST_NODE(&dentry->d_alias);
  1412. INIT_LIST_HEAD(&dentry->d_u.d_child);
  1413. d_set_d_op(dentry, dentry->d_sb->s_d_op);
  1414. this_cpu_inc(nr_dentry);
  1415. return dentry;
  1416. }
  1417. /**
  1418. * d_alloc - allocate a dcache entry
  1419. * @parent: parent of entry to allocate
  1420. * @name: qstr of the name
  1421. *
  1422. * Allocates a dentry. It returns %NULL if there is insufficient memory
  1423. * available. On a success the dentry is returned. The name passed in is
  1424. * copied and the copy passed in may be reused after this call.
  1425. */
  1426. struct dentry *d_alloc(struct dentry * parent, const struct qstr *name)
  1427. {
  1428. struct dentry *dentry = __d_alloc(parent->d_sb, name);
  1429. if (!dentry)
  1430. return NULL;
  1431. spin_lock(&parent->d_lock);
  1432. /*
  1433. * don't need child lock because it is not subject
  1434. * to concurrency here
  1435. */
  1436. __dget_dlock(parent);
  1437. dentry->d_parent = parent;
  1438. list_add(&dentry->d_u.d_child, &parent->d_subdirs);
  1439. spin_unlock(&parent->d_lock);
  1440. return dentry;
  1441. }
  1442. EXPORT_SYMBOL(d_alloc);
  1443. /**
  1444. * d_alloc_pseudo - allocate a dentry (for lookup-less filesystems)
  1445. * @sb: the superblock
  1446. * @name: qstr of the name
  1447. *
  1448. * For a filesystem that just pins its dentries in memory and never
  1449. * performs lookups at all, return an unhashed IS_ROOT dentry.
  1450. */
  1451. struct dentry *d_alloc_pseudo(struct super_block *sb, const struct qstr *name)
  1452. {
  1453. return __d_alloc(sb, name);
  1454. }
  1455. EXPORT_SYMBOL(d_alloc_pseudo);
  1456. struct dentry *d_alloc_name(struct dentry *parent, const char *name)
  1457. {
  1458. struct qstr q;
  1459. q.name = name;
  1460. q.len = strlen(name);
  1461. q.hash = full_name_hash(q.name, q.len);
  1462. return d_alloc(parent, &q);
  1463. }
  1464. EXPORT_SYMBOL(d_alloc_name);
  1465. void d_set_d_op(struct dentry *dentry, const struct dentry_operations *op)
  1466. {
  1467. WARN_ON_ONCE(dentry->d_op);
  1468. WARN_ON_ONCE(dentry->d_flags & (DCACHE_OP_HASH |
  1469. DCACHE_OP_COMPARE |
  1470. DCACHE_OP_REVALIDATE |
  1471. DCACHE_OP_WEAK_REVALIDATE |
  1472. DCACHE_OP_DELETE ));
  1473. dentry->d_op = op;
  1474. if (!op)
  1475. return;
  1476. if (op->d_hash)
  1477. dentry->d_flags |= DCACHE_OP_HASH;
  1478. if (op->d_compare)
  1479. dentry->d_flags |= DCACHE_OP_COMPARE;
  1480. if (op->d_revalidate)
  1481. dentry->d_flags |= DCACHE_OP_REVALIDATE;
  1482. if (op->d_weak_revalidate)
  1483. dentry->d_flags |= DCACHE_OP_WEAK_REVALIDATE;
  1484. if (op->d_delete)
  1485. dentry->d_flags |= DCACHE_OP_DELETE;
  1486. if (op->d_prune)
  1487. dentry->d_flags |= DCACHE_OP_PRUNE;
  1488. }
  1489. EXPORT_SYMBOL(d_set_d_op);
  1490. static unsigned d_flags_for_inode(struct inode *inode)
  1491. {
  1492. unsigned add_flags = DCACHE_FILE_TYPE;
  1493. if (!inode)
  1494. return DCACHE_MISS_TYPE;
  1495. if (S_ISDIR(inode->i_mode)) {
  1496. add_flags = DCACHE_DIRECTORY_TYPE;
  1497. if (unlikely(!(inode->i_opflags & IOP_LOOKUP))) {
  1498. if (unlikely(!inode->i_op->lookup))
  1499. add_flags = DCACHE_AUTODIR_TYPE;
  1500. else
  1501. inode->i_opflags |= IOP_LOOKUP;
  1502. }
  1503. } else if (unlikely(!(inode->i_opflags & IOP_NOFOLLOW))) {
  1504. if (unlikely(inode->i_op->follow_link))
  1505. add_flags = DCACHE_SYMLINK_TYPE;
  1506. else
  1507. inode->i_opflags |= IOP_NOFOLLOW;
  1508. }
  1509. if (unlikely(IS_AUTOMOUNT(inode)))
  1510. add_flags |= DCACHE_NEED_AUTOMOUNT;
  1511. return add_flags;
  1512. }
  1513. static void __d_instantiate(struct dentry *dentry, struct inode *inode)
  1514. {
  1515. unsigned add_flags = d_flags_for_inode(inode);
  1516. spin_lock(&dentry->d_lock);
  1517. dentry->d_flags &= ~DCACHE_ENTRY_TYPE;
  1518. dentry->d_flags |= add_flags;
  1519. if (inode)
  1520. hlist_add_head(&dentry->d_alias, &inode->i_dentry);
  1521. dentry->d_inode = inode;
  1522. dentry_rcuwalk_barrier(dentry);
  1523. spin_unlock(&dentry->d_lock);
  1524. fsnotify_d_instantiate(dentry, inode);
  1525. }
  1526. /**
  1527. * d_instantiate - fill in inode information for a dentry
  1528. * @entry: dentry to complete
  1529. * @inode: inode to attach to this dentry
  1530. *
  1531. * Fill in inode information in the entry.
  1532. *
  1533. * This turns negative dentries into productive full members
  1534. * of society.
  1535. *
  1536. * NOTE! This assumes that the inode count has been incremented
  1537. * (or otherwise set) by the caller to indicate that it is now
  1538. * in use by the dcache.
  1539. */
  1540. void d_instantiate(struct dentry *entry, struct inode * inode)
  1541. {
  1542. BUG_ON(!hlist_unhashed(&entry->d_alias));
  1543. if (inode)
  1544. spin_lock(&inode->i_lock);
  1545. __d_instantiate(entry, inode);
  1546. if (inode)
  1547. spin_unlock(&inode->i_lock);
  1548. security_d_instantiate(entry, inode);
  1549. }
  1550. EXPORT_SYMBOL(d_instantiate);
  1551. /**
  1552. * d_instantiate_unique - instantiate a non-aliased dentry
  1553. * @entry: dentry to instantiate
  1554. * @inode: inode to attach to this dentry
  1555. *
  1556. * Fill in inode information in the entry. On success, it returns NULL.
  1557. * If an unhashed alias of "entry" already exists, then we return the
  1558. * aliased dentry instead and drop one reference to inode.
  1559. *
  1560. * Note that in order to avoid conflicts with rename() etc, the caller
  1561. * had better be holding the parent directory semaphore.
  1562. *
  1563. * This also assumes that the inode count has been incremented
  1564. * (or otherwise set) by the caller to indicate that it is now
  1565. * in use by the dcache.
  1566. */
  1567. static struct dentry *__d_instantiate_unique(struct dentry *entry,
  1568. struct inode *inode)
  1569. {
  1570. struct dentry *alias;
  1571. int len = entry->d_name.len;
  1572. const char *name = entry->d_name.name;
  1573. unsigned int hash = entry->d_name.hash;
  1574. if (!inode) {
  1575. __d_instantiate(entry, NULL);
  1576. return NULL;
  1577. }
  1578. hlist_for_each_entry(alias, &inode->i_dentry, d_alias) {
  1579. /*
  1580. * Don't need alias->d_lock here, because aliases with
  1581. * d_parent == entry->d_parent are not subject to name or
  1582. * parent changes, because the parent inode i_mutex is held.
  1583. */
  1584. if (alias->d_name.hash != hash)
  1585. continue;
  1586. if (alias->d_parent != entry->d_parent)
  1587. continue;
  1588. if (alias->d_name.len != len)
  1589. continue;
  1590. if (dentry_cmp(alias, name, len))
  1591. continue;
  1592. __dget(alias);
  1593. return alias;
  1594. }
  1595. __d_instantiate(entry, inode);
  1596. return NULL;
  1597. }
  1598. struct dentry *d_instantiate_unique(struct dentry *entry, struct inode *inode)
  1599. {
  1600. struct dentry *result;
  1601. BUG_ON(!hlist_unhashed(&entry->d_alias));
  1602. if (inode)
  1603. spin_lock(&inode->i_lock);
  1604. result = __d_instantiate_unique(entry, inode);
  1605. if (inode)
  1606. spin_unlock(&inode->i_lock);
  1607. if (!result) {
  1608. security_d_instantiate(entry, inode);
  1609. return NULL;
  1610. }
  1611. BUG_ON(!d_unhashed(result));
  1612. iput(inode);
  1613. return result;
  1614. }
  1615. EXPORT_SYMBOL(d_instantiate_unique);
  1616. /**
  1617. * d_instantiate_no_diralias - instantiate a non-aliased dentry
  1618. * @entry: dentry to complete
  1619. * @inode: inode to attach to this dentry
  1620. *
  1621. * Fill in inode information in the entry. If a directory alias is found, then
  1622. * return an error (and drop inode). Together with d_materialise_unique() this
  1623. * guarantees that a directory inode may never have more than one alias.
  1624. */
  1625. int d_instantiate_no_diralias(struct dentry *entry, struct inode *inode)
  1626. {
  1627. BUG_ON(!hlist_unhashed(&entry->d_alias));
  1628. spin_lock(&inode->i_lock);
  1629. if (S_ISDIR(inode->i_mode) && !hlist_empty(&inode->i_dentry)) {
  1630. spin_unlock(&inode->i_lock);
  1631. iput(inode);
  1632. return -EBUSY;
  1633. }
  1634. __d_instantiate(entry, inode);
  1635. spin_unlock(&inode->i_lock);
  1636. security_d_instantiate(entry, inode);
  1637. return 0;
  1638. }
  1639. EXPORT_SYMBOL(d_instantiate_no_diralias);
  1640. struct dentry *d_make_root(struct inode *root_inode)
  1641. {
  1642. struct dentry *res = NULL;
  1643. if (root_inode) {
  1644. static const struct qstr name = QSTR_INIT("/", 1);
  1645. res = __d_alloc(root_inode->i_sb, &name);
  1646. if (res)
  1647. d_instantiate(res, root_inode);
  1648. else
  1649. iput(root_inode);
  1650. }
  1651. return res;
  1652. }
  1653. EXPORT_SYMBOL(d_make_root);
  1654. static struct dentry * __d_find_any_alias(struct inode *inode)
  1655. {
  1656. struct dentry *alias;
  1657. if (hlist_empty(&inode->i_dentry))
  1658. return NULL;
  1659. alias = hlist_entry(inode->i_dentry.first, struct dentry, d_alias);
  1660. __dget(alias);
  1661. return alias;
  1662. }
  1663. /**
  1664. * d_find_any_alias - find any alias for a given inode
  1665. * @inode: inode to find an alias for
  1666. *
  1667. * If any aliases exist for the given inode, take and return a
  1668. * reference for one of them. If no aliases exist, return %NULL.
  1669. */
  1670. struct dentry *d_find_any_alias(struct inode *inode)
  1671. {
  1672. struct dentry *de;
  1673. spin_lock(&inode->i_lock);
  1674. de = __d_find_any_alias(inode);
  1675. spin_unlock(&inode->i_lock);
  1676. return de;
  1677. }
  1678. EXPORT_SYMBOL(d_find_any_alias);
  1679. /**
  1680. * d_obtain_alias - find or allocate a dentry for a given inode
  1681. * @inode: inode to allocate the dentry for
  1682. *
  1683. * Obtain a dentry for an inode resulting from NFS filehandle conversion or
  1684. * similar open by handle operations. The returned dentry may be anonymous,
  1685. * or may have a full name (if the inode was already in the cache).
  1686. *
  1687. * When called on a directory inode, we must ensure that the inode only ever
  1688. * has one dentry. If a dentry is found, that is returned instead of
  1689. * allocating a new one.
  1690. *
  1691. * On successful return, the reference to the inode has been transferred
  1692. * to the dentry. In case of an error the reference on the inode is released.
  1693. * To make it easier to use in export operations a %NULL or IS_ERR inode may
  1694. * be passed in and will be the error will be propagate to the return value,
  1695. * with a %NULL @inode replaced by ERR_PTR(-ESTALE).
  1696. */
  1697. struct dentry *d_obtain_alias(struct inode *inode)
  1698. {
  1699. static const struct qstr anonstring = QSTR_INIT("/", 1);
  1700. struct dentry *tmp;
  1701. struct dentry *res;
  1702. unsigned add_flags;
  1703. if (!inode)
  1704. return ERR_PTR(-ESTALE);
  1705. if (IS_ERR(inode))
  1706. return ERR_CAST(inode);
  1707. res = d_find_any_alias(inode);
  1708. if (res)
  1709. goto out_iput;
  1710. tmp = __d_alloc(inode->i_sb, &anonstring);
  1711. if (!tmp) {
  1712. res = ERR_PTR(-ENOMEM);
  1713. goto out_iput;
  1714. }
  1715. spin_lock(&inode->i_lock);
  1716. res = __d_find_any_alias(inode);
  1717. if (res) {
  1718. spin_unlock(&inode->i_lock);
  1719. dput(tmp);
  1720. goto out_iput;
  1721. }
  1722. /* attach a disconnected dentry */
  1723. add_flags = d_flags_for_inode(inode) | DCACHE_DISCONNECTED;
  1724. spin_lock(&tmp->d_lock);
  1725. tmp->d_inode = inode;
  1726. tmp->d_flags |= add_flags;
  1727. hlist_add_head(&tmp->d_alias, &inode->i_dentry);
  1728. hlist_bl_lock(&tmp->d_sb->s_anon);
  1729. hlist_bl_add_head(&tmp->d_hash, &tmp->d_sb->s_anon);
  1730. hlist_bl_unlock(&tmp->d_sb->s_anon);
  1731. spin_unlock(&tmp->d_lock);
  1732. spin_unlock(&inode->i_lock);
  1733. security_d_instantiate(tmp, inode);
  1734. return tmp;
  1735. out_iput:
  1736. if (res && !IS_ERR(res))
  1737. security_d_instantiate(res, inode);
  1738. iput(inode);
  1739. return res;
  1740. }
  1741. EXPORT_SYMBOL(d_obtain_alias);
  1742. /**
  1743. * d_splice_alias - splice a disconnected dentry into the tree if one exists
  1744. * @inode: the inode which may have a disconnected dentry
  1745. * @dentry: a negative dentry which we want to point to the inode.
  1746. *
  1747. * If inode is a directory and has a 'disconnected' dentry (i.e. IS_ROOT and
  1748. * DCACHE_DISCONNECTED), then d_move that in place of the given dentry
  1749. * and return it, else simply d_add the inode to the dentry and return NULL.
  1750. *
  1751. * This is needed in the lookup routine of any filesystem that is exportable
  1752. * (via knfsd) so that we can build dcache paths to directories effectively.
  1753. *
  1754. * If a dentry was found and moved, then it is returned. Otherwise NULL
  1755. * is returned. This matches the expected return value of ->lookup.
  1756. *
  1757. * Cluster filesystems may call this function with a negative, hashed dentry.
  1758. * In that case, we know that the inode will be a regular file, and also this
  1759. * will only occur during atomic_open. So we need to check for the dentry
  1760. * being already hashed only in the final case.
  1761. */
  1762. struct dentry *d_splice_alias(struct inode *inode, struct dentry *dentry)
  1763. {
  1764. struct dentry *new = NULL;
  1765. if (IS_ERR(inode))
  1766. return ERR_CAST(inode);
  1767. if (inode && S_ISDIR(inode->i_mode)) {
  1768. spin_lock(&inode->i_lock);
  1769. new = __d_find_alias(inode, 1);
  1770. if (new) {
  1771. BUG_ON(!(new->d_flags & DCACHE_DISCONNECTED));
  1772. spin_unlock(&inode->i_lock);
  1773. security_d_instantiate(new, inode);
  1774. d_move(new, dentry);
  1775. iput(inode);
  1776. } else {
  1777. /* already taking inode->i_lock, so d_add() by hand */
  1778. __d_instantiate(dentry, inode);
  1779. spin_unlock(&inode->i_lock);
  1780. security_d_instantiate(dentry, inode);
  1781. d_rehash(dentry);
  1782. }
  1783. } else {
  1784. d_instantiate(dentry, inode);
  1785. if (d_unhashed(dentry))
  1786. d_rehash(dentry);
  1787. }
  1788. return new;
  1789. }
  1790. EXPORT_SYMBOL(d_splice_alias);
  1791. /**
  1792. * d_add_ci - lookup or allocate new dentry with case-exact name
  1793. * @inode: the inode case-insensitive lookup has found
  1794. * @dentry: the negative dentry that was passed to the parent's lookup func
  1795. * @name: the case-exact name to be associated with the returned dentry
  1796. *
  1797. * This is to avoid filling the dcache with case-insensitive names to the
  1798. * same inode, only the actual correct case is stored in the dcache for
  1799. * case-insensitive filesystems.
  1800. *
  1801. * For a case-insensitive lookup match and if the the case-exact dentry
  1802. * already exists in in the dcache, use it and return it.
  1803. *
  1804. * If no entry exists with the exact case name, allocate new dentry with
  1805. * the exact case, and return the spliced entry.
  1806. */
  1807. struct dentry *d_add_ci(struct dentry *dentry, struct inode *inode,
  1808. struct qstr *name)
  1809. {
  1810. struct dentry *found;
  1811. struct dentry *new;
  1812. /*
  1813. * First check if a dentry matching the name already exists,
  1814. * if not go ahead and create it now.
  1815. */
  1816. found = d_hash_and_lookup(dentry->d_parent, name);
  1817. if (unlikely(IS_ERR(found)))
  1818. goto err_out;
  1819. if (!found) {
  1820. new = d_alloc(dentry->d_parent, name);
  1821. if (!new) {
  1822. found = ERR_PTR(-ENOMEM);
  1823. goto err_out;
  1824. }
  1825. found = d_splice_alias(inode, new);
  1826. if (found) {
  1827. dput(new);
  1828. return found;
  1829. }
  1830. return new;
  1831. }
  1832. /*
  1833. * If a matching dentry exists, and it's not negative use it.
  1834. *
  1835. * Decrement the reference count to balance the iget() done
  1836. * earlier on.
  1837. */
  1838. if (found->d_inode) {
  1839. if (unlikely(found->d_inode != inode)) {
  1840. /* This can't happen because bad inodes are unhashed. */
  1841. BUG_ON(!is_bad_inode(inode));
  1842. BUG_ON(!is_bad_inode(found->d_inode));
  1843. }
  1844. iput(inode);
  1845. return found;
  1846. }
  1847. /*
  1848. * Negative dentry: instantiate it unless the inode is a directory and
  1849. * already has a dentry.
  1850. */
  1851. new = d_splice_alias(inode, found);
  1852. if (new) {
  1853. dput(found);
  1854. found = new;
  1855. }
  1856. return found;
  1857. err_out:
  1858. iput(inode);
  1859. return found;
  1860. }
  1861. EXPORT_SYMBOL(d_add_ci);
  1862. /*
  1863. * Do the slow-case of the dentry name compare.
  1864. *
  1865. * Unlike the dentry_cmp() function, we need to atomically
  1866. * load the name and length information, so that the
  1867. * filesystem can rely on them, and can use the 'name' and
  1868. * 'len' information without worrying about walking off the
  1869. * end of memory etc.
  1870. *
  1871. * Thus the read_seqcount_retry() and the "duplicate" info
  1872. * in arguments (the low-level filesystem should not look
  1873. * at the dentry inode or name contents directly, since
  1874. * rename can change them while we're in RCU mode).
  1875. */
  1876. enum slow_d_compare {
  1877. D_COMP_OK,
  1878. D_COMP_NOMATCH,
  1879. D_COMP_SEQRETRY,
  1880. };
  1881. static noinline enum slow_d_compare slow_dentry_cmp(
  1882. const struct dentry *parent,
  1883. struct dentry *dentry,
  1884. unsigned int seq,
  1885. const struct qstr *name)
  1886. {
  1887. int tlen = dentry->d_name.len;
  1888. const char *tname = dentry->d_name.name;
  1889. if (read_seqcount_retry(&dentry->d_seq, seq)) {
  1890. cpu_relax();
  1891. return D_COMP_SEQRETRY;
  1892. }
  1893. if (parent->d_op->d_compare(parent, dentry, tlen, tname, name))
  1894. return D_COMP_NOMATCH;
  1895. return D_COMP_OK;
  1896. }
  1897. /**
  1898. * __d_lookup_rcu - search for a dentry (racy, store-free)
  1899. * @parent: parent dentry
  1900. * @name: qstr of name we wish to find
  1901. * @seqp: returns d_seq value at the point where the dentry was found
  1902. * Returns: dentry, or NULL
  1903. *
  1904. * __d_lookup_rcu is the dcache lookup function for rcu-walk name
  1905. * resolution (store-free path walking) design described in
  1906. * Documentation/filesystems/path-lookup.txt.
  1907. *
  1908. * This is not to be used outside core vfs.
  1909. *
  1910. * __d_lookup_rcu must only be used in rcu-walk mode, ie. with vfsmount lock
  1911. * held, and rcu_read_lock held. The returned dentry must not be stored into
  1912. * without taking d_lock and checking d_seq sequence count against @seq
  1913. * returned here.
  1914. *
  1915. * A refcount may be taken on the found dentry with the d_rcu_to_refcount
  1916. * function.
  1917. *
  1918. * Alternatively, __d_lookup_rcu may be called again to look up the child of
  1919. * the returned dentry, so long as its parent's seqlock is checked after the
  1920. * child is looked up. Thus, an interlocking stepping of sequence lock checks
  1921. * is formed, giving integrity down the path walk.
  1922. *
  1923. * NOTE! The caller *has* to check the resulting dentry against the sequence
  1924. * number we've returned before using any of the resulting dentry state!
  1925. */
  1926. struct dentry *__d_lookup_rcu(const struct dentry *parent,
  1927. const struct qstr *name,
  1928. unsigned *seqp)
  1929. {
  1930. u64 hashlen = name->hash_len;
  1931. const unsigned char *str = name->name;
  1932. struct hlist_bl_head *b = d_hash(parent, hashlen_hash(hashlen));
  1933. struct hlist_bl_node *node;
  1934. struct dentry *dentry;
  1935. /*
  1936. * Note: There is significant duplication with __d_lookup_rcu which is
  1937. * required to prevent single threaded performance regressions
  1938. * especially on architectures where smp_rmb (in seqcounts) are costly.
  1939. * Keep the two functions in sync.
  1940. */
  1941. /*
  1942. * The hash list is protected using RCU.
  1943. *
  1944. * Carefully use d_seq when comparing a candidate dentry, to avoid
  1945. * races with d_move().
  1946. *
  1947. * It is possible that concurrent renames can mess up our list
  1948. * walk here and result in missing our dentry, resulting in the
  1949. * false-negative result. d_lookup() protects against concurrent
  1950. * renames using rename_lock seqlock.
  1951. *
  1952. * See Documentation/filesystems/path-lookup.txt for more details.
  1953. */
  1954. hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) {
  1955. unsigned seq;
  1956. seqretry:
  1957. /*
  1958. * The dentry sequence count protects us from concurrent
  1959. * renames, and thus protects parent and name fields.
  1960. *
  1961. * The caller must perform a seqcount check in order
  1962. * to do anything useful with the returned dentry.
  1963. *
  1964. * NOTE! We do a "raw" seqcount_begin here. That means that
  1965. * we don't wait for the sequence count to stabilize if it
  1966. * is in the middle of a sequence change. If we do the slow
  1967. * dentry compare, we will do seqretries until it is stable,
  1968. * and if we end up with a successful lookup, we actually
  1969. * want to exit RCU lookup anyway.
  1970. */
  1971. seq = raw_seqcount_begin(&dentry->d_seq);
  1972. if (dentry->d_parent != parent)
  1973. continue;
  1974. if (d_unhashed(dentry))
  1975. continue;
  1976. if (unlikely(parent->d_flags & DCACHE_OP_COMPARE)) {
  1977. if (dentry->d_name.hash != hashlen_hash(hashlen))
  1978. continue;
  1979. *seqp = seq;
  1980. switch (slow_dentry_cmp(parent, dentry, seq, name)) {
  1981. case D_COMP_OK:
  1982. return dentry;
  1983. case D_COMP_NOMATCH:
  1984. continue;
  1985. default:
  1986. goto seqretry;
  1987. }
  1988. }
  1989. if (dentry->d_name.hash_len != hashlen)
  1990. continue;
  1991. *seqp = seq;
  1992. if (!dentry_cmp(dentry, str, hashlen_len(hashlen)))
  1993. return dentry;
  1994. }
  1995. return NULL;
  1996. }
  1997. /**
  1998. * d_lookup - search for a dentry
  1999. * @parent: parent dentry
  2000. * @name: qstr of name we wish to find
  2001. * Returns: dentry, or NULL
  2002. *
  2003. * d_lookup searches the children of the parent dentry for the name in
  2004. * question. If the dentry is found its reference count is incremented and the
  2005. * dentry is returned. The caller must use dput to free the entry when it has
  2006. * finished using it. %NULL is returned if the dentry does not exist.
  2007. */
  2008. struct dentry *d_lookup(const struct dentry *parent, const struct qstr *name)
  2009. {
  2010. struct dentry *dentry;
  2011. unsigned seq;
  2012. do {
  2013. seq = read_seqbegin(&rename_lock);
  2014. dentry = __d_lookup(parent, name);
  2015. if (dentry)
  2016. break;
  2017. } while (read_seqretry(&rename_lock, seq));
  2018. return dentry;
  2019. }
  2020. EXPORT_SYMBOL(d_lookup);
  2021. /**
  2022. * __d_lookup - search for a dentry (racy)
  2023. * @parent: parent dentry
  2024. * @name: qstr of name we wish to find
  2025. * Returns: dentry, or NULL
  2026. *
  2027. * __d_lookup is like d_lookup, however it may (rarely) return a
  2028. * false-negative result due to unrelated rename activity.
  2029. *
  2030. * __d_lookup is slightly faster by avoiding rename_lock read seqlock,
  2031. * however it must be used carefully, eg. with a following d_lookup in
  2032. * the case of failure.
  2033. *
  2034. * __d_lookup callers must be commented.
  2035. */
  2036. struct dentry *__d_lookup(const struct dentry *parent, const struct qstr *name)
  2037. {
  2038. unsigned int len = name->len;
  2039. unsigned int hash = name->hash;
  2040. const unsigned char *str = name->name;
  2041. struct hlist_bl_head *b = d_hash(parent, hash);
  2042. struct hlist_bl_node *node;
  2043. struct dentry *found = NULL;
  2044. struct dentry *dentry;
  2045. /*
  2046. * Note: There is significant duplication with __d_lookup_rcu which is
  2047. * required to prevent single threaded performance regressions
  2048. * especially on architectures where smp_rmb (in seqcounts) are costly.
  2049. * Keep the two functions in sync.
  2050. */
  2051. /*
  2052. * The hash list is protected using RCU.
  2053. *
  2054. * Take d_lock when comparing a candidate dentry, to avoid races
  2055. * with d_move().
  2056. *
  2057. * It is possible that concurrent renames can mess up our list
  2058. * walk here and result in missing our dentry, resulting in the
  2059. * false-negative result. d_lookup() protects against concurrent
  2060. * renames using rename_lock seqlock.
  2061. *
  2062. * See Documentation/filesystems/path-lookup.txt for more details.
  2063. */
  2064. rcu_read_lock();
  2065. hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) {
  2066. if (dentry->d_name.hash != hash)
  2067. continue;
  2068. spin_lock(&dentry->d_lock);
  2069. if (dentry->d_parent != parent)
  2070. goto next;
  2071. if (d_unhashed(dentry))
  2072. goto next;
  2073. /*
  2074. * It is safe to compare names since d_move() cannot
  2075. * change the qstr (protected by d_lock).
  2076. */
  2077. if (parent->d_flags & DCACHE_OP_COMPARE) {
  2078. int tlen = dentry->d_name.len;
  2079. const char *tname = dentry->d_name.name;
  2080. if (parent->d_op->d_compare(parent, dentry, tlen, tname, name))
  2081. goto next;
  2082. } else {
  2083. if (dentry->d_name.len != len)
  2084. goto next;
  2085. if (dentry_cmp(dentry, str, len))
  2086. goto next;
  2087. }
  2088. dentry->d_lockref.count++;
  2089. found = dentry;
  2090. spin_unlock(&dentry->d_lock);
  2091. break;
  2092. next:
  2093. spin_unlock(&dentry->d_lock);
  2094. }
  2095. rcu_read_unlock();
  2096. return found;
  2097. }
  2098. /**
  2099. * d_hash_and_lookup - hash the qstr then search for a dentry
  2100. * @dir: Directory to search in
  2101. * @name: qstr of name we wish to find
  2102. *
  2103. * On lookup failure NULL is returned; on bad name - ERR_PTR(-error)
  2104. */
  2105. struct dentry *d_hash_and_lookup(struct dentry *dir, struct qstr *name)
  2106. {
  2107. /*
  2108. * Check for a fs-specific hash function. Note that we must
  2109. * calculate the standard hash first, as the d_op->d_hash()
  2110. * routine may choose to leave the hash value unchanged.
  2111. */
  2112. name->hash = full_name_hash(name->name, name->len);
  2113. if (dir->d_flags & DCACHE_OP_HASH) {
  2114. int err = dir->d_op->d_hash(dir, name);
  2115. if (unlikely(err < 0))
  2116. return ERR_PTR(err);
  2117. }
  2118. return d_lookup(dir, name);
  2119. }
  2120. EXPORT_SYMBOL(d_hash_and_lookup);
  2121. /**
  2122. * d_validate - verify dentry provided from insecure source (deprecated)
  2123. * @dentry: The dentry alleged to be valid child of @dparent
  2124. * @dparent: The parent dentry (known to be valid)
  2125. *
  2126. * An insecure source has sent us a dentry, here we verify it and dget() it.
  2127. * This is used by ncpfs in its readdir implementation.
  2128. * Zero is returned in the dentry is invalid.
  2129. *
  2130. * This function is slow for big directories, and deprecated, do not use it.
  2131. */
  2132. int d_validate(struct dentry *dentry, struct dentry *dparent)
  2133. {
  2134. struct dentry *child;
  2135. spin_lock(&dparent->d_lock);
  2136. list_for_each_entry(child, &dparent->d_subdirs, d_u.d_child) {
  2137. if (dentry == child) {
  2138. spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
  2139. __dget_dlock(dentry);
  2140. spin_unlock(&dentry->d_lock);
  2141. spin_unlock(&dparent->d_lock);
  2142. return 1;
  2143. }
  2144. }
  2145. spin_unlock(&dparent->d_lock);
  2146. return 0;
  2147. }
  2148. EXPORT_SYMBOL(d_validate);
  2149. /*
  2150. * When a file is deleted, we have two options:
  2151. * - turn this dentry into a negative dentry
  2152. * - unhash this dentry and free it.
  2153. *
  2154. * Usually, we want to just turn this into
  2155. * a negative dentry, but if anybody else is
  2156. * currently using the dentry or the inode
  2157. * we can't do that and we fall back on removing
  2158. * it from the hash queues and waiting for
  2159. * it to be deleted later when it has no users
  2160. */
  2161. /**
  2162. * d_delete - delete a dentry
  2163. * @dentry: The dentry to delete
  2164. *
  2165. * Turn the dentry into a negative dentry if possible, otherwise
  2166. * remove it from the hash queues so it can be deleted later
  2167. */
  2168. void d_delete(struct dentry * dentry)
  2169. {
  2170. struct inode *inode;
  2171. int isdir = 0;
  2172. /*
  2173. * Are we the only user?
  2174. */
  2175. again:
  2176. spin_lock(&dentry->d_lock);
  2177. inode = dentry->d_inode;
  2178. isdir = S_ISDIR(inode->i_mode);
  2179. if (dentry->d_lockref.count == 1) {
  2180. if (!spin_trylock(&inode->i_lock)) {
  2181. spin_unlock(&dentry->d_lock);
  2182. cpu_relax();
  2183. goto again;
  2184. }
  2185. dentry->d_flags &= ~DCACHE_CANT_MOUNT;
  2186. dentry_unlink_inode(dentry);
  2187. fsnotify_nameremove(dentry, isdir);
  2188. return;
  2189. }
  2190. if (!d_unhashed(dentry))
  2191. __d_drop(dentry);
  2192. spin_unlock(&dentry->d_lock);
  2193. fsnotify_nameremove(dentry, isdir);
  2194. }
  2195. EXPORT_SYMBOL(d_delete);
  2196. static void __d_rehash(struct dentry * entry, struct hlist_bl_head *b)
  2197. {
  2198. BUG_ON(!d_unhashed(entry));
  2199. hlist_bl_lock(b);
  2200. entry->d_flags |= DCACHE_RCUACCESS;
  2201. hlist_bl_add_head_rcu(&entry->d_hash, b);
  2202. hlist_bl_unlock(b);
  2203. }
  2204. static void _d_rehash(struct dentry * entry)
  2205. {
  2206. __d_rehash(entry, d_hash(entry->d_parent, entry->d_name.hash));
  2207. }
  2208. /**
  2209. * d_rehash - add an entry back to the hash
  2210. * @entry: dentry to add to the hash
  2211. *
  2212. * Adds a dentry to the hash according to its name.
  2213. */
  2214. void d_rehash(struct dentry * entry)
  2215. {
  2216. spin_lock(&entry->d_lock);
  2217. _d_rehash(entry);
  2218. spin_unlock(&entry->d_lock);
  2219. }
  2220. EXPORT_SYMBOL(d_rehash);
  2221. /**
  2222. * dentry_update_name_case - update case insensitive dentry with a new name
  2223. * @dentry: dentry to be updated
  2224. * @name: new name
  2225. *
  2226. * Update a case insensitive dentry with new case of name.
  2227. *
  2228. * dentry must have been returned by d_lookup with name @name. Old and new
  2229. * name lengths must match (ie. no d_compare which allows mismatched name
  2230. * lengths).
  2231. *
  2232. * Parent inode i_mutex must be held over d_lookup and into this call (to
  2233. * keep renames and concurrent inserts, and readdir(2) away).
  2234. */
  2235. void dentry_update_name_case(struct dentry *dentry, struct qstr *name)
  2236. {
  2237. BUG_ON(!mutex_is_locked(&dentry->d_parent->d_inode->i_mutex));
  2238. BUG_ON(dentry->d_name.len != name->len); /* d_lookup gives this */
  2239. spin_lock(&dentry->d_lock);
  2240. write_seqcount_begin(&dentry->d_seq);
  2241. memcpy((unsigned char *)dentry->d_name.name, name->name, name->len);
  2242. write_seqcount_end(&dentry->d_seq);
  2243. spin_unlock(&dentry->d_lock);
  2244. }
  2245. EXPORT_SYMBOL(dentry_update_name_case);
  2246. static void switch_names(struct dentry *dentry, struct dentry *target)
  2247. {
  2248. if (dname_external(target)) {
  2249. if (dname_external(dentry)) {
  2250. /*
  2251. * Both external: swap the pointers
  2252. */
  2253. swap(target->d_name.name, dentry->d_name.name);
  2254. } else {
  2255. /*
  2256. * dentry:internal, target:external. Steal target's
  2257. * storage and make target internal.
  2258. */
  2259. memcpy(target->d_iname, dentry->d_name.name,
  2260. dentry->d_name.len + 1);
  2261. dentry->d_name.name = target->d_name.name;
  2262. target->d_name.name = target->d_iname;
  2263. }
  2264. } else {
  2265. if (dname_external(dentry)) {
  2266. /*
  2267. * dentry:external, target:internal. Give dentry's
  2268. * storage to target and make dentry internal
  2269. */
  2270. memcpy(dentry->d_iname, target->d_name.name,
  2271. target->d_name.len + 1);
  2272. target->d_name.name = dentry->d_name.name;
  2273. dentry->d_name.name = dentry->d_iname;
  2274. } else {
  2275. /*
  2276. * Both are internal. Just copy target to dentry
  2277. */
  2278. memcpy(dentry->d_iname, target->d_name.name,
  2279. target->d_name.len + 1);
  2280. dentry->d_name.len = target->d_name.len;
  2281. return;
  2282. }
  2283. }
  2284. swap(dentry->d_name.len, target->d_name.len);
  2285. }
  2286. static void dentry_lock_for_move(struct dentry *dentry, struct dentry *target)
  2287. {
  2288. /*
  2289. * XXXX: do we really need to take target->d_lock?
  2290. */
  2291. if (IS_ROOT(dentry) || dentry->d_parent == target->d_parent)
  2292. spin_lock(&target->d_parent->d_lock);
  2293. else {
  2294. if (d_ancestor(dentry->d_parent, target->d_parent)) {
  2295. spin_lock(&dentry->d_parent->d_lock);
  2296. spin_lock_nested(&target->d_parent->d_lock,
  2297. DENTRY_D_LOCK_NESTED);
  2298. } else {
  2299. spin_lock(&target->d_parent->d_lock);
  2300. spin_lock_nested(&dentry->d_parent->d_lock,
  2301. DENTRY_D_LOCK_NESTED);
  2302. }
  2303. }
  2304. if (target < dentry) {
  2305. spin_lock_nested(&target->d_lock, 2);
  2306. spin_lock_nested(&dentry->d_lock, 3);
  2307. } else {
  2308. spin_lock_nested(&dentry->d_lock, 2);
  2309. spin_lock_nested(&target->d_lock, 3);
  2310. }
  2311. }
  2312. static void dentry_unlock_parents_for_move(struct dentry *dentry,
  2313. struct dentry *target)
  2314. {
  2315. if (target->d_parent != dentry->d_parent)
  2316. spin_unlock(&dentry->d_parent->d_lock);
  2317. if (target->d_parent != target)
  2318. spin_unlock(&target->d_parent->d_lock);
  2319. }
  2320. /*
  2321. * When switching names, the actual string doesn't strictly have to
  2322. * be preserved in the target - because we're dropping the target
  2323. * anyway. As such, we can just do a simple memcpy() to copy over
  2324. * the new name before we switch.
  2325. *
  2326. * Note that we have to be a lot more careful about getting the hash
  2327. * switched - we have to switch the hash value properly even if it
  2328. * then no longer matches the actual (corrupted) string of the target.
  2329. * The hash value has to match the hash queue that the dentry is on..
  2330. */
  2331. /*
  2332. * __d_move - move a dentry
  2333. * @dentry: entry to move
  2334. * @target: new dentry
  2335. *
  2336. * Update the dcache to reflect the move of a file name. Negative
  2337. * dcache entries should not be moved in this way. Caller must hold
  2338. * rename_lock, the i_mutex of the source and target directories,
  2339. * and the sb->s_vfs_rename_mutex if they differ. See lock_rename().
  2340. */
  2341. static void __d_move(struct dentry * dentry, struct dentry * target)
  2342. {
  2343. if (!dentry->d_inode)
  2344. printk(KERN_WARNING "VFS: moving negative dcache entry\n");
  2345. BUG_ON(d_ancestor(dentry, target));
  2346. BUG_ON(d_ancestor(target, dentry));
  2347. dentry_lock_for_move(dentry, target);
  2348. write_seqcount_begin(&dentry->d_seq);
  2349. write_seqcount_begin(&target->d_seq);
  2350. /* __d_drop does write_seqcount_barrier, but they're OK to nest. */
  2351. /*
  2352. * Move the dentry to the target hash queue. Don't bother checking
  2353. * for the same hash queue because of how unlikely it is.
  2354. */
  2355. __d_drop(dentry);
  2356. __d_rehash(dentry, d_hash(target->d_parent, target->d_name.hash));
  2357. /* Unhash the target: dput() will then get rid of it */
  2358. __d_drop(target);
  2359. list_del(&dentry->d_u.d_child);
  2360. list_del(&target->d_u.d_child);
  2361. /* Switch the names.. */
  2362. switch_names(dentry, target);
  2363. swap(dentry->d_name.hash, target->d_name.hash);
  2364. /* ... and switch the parents */
  2365. if (IS_ROOT(dentry)) {
  2366. dentry->d_parent = target->d_parent;
  2367. target->d_parent = target;
  2368. INIT_LIST_HEAD(&target->d_u.d_child);
  2369. } else {
  2370. swap(dentry->d_parent, target->d_parent);
  2371. /* And add them back to the (new) parent lists */
  2372. list_add(&target->d_u.d_child, &target->d_parent->d_subdirs);
  2373. }
  2374. list_add(&dentry->d_u.d_child, &dentry->d_parent->d_subdirs);
  2375. write_seqcount_end(&target->d_seq);
  2376. write_seqcount_end(&dentry->d_seq);
  2377. dentry_unlock_parents_for_move(dentry, target);
  2378. spin_unlock(&target->d_lock);
  2379. fsnotify_d_move(dentry);
  2380. spin_unlock(&dentry->d_lock);
  2381. }
  2382. /*
  2383. * d_move - move a dentry
  2384. * @dentry: entry to move
  2385. * @target: new dentry
  2386. *
  2387. * Update the dcache to reflect the move of a file name. Negative
  2388. * dcache entries should not be moved in this way. See the locking
  2389. * requirements for __d_move.
  2390. */
  2391. void d_move(struct dentry *dentry, struct dentry *target)
  2392. {
  2393. write_seqlock(&rename_lock);
  2394. __d_move(dentry, target);
  2395. write_sequnlock(&rename_lock);
  2396. }
  2397. EXPORT_SYMBOL(d_move);
  2398. /**
  2399. * d_ancestor - search for an ancestor
  2400. * @p1: ancestor dentry
  2401. * @p2: child dentry
  2402. *
  2403. * Returns the ancestor dentry of p2 which is a child of p1, if p1 is
  2404. * an ancestor of p2, else NULL.
  2405. */
  2406. struct dentry *d_ancestor(struct dentry *p1, struct dentry *p2)
  2407. {
  2408. struct dentry *p;
  2409. for (p = p2; !IS_ROOT(p); p = p->d_parent) {
  2410. if (p->d_parent == p1)
  2411. return p;
  2412. }
  2413. return NULL;
  2414. }
  2415. /*
  2416. * This helper attempts to cope with remotely renamed directories
  2417. *
  2418. * It assumes that the caller is already holding
  2419. * dentry->d_parent->d_inode->i_mutex, inode->i_lock and rename_lock
  2420. *
  2421. * Note: If ever the locking in lock_rename() changes, then please
  2422. * remember to update this too...
  2423. */
  2424. static struct dentry *__d_unalias(struct inode *inode,
  2425. struct dentry *dentry, struct dentry *alias)
  2426. {
  2427. struct mutex *m1 = NULL, *m2 = NULL;
  2428. struct dentry *ret = ERR_PTR(-EBUSY);
  2429. /* If alias and dentry share a parent, then no extra locks required */
  2430. if (alias->d_parent == dentry->d_parent)
  2431. goto out_unalias;
  2432. /* See lock_rename() */
  2433. if (!mutex_trylock(&dentry->d_sb->s_vfs_rename_mutex))
  2434. goto out_err;
  2435. m1 = &dentry->d_sb->s_vfs_rename_mutex;
  2436. if (!mutex_trylock(&alias->d_parent->d_inode->i_mutex))
  2437. goto out_err;
  2438. m2 = &alias->d_parent->d_inode->i_mutex;
  2439. out_unalias:
  2440. if (likely(!d_mountpoint(alias))) {
  2441. __d_move(alias, dentry);
  2442. ret = alias;
  2443. }
  2444. out_err:
  2445. spin_unlock(&inode->i_lock);
  2446. if (m2)
  2447. mutex_unlock(m2);
  2448. if (m1)
  2449. mutex_unlock(m1);
  2450. return ret;
  2451. }
  2452. /*
  2453. * Prepare an anonymous dentry for life in the superblock's dentry tree as a
  2454. * named dentry in place of the dentry to be replaced.
  2455. * returns with anon->d_lock held!
  2456. */
  2457. static void __d_materialise_dentry(struct dentry *dentry, struct dentry *anon)
  2458. {
  2459. struct dentry *dparent;
  2460. dentry_lock_for_move(anon, dentry);
  2461. write_seqcount_begin(&dentry->d_seq);
  2462. write_seqcount_begin(&anon->d_seq);
  2463. dparent = dentry->d_parent;
  2464. switch_names(dentry, anon);
  2465. swap(dentry->d_name.hash, anon->d_name.hash);
  2466. dentry->d_parent = dentry;
  2467. list_del_init(&dentry->d_u.d_child);
  2468. anon->d_parent = dparent;
  2469. list_move(&anon->d_u.d_child, &dparent->d_subdirs);
  2470. write_seqcount_end(&dentry->d_seq);
  2471. write_seqcount_end(&anon->d_seq);
  2472. dentry_unlock_parents_for_move(anon, dentry);
  2473. spin_unlock(&dentry->d_lock);
  2474. /* anon->d_lock still locked, returns locked */
  2475. }
  2476. /**
  2477. * d_materialise_unique - introduce an inode into the tree
  2478. * @dentry: candidate dentry
  2479. * @inode: inode to bind to the dentry, to which aliases may be attached
  2480. *
  2481. * Introduces an dentry into the tree, substituting an extant disconnected
  2482. * root directory alias in its place if there is one. Caller must hold the
  2483. * i_mutex of the parent directory.
  2484. */
  2485. struct dentry *d_materialise_unique(struct dentry *dentry, struct inode *inode)
  2486. {
  2487. struct dentry *actual;
  2488. BUG_ON(!d_unhashed(dentry));
  2489. if (!inode) {
  2490. actual = dentry;
  2491. __d_instantiate(dentry, NULL);
  2492. d_rehash(actual);
  2493. goto out_nolock;
  2494. }
  2495. spin_lock(&inode->i_lock);
  2496. if (S_ISDIR(inode->i_mode)) {
  2497. struct dentry *alias;
  2498. /* Does an aliased dentry already exist? */
  2499. alias = __d_find_alias(inode, 0);
  2500. if (alias) {
  2501. actual = alias;
  2502. write_seqlock(&rename_lock);
  2503. if (d_ancestor(alias, dentry)) {
  2504. /* Check for loops */
  2505. actual = ERR_PTR(-ELOOP);
  2506. spin_unlock(&inode->i_lock);
  2507. } else if (IS_ROOT(alias)) {
  2508. /* Is this an anonymous mountpoint that we
  2509. * could splice into our tree? */
  2510. __d_materialise_dentry(dentry, alias);
  2511. write_sequnlock(&rename_lock);
  2512. __d_drop(alias);
  2513. goto found;
  2514. } else {
  2515. /* Nope, but we must(!) avoid directory
  2516. * aliasing. This drops inode->i_lock */
  2517. actual = __d_unalias(inode, dentry, alias);
  2518. }
  2519. write_sequnlock(&rename_lock);
  2520. if (IS_ERR(actual)) {
  2521. if (PTR_ERR(actual) == -ELOOP)
  2522. pr_warn_ratelimited(
  2523. "VFS: Lookup of '%s' in %s %s"
  2524. " would have caused loop\n",
  2525. dentry->d_name.name,
  2526. inode->i_sb->s_type->name,
  2527. inode->i_sb->s_id);
  2528. dput(alias);
  2529. }
  2530. goto out_nolock;
  2531. }
  2532. }
  2533. /* Add a unique reference */
  2534. actual = __d_instantiate_unique(dentry, inode);
  2535. if (!actual)
  2536. actual = dentry;
  2537. else
  2538. BUG_ON(!d_unhashed(actual));
  2539. spin_lock(&actual->d_lock);
  2540. found:
  2541. _d_rehash(actual);
  2542. spin_unlock(&actual->d_lock);
  2543. spin_unlock(&inode->i_lock);
  2544. out_nolock:
  2545. if (actual == dentry) {
  2546. security_d_instantiate(dentry, inode);
  2547. return NULL;
  2548. }
  2549. iput(inode);
  2550. return actual;
  2551. }
  2552. EXPORT_SYMBOL_GPL(d_materialise_unique);
  2553. static int prepend(char **buffer, int *buflen, const char *str, int namelen)
  2554. {
  2555. *buflen -= namelen;
  2556. if (*buflen < 0)
  2557. return -ENAMETOOLONG;
  2558. *buffer -= namelen;
  2559. memcpy(*buffer, str, namelen);
  2560. return 0;
  2561. }
  2562. /**
  2563. * prepend_name - prepend a pathname in front of current buffer pointer
  2564. * @buffer: buffer pointer
  2565. * @buflen: allocated length of the buffer
  2566. * @name: name string and length qstr structure
  2567. *
  2568. * With RCU path tracing, it may race with d_move(). Use ACCESS_ONCE() to
  2569. * make sure that either the old or the new name pointer and length are
  2570. * fetched. However, there may be mismatch between length and pointer.
  2571. * The length cannot be trusted, we need to copy it byte-by-byte until
  2572. * the length is reached or a null byte is found. It also prepends "/" at
  2573. * the beginning of the name. The sequence number check at the caller will
  2574. * retry it again when a d_move() does happen. So any garbage in the buffer
  2575. * due to mismatched pointer and length will be discarded.
  2576. */
  2577. static int prepend_name(char **buffer, int *buflen, struct qstr *name)
  2578. {
  2579. const char *dname = ACCESS_ONCE(name->name);
  2580. u32 dlen = ACCESS_ONCE(name->len);
  2581. char *p;
  2582. if (*buflen < dlen + 1)
  2583. return -ENAMETOOLONG;
  2584. *buflen -= dlen + 1;
  2585. p = *buffer -= dlen + 1;
  2586. *p++ = '/';
  2587. while (dlen--) {
  2588. char c = *dname++;
  2589. if (!c)
  2590. break;
  2591. *p++ = c;
  2592. }
  2593. return 0;
  2594. }
  2595. /**
  2596. * prepend_path - Prepend path string to a buffer
  2597. * @path: the dentry/vfsmount to report
  2598. * @root: root vfsmnt/dentry
  2599. * @buffer: pointer to the end of the buffer
  2600. * @buflen: pointer to buffer length
  2601. *
  2602. * The function will first try to write out the pathname without taking any
  2603. * lock other than the RCU read lock to make sure that dentries won't go away.
  2604. * It only checks the sequence number of the global rename_lock as any change
  2605. * in the dentry's d_seq will be preceded by changes in the rename_lock
  2606. * sequence number. If the sequence number had been changed, it will restart
  2607. * the whole pathname back-tracing sequence again by taking the rename_lock.
  2608. * In this case, there is no need to take the RCU read lock as the recursive
  2609. * parent pointer references will keep the dentry chain alive as long as no
  2610. * rename operation is performed.
  2611. */
  2612. static int prepend_path(const struct path *path,
  2613. const struct path *root,
  2614. char **buffer, int *buflen)
  2615. {
  2616. struct dentry *dentry;
  2617. struct vfsmount *vfsmnt;
  2618. struct mount *mnt;
  2619. int error = 0;
  2620. unsigned seq, m_seq = 0;
  2621. char *bptr;
  2622. int blen;
  2623. rcu_read_lock();
  2624. restart_mnt:
  2625. read_seqbegin_or_lock(&mount_lock, &m_seq);
  2626. seq = 0;
  2627. rcu_read_lock();
  2628. restart:
  2629. bptr = *buffer;
  2630. blen = *buflen;
  2631. error = 0;
  2632. dentry = path->dentry;
  2633. vfsmnt = path->mnt;
  2634. mnt = real_mount(vfsmnt);
  2635. read_seqbegin_or_lock(&rename_lock, &seq);
  2636. while (dentry != root->dentry || vfsmnt != root->mnt) {
  2637. struct dentry * parent;
  2638. if (dentry == vfsmnt->mnt_root || IS_ROOT(dentry)) {
  2639. struct mount *parent = ACCESS_ONCE(mnt->mnt_parent);
  2640. /* Global root? */
  2641. if (mnt != parent) {
  2642. dentry = ACCESS_ONCE(mnt->mnt_mountpoint);
  2643. mnt = parent;
  2644. vfsmnt = &mnt->mnt;
  2645. continue;
  2646. }
  2647. /*
  2648. * Filesystems needing to implement special "root names"
  2649. * should do so with ->d_dname()
  2650. */
  2651. if (IS_ROOT(dentry) &&
  2652. (dentry->d_name.len != 1 ||
  2653. dentry->d_name.name[0] != '/')) {
  2654. WARN(1, "Root dentry has weird name <%.*s>\n",
  2655. (int) dentry->d_name.len,
  2656. dentry->d_name.name);
  2657. }
  2658. if (!error)
  2659. error = is_mounted(vfsmnt) ? 1 : 2;
  2660. break;
  2661. }
  2662. parent = dentry->d_parent;
  2663. prefetch(parent);
  2664. error = prepend_name(&bptr, &blen, &dentry->d_name);
  2665. if (error)
  2666. break;
  2667. dentry = parent;
  2668. }
  2669. if (!(seq & 1))
  2670. rcu_read_unlock();
  2671. if (need_seqretry(&rename_lock, seq)) {
  2672. seq = 1;
  2673. goto restart;
  2674. }
  2675. done_seqretry(&rename_lock, seq);
  2676. if (!(m_seq & 1))
  2677. rcu_read_unlock();
  2678. if (need_seqretry(&mount_lock, m_seq)) {
  2679. m_seq = 1;
  2680. goto restart_mnt;
  2681. }
  2682. done_seqretry(&mount_lock, m_seq);
  2683. if (error >= 0 && bptr == *buffer) {
  2684. if (--blen < 0)
  2685. error = -ENAMETOOLONG;
  2686. else
  2687. *--bptr = '/';
  2688. }
  2689. *buffer = bptr;
  2690. *buflen = blen;
  2691. return error;
  2692. }
  2693. /**
  2694. * __d_path - return the path of a dentry
  2695. * @path: the dentry/vfsmount to report
  2696. * @root: root vfsmnt/dentry
  2697. * @buf: buffer to return value in
  2698. * @buflen: buffer length
  2699. *
  2700. * Convert a dentry into an ASCII path name.
  2701. *
  2702. * Returns a pointer into the buffer or an error code if the
  2703. * path was too long.
  2704. *
  2705. * "buflen" should be positive.
  2706. *
  2707. * If the path is not reachable from the supplied root, return %NULL.
  2708. */
  2709. char *__d_path(const struct path *path,
  2710. const struct path *root,
  2711. char *buf, int buflen)
  2712. {
  2713. char *res = buf + buflen;
  2714. int error;
  2715. prepend(&res, &buflen, "\0", 1);
  2716. error = prepend_path(path, root, &res, &buflen);
  2717. if (error < 0)
  2718. return ERR_PTR(error);
  2719. if (error > 0)
  2720. return NULL;
  2721. return res;
  2722. }
  2723. char *d_absolute_path(const struct path *path,
  2724. char *buf, int buflen)
  2725. {
  2726. struct path root = {};
  2727. char *res = buf + buflen;
  2728. int error;
  2729. prepend(&res, &buflen, "\0", 1);
  2730. error = prepend_path(path, &root, &res, &buflen);
  2731. if (error > 1)
  2732. error = -EINVAL;
  2733. if (error < 0)
  2734. return ERR_PTR(error);
  2735. return res;
  2736. }
  2737. /*
  2738. * same as __d_path but appends "(deleted)" for unlinked files.
  2739. */
  2740. static int path_with_deleted(const struct path *path,
  2741. const struct path *root,
  2742. char **buf, int *buflen)
  2743. {
  2744. prepend(buf, buflen, "\0", 1);
  2745. if (d_unlinked(path->dentry)) {
  2746. int error = prepend(buf, buflen, " (deleted)", 10);
  2747. if (error)
  2748. return error;
  2749. }
  2750. return prepend_path(path, root, buf, buflen);
  2751. }
  2752. static int prepend_unreachable(char **buffer, int *buflen)
  2753. {
  2754. return prepend(buffer, buflen, "(unreachable)", 13);
  2755. }
  2756. static void get_fs_root_rcu(struct fs_struct *fs, struct path *root)
  2757. {
  2758. unsigned seq;
  2759. do {
  2760. seq = read_seqcount_begin(&fs->seq);
  2761. *root = fs->root;
  2762. } while (read_seqcount_retry(&fs->seq, seq));
  2763. }
  2764. /**
  2765. * d_path - return the path of a dentry
  2766. * @path: path to report
  2767. * @buf: buffer to return value in
  2768. * @buflen: buffer length
  2769. *
  2770. * Convert a dentry into an ASCII path name. If the entry has been deleted
  2771. * the string " (deleted)" is appended. Note that this is ambiguous.
  2772. *
  2773. * Returns a pointer into the buffer or an error code if the path was
  2774. * too long. Note: Callers should use the returned pointer, not the passed
  2775. * in buffer, to use the name! The implementation often starts at an offset
  2776. * into the buffer, and may leave 0 bytes at the start.
  2777. *
  2778. * "buflen" should be positive.
  2779. */
  2780. char *d_path(const struct path *path, char *buf, int buflen)
  2781. {
  2782. char *res = buf + buflen;
  2783. struct path root;
  2784. int error;
  2785. /*
  2786. * We have various synthetic filesystems that never get mounted. On
  2787. * these filesystems dentries are never used for lookup purposes, and
  2788. * thus don't need to be hashed. They also don't need a name until a
  2789. * user wants to identify the object in /proc/pid/fd/. The little hack
  2790. * below allows us to generate a name for these objects on demand:
  2791. */
  2792. if (path->dentry->d_op && path->dentry->d_op->d_dname)
  2793. return path->dentry->d_op->d_dname(path->dentry, buf, buflen);
  2794. rcu_read_lock();
  2795. get_fs_root_rcu(current->fs, &root);
  2796. error = path_with_deleted(path, &root, &res, &buflen);
  2797. rcu_read_unlock();
  2798. if (error < 0)
  2799. res = ERR_PTR(error);
  2800. return res;
  2801. }
  2802. EXPORT_SYMBOL(d_path);
  2803. /*
  2804. * Helper function for dentry_operations.d_dname() members
  2805. */
  2806. char *dynamic_dname(struct dentry *dentry, char *buffer, int buflen,
  2807. const char *fmt, ...)
  2808. {
  2809. va_list args;
  2810. char temp[64];
  2811. int sz;
  2812. va_start(args, fmt);
  2813. sz = vsnprintf(temp, sizeof(temp), fmt, args) + 1;
  2814. va_end(args);
  2815. if (sz > sizeof(temp) || sz > buflen)
  2816. return ERR_PTR(-ENAMETOOLONG);
  2817. buffer += buflen - sz;
  2818. return memcpy(buffer, temp, sz);
  2819. }
  2820. char *simple_dname(struct dentry *dentry, char *buffer, int buflen)
  2821. {
  2822. char *end = buffer + buflen;
  2823. /* these dentries are never renamed, so d_lock is not needed */
  2824. if (prepend(&end, &buflen, " (deleted)", 11) ||
  2825. prepend(&end, &buflen, dentry->d_name.name, dentry->d_name.len) ||
  2826. prepend(&end, &buflen, "/", 1))
  2827. end = ERR_PTR(-ENAMETOOLONG);
  2828. return end;
  2829. }
  2830. /*
  2831. * Write full pathname from the root of the filesystem into the buffer.
  2832. */
  2833. static char *__dentry_path(struct dentry *dentry, char *buf, int buflen)
  2834. {
  2835. char *end, *retval;
  2836. int len, seq = 0;
  2837. int error = 0;
  2838. rcu_read_lock();
  2839. restart:
  2840. end = buf + buflen;
  2841. len = buflen;
  2842. prepend(&end, &len, "\0", 1);
  2843. if (buflen < 1)
  2844. goto Elong;
  2845. /* Get '/' right */
  2846. retval = end-1;
  2847. *retval = '/';
  2848. read_seqbegin_or_lock(&rename_lock, &seq);
  2849. while (!IS_ROOT(dentry)) {
  2850. struct dentry *parent = dentry->d_parent;
  2851. int error;
  2852. prefetch(parent);
  2853. error = prepend_name(&end, &len, &dentry->d_name);
  2854. if (error)
  2855. break;
  2856. retval = end;
  2857. dentry = parent;
  2858. }
  2859. if (!(seq & 1))
  2860. rcu_read_unlock();
  2861. if (need_seqretry(&rename_lock, seq)) {
  2862. seq = 1;
  2863. goto restart;
  2864. }
  2865. done_seqretry(&rename_lock, seq);
  2866. if (error)
  2867. goto Elong;
  2868. return retval;
  2869. Elong:
  2870. return ERR_PTR(-ENAMETOOLONG);
  2871. }
  2872. char *dentry_path_raw(struct dentry *dentry, char *buf, int buflen)
  2873. {
  2874. return __dentry_path(dentry, buf, buflen);
  2875. }
  2876. EXPORT_SYMBOL(dentry_path_raw);
  2877. char *dentry_path(struct dentry *dentry, char *buf, int buflen)
  2878. {
  2879. char *p = NULL;
  2880. char *retval;
  2881. if (d_unlinked(dentry)) {
  2882. p = buf + buflen;
  2883. if (prepend(&p, &buflen, "//deleted", 10) != 0)
  2884. goto Elong;
  2885. buflen++;
  2886. }
  2887. retval = __dentry_path(dentry, buf, buflen);
  2888. if (!IS_ERR(retval) && p)
  2889. *p = '/'; /* restore '/' overriden with '\0' */
  2890. return retval;
  2891. Elong:
  2892. return ERR_PTR(-ENAMETOOLONG);
  2893. }
  2894. static void get_fs_root_and_pwd_rcu(struct fs_struct *fs, struct path *root,
  2895. struct path *pwd)
  2896. {
  2897. unsigned seq;
  2898. do {
  2899. seq = read_seqcount_begin(&fs->seq);
  2900. *root = fs->root;
  2901. *pwd = fs->pwd;
  2902. } while (read_seqcount_retry(&fs->seq, seq));
  2903. }
  2904. /*
  2905. * NOTE! The user-level library version returns a
  2906. * character pointer. The kernel system call just
  2907. * returns the length of the buffer filled (which
  2908. * includes the ending '\0' character), or a negative
  2909. * error value. So libc would do something like
  2910. *
  2911. * char *getcwd(char * buf, size_t size)
  2912. * {
  2913. * int retval;
  2914. *
  2915. * retval = sys_getcwd(buf, size);
  2916. * if (retval >= 0)
  2917. * return buf;
  2918. * errno = -retval;
  2919. * return NULL;
  2920. * }
  2921. */
  2922. SYSCALL_DEFINE2(getcwd, char __user *, buf, unsigned long, size)
  2923. {
  2924. int error;
  2925. struct path pwd, root;
  2926. char *page = __getname();
  2927. if (!page)
  2928. return -ENOMEM;
  2929. rcu_read_lock();
  2930. get_fs_root_and_pwd_rcu(current->fs, &root, &pwd);
  2931. error = -ENOENT;
  2932. if (!d_unlinked(pwd.dentry)) {
  2933. unsigned long len;
  2934. char *cwd = page + PATH_MAX;
  2935. int buflen = PATH_MAX;
  2936. prepend(&cwd, &buflen, "\0", 1);
  2937. error = prepend_path(&pwd, &root, &cwd, &buflen);
  2938. rcu_read_unlock();
  2939. if (error < 0)
  2940. goto out;
  2941. /* Unreachable from current root */
  2942. if (error > 0) {
  2943. error = prepend_unreachable(&cwd, &buflen);
  2944. if (error)
  2945. goto out;
  2946. }
  2947. error = -ERANGE;
  2948. len = PATH_MAX + page - cwd;
  2949. if (len <= size) {
  2950. error = len;
  2951. if (copy_to_user(buf, cwd, len))
  2952. error = -EFAULT;
  2953. }
  2954. } else {
  2955. rcu_read_unlock();
  2956. }
  2957. out:
  2958. __putname(page);
  2959. return error;
  2960. }
  2961. /*
  2962. * Test whether new_dentry is a subdirectory of old_dentry.
  2963. *
  2964. * Trivially implemented using the dcache structure
  2965. */
  2966. /**
  2967. * is_subdir - is new dentry a subdirectory of old_dentry
  2968. * @new_dentry: new dentry
  2969. * @old_dentry: old dentry
  2970. *
  2971. * Returns 1 if new_dentry is a subdirectory of the parent (at any depth).
  2972. * Returns 0 otherwise.
  2973. * Caller must ensure that "new_dentry" is pinned before calling is_subdir()
  2974. */
  2975. int is_subdir(struct dentry *new_dentry, struct dentry *old_dentry)
  2976. {
  2977. int result;
  2978. unsigned seq;
  2979. if (new_dentry == old_dentry)
  2980. return 1;
  2981. do {
  2982. /* for restarting inner loop in case of seq retry */
  2983. seq = read_seqbegin(&rename_lock);
  2984. /*
  2985. * Need rcu_readlock to protect against the d_parent trashing
  2986. * due to d_move
  2987. */
  2988. rcu_read_lock();
  2989. if (d_ancestor(old_dentry, new_dentry))
  2990. result = 1;
  2991. else
  2992. result = 0;
  2993. rcu_read_unlock();
  2994. } while (read_seqretry(&rename_lock, seq));
  2995. return result;
  2996. }
  2997. static enum d_walk_ret d_genocide_kill(void *data, struct dentry *dentry)
  2998. {
  2999. struct dentry *root = data;
  3000. if (dentry != root) {
  3001. if (d_unhashed(dentry) || !dentry->d_inode)
  3002. return D_WALK_SKIP;
  3003. if (!(dentry->d_flags & DCACHE_GENOCIDE)) {
  3004. dentry->d_flags |= DCACHE_GENOCIDE;
  3005. dentry->d_lockref.count--;
  3006. }
  3007. }
  3008. return D_WALK_CONTINUE;
  3009. }
  3010. void d_genocide(struct dentry *parent)
  3011. {
  3012. d_walk(parent, parent, d_genocide_kill, NULL);
  3013. }
  3014. void d_tmpfile(struct dentry *dentry, struct inode *inode)
  3015. {
  3016. inode_dec_link_count(inode);
  3017. BUG_ON(dentry->d_name.name != dentry->d_iname ||
  3018. !hlist_unhashed(&dentry->d_alias) ||
  3019. !d_unlinked(dentry));
  3020. spin_lock(&dentry->d_parent->d_lock);
  3021. spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
  3022. dentry->d_name.len = sprintf(dentry->d_iname, "#%llu",
  3023. (unsigned long long)inode->i_ino);
  3024. spin_unlock(&dentry->d_lock);
  3025. spin_unlock(&dentry->d_parent->d_lock);
  3026. d_instantiate(dentry, inode);
  3027. }
  3028. EXPORT_SYMBOL(d_tmpfile);
  3029. static __initdata unsigned long dhash_entries;
  3030. static int __init set_dhash_entries(char *str)
  3031. {
  3032. if (!str)
  3033. return 0;
  3034. dhash_entries = simple_strtoul(str, &str, 0);
  3035. return 1;
  3036. }
  3037. __setup("dhash_entries=", set_dhash_entries);
  3038. static void __init dcache_init_early(void)
  3039. {
  3040. unsigned int loop;
  3041. /* If hashes are distributed across NUMA nodes, defer
  3042. * hash allocation until vmalloc space is available.
  3043. */
  3044. if (hashdist)
  3045. return;
  3046. dentry_hashtable =
  3047. alloc_large_system_hash("Dentry cache",
  3048. sizeof(struct hlist_bl_head),
  3049. dhash_entries,
  3050. 13,
  3051. HASH_EARLY,
  3052. &d_hash_shift,
  3053. &d_hash_mask,
  3054. 0,
  3055. 0);
  3056. for (loop = 0; loop < (1U << d_hash_shift); loop++)
  3057. INIT_HLIST_BL_HEAD(dentry_hashtable + loop);
  3058. }
  3059. static void __init dcache_init(void)
  3060. {
  3061. unsigned int loop;
  3062. /*
  3063. * A constructor could be added for stable state like the lists,
  3064. * but it is probably not worth it because of the cache nature
  3065. * of the dcache.
  3066. */
  3067. dentry_cache = KMEM_CACHE(dentry,
  3068. SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|SLAB_MEM_SPREAD);
  3069. /* Hash may have been set up in dcache_init_early */
  3070. if (!hashdist)
  3071. return;
  3072. dentry_hashtable =
  3073. alloc_large_system_hash("Dentry cache",
  3074. sizeof(struct hlist_bl_head),
  3075. dhash_entries,
  3076. 13,
  3077. 0,
  3078. &d_hash_shift,
  3079. &d_hash_mask,
  3080. 0,
  3081. 0);
  3082. for (loop = 0; loop < (1U << d_hash_shift); loop++)
  3083. INIT_HLIST_BL_HEAD(dentry_hashtable + loop);
  3084. }
  3085. /* SLAB cache for __getname() consumers */
  3086. struct kmem_cache *names_cachep __read_mostly;
  3087. EXPORT_SYMBOL(names_cachep);
  3088. EXPORT_SYMBOL(d_genocide);
  3089. void __init vfs_caches_init_early(void)
  3090. {
  3091. dcache_init_early();
  3092. inode_init_early();
  3093. }
  3094. void __init vfs_caches_init(unsigned long mempages)
  3095. {
  3096. unsigned long reserve;
  3097. /* Base hash sizes on available memory, with a reserve equal to
  3098. 150% of current kernel size */
  3099. reserve = min((mempages - nr_free_pages()) * 3/2, mempages - 1);
  3100. mempages -= reserve;
  3101. names_cachep = kmem_cache_create("names_cache", PATH_MAX, 0,
  3102. SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
  3103. dcache_init();
  3104. inode_init();
  3105. files_init(mempages);
  3106. mnt_init();
  3107. bdev_cache_init();
  3108. chrdev_init();
  3109. }