extent_io.c 85 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489
  1. #include <linux/bitops.h>
  2. #include <linux/slab.h>
  3. #include <linux/bio.h>
  4. #include <linux/mm.h>
  5. #include <linux/gfp.h>
  6. #include <linux/pagemap.h>
  7. #include <linux/page-flags.h>
  8. #include <linux/module.h>
  9. #include <linux/spinlock.h>
  10. #include <linux/blkdev.h>
  11. #include <linux/swap.h>
  12. #include <linux/version.h>
  13. #include <linux/writeback.h>
  14. #include <linux/pagevec.h>
  15. #include "extent_io.h"
  16. #include "extent_map.h"
  17. /* temporary define until extent_map moves out of btrfs */
  18. struct kmem_cache *btrfs_cache_create(const char *name, size_t size,
  19. unsigned long extra_flags,
  20. void (*ctor)(void *, struct kmem_cache *,
  21. unsigned long));
  22. static struct kmem_cache *extent_state_cache;
  23. static struct kmem_cache *extent_buffer_cache;
  24. static LIST_HEAD(buffers);
  25. static LIST_HEAD(states);
  26. static spinlock_t leak_lock = SPIN_LOCK_UNLOCKED;
  27. #define BUFFER_LRU_MAX 64
  28. struct tree_entry {
  29. u64 start;
  30. u64 end;
  31. struct rb_node rb_node;
  32. };
  33. struct extent_page_data {
  34. struct bio *bio;
  35. struct extent_io_tree *tree;
  36. get_extent_t *get_extent;
  37. };
  38. int __init extent_io_init(void)
  39. {
  40. extent_state_cache = btrfs_cache_create("extent_state",
  41. sizeof(struct extent_state), 0,
  42. NULL);
  43. if (!extent_state_cache)
  44. return -ENOMEM;
  45. extent_buffer_cache = btrfs_cache_create("extent_buffers",
  46. sizeof(struct extent_buffer), 0,
  47. NULL);
  48. if (!extent_buffer_cache)
  49. goto free_state_cache;
  50. return 0;
  51. free_state_cache:
  52. kmem_cache_destroy(extent_state_cache);
  53. return -ENOMEM;
  54. }
  55. void extent_io_exit(void)
  56. {
  57. struct extent_state *state;
  58. struct extent_buffer *eb;
  59. while (!list_empty(&states)) {
  60. state = list_entry(states.next, struct extent_state, leak_list);
  61. printk("state leak: start %Lu end %Lu state %lu in tree %p refs %d\n", state->start, state->end, state->state, state->tree, atomic_read(&state->refs));
  62. list_del(&state->leak_list);
  63. kmem_cache_free(extent_state_cache, state);
  64. }
  65. while (!list_empty(&buffers)) {
  66. eb = list_entry(buffers.next, struct extent_buffer, leak_list);
  67. printk("buffer leak start %Lu len %lu refs %d\n", eb->start, eb->len, atomic_read(&eb->refs));
  68. list_del(&eb->leak_list);
  69. kmem_cache_free(extent_buffer_cache, eb);
  70. }
  71. if (extent_state_cache)
  72. kmem_cache_destroy(extent_state_cache);
  73. if (extent_buffer_cache)
  74. kmem_cache_destroy(extent_buffer_cache);
  75. }
  76. void extent_io_tree_init(struct extent_io_tree *tree,
  77. struct address_space *mapping, gfp_t mask)
  78. {
  79. tree->state.rb_node = NULL;
  80. tree->ops = NULL;
  81. tree->dirty_bytes = 0;
  82. spin_lock_init(&tree->lock);
  83. spin_lock_init(&tree->lru_lock);
  84. tree->mapping = mapping;
  85. INIT_LIST_HEAD(&tree->buffer_lru);
  86. tree->lru_size = 0;
  87. tree->last = NULL;
  88. }
  89. EXPORT_SYMBOL(extent_io_tree_init);
  90. void extent_io_tree_empty_lru(struct extent_io_tree *tree)
  91. {
  92. struct extent_buffer *eb;
  93. while(!list_empty(&tree->buffer_lru)) {
  94. eb = list_entry(tree->buffer_lru.next, struct extent_buffer,
  95. lru);
  96. list_del_init(&eb->lru);
  97. free_extent_buffer(eb);
  98. }
  99. }
  100. EXPORT_SYMBOL(extent_io_tree_empty_lru);
  101. struct extent_state *alloc_extent_state(gfp_t mask)
  102. {
  103. struct extent_state *state;
  104. unsigned long flags;
  105. state = kmem_cache_alloc(extent_state_cache, mask);
  106. if (!state)
  107. return state;
  108. state->state = 0;
  109. state->private = 0;
  110. state->tree = NULL;
  111. spin_lock_irqsave(&leak_lock, flags);
  112. list_add(&state->leak_list, &states);
  113. spin_unlock_irqrestore(&leak_lock, flags);
  114. atomic_set(&state->refs, 1);
  115. init_waitqueue_head(&state->wq);
  116. return state;
  117. }
  118. EXPORT_SYMBOL(alloc_extent_state);
  119. void free_extent_state(struct extent_state *state)
  120. {
  121. if (!state)
  122. return;
  123. if (atomic_dec_and_test(&state->refs)) {
  124. unsigned long flags;
  125. WARN_ON(state->tree);
  126. spin_lock_irqsave(&leak_lock, flags);
  127. list_del(&state->leak_list);
  128. spin_unlock_irqrestore(&leak_lock, flags);
  129. kmem_cache_free(extent_state_cache, state);
  130. }
  131. }
  132. EXPORT_SYMBOL(free_extent_state);
  133. static struct rb_node *tree_insert(struct rb_root *root, u64 offset,
  134. struct rb_node *node)
  135. {
  136. struct rb_node ** p = &root->rb_node;
  137. struct rb_node * parent = NULL;
  138. struct tree_entry *entry;
  139. while(*p) {
  140. parent = *p;
  141. entry = rb_entry(parent, struct tree_entry, rb_node);
  142. if (offset < entry->start)
  143. p = &(*p)->rb_left;
  144. else if (offset > entry->end)
  145. p = &(*p)->rb_right;
  146. else
  147. return parent;
  148. }
  149. entry = rb_entry(node, struct tree_entry, rb_node);
  150. rb_link_node(node, parent, p);
  151. rb_insert_color(node, root);
  152. return NULL;
  153. }
  154. static struct rb_node *__etree_search(struct extent_io_tree *tree, u64 offset,
  155. struct rb_node **prev_ret,
  156. struct rb_node **next_ret)
  157. {
  158. struct rb_root *root = &tree->state;
  159. struct rb_node * n = root->rb_node;
  160. struct rb_node *prev = NULL;
  161. struct rb_node *orig_prev = NULL;
  162. struct tree_entry *entry;
  163. struct tree_entry *prev_entry = NULL;
  164. if (tree->last) {
  165. struct extent_state *state;
  166. state = tree->last;
  167. if (state->start <= offset && offset <= state->end)
  168. return &tree->last->rb_node;
  169. }
  170. while(n) {
  171. entry = rb_entry(n, struct tree_entry, rb_node);
  172. prev = n;
  173. prev_entry = entry;
  174. if (offset < entry->start)
  175. n = n->rb_left;
  176. else if (offset > entry->end)
  177. n = n->rb_right;
  178. else {
  179. tree->last = rb_entry(n, struct extent_state, rb_node);
  180. return n;
  181. }
  182. }
  183. if (prev_ret) {
  184. orig_prev = prev;
  185. while(prev && offset > prev_entry->end) {
  186. prev = rb_next(prev);
  187. prev_entry = rb_entry(prev, struct tree_entry, rb_node);
  188. }
  189. *prev_ret = prev;
  190. prev = orig_prev;
  191. }
  192. if (next_ret) {
  193. prev_entry = rb_entry(prev, struct tree_entry, rb_node);
  194. while(prev && offset < prev_entry->start) {
  195. prev = rb_prev(prev);
  196. prev_entry = rb_entry(prev, struct tree_entry, rb_node);
  197. }
  198. *next_ret = prev;
  199. }
  200. return NULL;
  201. }
  202. static inline struct rb_node *tree_search(struct extent_io_tree *tree,
  203. u64 offset)
  204. {
  205. struct rb_node *prev = NULL;
  206. struct rb_node *ret;
  207. ret = __etree_search(tree, offset, &prev, NULL);
  208. if (!ret) {
  209. if (prev) {
  210. tree->last = rb_entry(prev, struct extent_state,
  211. rb_node);
  212. }
  213. return prev;
  214. }
  215. return ret;
  216. }
  217. /*
  218. * utility function to look for merge candidates inside a given range.
  219. * Any extents with matching state are merged together into a single
  220. * extent in the tree. Extents with EXTENT_IO in their state field
  221. * are not merged because the end_io handlers need to be able to do
  222. * operations on them without sleeping (or doing allocations/splits).
  223. *
  224. * This should be called with the tree lock held.
  225. */
  226. static int merge_state(struct extent_io_tree *tree,
  227. struct extent_state *state)
  228. {
  229. struct extent_state *other;
  230. struct rb_node *other_node;
  231. if (state->state & EXTENT_IOBITS)
  232. return 0;
  233. other_node = rb_prev(&state->rb_node);
  234. if (other_node) {
  235. other = rb_entry(other_node, struct extent_state, rb_node);
  236. if (other->end == state->start - 1 &&
  237. other->state == state->state) {
  238. state->start = other->start;
  239. other->tree = NULL;
  240. if (tree->last == other)
  241. tree->last = state;
  242. rb_erase(&other->rb_node, &tree->state);
  243. free_extent_state(other);
  244. }
  245. }
  246. other_node = rb_next(&state->rb_node);
  247. if (other_node) {
  248. other = rb_entry(other_node, struct extent_state, rb_node);
  249. if (other->start == state->end + 1 &&
  250. other->state == state->state) {
  251. other->start = state->start;
  252. state->tree = NULL;
  253. if (tree->last == state)
  254. tree->last = other;
  255. rb_erase(&state->rb_node, &tree->state);
  256. free_extent_state(state);
  257. }
  258. }
  259. return 0;
  260. }
  261. static void set_state_cb(struct extent_io_tree *tree,
  262. struct extent_state *state,
  263. unsigned long bits)
  264. {
  265. if (tree->ops && tree->ops->set_bit_hook) {
  266. tree->ops->set_bit_hook(tree->mapping->host, state->start,
  267. state->end, state->state, bits);
  268. }
  269. }
  270. static void clear_state_cb(struct extent_io_tree *tree,
  271. struct extent_state *state,
  272. unsigned long bits)
  273. {
  274. if (tree->ops && tree->ops->set_bit_hook) {
  275. tree->ops->clear_bit_hook(tree->mapping->host, state->start,
  276. state->end, state->state, bits);
  277. }
  278. }
  279. /*
  280. * insert an extent_state struct into the tree. 'bits' are set on the
  281. * struct before it is inserted.
  282. *
  283. * This may return -EEXIST if the extent is already there, in which case the
  284. * state struct is freed.
  285. *
  286. * The tree lock is not taken internally. This is a utility function and
  287. * probably isn't what you want to call (see set/clear_extent_bit).
  288. */
  289. static int insert_state(struct extent_io_tree *tree,
  290. struct extent_state *state, u64 start, u64 end,
  291. int bits)
  292. {
  293. struct rb_node *node;
  294. if (end < start) {
  295. printk("end < start %Lu %Lu\n", end, start);
  296. WARN_ON(1);
  297. }
  298. if (bits & EXTENT_DIRTY)
  299. tree->dirty_bytes += end - start + 1;
  300. set_state_cb(tree, state, bits);
  301. state->state |= bits;
  302. state->start = start;
  303. state->end = end;
  304. node = tree_insert(&tree->state, end, &state->rb_node);
  305. if (node) {
  306. struct extent_state *found;
  307. found = rb_entry(node, struct extent_state, rb_node);
  308. printk("found node %Lu %Lu on insert of %Lu %Lu\n", found->start, found->end, start, end);
  309. free_extent_state(state);
  310. return -EEXIST;
  311. }
  312. state->tree = tree;
  313. tree->last = state;
  314. merge_state(tree, state);
  315. return 0;
  316. }
  317. /*
  318. * split a given extent state struct in two, inserting the preallocated
  319. * struct 'prealloc' as the newly created second half. 'split' indicates an
  320. * offset inside 'orig' where it should be split.
  321. *
  322. * Before calling,
  323. * the tree has 'orig' at [orig->start, orig->end]. After calling, there
  324. * are two extent state structs in the tree:
  325. * prealloc: [orig->start, split - 1]
  326. * orig: [ split, orig->end ]
  327. *
  328. * The tree locks are not taken by this function. They need to be held
  329. * by the caller.
  330. */
  331. static int split_state(struct extent_io_tree *tree, struct extent_state *orig,
  332. struct extent_state *prealloc, u64 split)
  333. {
  334. struct rb_node *node;
  335. prealloc->start = orig->start;
  336. prealloc->end = split - 1;
  337. prealloc->state = orig->state;
  338. orig->start = split;
  339. node = tree_insert(&tree->state, prealloc->end, &prealloc->rb_node);
  340. if (node) {
  341. struct extent_state *found;
  342. found = rb_entry(node, struct extent_state, rb_node);
  343. printk("found node %Lu %Lu on insert of %Lu %Lu\n", found->start, found->end, prealloc->start, prealloc->end);
  344. free_extent_state(prealloc);
  345. return -EEXIST;
  346. }
  347. prealloc->tree = tree;
  348. return 0;
  349. }
  350. /*
  351. * utility function to clear some bits in an extent state struct.
  352. * it will optionally wake up any one waiting on this state (wake == 1), or
  353. * forcibly remove the state from the tree (delete == 1).
  354. *
  355. * If no bits are set on the state struct after clearing things, the
  356. * struct is freed and removed from the tree
  357. */
  358. static int clear_state_bit(struct extent_io_tree *tree,
  359. struct extent_state *state, int bits, int wake,
  360. int delete)
  361. {
  362. int ret = state->state & bits;
  363. if ((bits & EXTENT_DIRTY) && (state->state & EXTENT_DIRTY)) {
  364. u64 range = state->end - state->start + 1;
  365. WARN_ON(range > tree->dirty_bytes);
  366. tree->dirty_bytes -= range;
  367. }
  368. clear_state_cb(tree, state, bits);
  369. state->state &= ~bits;
  370. if (wake)
  371. wake_up(&state->wq);
  372. if (delete || state->state == 0) {
  373. if (state->tree) {
  374. clear_state_cb(tree, state, state->state);
  375. if (tree->last == state) {
  376. tree->last = extent_state_next(state);
  377. }
  378. rb_erase(&state->rb_node, &tree->state);
  379. state->tree = NULL;
  380. free_extent_state(state);
  381. } else {
  382. WARN_ON(1);
  383. }
  384. } else {
  385. merge_state(tree, state);
  386. }
  387. return ret;
  388. }
  389. /*
  390. * clear some bits on a range in the tree. This may require splitting
  391. * or inserting elements in the tree, so the gfp mask is used to
  392. * indicate which allocations or sleeping are allowed.
  393. *
  394. * pass 'wake' == 1 to kick any sleepers, and 'delete' == 1 to remove
  395. * the given range from the tree regardless of state (ie for truncate).
  396. *
  397. * the range [start, end] is inclusive.
  398. *
  399. * This takes the tree lock, and returns < 0 on error, > 0 if any of the
  400. * bits were already set, or zero if none of the bits were already set.
  401. */
  402. int clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
  403. int bits, int wake, int delete, gfp_t mask)
  404. {
  405. struct extent_state *state;
  406. struct extent_state *prealloc = NULL;
  407. struct rb_node *node;
  408. unsigned long flags;
  409. int err;
  410. int set = 0;
  411. again:
  412. if (!prealloc && (mask & __GFP_WAIT)) {
  413. prealloc = alloc_extent_state(mask);
  414. if (!prealloc)
  415. return -ENOMEM;
  416. }
  417. spin_lock_irqsave(&tree->lock, flags);
  418. /*
  419. * this search will find the extents that end after
  420. * our range starts
  421. */
  422. node = tree_search(tree, start);
  423. if (!node)
  424. goto out;
  425. state = rb_entry(node, struct extent_state, rb_node);
  426. if (state->start > end)
  427. goto out;
  428. WARN_ON(state->end < start);
  429. /*
  430. * | ---- desired range ---- |
  431. * | state | or
  432. * | ------------- state -------------- |
  433. *
  434. * We need to split the extent we found, and may flip
  435. * bits on second half.
  436. *
  437. * If the extent we found extends past our range, we
  438. * just split and search again. It'll get split again
  439. * the next time though.
  440. *
  441. * If the extent we found is inside our range, we clear
  442. * the desired bit on it.
  443. */
  444. if (state->start < start) {
  445. if (!prealloc)
  446. prealloc = alloc_extent_state(GFP_ATOMIC);
  447. err = split_state(tree, state, prealloc, start);
  448. BUG_ON(err == -EEXIST);
  449. prealloc = NULL;
  450. if (err)
  451. goto out;
  452. if (state->end <= end) {
  453. start = state->end + 1;
  454. set |= clear_state_bit(tree, state, bits,
  455. wake, delete);
  456. } else {
  457. start = state->start;
  458. }
  459. goto search_again;
  460. }
  461. /*
  462. * | ---- desired range ---- |
  463. * | state |
  464. * We need to split the extent, and clear the bit
  465. * on the first half
  466. */
  467. if (state->start <= end && state->end > end) {
  468. if (!prealloc)
  469. prealloc = alloc_extent_state(GFP_ATOMIC);
  470. err = split_state(tree, state, prealloc, end + 1);
  471. BUG_ON(err == -EEXIST);
  472. if (wake)
  473. wake_up(&state->wq);
  474. set |= clear_state_bit(tree, prealloc, bits,
  475. wake, delete);
  476. prealloc = NULL;
  477. goto out;
  478. }
  479. start = state->end + 1;
  480. set |= clear_state_bit(tree, state, bits, wake, delete);
  481. goto search_again;
  482. out:
  483. spin_unlock_irqrestore(&tree->lock, flags);
  484. if (prealloc)
  485. free_extent_state(prealloc);
  486. return set;
  487. search_again:
  488. if (start > end)
  489. goto out;
  490. spin_unlock_irqrestore(&tree->lock, flags);
  491. if (mask & __GFP_WAIT)
  492. cond_resched();
  493. goto again;
  494. }
  495. EXPORT_SYMBOL(clear_extent_bit);
  496. static int wait_on_state(struct extent_io_tree *tree,
  497. struct extent_state *state)
  498. {
  499. DEFINE_WAIT(wait);
  500. prepare_to_wait(&state->wq, &wait, TASK_UNINTERRUPTIBLE);
  501. spin_unlock_irq(&tree->lock);
  502. schedule();
  503. spin_lock_irq(&tree->lock);
  504. finish_wait(&state->wq, &wait);
  505. return 0;
  506. }
  507. /*
  508. * waits for one or more bits to clear on a range in the state tree.
  509. * The range [start, end] is inclusive.
  510. * The tree lock is taken by this function
  511. */
  512. int wait_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, int bits)
  513. {
  514. struct extent_state *state;
  515. struct rb_node *node;
  516. spin_lock_irq(&tree->lock);
  517. again:
  518. while (1) {
  519. /*
  520. * this search will find all the extents that end after
  521. * our range starts
  522. */
  523. node = tree_search(tree, start);
  524. if (!node)
  525. break;
  526. state = rb_entry(node, struct extent_state, rb_node);
  527. if (state->start > end)
  528. goto out;
  529. if (state->state & bits) {
  530. start = state->start;
  531. atomic_inc(&state->refs);
  532. wait_on_state(tree, state);
  533. free_extent_state(state);
  534. goto again;
  535. }
  536. start = state->end + 1;
  537. if (start > end)
  538. break;
  539. if (need_resched()) {
  540. spin_unlock_irq(&tree->lock);
  541. cond_resched();
  542. spin_lock_irq(&tree->lock);
  543. }
  544. }
  545. out:
  546. spin_unlock_irq(&tree->lock);
  547. return 0;
  548. }
  549. EXPORT_SYMBOL(wait_extent_bit);
  550. static void set_state_bits(struct extent_io_tree *tree,
  551. struct extent_state *state,
  552. int bits)
  553. {
  554. if ((bits & EXTENT_DIRTY) && !(state->state & EXTENT_DIRTY)) {
  555. u64 range = state->end - state->start + 1;
  556. tree->dirty_bytes += range;
  557. }
  558. set_state_cb(tree, state, bits);
  559. state->state |= bits;
  560. }
  561. /*
  562. * set some bits on a range in the tree. This may require allocations
  563. * or sleeping, so the gfp mask is used to indicate what is allowed.
  564. *
  565. * If 'exclusive' == 1, this will fail with -EEXIST if some part of the
  566. * range already has the desired bits set. The start of the existing
  567. * range is returned in failed_start in this case.
  568. *
  569. * [start, end] is inclusive
  570. * This takes the tree lock.
  571. */
  572. int set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, int bits,
  573. int exclusive, u64 *failed_start, gfp_t mask)
  574. {
  575. struct extent_state *state;
  576. struct extent_state *prealloc = NULL;
  577. struct rb_node *node;
  578. unsigned long flags;
  579. int err = 0;
  580. int set;
  581. u64 last_start;
  582. u64 last_end;
  583. again:
  584. if (!prealloc && (mask & __GFP_WAIT)) {
  585. prealloc = alloc_extent_state(mask);
  586. if (!prealloc)
  587. return -ENOMEM;
  588. }
  589. spin_lock_irqsave(&tree->lock, flags);
  590. /*
  591. * this search will find all the extents that end after
  592. * our range starts.
  593. */
  594. node = tree_search(tree, start);
  595. if (!node) {
  596. err = insert_state(tree, prealloc, start, end, bits);
  597. prealloc = NULL;
  598. BUG_ON(err == -EEXIST);
  599. goto out;
  600. }
  601. state = rb_entry(node, struct extent_state, rb_node);
  602. last_start = state->start;
  603. last_end = state->end;
  604. /*
  605. * | ---- desired range ---- |
  606. * | state |
  607. *
  608. * Just lock what we found and keep going
  609. */
  610. if (state->start == start && state->end <= end) {
  611. set = state->state & bits;
  612. if (set && exclusive) {
  613. *failed_start = state->start;
  614. err = -EEXIST;
  615. goto out;
  616. }
  617. set_state_bits(tree, state, bits);
  618. start = state->end + 1;
  619. merge_state(tree, state);
  620. goto search_again;
  621. }
  622. /*
  623. * | ---- desired range ---- |
  624. * | state |
  625. * or
  626. * | ------------- state -------------- |
  627. *
  628. * We need to split the extent we found, and may flip bits on
  629. * second half.
  630. *
  631. * If the extent we found extends past our
  632. * range, we just split and search again. It'll get split
  633. * again the next time though.
  634. *
  635. * If the extent we found is inside our range, we set the
  636. * desired bit on it.
  637. */
  638. if (state->start < start) {
  639. set = state->state & bits;
  640. if (exclusive && set) {
  641. *failed_start = start;
  642. err = -EEXIST;
  643. goto out;
  644. }
  645. err = split_state(tree, state, prealloc, start);
  646. BUG_ON(err == -EEXIST);
  647. prealloc = NULL;
  648. if (err)
  649. goto out;
  650. if (state->end <= end) {
  651. set_state_bits(tree, state, bits);
  652. start = state->end + 1;
  653. merge_state(tree, state);
  654. } else {
  655. start = state->start;
  656. }
  657. goto search_again;
  658. }
  659. /*
  660. * | ---- desired range ---- |
  661. * | state | or | state |
  662. *
  663. * There's a hole, we need to insert something in it and
  664. * ignore the extent we found.
  665. */
  666. if (state->start > start) {
  667. u64 this_end;
  668. if (end < last_start)
  669. this_end = end;
  670. else
  671. this_end = last_start -1;
  672. err = insert_state(tree, prealloc, start, this_end,
  673. bits);
  674. prealloc = NULL;
  675. BUG_ON(err == -EEXIST);
  676. if (err)
  677. goto out;
  678. start = this_end + 1;
  679. goto search_again;
  680. }
  681. /*
  682. * | ---- desired range ---- |
  683. * | state |
  684. * We need to split the extent, and set the bit
  685. * on the first half
  686. */
  687. if (state->start <= end && state->end > end) {
  688. set = state->state & bits;
  689. if (exclusive && set) {
  690. *failed_start = start;
  691. err = -EEXIST;
  692. goto out;
  693. }
  694. err = split_state(tree, state, prealloc, end + 1);
  695. BUG_ON(err == -EEXIST);
  696. set_state_bits(tree, prealloc, bits);
  697. merge_state(tree, prealloc);
  698. prealloc = NULL;
  699. goto out;
  700. }
  701. goto search_again;
  702. out:
  703. spin_unlock_irqrestore(&tree->lock, flags);
  704. if (prealloc)
  705. free_extent_state(prealloc);
  706. return err;
  707. search_again:
  708. if (start > end)
  709. goto out;
  710. spin_unlock_irqrestore(&tree->lock, flags);
  711. if (mask & __GFP_WAIT)
  712. cond_resched();
  713. goto again;
  714. }
  715. EXPORT_SYMBOL(set_extent_bit);
  716. /* wrappers around set/clear extent bit */
  717. int set_extent_dirty(struct extent_io_tree *tree, u64 start, u64 end,
  718. gfp_t mask)
  719. {
  720. return set_extent_bit(tree, start, end, EXTENT_DIRTY, 0, NULL,
  721. mask);
  722. }
  723. EXPORT_SYMBOL(set_extent_dirty);
  724. int set_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
  725. int bits, gfp_t mask)
  726. {
  727. return set_extent_bit(tree, start, end, bits, 0, NULL,
  728. mask);
  729. }
  730. EXPORT_SYMBOL(set_extent_bits);
  731. int clear_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
  732. int bits, gfp_t mask)
  733. {
  734. return clear_extent_bit(tree, start, end, bits, 0, 0, mask);
  735. }
  736. EXPORT_SYMBOL(clear_extent_bits);
  737. int set_extent_delalloc(struct extent_io_tree *tree, u64 start, u64 end,
  738. gfp_t mask)
  739. {
  740. return set_extent_bit(tree, start, end,
  741. EXTENT_DELALLOC | EXTENT_DIRTY, 0, NULL,
  742. mask);
  743. }
  744. EXPORT_SYMBOL(set_extent_delalloc);
  745. int clear_extent_dirty(struct extent_io_tree *tree, u64 start, u64 end,
  746. gfp_t mask)
  747. {
  748. return clear_extent_bit(tree, start, end,
  749. EXTENT_DIRTY | EXTENT_DELALLOC, 0, 0, mask);
  750. }
  751. EXPORT_SYMBOL(clear_extent_dirty);
  752. int set_extent_new(struct extent_io_tree *tree, u64 start, u64 end,
  753. gfp_t mask)
  754. {
  755. return set_extent_bit(tree, start, end, EXTENT_NEW, 0, NULL,
  756. mask);
  757. }
  758. EXPORT_SYMBOL(set_extent_new);
  759. int clear_extent_new(struct extent_io_tree *tree, u64 start, u64 end,
  760. gfp_t mask)
  761. {
  762. return clear_extent_bit(tree, start, end, EXTENT_NEW, 0, 0, mask);
  763. }
  764. EXPORT_SYMBOL(clear_extent_new);
  765. int set_extent_uptodate(struct extent_io_tree *tree, u64 start, u64 end,
  766. gfp_t mask)
  767. {
  768. return set_extent_bit(tree, start, end, EXTENT_UPTODATE, 0, NULL,
  769. mask);
  770. }
  771. EXPORT_SYMBOL(set_extent_uptodate);
  772. int clear_extent_uptodate(struct extent_io_tree *tree, u64 start, u64 end,
  773. gfp_t mask)
  774. {
  775. return clear_extent_bit(tree, start, end, EXTENT_UPTODATE, 0, 0, mask);
  776. }
  777. EXPORT_SYMBOL(clear_extent_uptodate);
  778. int set_extent_writeback(struct extent_io_tree *tree, u64 start, u64 end,
  779. gfp_t mask)
  780. {
  781. return set_extent_bit(tree, start, end, EXTENT_WRITEBACK,
  782. 0, NULL, mask);
  783. }
  784. EXPORT_SYMBOL(set_extent_writeback);
  785. int clear_extent_writeback(struct extent_io_tree *tree, u64 start, u64 end,
  786. gfp_t mask)
  787. {
  788. return clear_extent_bit(tree, start, end, EXTENT_WRITEBACK, 1, 0, mask);
  789. }
  790. EXPORT_SYMBOL(clear_extent_writeback);
  791. int wait_on_extent_writeback(struct extent_io_tree *tree, u64 start, u64 end)
  792. {
  793. return wait_extent_bit(tree, start, end, EXTENT_WRITEBACK);
  794. }
  795. EXPORT_SYMBOL(wait_on_extent_writeback);
  796. int lock_extent(struct extent_io_tree *tree, u64 start, u64 end, gfp_t mask)
  797. {
  798. int err;
  799. u64 failed_start;
  800. while (1) {
  801. err = set_extent_bit(tree, start, end, EXTENT_LOCKED, 1,
  802. &failed_start, mask);
  803. if (err == -EEXIST && (mask & __GFP_WAIT)) {
  804. wait_extent_bit(tree, failed_start, end, EXTENT_LOCKED);
  805. start = failed_start;
  806. } else {
  807. break;
  808. }
  809. WARN_ON(start > end);
  810. }
  811. return err;
  812. }
  813. EXPORT_SYMBOL(lock_extent);
  814. int unlock_extent(struct extent_io_tree *tree, u64 start, u64 end,
  815. gfp_t mask)
  816. {
  817. return clear_extent_bit(tree, start, end, EXTENT_LOCKED, 1, 0, mask);
  818. }
  819. EXPORT_SYMBOL(unlock_extent);
  820. /*
  821. * helper function to set pages and extents in the tree dirty
  822. */
  823. int set_range_dirty(struct extent_io_tree *tree, u64 start, u64 end)
  824. {
  825. unsigned long index = start >> PAGE_CACHE_SHIFT;
  826. unsigned long end_index = end >> PAGE_CACHE_SHIFT;
  827. struct page *page;
  828. while (index <= end_index) {
  829. page = find_get_page(tree->mapping, index);
  830. BUG_ON(!page);
  831. __set_page_dirty_nobuffers(page);
  832. page_cache_release(page);
  833. index++;
  834. }
  835. set_extent_dirty(tree, start, end, GFP_NOFS);
  836. return 0;
  837. }
  838. EXPORT_SYMBOL(set_range_dirty);
  839. /*
  840. * helper function to set both pages and extents in the tree writeback
  841. */
  842. int set_range_writeback(struct extent_io_tree *tree, u64 start, u64 end)
  843. {
  844. unsigned long index = start >> PAGE_CACHE_SHIFT;
  845. unsigned long end_index = end >> PAGE_CACHE_SHIFT;
  846. struct page *page;
  847. while (index <= end_index) {
  848. page = find_get_page(tree->mapping, index);
  849. BUG_ON(!page);
  850. set_page_writeback(page);
  851. page_cache_release(page);
  852. index++;
  853. }
  854. set_extent_writeback(tree, start, end, GFP_NOFS);
  855. return 0;
  856. }
  857. EXPORT_SYMBOL(set_range_writeback);
  858. int find_first_extent_bit(struct extent_io_tree *tree, u64 start,
  859. u64 *start_ret, u64 *end_ret, int bits)
  860. {
  861. struct rb_node *node;
  862. struct extent_state *state;
  863. int ret = 1;
  864. spin_lock_irq(&tree->lock);
  865. /*
  866. * this search will find all the extents that end after
  867. * our range starts.
  868. */
  869. node = tree_search(tree, start);
  870. if (!node) {
  871. goto out;
  872. }
  873. while(1) {
  874. state = rb_entry(node, struct extent_state, rb_node);
  875. if (state->end >= start && (state->state & bits)) {
  876. *start_ret = state->start;
  877. *end_ret = state->end;
  878. ret = 0;
  879. break;
  880. }
  881. node = rb_next(node);
  882. if (!node)
  883. break;
  884. }
  885. out:
  886. spin_unlock_irq(&tree->lock);
  887. return ret;
  888. }
  889. EXPORT_SYMBOL(find_first_extent_bit);
  890. struct extent_state *find_first_extent_bit_state(struct extent_io_tree *tree,
  891. u64 start, int bits)
  892. {
  893. struct rb_node *node;
  894. struct extent_state *state;
  895. /*
  896. * this search will find all the extents that end after
  897. * our range starts.
  898. */
  899. node = tree_search(tree, start);
  900. if (!node) {
  901. goto out;
  902. }
  903. while(1) {
  904. state = rb_entry(node, struct extent_state, rb_node);
  905. if (state->end >= start && (state->state & bits)) {
  906. return state;
  907. }
  908. node = rb_next(node);
  909. if (!node)
  910. break;
  911. }
  912. out:
  913. return NULL;
  914. }
  915. EXPORT_SYMBOL(find_first_extent_bit_state);
  916. u64 find_lock_delalloc_range(struct extent_io_tree *tree,
  917. u64 *start, u64 *end, u64 max_bytes)
  918. {
  919. struct rb_node *node;
  920. struct extent_state *state;
  921. u64 cur_start = *start;
  922. u64 found = 0;
  923. u64 total_bytes = 0;
  924. spin_lock_irq(&tree->lock);
  925. /*
  926. * this search will find all the extents that end after
  927. * our range starts.
  928. */
  929. search_again:
  930. node = tree_search(tree, cur_start);
  931. if (!node) {
  932. *end = (u64)-1;
  933. goto out;
  934. }
  935. while(1) {
  936. state = rb_entry(node, struct extent_state, rb_node);
  937. if (found && state->start != cur_start) {
  938. goto out;
  939. }
  940. if (!(state->state & EXTENT_DELALLOC)) {
  941. if (!found)
  942. *end = state->end;
  943. goto out;
  944. }
  945. if (!found) {
  946. struct extent_state *prev_state;
  947. struct rb_node *prev_node = node;
  948. while(1) {
  949. prev_node = rb_prev(prev_node);
  950. if (!prev_node)
  951. break;
  952. prev_state = rb_entry(prev_node,
  953. struct extent_state,
  954. rb_node);
  955. if (!(prev_state->state & EXTENT_DELALLOC))
  956. break;
  957. state = prev_state;
  958. node = prev_node;
  959. }
  960. }
  961. if (state->state & EXTENT_LOCKED) {
  962. DEFINE_WAIT(wait);
  963. atomic_inc(&state->refs);
  964. prepare_to_wait(&state->wq, &wait,
  965. TASK_UNINTERRUPTIBLE);
  966. spin_unlock_irq(&tree->lock);
  967. schedule();
  968. spin_lock_irq(&tree->lock);
  969. finish_wait(&state->wq, &wait);
  970. free_extent_state(state);
  971. goto search_again;
  972. }
  973. set_state_cb(tree, state, EXTENT_LOCKED);
  974. state->state |= EXTENT_LOCKED;
  975. if (!found)
  976. *start = state->start;
  977. found++;
  978. *end = state->end;
  979. cur_start = state->end + 1;
  980. node = rb_next(node);
  981. if (!node)
  982. break;
  983. total_bytes += state->end - state->start + 1;
  984. if (total_bytes >= max_bytes)
  985. break;
  986. }
  987. out:
  988. spin_unlock_irq(&tree->lock);
  989. return found;
  990. }
  991. u64 count_range_bits(struct extent_io_tree *tree,
  992. u64 *start, u64 search_end, u64 max_bytes,
  993. unsigned long bits)
  994. {
  995. struct rb_node *node;
  996. struct extent_state *state;
  997. u64 cur_start = *start;
  998. u64 total_bytes = 0;
  999. int found = 0;
  1000. if (search_end <= cur_start) {
  1001. printk("search_end %Lu start %Lu\n", search_end, cur_start);
  1002. WARN_ON(1);
  1003. return 0;
  1004. }
  1005. spin_lock_irq(&tree->lock);
  1006. if (cur_start == 0 && bits == EXTENT_DIRTY) {
  1007. total_bytes = tree->dirty_bytes;
  1008. goto out;
  1009. }
  1010. /*
  1011. * this search will find all the extents that end after
  1012. * our range starts.
  1013. */
  1014. node = tree_search(tree, cur_start);
  1015. if (!node) {
  1016. goto out;
  1017. }
  1018. while(1) {
  1019. state = rb_entry(node, struct extent_state, rb_node);
  1020. if (state->start > search_end)
  1021. break;
  1022. if (state->end >= cur_start && (state->state & bits)) {
  1023. total_bytes += min(search_end, state->end) + 1 -
  1024. max(cur_start, state->start);
  1025. if (total_bytes >= max_bytes)
  1026. break;
  1027. if (!found) {
  1028. *start = state->start;
  1029. found = 1;
  1030. }
  1031. }
  1032. node = rb_next(node);
  1033. if (!node)
  1034. break;
  1035. }
  1036. out:
  1037. spin_unlock_irq(&tree->lock);
  1038. return total_bytes;
  1039. }
  1040. /*
  1041. * helper function to lock both pages and extents in the tree.
  1042. * pages must be locked first.
  1043. */
  1044. int lock_range(struct extent_io_tree *tree, u64 start, u64 end)
  1045. {
  1046. unsigned long index = start >> PAGE_CACHE_SHIFT;
  1047. unsigned long end_index = end >> PAGE_CACHE_SHIFT;
  1048. struct page *page;
  1049. int err;
  1050. while (index <= end_index) {
  1051. page = grab_cache_page(tree->mapping, index);
  1052. if (!page) {
  1053. err = -ENOMEM;
  1054. goto failed;
  1055. }
  1056. if (IS_ERR(page)) {
  1057. err = PTR_ERR(page);
  1058. goto failed;
  1059. }
  1060. index++;
  1061. }
  1062. lock_extent(tree, start, end, GFP_NOFS);
  1063. return 0;
  1064. failed:
  1065. /*
  1066. * we failed above in getting the page at 'index', so we undo here
  1067. * up to but not including the page at 'index'
  1068. */
  1069. end_index = index;
  1070. index = start >> PAGE_CACHE_SHIFT;
  1071. while (index < end_index) {
  1072. page = find_get_page(tree->mapping, index);
  1073. unlock_page(page);
  1074. page_cache_release(page);
  1075. index++;
  1076. }
  1077. return err;
  1078. }
  1079. EXPORT_SYMBOL(lock_range);
  1080. /*
  1081. * helper function to unlock both pages and extents in the tree.
  1082. */
  1083. int unlock_range(struct extent_io_tree *tree, u64 start, u64 end)
  1084. {
  1085. unsigned long index = start >> PAGE_CACHE_SHIFT;
  1086. unsigned long end_index = end >> PAGE_CACHE_SHIFT;
  1087. struct page *page;
  1088. while (index <= end_index) {
  1089. page = find_get_page(tree->mapping, index);
  1090. unlock_page(page);
  1091. page_cache_release(page);
  1092. index++;
  1093. }
  1094. unlock_extent(tree, start, end, GFP_NOFS);
  1095. return 0;
  1096. }
  1097. EXPORT_SYMBOL(unlock_range);
  1098. int set_state_private(struct extent_io_tree *tree, u64 start, u64 private)
  1099. {
  1100. struct rb_node *node;
  1101. struct extent_state *state;
  1102. int ret = 0;
  1103. spin_lock_irq(&tree->lock);
  1104. /*
  1105. * this search will find all the extents that end after
  1106. * our range starts.
  1107. */
  1108. node = tree_search(tree, start);
  1109. if (!node) {
  1110. ret = -ENOENT;
  1111. goto out;
  1112. }
  1113. state = rb_entry(node, struct extent_state, rb_node);
  1114. if (state->start != start) {
  1115. ret = -ENOENT;
  1116. goto out;
  1117. }
  1118. state->private = private;
  1119. out:
  1120. spin_unlock_irq(&tree->lock);
  1121. return ret;
  1122. }
  1123. int get_state_private(struct extent_io_tree *tree, u64 start, u64 *private)
  1124. {
  1125. struct rb_node *node;
  1126. struct extent_state *state;
  1127. int ret = 0;
  1128. spin_lock_irq(&tree->lock);
  1129. /*
  1130. * this search will find all the extents that end after
  1131. * our range starts.
  1132. */
  1133. node = tree_search(tree, start);
  1134. if (!node) {
  1135. ret = -ENOENT;
  1136. goto out;
  1137. }
  1138. state = rb_entry(node, struct extent_state, rb_node);
  1139. if (state->start != start) {
  1140. ret = -ENOENT;
  1141. goto out;
  1142. }
  1143. *private = state->private;
  1144. out:
  1145. spin_unlock_irq(&tree->lock);
  1146. return ret;
  1147. }
  1148. /*
  1149. * searches a range in the state tree for a given mask.
  1150. * If 'filled' == 1, this returns 1 only if every extent in the tree
  1151. * has the bits set. Otherwise, 1 is returned if any bit in the
  1152. * range is found set.
  1153. */
  1154. int test_range_bit(struct extent_io_tree *tree, u64 start, u64 end,
  1155. int bits, int filled)
  1156. {
  1157. struct extent_state *state = NULL;
  1158. struct rb_node *node;
  1159. int bitset = 0;
  1160. unsigned long flags;
  1161. spin_lock_irqsave(&tree->lock, flags);
  1162. node = tree_search(tree, start);
  1163. while (node && start <= end) {
  1164. state = rb_entry(node, struct extent_state, rb_node);
  1165. if (filled && state->start > start) {
  1166. bitset = 0;
  1167. break;
  1168. }
  1169. if (state->start > end)
  1170. break;
  1171. if (state->state & bits) {
  1172. bitset = 1;
  1173. if (!filled)
  1174. break;
  1175. } else if (filled) {
  1176. bitset = 0;
  1177. break;
  1178. }
  1179. start = state->end + 1;
  1180. if (start > end)
  1181. break;
  1182. node = rb_next(node);
  1183. if (!node) {
  1184. if (filled)
  1185. bitset = 0;
  1186. break;
  1187. }
  1188. }
  1189. spin_unlock_irqrestore(&tree->lock, flags);
  1190. return bitset;
  1191. }
  1192. EXPORT_SYMBOL(test_range_bit);
  1193. /*
  1194. * helper function to set a given page up to date if all the
  1195. * extents in the tree for that page are up to date
  1196. */
  1197. static int check_page_uptodate(struct extent_io_tree *tree,
  1198. struct page *page)
  1199. {
  1200. u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
  1201. u64 end = start + PAGE_CACHE_SIZE - 1;
  1202. if (test_range_bit(tree, start, end, EXTENT_UPTODATE, 1))
  1203. SetPageUptodate(page);
  1204. return 0;
  1205. }
  1206. /*
  1207. * helper function to unlock a page if all the extents in the tree
  1208. * for that page are unlocked
  1209. */
  1210. static int check_page_locked(struct extent_io_tree *tree,
  1211. struct page *page)
  1212. {
  1213. u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
  1214. u64 end = start + PAGE_CACHE_SIZE - 1;
  1215. if (!test_range_bit(tree, start, end, EXTENT_LOCKED, 0))
  1216. unlock_page(page);
  1217. return 0;
  1218. }
  1219. /*
  1220. * helper function to end page writeback if all the extents
  1221. * in the tree for that page are done with writeback
  1222. */
  1223. static int check_page_writeback(struct extent_io_tree *tree,
  1224. struct page *page)
  1225. {
  1226. u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
  1227. u64 end = start + PAGE_CACHE_SIZE - 1;
  1228. if (!test_range_bit(tree, start, end, EXTENT_WRITEBACK, 0))
  1229. end_page_writeback(page);
  1230. return 0;
  1231. }
  1232. /* lots and lots of room for performance fixes in the end_bio funcs */
  1233. /*
  1234. * after a writepage IO is done, we need to:
  1235. * clear the uptodate bits on error
  1236. * clear the writeback bits in the extent tree for this IO
  1237. * end_page_writeback if the page has no more pending IO
  1238. *
  1239. * Scheduling is not allowed, so the extent state tree is expected
  1240. * to have one and only one object corresponding to this IO.
  1241. */
  1242. #if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,23)
  1243. static void end_bio_extent_writepage(struct bio *bio, int err)
  1244. #else
  1245. static int end_bio_extent_writepage(struct bio *bio,
  1246. unsigned int bytes_done, int err)
  1247. #endif
  1248. {
  1249. const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
  1250. struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
  1251. struct extent_state *state = bio->bi_private;
  1252. struct extent_io_tree *tree = state->tree;
  1253. struct rb_node *node;
  1254. u64 start;
  1255. u64 end;
  1256. u64 cur;
  1257. int whole_page;
  1258. unsigned long flags;
  1259. #if LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,23)
  1260. if (bio->bi_size)
  1261. return 1;
  1262. #endif
  1263. do {
  1264. struct page *page = bvec->bv_page;
  1265. start = ((u64)page->index << PAGE_CACHE_SHIFT) +
  1266. bvec->bv_offset;
  1267. end = start + bvec->bv_len - 1;
  1268. if (bvec->bv_offset == 0 && bvec->bv_len == PAGE_CACHE_SIZE)
  1269. whole_page = 1;
  1270. else
  1271. whole_page = 0;
  1272. if (--bvec >= bio->bi_io_vec)
  1273. prefetchw(&bvec->bv_page->flags);
  1274. if (!uptodate) {
  1275. clear_extent_uptodate(tree, start, end, GFP_ATOMIC);
  1276. ClearPageUptodate(page);
  1277. SetPageError(page);
  1278. }
  1279. if (tree->ops && tree->ops->writepage_end_io_hook) {
  1280. tree->ops->writepage_end_io_hook(page, start, end,
  1281. state);
  1282. }
  1283. /*
  1284. * bios can get merged in funny ways, and so we need to
  1285. * be careful with the state variable. We know the
  1286. * state won't be merged with others because it has
  1287. * WRITEBACK set, but we can't be sure each biovec is
  1288. * sequential in the file. So, if our cached state
  1289. * doesn't match the expected end, search the tree
  1290. * for the correct one.
  1291. */
  1292. spin_lock_irqsave(&tree->lock, flags);
  1293. if (!state || state->end != end) {
  1294. state = NULL;
  1295. node = __etree_search(tree, start, NULL, NULL);
  1296. if (node) {
  1297. state = rb_entry(node, struct extent_state,
  1298. rb_node);
  1299. if (state->end != end ||
  1300. !(state->state & EXTENT_WRITEBACK))
  1301. state = NULL;
  1302. }
  1303. if (!state) {
  1304. spin_unlock_irqrestore(&tree->lock, flags);
  1305. clear_extent_writeback(tree, start,
  1306. end, GFP_ATOMIC);
  1307. goto next_io;
  1308. }
  1309. }
  1310. cur = end;
  1311. while(1) {
  1312. struct extent_state *clear = state;
  1313. cur = state->start;
  1314. node = rb_prev(&state->rb_node);
  1315. if (node) {
  1316. state = rb_entry(node,
  1317. struct extent_state,
  1318. rb_node);
  1319. } else {
  1320. state = NULL;
  1321. }
  1322. clear_state_bit(tree, clear, EXTENT_WRITEBACK,
  1323. 1, 0);
  1324. if (cur == start)
  1325. break;
  1326. if (cur < start) {
  1327. WARN_ON(1);
  1328. break;
  1329. }
  1330. if (!node)
  1331. break;
  1332. }
  1333. /* before releasing the lock, make sure the next state
  1334. * variable has the expected bits set and corresponds
  1335. * to the correct offsets in the file
  1336. */
  1337. if (state && (state->end + 1 != start ||
  1338. !(state->state & EXTENT_WRITEBACK))) {
  1339. state = NULL;
  1340. }
  1341. spin_unlock_irqrestore(&tree->lock, flags);
  1342. next_io:
  1343. if (whole_page)
  1344. end_page_writeback(page);
  1345. else
  1346. check_page_writeback(tree, page);
  1347. } while (bvec >= bio->bi_io_vec);
  1348. bio_put(bio);
  1349. #if LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,23)
  1350. return 0;
  1351. #endif
  1352. }
  1353. /*
  1354. * after a readpage IO is done, we need to:
  1355. * clear the uptodate bits on error
  1356. * set the uptodate bits if things worked
  1357. * set the page up to date if all extents in the tree are uptodate
  1358. * clear the lock bit in the extent tree
  1359. * unlock the page if there are no other extents locked for it
  1360. *
  1361. * Scheduling is not allowed, so the extent state tree is expected
  1362. * to have one and only one object corresponding to this IO.
  1363. */
  1364. #if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,23)
  1365. static void end_bio_extent_readpage(struct bio *bio, int err)
  1366. #else
  1367. static int end_bio_extent_readpage(struct bio *bio,
  1368. unsigned int bytes_done, int err)
  1369. #endif
  1370. {
  1371. int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
  1372. struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
  1373. struct extent_state *state = bio->bi_private;
  1374. struct extent_io_tree *tree = state->tree;
  1375. struct rb_node *node;
  1376. u64 start;
  1377. u64 end;
  1378. u64 cur;
  1379. unsigned long flags;
  1380. int whole_page;
  1381. int ret;
  1382. #if LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,23)
  1383. if (bio->bi_size)
  1384. return 1;
  1385. #endif
  1386. do {
  1387. struct page *page = bvec->bv_page;
  1388. start = ((u64)page->index << PAGE_CACHE_SHIFT) +
  1389. bvec->bv_offset;
  1390. end = start + bvec->bv_len - 1;
  1391. if (bvec->bv_offset == 0 && bvec->bv_len == PAGE_CACHE_SIZE)
  1392. whole_page = 1;
  1393. else
  1394. whole_page = 0;
  1395. if (--bvec >= bio->bi_io_vec)
  1396. prefetchw(&bvec->bv_page->flags);
  1397. if (uptodate && tree->ops && tree->ops->readpage_end_io_hook) {
  1398. ret = tree->ops->readpage_end_io_hook(page, start, end,
  1399. state);
  1400. if (ret)
  1401. uptodate = 0;
  1402. }
  1403. if (!uptodate && tree->ops &&
  1404. tree->ops->readpage_io_failed_hook) {
  1405. ret = tree->ops->readpage_io_failed_hook(bio, page,
  1406. start, end, state);
  1407. if (ret == 0) {
  1408. state = NULL;
  1409. continue;
  1410. }
  1411. }
  1412. spin_lock_irqsave(&tree->lock, flags);
  1413. if (!state || state->end != end) {
  1414. state = NULL;
  1415. node = __etree_search(tree, start, NULL, NULL);
  1416. if (node) {
  1417. state = rb_entry(node, struct extent_state,
  1418. rb_node);
  1419. if (state->end != end ||
  1420. !(state->state & EXTENT_LOCKED))
  1421. state = NULL;
  1422. }
  1423. if (!state && uptodate) {
  1424. spin_unlock_irqrestore(&tree->lock, flags);
  1425. set_extent_uptodate(tree, start, end,
  1426. GFP_ATOMIC);
  1427. unlock_extent(tree, start, end, GFP_ATOMIC);
  1428. goto next_io;
  1429. }
  1430. }
  1431. cur = end;
  1432. while(1) {
  1433. struct extent_state *clear = state;
  1434. cur = state->start;
  1435. node = rb_prev(&state->rb_node);
  1436. if (node) {
  1437. state = rb_entry(node,
  1438. struct extent_state,
  1439. rb_node);
  1440. } else {
  1441. state = NULL;
  1442. }
  1443. if (uptodate) {
  1444. set_state_cb(tree, clear, EXTENT_UPTODATE);
  1445. clear->state |= EXTENT_UPTODATE;
  1446. }
  1447. clear_state_bit(tree, clear, EXTENT_LOCKED,
  1448. 1, 0);
  1449. if (cur == start)
  1450. break;
  1451. if (cur < start) {
  1452. WARN_ON(1);
  1453. break;
  1454. }
  1455. if (!node)
  1456. break;
  1457. }
  1458. /* before releasing the lock, make sure the next state
  1459. * variable has the expected bits set and corresponds
  1460. * to the correct offsets in the file
  1461. */
  1462. if (state && (state->end + 1 != start ||
  1463. !(state->state & EXTENT_LOCKED))) {
  1464. state = NULL;
  1465. }
  1466. spin_unlock_irqrestore(&tree->lock, flags);
  1467. next_io:
  1468. if (whole_page) {
  1469. if (uptodate) {
  1470. SetPageUptodate(page);
  1471. } else {
  1472. ClearPageUptodate(page);
  1473. SetPageError(page);
  1474. }
  1475. unlock_page(page);
  1476. } else {
  1477. if (uptodate) {
  1478. check_page_uptodate(tree, page);
  1479. } else {
  1480. ClearPageUptodate(page);
  1481. SetPageError(page);
  1482. }
  1483. check_page_locked(tree, page);
  1484. }
  1485. } while (bvec >= bio->bi_io_vec);
  1486. bio_put(bio);
  1487. #if LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,23)
  1488. return 0;
  1489. #endif
  1490. }
  1491. /*
  1492. * IO done from prepare_write is pretty simple, we just unlock
  1493. * the structs in the extent tree when done, and set the uptodate bits
  1494. * as appropriate.
  1495. */
  1496. #if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,23)
  1497. static void end_bio_extent_preparewrite(struct bio *bio, int err)
  1498. #else
  1499. static int end_bio_extent_preparewrite(struct bio *bio,
  1500. unsigned int bytes_done, int err)
  1501. #endif
  1502. {
  1503. const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
  1504. struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
  1505. struct extent_state *state = bio->bi_private;
  1506. struct extent_io_tree *tree = state->tree;
  1507. u64 start;
  1508. u64 end;
  1509. #if LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,23)
  1510. if (bio->bi_size)
  1511. return 1;
  1512. #endif
  1513. do {
  1514. struct page *page = bvec->bv_page;
  1515. start = ((u64)page->index << PAGE_CACHE_SHIFT) +
  1516. bvec->bv_offset;
  1517. end = start + bvec->bv_len - 1;
  1518. if (--bvec >= bio->bi_io_vec)
  1519. prefetchw(&bvec->bv_page->flags);
  1520. if (uptodate) {
  1521. set_extent_uptodate(tree, start, end, GFP_ATOMIC);
  1522. } else {
  1523. ClearPageUptodate(page);
  1524. SetPageError(page);
  1525. }
  1526. unlock_extent(tree, start, end, GFP_ATOMIC);
  1527. } while (bvec >= bio->bi_io_vec);
  1528. bio_put(bio);
  1529. #if LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,23)
  1530. return 0;
  1531. #endif
  1532. }
  1533. static struct bio *
  1534. extent_bio_alloc(struct block_device *bdev, u64 first_sector, int nr_vecs,
  1535. gfp_t gfp_flags)
  1536. {
  1537. struct bio *bio;
  1538. bio = bio_alloc(gfp_flags, nr_vecs);
  1539. if (bio == NULL && (current->flags & PF_MEMALLOC)) {
  1540. while (!bio && (nr_vecs /= 2))
  1541. bio = bio_alloc(gfp_flags, nr_vecs);
  1542. }
  1543. if (bio) {
  1544. bio->bi_bdev = bdev;
  1545. bio->bi_sector = first_sector;
  1546. }
  1547. return bio;
  1548. }
  1549. static int submit_one_bio(int rw, struct bio *bio, int mirror_num)
  1550. {
  1551. u64 maxsector;
  1552. int ret = 0;
  1553. struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
  1554. struct page *page = bvec->bv_page;
  1555. struct extent_io_tree *tree = bio->bi_private;
  1556. struct rb_node *node;
  1557. struct extent_state *state;
  1558. u64 start;
  1559. u64 end;
  1560. start = ((u64)page->index << PAGE_CACHE_SHIFT) + bvec->bv_offset;
  1561. end = start + bvec->bv_len - 1;
  1562. spin_lock_irq(&tree->lock);
  1563. node = __etree_search(tree, start, NULL, NULL);
  1564. BUG_ON(!node);
  1565. state = rb_entry(node, struct extent_state, rb_node);
  1566. while(state->end < end) {
  1567. node = rb_next(node);
  1568. state = rb_entry(node, struct extent_state, rb_node);
  1569. }
  1570. BUG_ON(state->end != end);
  1571. spin_unlock_irq(&tree->lock);
  1572. bio->bi_private = state;
  1573. bio_get(bio);
  1574. maxsector = bio->bi_bdev->bd_inode->i_size >> 9;
  1575. if (maxsector < bio->bi_sector) {
  1576. printk("sector too large max %Lu got %llu\n", maxsector,
  1577. (unsigned long long)bio->bi_sector);
  1578. WARN_ON(1);
  1579. }
  1580. if (tree->ops && tree->ops->submit_bio_hook)
  1581. tree->ops->submit_bio_hook(page->mapping->host, rw, bio,
  1582. mirror_num);
  1583. else
  1584. submit_bio(rw, bio);
  1585. if (bio_flagged(bio, BIO_EOPNOTSUPP))
  1586. ret = -EOPNOTSUPP;
  1587. bio_put(bio);
  1588. return ret;
  1589. }
  1590. static int submit_extent_page(int rw, struct extent_io_tree *tree,
  1591. struct page *page, sector_t sector,
  1592. size_t size, unsigned long offset,
  1593. struct block_device *bdev,
  1594. struct bio **bio_ret,
  1595. unsigned long max_pages,
  1596. bio_end_io_t end_io_func,
  1597. int mirror_num)
  1598. {
  1599. int ret = 0;
  1600. struct bio *bio;
  1601. int nr;
  1602. if (bio_ret && *bio_ret) {
  1603. bio = *bio_ret;
  1604. if (bio->bi_sector + (bio->bi_size >> 9) != sector ||
  1605. (tree->ops && tree->ops->merge_bio_hook &&
  1606. tree->ops->merge_bio_hook(page, offset, size, bio)) ||
  1607. bio_add_page(bio, page, size, offset) < size) {
  1608. ret = submit_one_bio(rw, bio, mirror_num);
  1609. bio = NULL;
  1610. } else {
  1611. return 0;
  1612. }
  1613. }
  1614. nr = bio_get_nr_vecs(bdev);
  1615. bio = extent_bio_alloc(bdev, sector, nr, GFP_NOFS | __GFP_HIGH);
  1616. if (!bio) {
  1617. printk("failed to allocate bio nr %d\n", nr);
  1618. }
  1619. bio_add_page(bio, page, size, offset);
  1620. bio->bi_end_io = end_io_func;
  1621. bio->bi_private = tree;
  1622. if (bio_ret) {
  1623. *bio_ret = bio;
  1624. } else {
  1625. ret = submit_one_bio(rw, bio, mirror_num);
  1626. }
  1627. return ret;
  1628. }
  1629. void set_page_extent_mapped(struct page *page)
  1630. {
  1631. if (!PagePrivate(page)) {
  1632. SetPagePrivate(page);
  1633. WARN_ON(!page->mapping->a_ops->invalidatepage);
  1634. set_page_private(page, EXTENT_PAGE_PRIVATE);
  1635. page_cache_get(page);
  1636. }
  1637. }
  1638. void set_page_extent_head(struct page *page, unsigned long len)
  1639. {
  1640. set_page_private(page, EXTENT_PAGE_PRIVATE_FIRST_PAGE | len << 2);
  1641. }
  1642. /*
  1643. * basic readpage implementation. Locked extent state structs are inserted
  1644. * into the tree that are removed when the IO is done (by the end_io
  1645. * handlers)
  1646. */
  1647. static int __extent_read_full_page(struct extent_io_tree *tree,
  1648. struct page *page,
  1649. get_extent_t *get_extent,
  1650. struct bio **bio, int mirror_num)
  1651. {
  1652. struct inode *inode = page->mapping->host;
  1653. u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
  1654. u64 page_end = start + PAGE_CACHE_SIZE - 1;
  1655. u64 end;
  1656. u64 cur = start;
  1657. u64 extent_offset;
  1658. u64 last_byte = i_size_read(inode);
  1659. u64 block_start;
  1660. u64 cur_end;
  1661. sector_t sector;
  1662. struct extent_map *em;
  1663. struct block_device *bdev;
  1664. int ret;
  1665. int nr = 0;
  1666. size_t page_offset = 0;
  1667. size_t iosize;
  1668. size_t blocksize = inode->i_sb->s_blocksize;
  1669. set_page_extent_mapped(page);
  1670. end = page_end;
  1671. lock_extent(tree, start, end, GFP_NOFS);
  1672. while (cur <= end) {
  1673. if (cur >= last_byte) {
  1674. char *userpage;
  1675. iosize = PAGE_CACHE_SIZE - page_offset;
  1676. userpage = kmap_atomic(page, KM_USER0);
  1677. memset(userpage + page_offset, 0, iosize);
  1678. flush_dcache_page(page);
  1679. kunmap_atomic(userpage, KM_USER0);
  1680. set_extent_uptodate(tree, cur, cur + iosize - 1,
  1681. GFP_NOFS);
  1682. unlock_extent(tree, cur, cur + iosize - 1, GFP_NOFS);
  1683. break;
  1684. }
  1685. em = get_extent(inode, page, page_offset, cur,
  1686. end - cur + 1, 0);
  1687. if (IS_ERR(em) || !em) {
  1688. SetPageError(page);
  1689. unlock_extent(tree, cur, end, GFP_NOFS);
  1690. break;
  1691. }
  1692. extent_offset = cur - em->start;
  1693. BUG_ON(extent_map_end(em) <= cur);
  1694. BUG_ON(end < cur);
  1695. iosize = min(extent_map_end(em) - cur, end - cur + 1);
  1696. cur_end = min(extent_map_end(em) - 1, end);
  1697. iosize = (iosize + blocksize - 1) & ~((u64)blocksize - 1);
  1698. sector = (em->block_start + extent_offset) >> 9;
  1699. bdev = em->bdev;
  1700. block_start = em->block_start;
  1701. free_extent_map(em);
  1702. em = NULL;
  1703. /* we've found a hole, just zero and go on */
  1704. if (block_start == EXTENT_MAP_HOLE) {
  1705. char *userpage;
  1706. userpage = kmap_atomic(page, KM_USER0);
  1707. memset(userpage + page_offset, 0, iosize);
  1708. flush_dcache_page(page);
  1709. kunmap_atomic(userpage, KM_USER0);
  1710. set_extent_uptodate(tree, cur, cur + iosize - 1,
  1711. GFP_NOFS);
  1712. unlock_extent(tree, cur, cur + iosize - 1, GFP_NOFS);
  1713. cur = cur + iosize;
  1714. page_offset += iosize;
  1715. continue;
  1716. }
  1717. /* the get_extent function already copied into the page */
  1718. if (test_range_bit(tree, cur, cur_end, EXTENT_UPTODATE, 1)) {
  1719. unlock_extent(tree, cur, cur + iosize - 1, GFP_NOFS);
  1720. cur = cur + iosize;
  1721. page_offset += iosize;
  1722. continue;
  1723. }
  1724. /* we have an inline extent but it didn't get marked up
  1725. * to date. Error out
  1726. */
  1727. if (block_start == EXTENT_MAP_INLINE) {
  1728. SetPageError(page);
  1729. unlock_extent(tree, cur, cur + iosize - 1, GFP_NOFS);
  1730. cur = cur + iosize;
  1731. page_offset += iosize;
  1732. continue;
  1733. }
  1734. ret = 0;
  1735. if (tree->ops && tree->ops->readpage_io_hook) {
  1736. ret = tree->ops->readpage_io_hook(page, cur,
  1737. cur + iosize - 1);
  1738. }
  1739. if (!ret) {
  1740. unsigned long nr = (last_byte >> PAGE_CACHE_SHIFT) + 1;
  1741. nr -= page->index;
  1742. ret = submit_extent_page(READ, tree, page,
  1743. sector, iosize, page_offset,
  1744. bdev, bio, nr,
  1745. end_bio_extent_readpage, mirror_num);
  1746. }
  1747. if (ret)
  1748. SetPageError(page);
  1749. cur = cur + iosize;
  1750. page_offset += iosize;
  1751. nr++;
  1752. }
  1753. if (!nr) {
  1754. if (!PageError(page))
  1755. SetPageUptodate(page);
  1756. unlock_page(page);
  1757. }
  1758. return 0;
  1759. }
  1760. int extent_read_full_page(struct extent_io_tree *tree, struct page *page,
  1761. get_extent_t *get_extent)
  1762. {
  1763. struct bio *bio = NULL;
  1764. int ret;
  1765. ret = __extent_read_full_page(tree, page, get_extent, &bio, 0);
  1766. if (bio)
  1767. submit_one_bio(READ, bio, 0);
  1768. return ret;
  1769. }
  1770. EXPORT_SYMBOL(extent_read_full_page);
  1771. /*
  1772. * the writepage semantics are similar to regular writepage. extent
  1773. * records are inserted to lock ranges in the tree, and as dirty areas
  1774. * are found, they are marked writeback. Then the lock bits are removed
  1775. * and the end_io handler clears the writeback ranges
  1776. */
  1777. static int __extent_writepage(struct page *page, struct writeback_control *wbc,
  1778. void *data)
  1779. {
  1780. struct inode *inode = page->mapping->host;
  1781. struct extent_page_data *epd = data;
  1782. struct extent_io_tree *tree = epd->tree;
  1783. u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
  1784. u64 delalloc_start;
  1785. u64 page_end = start + PAGE_CACHE_SIZE - 1;
  1786. u64 end;
  1787. u64 cur = start;
  1788. u64 extent_offset;
  1789. u64 last_byte = i_size_read(inode);
  1790. u64 block_start;
  1791. u64 iosize;
  1792. sector_t sector;
  1793. struct extent_map *em;
  1794. struct block_device *bdev;
  1795. int ret;
  1796. int nr = 0;
  1797. size_t page_offset = 0;
  1798. size_t blocksize;
  1799. loff_t i_size = i_size_read(inode);
  1800. unsigned long end_index = i_size >> PAGE_CACHE_SHIFT;
  1801. u64 nr_delalloc;
  1802. u64 delalloc_end;
  1803. WARN_ON(!PageLocked(page));
  1804. if (page->index > end_index) {
  1805. clear_extent_dirty(tree, start, page_end, GFP_NOFS);
  1806. unlock_page(page);
  1807. return 0;
  1808. }
  1809. if (page->index == end_index) {
  1810. char *userpage;
  1811. size_t offset = i_size & (PAGE_CACHE_SIZE - 1);
  1812. userpage = kmap_atomic(page, KM_USER0);
  1813. memset(userpage + offset, 0, PAGE_CACHE_SIZE - offset);
  1814. flush_dcache_page(page);
  1815. kunmap_atomic(userpage, KM_USER0);
  1816. }
  1817. set_page_extent_mapped(page);
  1818. delalloc_start = start;
  1819. delalloc_end = 0;
  1820. while(delalloc_end < page_end) {
  1821. nr_delalloc = find_lock_delalloc_range(tree, &delalloc_start,
  1822. &delalloc_end,
  1823. 128 * 1024 * 1024);
  1824. if (nr_delalloc == 0) {
  1825. delalloc_start = delalloc_end + 1;
  1826. continue;
  1827. }
  1828. tree->ops->fill_delalloc(inode, delalloc_start,
  1829. delalloc_end);
  1830. clear_extent_bit(tree, delalloc_start,
  1831. delalloc_end,
  1832. EXTENT_LOCKED | EXTENT_DELALLOC,
  1833. 1, 0, GFP_NOFS);
  1834. delalloc_start = delalloc_end + 1;
  1835. }
  1836. lock_extent(tree, start, page_end, GFP_NOFS);
  1837. end = page_end;
  1838. if (test_range_bit(tree, start, page_end, EXTENT_DELALLOC, 0)) {
  1839. printk("found delalloc bits after lock_extent\n");
  1840. }
  1841. if (last_byte <= start) {
  1842. clear_extent_dirty(tree, start, page_end, GFP_NOFS);
  1843. goto done;
  1844. }
  1845. set_extent_uptodate(tree, start, page_end, GFP_NOFS);
  1846. blocksize = inode->i_sb->s_blocksize;
  1847. while (cur <= end) {
  1848. if (cur >= last_byte) {
  1849. clear_extent_dirty(tree, cur, page_end, GFP_NOFS);
  1850. break;
  1851. }
  1852. em = epd->get_extent(inode, page, page_offset, cur,
  1853. end - cur + 1, 1);
  1854. if (IS_ERR(em) || !em) {
  1855. SetPageError(page);
  1856. break;
  1857. }
  1858. extent_offset = cur - em->start;
  1859. BUG_ON(extent_map_end(em) <= cur);
  1860. BUG_ON(end < cur);
  1861. iosize = min(extent_map_end(em) - cur, end - cur + 1);
  1862. iosize = (iosize + blocksize - 1) & ~((u64)blocksize - 1);
  1863. sector = (em->block_start + extent_offset) >> 9;
  1864. bdev = em->bdev;
  1865. block_start = em->block_start;
  1866. free_extent_map(em);
  1867. em = NULL;
  1868. if (block_start == EXTENT_MAP_HOLE ||
  1869. block_start == EXTENT_MAP_INLINE) {
  1870. clear_extent_dirty(tree, cur,
  1871. cur + iosize - 1, GFP_NOFS);
  1872. cur = cur + iosize;
  1873. page_offset += iosize;
  1874. continue;
  1875. }
  1876. /* leave this out until we have a page_mkwrite call */
  1877. if (0 && !test_range_bit(tree, cur, cur + iosize - 1,
  1878. EXTENT_DIRTY, 0)) {
  1879. cur = cur + iosize;
  1880. page_offset += iosize;
  1881. continue;
  1882. }
  1883. clear_extent_dirty(tree, cur, cur + iosize - 1, GFP_NOFS);
  1884. if (tree->ops && tree->ops->writepage_io_hook) {
  1885. ret = tree->ops->writepage_io_hook(page, cur,
  1886. cur + iosize - 1);
  1887. } else {
  1888. ret = 0;
  1889. }
  1890. if (ret)
  1891. SetPageError(page);
  1892. else {
  1893. unsigned long max_nr = end_index + 1;
  1894. set_range_writeback(tree, cur, cur + iosize - 1);
  1895. if (!PageWriteback(page)) {
  1896. printk("warning page %lu not writeback, "
  1897. "cur %llu end %llu\n", page->index,
  1898. (unsigned long long)cur,
  1899. (unsigned long long)end);
  1900. }
  1901. ret = submit_extent_page(WRITE, tree, page, sector,
  1902. iosize, page_offset, bdev,
  1903. &epd->bio, max_nr,
  1904. end_bio_extent_writepage, 0);
  1905. if (ret)
  1906. SetPageError(page);
  1907. }
  1908. cur = cur + iosize;
  1909. page_offset += iosize;
  1910. nr++;
  1911. }
  1912. done:
  1913. if (nr == 0) {
  1914. /* make sure the mapping tag for page dirty gets cleared */
  1915. set_page_writeback(page);
  1916. end_page_writeback(page);
  1917. }
  1918. unlock_extent(tree, start, page_end, GFP_NOFS);
  1919. unlock_page(page);
  1920. return 0;
  1921. }
  1922. #if LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,20)
  1923. /* Taken directly from 2.6.23 for 2.6.18 back port */
  1924. typedef int (*writepage_t)(struct page *page, struct writeback_control *wbc,
  1925. void *data);
  1926. /**
  1927. * write_cache_pages - walk the list of dirty pages of the given address space
  1928. * and write all of them.
  1929. * @mapping: address space structure to write
  1930. * @wbc: subtract the number of written pages from *@wbc->nr_to_write
  1931. * @writepage: function called for each page
  1932. * @data: data passed to writepage function
  1933. *
  1934. * If a page is already under I/O, write_cache_pages() skips it, even
  1935. * if it's dirty. This is desirable behaviour for memory-cleaning writeback,
  1936. * but it is INCORRECT for data-integrity system calls such as fsync(). fsync()
  1937. * and msync() need to guarantee that all the data which was dirty at the time
  1938. * the call was made get new I/O started against them. If wbc->sync_mode is
  1939. * WB_SYNC_ALL then we were called for data integrity and we must wait for
  1940. * existing IO to complete.
  1941. */
  1942. static int write_cache_pages(struct address_space *mapping,
  1943. struct writeback_control *wbc, writepage_t writepage,
  1944. void *data)
  1945. {
  1946. struct backing_dev_info *bdi = mapping->backing_dev_info;
  1947. int ret = 0;
  1948. int done = 0;
  1949. struct pagevec pvec;
  1950. int nr_pages;
  1951. pgoff_t index;
  1952. pgoff_t end; /* Inclusive */
  1953. int scanned = 0;
  1954. int range_whole = 0;
  1955. if (wbc->nonblocking && bdi_write_congested(bdi)) {
  1956. wbc->encountered_congestion = 1;
  1957. return 0;
  1958. }
  1959. pagevec_init(&pvec, 0);
  1960. if (wbc->range_cyclic) {
  1961. index = mapping->writeback_index; /* Start from prev offset */
  1962. end = -1;
  1963. } else {
  1964. index = wbc->range_start >> PAGE_CACHE_SHIFT;
  1965. end = wbc->range_end >> PAGE_CACHE_SHIFT;
  1966. if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
  1967. range_whole = 1;
  1968. scanned = 1;
  1969. }
  1970. retry:
  1971. while (!done && (index <= end) &&
  1972. (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index,
  1973. PAGECACHE_TAG_DIRTY,
  1974. min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1))) {
  1975. unsigned i;
  1976. scanned = 1;
  1977. for (i = 0; i < nr_pages; i++) {
  1978. struct page *page = pvec.pages[i];
  1979. /*
  1980. * At this point we hold neither mapping->tree_lock nor
  1981. * lock on the page itself: the page may be truncated or
  1982. * invalidated (changing page->mapping to NULL), or even
  1983. * swizzled back from swapper_space to tmpfs file
  1984. * mapping
  1985. */
  1986. lock_page(page);
  1987. if (unlikely(page->mapping != mapping)) {
  1988. unlock_page(page);
  1989. continue;
  1990. }
  1991. if (!wbc->range_cyclic && page->index > end) {
  1992. done = 1;
  1993. unlock_page(page);
  1994. continue;
  1995. }
  1996. if (wbc->sync_mode != WB_SYNC_NONE)
  1997. wait_on_page_writeback(page);
  1998. if (PageWriteback(page) ||
  1999. !clear_page_dirty_for_io(page)) {
  2000. unlock_page(page);
  2001. continue;
  2002. }
  2003. ret = (*writepage)(page, wbc, data);
  2004. if (unlikely(ret == AOP_WRITEPAGE_ACTIVATE)) {
  2005. unlock_page(page);
  2006. ret = 0;
  2007. }
  2008. if (ret || (--(wbc->nr_to_write) <= 0))
  2009. done = 1;
  2010. if (wbc->nonblocking && bdi_write_congested(bdi)) {
  2011. wbc->encountered_congestion = 1;
  2012. done = 1;
  2013. }
  2014. }
  2015. pagevec_release(&pvec);
  2016. cond_resched();
  2017. }
  2018. if (!scanned && !done) {
  2019. /*
  2020. * We hit the last page and there is more work to be done: wrap
  2021. * back to the start of the file
  2022. */
  2023. scanned = 1;
  2024. index = 0;
  2025. goto retry;
  2026. }
  2027. if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0))
  2028. mapping->writeback_index = index;
  2029. return ret;
  2030. }
  2031. #endif
  2032. int extent_write_full_page(struct extent_io_tree *tree, struct page *page,
  2033. get_extent_t *get_extent,
  2034. struct writeback_control *wbc)
  2035. {
  2036. int ret;
  2037. struct address_space *mapping = page->mapping;
  2038. struct extent_page_data epd = {
  2039. .bio = NULL,
  2040. .tree = tree,
  2041. .get_extent = get_extent,
  2042. };
  2043. struct writeback_control wbc_writepages = {
  2044. .bdi = wbc->bdi,
  2045. .sync_mode = WB_SYNC_NONE,
  2046. .older_than_this = NULL,
  2047. .nr_to_write = 64,
  2048. .range_start = page_offset(page) + PAGE_CACHE_SIZE,
  2049. .range_end = (loff_t)-1,
  2050. };
  2051. ret = __extent_writepage(page, wbc, &epd);
  2052. write_cache_pages(mapping, &wbc_writepages, __extent_writepage, &epd);
  2053. if (epd.bio) {
  2054. submit_one_bio(WRITE, epd.bio, 0);
  2055. }
  2056. return ret;
  2057. }
  2058. EXPORT_SYMBOL(extent_write_full_page);
  2059. int extent_writepages(struct extent_io_tree *tree,
  2060. struct address_space *mapping,
  2061. get_extent_t *get_extent,
  2062. struct writeback_control *wbc)
  2063. {
  2064. int ret = 0;
  2065. struct extent_page_data epd = {
  2066. .bio = NULL,
  2067. .tree = tree,
  2068. .get_extent = get_extent,
  2069. };
  2070. ret = write_cache_pages(mapping, wbc, __extent_writepage, &epd);
  2071. if (epd.bio) {
  2072. submit_one_bio(WRITE, epd.bio, 0);
  2073. }
  2074. return ret;
  2075. }
  2076. EXPORT_SYMBOL(extent_writepages);
  2077. int extent_readpages(struct extent_io_tree *tree,
  2078. struct address_space *mapping,
  2079. struct list_head *pages, unsigned nr_pages,
  2080. get_extent_t get_extent)
  2081. {
  2082. struct bio *bio = NULL;
  2083. unsigned page_idx;
  2084. struct pagevec pvec;
  2085. pagevec_init(&pvec, 0);
  2086. for (page_idx = 0; page_idx < nr_pages; page_idx++) {
  2087. struct page *page = list_entry(pages->prev, struct page, lru);
  2088. prefetchw(&page->flags);
  2089. list_del(&page->lru);
  2090. /*
  2091. * what we want to do here is call add_to_page_cache_lru,
  2092. * but that isn't exported, so we reproduce it here
  2093. */
  2094. if (!add_to_page_cache(page, mapping,
  2095. page->index, GFP_KERNEL)) {
  2096. /* open coding of lru_cache_add, also not exported */
  2097. page_cache_get(page);
  2098. if (!pagevec_add(&pvec, page))
  2099. __pagevec_lru_add(&pvec);
  2100. __extent_read_full_page(tree, page, get_extent,
  2101. &bio, 0);
  2102. }
  2103. page_cache_release(page);
  2104. }
  2105. if (pagevec_count(&pvec))
  2106. __pagevec_lru_add(&pvec);
  2107. BUG_ON(!list_empty(pages));
  2108. if (bio)
  2109. submit_one_bio(READ, bio, 0);
  2110. return 0;
  2111. }
  2112. EXPORT_SYMBOL(extent_readpages);
  2113. /*
  2114. * basic invalidatepage code, this waits on any locked or writeback
  2115. * ranges corresponding to the page, and then deletes any extent state
  2116. * records from the tree
  2117. */
  2118. int extent_invalidatepage(struct extent_io_tree *tree,
  2119. struct page *page, unsigned long offset)
  2120. {
  2121. u64 start = ((u64)page->index << PAGE_CACHE_SHIFT);
  2122. u64 end = start + PAGE_CACHE_SIZE - 1;
  2123. size_t blocksize = page->mapping->host->i_sb->s_blocksize;
  2124. start += (offset + blocksize -1) & ~(blocksize - 1);
  2125. if (start > end)
  2126. return 0;
  2127. lock_extent(tree, start, end, GFP_NOFS);
  2128. wait_on_extent_writeback(tree, start, end);
  2129. clear_extent_bit(tree, start, end,
  2130. EXTENT_LOCKED | EXTENT_DIRTY | EXTENT_DELALLOC,
  2131. 1, 1, GFP_NOFS);
  2132. return 0;
  2133. }
  2134. EXPORT_SYMBOL(extent_invalidatepage);
  2135. /*
  2136. * simple commit_write call, set_range_dirty is used to mark both
  2137. * the pages and the extent records as dirty
  2138. */
  2139. int extent_commit_write(struct extent_io_tree *tree,
  2140. struct inode *inode, struct page *page,
  2141. unsigned from, unsigned to)
  2142. {
  2143. loff_t pos = ((loff_t)page->index << PAGE_CACHE_SHIFT) + to;
  2144. set_page_extent_mapped(page);
  2145. set_page_dirty(page);
  2146. if (pos > inode->i_size) {
  2147. i_size_write(inode, pos);
  2148. mark_inode_dirty(inode);
  2149. }
  2150. return 0;
  2151. }
  2152. EXPORT_SYMBOL(extent_commit_write);
  2153. int extent_prepare_write(struct extent_io_tree *tree,
  2154. struct inode *inode, struct page *page,
  2155. unsigned from, unsigned to, get_extent_t *get_extent)
  2156. {
  2157. u64 page_start = (u64)page->index << PAGE_CACHE_SHIFT;
  2158. u64 page_end = page_start + PAGE_CACHE_SIZE - 1;
  2159. u64 block_start;
  2160. u64 orig_block_start;
  2161. u64 block_end;
  2162. u64 cur_end;
  2163. struct extent_map *em;
  2164. unsigned blocksize = 1 << inode->i_blkbits;
  2165. size_t page_offset = 0;
  2166. size_t block_off_start;
  2167. size_t block_off_end;
  2168. int err = 0;
  2169. int iocount = 0;
  2170. int ret = 0;
  2171. int isnew;
  2172. set_page_extent_mapped(page);
  2173. block_start = (page_start + from) & ~((u64)blocksize - 1);
  2174. block_end = (page_start + to - 1) | (blocksize - 1);
  2175. orig_block_start = block_start;
  2176. lock_extent(tree, page_start, page_end, GFP_NOFS);
  2177. while(block_start <= block_end) {
  2178. em = get_extent(inode, page, page_offset, block_start,
  2179. block_end - block_start + 1, 1);
  2180. if (IS_ERR(em) || !em) {
  2181. goto err;
  2182. }
  2183. cur_end = min(block_end, extent_map_end(em) - 1);
  2184. block_off_start = block_start & (PAGE_CACHE_SIZE - 1);
  2185. block_off_end = block_off_start + blocksize;
  2186. isnew = clear_extent_new(tree, block_start, cur_end, GFP_NOFS);
  2187. if (!PageUptodate(page) && isnew &&
  2188. (block_off_end > to || block_off_start < from)) {
  2189. void *kaddr;
  2190. kaddr = kmap_atomic(page, KM_USER0);
  2191. if (block_off_end > to)
  2192. memset(kaddr + to, 0, block_off_end - to);
  2193. if (block_off_start < from)
  2194. memset(kaddr + block_off_start, 0,
  2195. from - block_off_start);
  2196. flush_dcache_page(page);
  2197. kunmap_atomic(kaddr, KM_USER0);
  2198. }
  2199. if ((em->block_start != EXTENT_MAP_HOLE &&
  2200. em->block_start != EXTENT_MAP_INLINE) &&
  2201. !isnew && !PageUptodate(page) &&
  2202. (block_off_end > to || block_off_start < from) &&
  2203. !test_range_bit(tree, block_start, cur_end,
  2204. EXTENT_UPTODATE, 1)) {
  2205. u64 sector;
  2206. u64 extent_offset = block_start - em->start;
  2207. size_t iosize;
  2208. sector = (em->block_start + extent_offset) >> 9;
  2209. iosize = (cur_end - block_start + blocksize) &
  2210. ~((u64)blocksize - 1);
  2211. /*
  2212. * we've already got the extent locked, but we
  2213. * need to split the state such that our end_bio
  2214. * handler can clear the lock.
  2215. */
  2216. set_extent_bit(tree, block_start,
  2217. block_start + iosize - 1,
  2218. EXTENT_LOCKED, 0, NULL, GFP_NOFS);
  2219. ret = submit_extent_page(READ, tree, page,
  2220. sector, iosize, page_offset, em->bdev,
  2221. NULL, 1,
  2222. end_bio_extent_preparewrite, 0);
  2223. iocount++;
  2224. block_start = block_start + iosize;
  2225. } else {
  2226. set_extent_uptodate(tree, block_start, cur_end,
  2227. GFP_NOFS);
  2228. unlock_extent(tree, block_start, cur_end, GFP_NOFS);
  2229. block_start = cur_end + 1;
  2230. }
  2231. page_offset = block_start & (PAGE_CACHE_SIZE - 1);
  2232. free_extent_map(em);
  2233. }
  2234. if (iocount) {
  2235. wait_extent_bit(tree, orig_block_start,
  2236. block_end, EXTENT_LOCKED);
  2237. }
  2238. check_page_uptodate(tree, page);
  2239. err:
  2240. /* FIXME, zero out newly allocated blocks on error */
  2241. return err;
  2242. }
  2243. EXPORT_SYMBOL(extent_prepare_write);
  2244. /*
  2245. * a helper for releasepage. As long as there are no locked extents
  2246. * in the range corresponding to the page, both state records and extent
  2247. * map records are removed
  2248. */
  2249. int try_release_extent_mapping(struct extent_map_tree *map,
  2250. struct extent_io_tree *tree, struct page *page,
  2251. gfp_t mask)
  2252. {
  2253. struct extent_map *em;
  2254. u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
  2255. u64 end = start + PAGE_CACHE_SIZE - 1;
  2256. u64 orig_start = start;
  2257. int ret = 1;
  2258. if ((mask & __GFP_WAIT) &&
  2259. page->mapping->host->i_size > 16 * 1024 * 1024) {
  2260. u64 len;
  2261. while (start <= end) {
  2262. len = end - start + 1;
  2263. spin_lock(&map->lock);
  2264. em = lookup_extent_mapping(map, start, len);
  2265. if (!em || IS_ERR(em)) {
  2266. spin_unlock(&map->lock);
  2267. break;
  2268. }
  2269. if (em->start != start) {
  2270. spin_unlock(&map->lock);
  2271. free_extent_map(em);
  2272. break;
  2273. }
  2274. if (!test_range_bit(tree, em->start,
  2275. extent_map_end(em) - 1,
  2276. EXTENT_LOCKED, 0)) {
  2277. remove_extent_mapping(map, em);
  2278. /* once for the rb tree */
  2279. free_extent_map(em);
  2280. }
  2281. start = extent_map_end(em);
  2282. spin_unlock(&map->lock);
  2283. /* once for us */
  2284. free_extent_map(em);
  2285. }
  2286. }
  2287. if (test_range_bit(tree, orig_start, end, EXTENT_IOBITS, 0))
  2288. ret = 0;
  2289. else {
  2290. if ((mask & GFP_NOFS) == GFP_NOFS)
  2291. mask = GFP_NOFS;
  2292. clear_extent_bit(tree, orig_start, end, EXTENT_UPTODATE,
  2293. 1, 1, mask);
  2294. }
  2295. return ret;
  2296. }
  2297. EXPORT_SYMBOL(try_release_extent_mapping);
  2298. sector_t extent_bmap(struct address_space *mapping, sector_t iblock,
  2299. get_extent_t *get_extent)
  2300. {
  2301. struct inode *inode = mapping->host;
  2302. u64 start = iblock << inode->i_blkbits;
  2303. sector_t sector = 0;
  2304. struct extent_map *em;
  2305. em = get_extent(inode, NULL, 0, start, (1 << inode->i_blkbits), 0);
  2306. if (!em || IS_ERR(em))
  2307. return 0;
  2308. if (em->block_start == EXTENT_MAP_INLINE ||
  2309. em->block_start == EXTENT_MAP_HOLE)
  2310. goto out;
  2311. sector = (em->block_start + start - em->start) >> inode->i_blkbits;
  2312. out:
  2313. free_extent_map(em);
  2314. return sector;
  2315. }
  2316. static int add_lru(struct extent_io_tree *tree, struct extent_buffer *eb)
  2317. {
  2318. if (list_empty(&eb->lru)) {
  2319. extent_buffer_get(eb);
  2320. list_add(&eb->lru, &tree->buffer_lru);
  2321. tree->lru_size++;
  2322. if (tree->lru_size >= BUFFER_LRU_MAX) {
  2323. struct extent_buffer *rm;
  2324. rm = list_entry(tree->buffer_lru.prev,
  2325. struct extent_buffer, lru);
  2326. tree->lru_size--;
  2327. list_del_init(&rm->lru);
  2328. free_extent_buffer(rm);
  2329. }
  2330. } else
  2331. list_move(&eb->lru, &tree->buffer_lru);
  2332. return 0;
  2333. }
  2334. static struct extent_buffer *find_lru(struct extent_io_tree *tree,
  2335. u64 start, unsigned long len)
  2336. {
  2337. struct list_head *lru = &tree->buffer_lru;
  2338. struct list_head *cur = lru->next;
  2339. struct extent_buffer *eb;
  2340. if (list_empty(lru))
  2341. return NULL;
  2342. do {
  2343. eb = list_entry(cur, struct extent_buffer, lru);
  2344. if (eb->start == start && eb->len == len) {
  2345. extent_buffer_get(eb);
  2346. return eb;
  2347. }
  2348. cur = cur->next;
  2349. } while (cur != lru);
  2350. return NULL;
  2351. }
  2352. static inline unsigned long num_extent_pages(u64 start, u64 len)
  2353. {
  2354. return ((start + len + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT) -
  2355. (start >> PAGE_CACHE_SHIFT);
  2356. }
  2357. static inline struct page *extent_buffer_page(struct extent_buffer *eb,
  2358. unsigned long i)
  2359. {
  2360. struct page *p;
  2361. struct address_space *mapping;
  2362. if (i == 0)
  2363. return eb->first_page;
  2364. i += eb->start >> PAGE_CACHE_SHIFT;
  2365. mapping = eb->first_page->mapping;
  2366. read_lock_irq(&mapping->tree_lock);
  2367. p = radix_tree_lookup(&mapping->page_tree, i);
  2368. read_unlock_irq(&mapping->tree_lock);
  2369. return p;
  2370. }
  2371. int release_extent_buffer_tail_pages(struct extent_buffer *eb)
  2372. {
  2373. unsigned long num_pages = num_extent_pages(eb->start, eb->len);
  2374. struct page *page;
  2375. unsigned long i;
  2376. if (num_pages == 1)
  2377. return 0;
  2378. for (i = 1; i < num_pages; i++) {
  2379. page = extent_buffer_page(eb, i);
  2380. page_cache_release(page);
  2381. }
  2382. return 0;
  2383. }
  2384. int invalidate_extent_lru(struct extent_io_tree *tree, u64 start,
  2385. unsigned long len)
  2386. {
  2387. struct list_head *lru = &tree->buffer_lru;
  2388. struct list_head *cur = lru->next;
  2389. struct extent_buffer *eb;
  2390. int found = 0;
  2391. spin_lock(&tree->lru_lock);
  2392. if (list_empty(lru))
  2393. goto out;
  2394. do {
  2395. eb = list_entry(cur, struct extent_buffer, lru);
  2396. if (eb->start <= start && eb->start + eb->len > start) {
  2397. eb->flags &= ~EXTENT_UPTODATE;
  2398. }
  2399. cur = cur->next;
  2400. } while (cur != lru);
  2401. out:
  2402. spin_unlock(&tree->lru_lock);
  2403. return found;
  2404. }
  2405. static struct extent_buffer *__alloc_extent_buffer(struct extent_io_tree *tree,
  2406. u64 start,
  2407. unsigned long len,
  2408. gfp_t mask)
  2409. {
  2410. struct extent_buffer *eb = NULL;
  2411. unsigned long flags;
  2412. spin_lock(&tree->lru_lock);
  2413. eb = find_lru(tree, start, len);
  2414. spin_unlock(&tree->lru_lock);
  2415. if (eb) {
  2416. return eb;
  2417. }
  2418. eb = kmem_cache_zalloc(extent_buffer_cache, mask);
  2419. INIT_LIST_HEAD(&eb->lru);
  2420. eb->start = start;
  2421. eb->len = len;
  2422. spin_lock_irqsave(&leak_lock, flags);
  2423. list_add(&eb->leak_list, &buffers);
  2424. spin_unlock_irqrestore(&leak_lock, flags);
  2425. atomic_set(&eb->refs, 1);
  2426. return eb;
  2427. }
  2428. static void __free_extent_buffer(struct extent_buffer *eb)
  2429. {
  2430. unsigned long flags;
  2431. spin_lock_irqsave(&leak_lock, flags);
  2432. list_del(&eb->leak_list);
  2433. spin_unlock_irqrestore(&leak_lock, flags);
  2434. kmem_cache_free(extent_buffer_cache, eb);
  2435. }
  2436. struct extent_buffer *alloc_extent_buffer(struct extent_io_tree *tree,
  2437. u64 start, unsigned long len,
  2438. struct page *page0,
  2439. gfp_t mask)
  2440. {
  2441. unsigned long num_pages = num_extent_pages(start, len);
  2442. unsigned long i;
  2443. unsigned long index = start >> PAGE_CACHE_SHIFT;
  2444. struct extent_buffer *eb;
  2445. struct page *p;
  2446. struct address_space *mapping = tree->mapping;
  2447. int uptodate = 1;
  2448. eb = __alloc_extent_buffer(tree, start, len, mask);
  2449. if (!eb)
  2450. return NULL;
  2451. if (eb->flags & EXTENT_BUFFER_FILLED)
  2452. goto lru_add;
  2453. if (page0) {
  2454. eb->first_page = page0;
  2455. i = 1;
  2456. index++;
  2457. page_cache_get(page0);
  2458. mark_page_accessed(page0);
  2459. set_page_extent_mapped(page0);
  2460. set_page_extent_head(page0, len);
  2461. uptodate = PageUptodate(page0);
  2462. } else {
  2463. i = 0;
  2464. }
  2465. for (; i < num_pages; i++, index++) {
  2466. p = find_or_create_page(mapping, index, mask | __GFP_HIGHMEM);
  2467. if (!p) {
  2468. WARN_ON(1);
  2469. goto fail;
  2470. }
  2471. set_page_extent_mapped(p);
  2472. mark_page_accessed(p);
  2473. if (i == 0) {
  2474. eb->first_page = p;
  2475. set_page_extent_head(p, len);
  2476. } else {
  2477. set_page_private(p, EXTENT_PAGE_PRIVATE);
  2478. }
  2479. if (!PageUptodate(p))
  2480. uptodate = 0;
  2481. unlock_page(p);
  2482. }
  2483. if (uptodate)
  2484. eb->flags |= EXTENT_UPTODATE;
  2485. eb->flags |= EXTENT_BUFFER_FILLED;
  2486. lru_add:
  2487. spin_lock(&tree->lru_lock);
  2488. add_lru(tree, eb);
  2489. spin_unlock(&tree->lru_lock);
  2490. return eb;
  2491. fail:
  2492. spin_lock(&tree->lru_lock);
  2493. list_del_init(&eb->lru);
  2494. spin_unlock(&tree->lru_lock);
  2495. if (!atomic_dec_and_test(&eb->refs))
  2496. return NULL;
  2497. for (index = 1; index < i; index++) {
  2498. page_cache_release(extent_buffer_page(eb, index));
  2499. }
  2500. if (i > 0)
  2501. page_cache_release(extent_buffer_page(eb, 0));
  2502. __free_extent_buffer(eb);
  2503. return NULL;
  2504. }
  2505. EXPORT_SYMBOL(alloc_extent_buffer);
  2506. struct extent_buffer *find_extent_buffer(struct extent_io_tree *tree,
  2507. u64 start, unsigned long len,
  2508. gfp_t mask)
  2509. {
  2510. unsigned long num_pages = num_extent_pages(start, len);
  2511. unsigned long i;
  2512. unsigned long index = start >> PAGE_CACHE_SHIFT;
  2513. struct extent_buffer *eb;
  2514. struct page *p;
  2515. struct address_space *mapping = tree->mapping;
  2516. int uptodate = 1;
  2517. eb = __alloc_extent_buffer(tree, start, len, mask);
  2518. if (!eb)
  2519. return NULL;
  2520. if (eb->flags & EXTENT_BUFFER_FILLED)
  2521. goto lru_add;
  2522. for (i = 0; i < num_pages; i++, index++) {
  2523. p = find_lock_page(mapping, index);
  2524. if (!p) {
  2525. goto fail;
  2526. }
  2527. set_page_extent_mapped(p);
  2528. mark_page_accessed(p);
  2529. if (i == 0) {
  2530. eb->first_page = p;
  2531. set_page_extent_head(p, len);
  2532. } else {
  2533. set_page_private(p, EXTENT_PAGE_PRIVATE);
  2534. }
  2535. if (!PageUptodate(p))
  2536. uptodate = 0;
  2537. unlock_page(p);
  2538. }
  2539. if (uptodate)
  2540. eb->flags |= EXTENT_UPTODATE;
  2541. eb->flags |= EXTENT_BUFFER_FILLED;
  2542. lru_add:
  2543. spin_lock(&tree->lru_lock);
  2544. add_lru(tree, eb);
  2545. spin_unlock(&tree->lru_lock);
  2546. return eb;
  2547. fail:
  2548. spin_lock(&tree->lru_lock);
  2549. list_del_init(&eb->lru);
  2550. spin_unlock(&tree->lru_lock);
  2551. if (!atomic_dec_and_test(&eb->refs))
  2552. return NULL;
  2553. for (index = 1; index < i; index++) {
  2554. page_cache_release(extent_buffer_page(eb, index));
  2555. }
  2556. if (i > 0)
  2557. page_cache_release(extent_buffer_page(eb, 0));
  2558. __free_extent_buffer(eb);
  2559. return NULL;
  2560. }
  2561. EXPORT_SYMBOL(find_extent_buffer);
  2562. void free_extent_buffer(struct extent_buffer *eb)
  2563. {
  2564. unsigned long i;
  2565. unsigned long num_pages;
  2566. if (!eb)
  2567. return;
  2568. if (!atomic_dec_and_test(&eb->refs))
  2569. return;
  2570. WARN_ON(!list_empty(&eb->lru));
  2571. num_pages = num_extent_pages(eb->start, eb->len);
  2572. for (i = 1; i < num_pages; i++) {
  2573. page_cache_release(extent_buffer_page(eb, i));
  2574. }
  2575. page_cache_release(extent_buffer_page(eb, 0));
  2576. __free_extent_buffer(eb);
  2577. }
  2578. EXPORT_SYMBOL(free_extent_buffer);
  2579. int clear_extent_buffer_dirty(struct extent_io_tree *tree,
  2580. struct extent_buffer *eb)
  2581. {
  2582. int set;
  2583. unsigned long i;
  2584. unsigned long num_pages;
  2585. struct page *page;
  2586. u64 start = eb->start;
  2587. u64 end = start + eb->len - 1;
  2588. set = clear_extent_dirty(tree, start, end, GFP_NOFS);
  2589. num_pages = num_extent_pages(eb->start, eb->len);
  2590. for (i = 0; i < num_pages; i++) {
  2591. page = extent_buffer_page(eb, i);
  2592. lock_page(page);
  2593. if (i == 0)
  2594. set_page_extent_head(page, eb->len);
  2595. else
  2596. set_page_private(page, EXTENT_PAGE_PRIVATE);
  2597. /*
  2598. * if we're on the last page or the first page and the
  2599. * block isn't aligned on a page boundary, do extra checks
  2600. * to make sure we don't clean page that is partially dirty
  2601. */
  2602. if ((i == 0 && (eb->start & (PAGE_CACHE_SIZE - 1))) ||
  2603. ((i == num_pages - 1) &&
  2604. ((eb->start + eb->len) & (PAGE_CACHE_SIZE - 1)))) {
  2605. start = (u64)page->index << PAGE_CACHE_SHIFT;
  2606. end = start + PAGE_CACHE_SIZE - 1;
  2607. if (test_range_bit(tree, start, end,
  2608. EXTENT_DIRTY, 0)) {
  2609. unlock_page(page);
  2610. continue;
  2611. }
  2612. }
  2613. clear_page_dirty_for_io(page);
  2614. read_lock_irq(&page->mapping->tree_lock);
  2615. if (!PageDirty(page)) {
  2616. radix_tree_tag_clear(&page->mapping->page_tree,
  2617. page_index(page),
  2618. PAGECACHE_TAG_DIRTY);
  2619. }
  2620. read_unlock_irq(&page->mapping->tree_lock);
  2621. unlock_page(page);
  2622. }
  2623. return 0;
  2624. }
  2625. EXPORT_SYMBOL(clear_extent_buffer_dirty);
  2626. int wait_on_extent_buffer_writeback(struct extent_io_tree *tree,
  2627. struct extent_buffer *eb)
  2628. {
  2629. return wait_on_extent_writeback(tree, eb->start,
  2630. eb->start + eb->len - 1);
  2631. }
  2632. EXPORT_SYMBOL(wait_on_extent_buffer_writeback);
  2633. int set_extent_buffer_dirty(struct extent_io_tree *tree,
  2634. struct extent_buffer *eb)
  2635. {
  2636. unsigned long i;
  2637. unsigned long num_pages;
  2638. num_pages = num_extent_pages(eb->start, eb->len);
  2639. for (i = 0; i < num_pages; i++) {
  2640. struct page *page = extent_buffer_page(eb, i);
  2641. /* writepage may need to do something special for the
  2642. * first page, we have to make sure page->private is
  2643. * properly set. releasepage may drop page->private
  2644. * on us if the page isn't already dirty.
  2645. */
  2646. if (i == 0) {
  2647. lock_page(page);
  2648. set_page_extent_head(page, eb->len);
  2649. } else if (PagePrivate(page) &&
  2650. page->private != EXTENT_PAGE_PRIVATE) {
  2651. lock_page(page);
  2652. set_page_extent_mapped(page);
  2653. unlock_page(page);
  2654. }
  2655. __set_page_dirty_nobuffers(extent_buffer_page(eb, i));
  2656. if (i == 0)
  2657. unlock_page(page);
  2658. }
  2659. return set_extent_dirty(tree, eb->start,
  2660. eb->start + eb->len - 1, GFP_NOFS);
  2661. }
  2662. EXPORT_SYMBOL(set_extent_buffer_dirty);
  2663. int set_extent_buffer_uptodate(struct extent_io_tree *tree,
  2664. struct extent_buffer *eb)
  2665. {
  2666. unsigned long i;
  2667. struct page *page;
  2668. unsigned long num_pages;
  2669. num_pages = num_extent_pages(eb->start, eb->len);
  2670. set_extent_uptodate(tree, eb->start, eb->start + eb->len - 1,
  2671. GFP_NOFS);
  2672. for (i = 0; i < num_pages; i++) {
  2673. page = extent_buffer_page(eb, i);
  2674. if ((i == 0 && (eb->start & (PAGE_CACHE_SIZE - 1))) ||
  2675. ((i == num_pages - 1) &&
  2676. ((eb->start + eb->len) & (PAGE_CACHE_SIZE - 1)))) {
  2677. check_page_uptodate(tree, page);
  2678. continue;
  2679. }
  2680. SetPageUptodate(page);
  2681. }
  2682. return 0;
  2683. }
  2684. EXPORT_SYMBOL(set_extent_buffer_uptodate);
  2685. int extent_range_uptodate(struct extent_io_tree *tree,
  2686. u64 start, u64 end)
  2687. {
  2688. struct page *page;
  2689. int ret;
  2690. int pg_uptodate = 1;
  2691. int uptodate;
  2692. unsigned long index;
  2693. ret = test_range_bit(tree, start, end, EXTENT_UPTODATE, 1);
  2694. if (ret)
  2695. return 1;
  2696. while(start <= end) {
  2697. index = start >> PAGE_CACHE_SHIFT;
  2698. page = find_get_page(tree->mapping, index);
  2699. uptodate = PageUptodate(page);
  2700. page_cache_release(page);
  2701. if (!uptodate) {
  2702. pg_uptodate = 0;
  2703. break;
  2704. }
  2705. start += PAGE_CACHE_SIZE;
  2706. }
  2707. return pg_uptodate;
  2708. }
  2709. int extent_buffer_uptodate(struct extent_io_tree *tree,
  2710. struct extent_buffer *eb)
  2711. {
  2712. int ret = 0;
  2713. int ret2;
  2714. unsigned long num_pages;
  2715. unsigned long i;
  2716. struct page *page;
  2717. int pg_uptodate = 1;
  2718. if (eb->flags & EXTENT_UPTODATE)
  2719. ret = 1;
  2720. ret2 = test_range_bit(tree, eb->start, eb->start + eb->len - 1,
  2721. EXTENT_UPTODATE, 1);
  2722. num_pages = num_extent_pages(eb->start, eb->len);
  2723. for (i = 0; i < num_pages; i++) {
  2724. page = extent_buffer_page(eb, i);
  2725. if (!PageUptodate(page)) {
  2726. pg_uptodate = 0;
  2727. break;
  2728. }
  2729. }
  2730. if ((ret || ret2) && !pg_uptodate) {
  2731. printk("uptodate error2 eb %Lu ret %d ret2 %d pg_uptodate %d\n", eb->start, ret, ret2, pg_uptodate);
  2732. WARN_ON(1);
  2733. }
  2734. return (ret || ret2);
  2735. }
  2736. EXPORT_SYMBOL(extent_buffer_uptodate);
  2737. int read_extent_buffer_pages(struct extent_io_tree *tree,
  2738. struct extent_buffer *eb,
  2739. u64 start, int wait,
  2740. get_extent_t *get_extent, int mirror_num)
  2741. {
  2742. unsigned long i;
  2743. unsigned long start_i;
  2744. struct page *page;
  2745. int err;
  2746. int ret = 0;
  2747. int locked_pages = 0;
  2748. int all_uptodate = 1;
  2749. int inc_all_pages = 0;
  2750. unsigned long num_pages;
  2751. struct bio *bio = NULL;
  2752. if (eb->flags & EXTENT_UPTODATE)
  2753. return 0;
  2754. if (test_range_bit(tree, eb->start, eb->start + eb->len - 1,
  2755. EXTENT_UPTODATE, 1)) {
  2756. return 0;
  2757. }
  2758. if (start) {
  2759. WARN_ON(start < eb->start);
  2760. start_i = (start >> PAGE_CACHE_SHIFT) -
  2761. (eb->start >> PAGE_CACHE_SHIFT);
  2762. } else {
  2763. start_i = 0;
  2764. }
  2765. num_pages = num_extent_pages(eb->start, eb->len);
  2766. for (i = start_i; i < num_pages; i++) {
  2767. page = extent_buffer_page(eb, i);
  2768. if (!wait) {
  2769. if (TestSetPageLocked(page))
  2770. goto unlock_exit;
  2771. } else {
  2772. lock_page(page);
  2773. }
  2774. locked_pages++;
  2775. if (!PageUptodate(page)) {
  2776. all_uptodate = 0;
  2777. }
  2778. }
  2779. if (all_uptodate) {
  2780. if (start_i == 0)
  2781. eb->flags |= EXTENT_UPTODATE;
  2782. goto unlock_exit;
  2783. }
  2784. for (i = start_i; i < num_pages; i++) {
  2785. page = extent_buffer_page(eb, i);
  2786. if (inc_all_pages)
  2787. page_cache_get(page);
  2788. if (!PageUptodate(page)) {
  2789. if (start_i == 0)
  2790. inc_all_pages = 1;
  2791. ClearPageError(page);
  2792. err = __extent_read_full_page(tree, page,
  2793. get_extent, &bio,
  2794. mirror_num);
  2795. if (err) {
  2796. ret = err;
  2797. }
  2798. } else {
  2799. unlock_page(page);
  2800. }
  2801. }
  2802. if (bio)
  2803. submit_one_bio(READ, bio, mirror_num);
  2804. if (ret || !wait) {
  2805. return ret;
  2806. }
  2807. for (i = start_i; i < num_pages; i++) {
  2808. page = extent_buffer_page(eb, i);
  2809. wait_on_page_locked(page);
  2810. if (!PageUptodate(page)) {
  2811. ret = -EIO;
  2812. }
  2813. }
  2814. if (!ret)
  2815. eb->flags |= EXTENT_UPTODATE;
  2816. return ret;
  2817. unlock_exit:
  2818. i = start_i;
  2819. while(locked_pages > 0) {
  2820. page = extent_buffer_page(eb, i);
  2821. i++;
  2822. unlock_page(page);
  2823. locked_pages--;
  2824. }
  2825. return ret;
  2826. }
  2827. EXPORT_SYMBOL(read_extent_buffer_pages);
  2828. void read_extent_buffer(struct extent_buffer *eb, void *dstv,
  2829. unsigned long start,
  2830. unsigned long len)
  2831. {
  2832. size_t cur;
  2833. size_t offset;
  2834. struct page *page;
  2835. char *kaddr;
  2836. char *dst = (char *)dstv;
  2837. size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
  2838. unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
  2839. WARN_ON(start > eb->len);
  2840. WARN_ON(start + len > eb->start + eb->len);
  2841. offset = (start_offset + start) & ((unsigned long)PAGE_CACHE_SIZE - 1);
  2842. while(len > 0) {
  2843. page = extent_buffer_page(eb, i);
  2844. cur = min(len, (PAGE_CACHE_SIZE - offset));
  2845. kaddr = kmap_atomic(page, KM_USER1);
  2846. memcpy(dst, kaddr + offset, cur);
  2847. kunmap_atomic(kaddr, KM_USER1);
  2848. dst += cur;
  2849. len -= cur;
  2850. offset = 0;
  2851. i++;
  2852. }
  2853. }
  2854. EXPORT_SYMBOL(read_extent_buffer);
  2855. int map_private_extent_buffer(struct extent_buffer *eb, unsigned long start,
  2856. unsigned long min_len, char **token, char **map,
  2857. unsigned long *map_start,
  2858. unsigned long *map_len, int km)
  2859. {
  2860. size_t offset = start & (PAGE_CACHE_SIZE - 1);
  2861. char *kaddr;
  2862. struct page *p;
  2863. size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
  2864. unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
  2865. unsigned long end_i = (start_offset + start + min_len - 1) >>
  2866. PAGE_CACHE_SHIFT;
  2867. if (i != end_i)
  2868. return -EINVAL;
  2869. if (i == 0) {
  2870. offset = start_offset;
  2871. *map_start = 0;
  2872. } else {
  2873. offset = 0;
  2874. *map_start = ((u64)i << PAGE_CACHE_SHIFT) - start_offset;
  2875. }
  2876. if (start + min_len > eb->len) {
  2877. printk("bad mapping eb start %Lu len %lu, wanted %lu %lu\n", eb->start, eb->len, start, min_len);
  2878. WARN_ON(1);
  2879. }
  2880. p = extent_buffer_page(eb, i);
  2881. kaddr = kmap_atomic(p, km);
  2882. *token = kaddr;
  2883. *map = kaddr + offset;
  2884. *map_len = PAGE_CACHE_SIZE - offset;
  2885. return 0;
  2886. }
  2887. EXPORT_SYMBOL(map_private_extent_buffer);
  2888. int map_extent_buffer(struct extent_buffer *eb, unsigned long start,
  2889. unsigned long min_len,
  2890. char **token, char **map,
  2891. unsigned long *map_start,
  2892. unsigned long *map_len, int km)
  2893. {
  2894. int err;
  2895. int save = 0;
  2896. if (eb->map_token) {
  2897. unmap_extent_buffer(eb, eb->map_token, km);
  2898. eb->map_token = NULL;
  2899. save = 1;
  2900. }
  2901. err = map_private_extent_buffer(eb, start, min_len, token, map,
  2902. map_start, map_len, km);
  2903. if (!err && save) {
  2904. eb->map_token = *token;
  2905. eb->kaddr = *map;
  2906. eb->map_start = *map_start;
  2907. eb->map_len = *map_len;
  2908. }
  2909. return err;
  2910. }
  2911. EXPORT_SYMBOL(map_extent_buffer);
  2912. void unmap_extent_buffer(struct extent_buffer *eb, char *token, int km)
  2913. {
  2914. kunmap_atomic(token, km);
  2915. }
  2916. EXPORT_SYMBOL(unmap_extent_buffer);
  2917. int memcmp_extent_buffer(struct extent_buffer *eb, const void *ptrv,
  2918. unsigned long start,
  2919. unsigned long len)
  2920. {
  2921. size_t cur;
  2922. size_t offset;
  2923. struct page *page;
  2924. char *kaddr;
  2925. char *ptr = (char *)ptrv;
  2926. size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
  2927. unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
  2928. int ret = 0;
  2929. WARN_ON(start > eb->len);
  2930. WARN_ON(start + len > eb->start + eb->len);
  2931. offset = (start_offset + start) & ((unsigned long)PAGE_CACHE_SIZE - 1);
  2932. while(len > 0) {
  2933. page = extent_buffer_page(eb, i);
  2934. cur = min(len, (PAGE_CACHE_SIZE - offset));
  2935. kaddr = kmap_atomic(page, KM_USER0);
  2936. ret = memcmp(ptr, kaddr + offset, cur);
  2937. kunmap_atomic(kaddr, KM_USER0);
  2938. if (ret)
  2939. break;
  2940. ptr += cur;
  2941. len -= cur;
  2942. offset = 0;
  2943. i++;
  2944. }
  2945. return ret;
  2946. }
  2947. EXPORT_SYMBOL(memcmp_extent_buffer);
  2948. void write_extent_buffer(struct extent_buffer *eb, const void *srcv,
  2949. unsigned long start, unsigned long len)
  2950. {
  2951. size_t cur;
  2952. size_t offset;
  2953. struct page *page;
  2954. char *kaddr;
  2955. char *src = (char *)srcv;
  2956. size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
  2957. unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
  2958. WARN_ON(start > eb->len);
  2959. WARN_ON(start + len > eb->start + eb->len);
  2960. offset = (start_offset + start) & ((unsigned long)PAGE_CACHE_SIZE - 1);
  2961. while(len > 0) {
  2962. page = extent_buffer_page(eb, i);
  2963. WARN_ON(!PageUptodate(page));
  2964. cur = min(len, PAGE_CACHE_SIZE - offset);
  2965. kaddr = kmap_atomic(page, KM_USER1);
  2966. memcpy(kaddr + offset, src, cur);
  2967. kunmap_atomic(kaddr, KM_USER1);
  2968. src += cur;
  2969. len -= cur;
  2970. offset = 0;
  2971. i++;
  2972. }
  2973. }
  2974. EXPORT_SYMBOL(write_extent_buffer);
  2975. void memset_extent_buffer(struct extent_buffer *eb, char c,
  2976. unsigned long start, unsigned long len)
  2977. {
  2978. size_t cur;
  2979. size_t offset;
  2980. struct page *page;
  2981. char *kaddr;
  2982. size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
  2983. unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
  2984. WARN_ON(start > eb->len);
  2985. WARN_ON(start + len > eb->start + eb->len);
  2986. offset = (start_offset + start) & ((unsigned long)PAGE_CACHE_SIZE - 1);
  2987. while(len > 0) {
  2988. page = extent_buffer_page(eb, i);
  2989. WARN_ON(!PageUptodate(page));
  2990. cur = min(len, PAGE_CACHE_SIZE - offset);
  2991. kaddr = kmap_atomic(page, KM_USER0);
  2992. memset(kaddr + offset, c, cur);
  2993. kunmap_atomic(kaddr, KM_USER0);
  2994. len -= cur;
  2995. offset = 0;
  2996. i++;
  2997. }
  2998. }
  2999. EXPORT_SYMBOL(memset_extent_buffer);
  3000. void copy_extent_buffer(struct extent_buffer *dst, struct extent_buffer *src,
  3001. unsigned long dst_offset, unsigned long src_offset,
  3002. unsigned long len)
  3003. {
  3004. u64 dst_len = dst->len;
  3005. size_t cur;
  3006. size_t offset;
  3007. struct page *page;
  3008. char *kaddr;
  3009. size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
  3010. unsigned long i = (start_offset + dst_offset) >> PAGE_CACHE_SHIFT;
  3011. WARN_ON(src->len != dst_len);
  3012. offset = (start_offset + dst_offset) &
  3013. ((unsigned long)PAGE_CACHE_SIZE - 1);
  3014. while(len > 0) {
  3015. page = extent_buffer_page(dst, i);
  3016. WARN_ON(!PageUptodate(page));
  3017. cur = min(len, (unsigned long)(PAGE_CACHE_SIZE - offset));
  3018. kaddr = kmap_atomic(page, KM_USER0);
  3019. read_extent_buffer(src, kaddr + offset, src_offset, cur);
  3020. kunmap_atomic(kaddr, KM_USER0);
  3021. src_offset += cur;
  3022. len -= cur;
  3023. offset = 0;
  3024. i++;
  3025. }
  3026. }
  3027. EXPORT_SYMBOL(copy_extent_buffer);
  3028. static void move_pages(struct page *dst_page, struct page *src_page,
  3029. unsigned long dst_off, unsigned long src_off,
  3030. unsigned long len)
  3031. {
  3032. char *dst_kaddr = kmap_atomic(dst_page, KM_USER0);
  3033. if (dst_page == src_page) {
  3034. memmove(dst_kaddr + dst_off, dst_kaddr + src_off, len);
  3035. } else {
  3036. char *src_kaddr = kmap_atomic(src_page, KM_USER1);
  3037. char *p = dst_kaddr + dst_off + len;
  3038. char *s = src_kaddr + src_off + len;
  3039. while (len--)
  3040. *--p = *--s;
  3041. kunmap_atomic(src_kaddr, KM_USER1);
  3042. }
  3043. kunmap_atomic(dst_kaddr, KM_USER0);
  3044. }
  3045. static void copy_pages(struct page *dst_page, struct page *src_page,
  3046. unsigned long dst_off, unsigned long src_off,
  3047. unsigned long len)
  3048. {
  3049. char *dst_kaddr = kmap_atomic(dst_page, KM_USER0);
  3050. char *src_kaddr;
  3051. if (dst_page != src_page)
  3052. src_kaddr = kmap_atomic(src_page, KM_USER1);
  3053. else
  3054. src_kaddr = dst_kaddr;
  3055. memcpy(dst_kaddr + dst_off, src_kaddr + src_off, len);
  3056. kunmap_atomic(dst_kaddr, KM_USER0);
  3057. if (dst_page != src_page)
  3058. kunmap_atomic(src_kaddr, KM_USER1);
  3059. }
  3060. void memcpy_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
  3061. unsigned long src_offset, unsigned long len)
  3062. {
  3063. size_t cur;
  3064. size_t dst_off_in_page;
  3065. size_t src_off_in_page;
  3066. size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
  3067. unsigned long dst_i;
  3068. unsigned long src_i;
  3069. if (src_offset + len > dst->len) {
  3070. printk("memmove bogus src_offset %lu move len %lu len %lu\n",
  3071. src_offset, len, dst->len);
  3072. BUG_ON(1);
  3073. }
  3074. if (dst_offset + len > dst->len) {
  3075. printk("memmove bogus dst_offset %lu move len %lu len %lu\n",
  3076. dst_offset, len, dst->len);
  3077. BUG_ON(1);
  3078. }
  3079. while(len > 0) {
  3080. dst_off_in_page = (start_offset + dst_offset) &
  3081. ((unsigned long)PAGE_CACHE_SIZE - 1);
  3082. src_off_in_page = (start_offset + src_offset) &
  3083. ((unsigned long)PAGE_CACHE_SIZE - 1);
  3084. dst_i = (start_offset + dst_offset) >> PAGE_CACHE_SHIFT;
  3085. src_i = (start_offset + src_offset) >> PAGE_CACHE_SHIFT;
  3086. cur = min(len, (unsigned long)(PAGE_CACHE_SIZE -
  3087. src_off_in_page));
  3088. cur = min_t(unsigned long, cur,
  3089. (unsigned long)(PAGE_CACHE_SIZE - dst_off_in_page));
  3090. copy_pages(extent_buffer_page(dst, dst_i),
  3091. extent_buffer_page(dst, src_i),
  3092. dst_off_in_page, src_off_in_page, cur);
  3093. src_offset += cur;
  3094. dst_offset += cur;
  3095. len -= cur;
  3096. }
  3097. }
  3098. EXPORT_SYMBOL(memcpy_extent_buffer);
  3099. void memmove_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
  3100. unsigned long src_offset, unsigned long len)
  3101. {
  3102. size_t cur;
  3103. size_t dst_off_in_page;
  3104. size_t src_off_in_page;
  3105. unsigned long dst_end = dst_offset + len - 1;
  3106. unsigned long src_end = src_offset + len - 1;
  3107. size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
  3108. unsigned long dst_i;
  3109. unsigned long src_i;
  3110. if (src_offset + len > dst->len) {
  3111. printk("memmove bogus src_offset %lu move len %lu len %lu\n",
  3112. src_offset, len, dst->len);
  3113. BUG_ON(1);
  3114. }
  3115. if (dst_offset + len > dst->len) {
  3116. printk("memmove bogus dst_offset %lu move len %lu len %lu\n",
  3117. dst_offset, len, dst->len);
  3118. BUG_ON(1);
  3119. }
  3120. if (dst_offset < src_offset) {
  3121. memcpy_extent_buffer(dst, dst_offset, src_offset, len);
  3122. return;
  3123. }
  3124. while(len > 0) {
  3125. dst_i = (start_offset + dst_end) >> PAGE_CACHE_SHIFT;
  3126. src_i = (start_offset + src_end) >> PAGE_CACHE_SHIFT;
  3127. dst_off_in_page = (start_offset + dst_end) &
  3128. ((unsigned long)PAGE_CACHE_SIZE - 1);
  3129. src_off_in_page = (start_offset + src_end) &
  3130. ((unsigned long)PAGE_CACHE_SIZE - 1);
  3131. cur = min_t(unsigned long, len, src_off_in_page + 1);
  3132. cur = min(cur, dst_off_in_page + 1);
  3133. move_pages(extent_buffer_page(dst, dst_i),
  3134. extent_buffer_page(dst, src_i),
  3135. dst_off_in_page - cur + 1,
  3136. src_off_in_page - cur + 1, cur);
  3137. dst_end -= cur;
  3138. src_end -= cur;
  3139. len -= cur;
  3140. }
  3141. }
  3142. EXPORT_SYMBOL(memmove_extent_buffer);