extent_map.c 72 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934
  1. #include <linux/bitops.h>
  2. #include <linux/slab.h>
  3. #include <linux/bio.h>
  4. #include <linux/mm.h>
  5. #include <linux/gfp.h>
  6. #include <linux/pagemap.h>
  7. #include <linux/page-flags.h>
  8. #include <linux/module.h>
  9. #include <linux/spinlock.h>
  10. #include <linux/blkdev.h>
  11. #include <linux/swap.h>
  12. #include <linux/version.h>
  13. #include <linux/writeback.h>
  14. #include <linux/pagevec.h>
  15. #include "extent_map.h"
  16. /* temporary define until extent_map moves out of btrfs */
  17. struct kmem_cache *btrfs_cache_create(const char *name, size_t size,
  18. unsigned long extra_flags,
  19. void (*ctor)(void *, struct kmem_cache *,
  20. unsigned long));
  21. static struct kmem_cache *extent_map_cache;
  22. static struct kmem_cache *extent_state_cache;
  23. static struct kmem_cache *extent_buffer_cache;
  24. static LIST_HEAD(buffers);
  25. static LIST_HEAD(states);
  26. static spinlock_t state_lock = SPIN_LOCK_UNLOCKED;
  27. #define BUFFER_LRU_MAX 64
  28. struct tree_entry {
  29. u64 start;
  30. u64 end;
  31. int in_tree;
  32. struct rb_node rb_node;
  33. };
  34. struct extent_page_data {
  35. struct bio *bio;
  36. struct extent_map_tree *tree;
  37. get_extent_t *get_extent;
  38. };
  39. int __init extent_map_init(void)
  40. {
  41. extent_map_cache = btrfs_cache_create("extent_map",
  42. sizeof(struct extent_map), 0,
  43. NULL);
  44. if (!extent_map_cache)
  45. return -ENOMEM;
  46. extent_state_cache = btrfs_cache_create("extent_state",
  47. sizeof(struct extent_state), 0,
  48. NULL);
  49. if (!extent_state_cache)
  50. goto free_map_cache;
  51. extent_buffer_cache = btrfs_cache_create("extent_buffers",
  52. sizeof(struct extent_buffer), 0,
  53. NULL);
  54. if (!extent_buffer_cache)
  55. goto free_state_cache;
  56. return 0;
  57. free_state_cache:
  58. kmem_cache_destroy(extent_state_cache);
  59. free_map_cache:
  60. kmem_cache_destroy(extent_map_cache);
  61. return -ENOMEM;
  62. }
  63. void __exit extent_map_exit(void)
  64. {
  65. struct extent_state *state;
  66. while (!list_empty(&states)) {
  67. state = list_entry(states.next, struct extent_state, list);
  68. printk("state leak: start %Lu end %Lu state %lu in tree %d refs %d\n", state->start, state->end, state->state, state->in_tree, atomic_read(&state->refs));
  69. list_del(&state->list);
  70. kmem_cache_free(extent_state_cache, state);
  71. }
  72. if (extent_map_cache)
  73. kmem_cache_destroy(extent_map_cache);
  74. if (extent_state_cache)
  75. kmem_cache_destroy(extent_state_cache);
  76. if (extent_buffer_cache)
  77. kmem_cache_destroy(extent_buffer_cache);
  78. }
  79. void extent_map_tree_init(struct extent_map_tree *tree,
  80. struct address_space *mapping, gfp_t mask)
  81. {
  82. tree->map.rb_node = NULL;
  83. tree->state.rb_node = NULL;
  84. tree->ops = NULL;
  85. rwlock_init(&tree->lock);
  86. spin_lock_init(&tree->lru_lock);
  87. tree->mapping = mapping;
  88. INIT_LIST_HEAD(&tree->buffer_lru);
  89. tree->lru_size = 0;
  90. }
  91. EXPORT_SYMBOL(extent_map_tree_init);
  92. void extent_map_tree_empty_lru(struct extent_map_tree *tree)
  93. {
  94. struct extent_buffer *eb;
  95. while(!list_empty(&tree->buffer_lru)) {
  96. eb = list_entry(tree->buffer_lru.next, struct extent_buffer,
  97. lru);
  98. list_del_init(&eb->lru);
  99. free_extent_buffer(eb);
  100. }
  101. }
  102. EXPORT_SYMBOL(extent_map_tree_empty_lru);
  103. struct extent_map *alloc_extent_map(gfp_t mask)
  104. {
  105. struct extent_map *em;
  106. em = kmem_cache_alloc(extent_map_cache, mask);
  107. if (!em || IS_ERR(em))
  108. return em;
  109. em->in_tree = 0;
  110. atomic_set(&em->refs, 1);
  111. return em;
  112. }
  113. EXPORT_SYMBOL(alloc_extent_map);
  114. void free_extent_map(struct extent_map *em)
  115. {
  116. if (!em)
  117. return;
  118. if (atomic_dec_and_test(&em->refs)) {
  119. WARN_ON(em->in_tree);
  120. kmem_cache_free(extent_map_cache, em);
  121. }
  122. }
  123. EXPORT_SYMBOL(free_extent_map);
  124. struct extent_state *alloc_extent_state(gfp_t mask)
  125. {
  126. struct extent_state *state;
  127. unsigned long flags;
  128. state = kmem_cache_alloc(extent_state_cache, mask);
  129. if (!state || IS_ERR(state))
  130. return state;
  131. state->state = 0;
  132. state->in_tree = 0;
  133. state->private = 0;
  134. spin_lock_irqsave(&state_lock, flags);
  135. list_add(&state->list, &states);
  136. spin_unlock_irqrestore(&state_lock, flags);
  137. atomic_set(&state->refs, 1);
  138. init_waitqueue_head(&state->wq);
  139. return state;
  140. }
  141. EXPORT_SYMBOL(alloc_extent_state);
  142. void free_extent_state(struct extent_state *state)
  143. {
  144. unsigned long flags;
  145. if (!state)
  146. return;
  147. if (atomic_dec_and_test(&state->refs)) {
  148. WARN_ON(state->in_tree);
  149. spin_lock_irqsave(&state_lock, flags);
  150. list_del(&state->list);
  151. spin_unlock_irqrestore(&state_lock, flags);
  152. kmem_cache_free(extent_state_cache, state);
  153. }
  154. }
  155. EXPORT_SYMBOL(free_extent_state);
  156. static struct rb_node *tree_insert(struct rb_root *root, u64 offset,
  157. struct rb_node *node)
  158. {
  159. struct rb_node ** p = &root->rb_node;
  160. struct rb_node * parent = NULL;
  161. struct tree_entry *entry;
  162. while(*p) {
  163. parent = *p;
  164. entry = rb_entry(parent, struct tree_entry, rb_node);
  165. if (offset < entry->start)
  166. p = &(*p)->rb_left;
  167. else if (offset > entry->end)
  168. p = &(*p)->rb_right;
  169. else
  170. return parent;
  171. }
  172. entry = rb_entry(node, struct tree_entry, rb_node);
  173. entry->in_tree = 1;
  174. rb_link_node(node, parent, p);
  175. rb_insert_color(node, root);
  176. return NULL;
  177. }
  178. static struct rb_node *__tree_search(struct rb_root *root, u64 offset,
  179. struct rb_node **prev_ret)
  180. {
  181. struct rb_node * n = root->rb_node;
  182. struct rb_node *prev = NULL;
  183. struct tree_entry *entry;
  184. struct tree_entry *prev_entry = NULL;
  185. while(n) {
  186. entry = rb_entry(n, struct tree_entry, rb_node);
  187. prev = n;
  188. prev_entry = entry;
  189. if (offset < entry->start)
  190. n = n->rb_left;
  191. else if (offset > entry->end)
  192. n = n->rb_right;
  193. else
  194. return n;
  195. }
  196. if (!prev_ret)
  197. return NULL;
  198. while(prev && offset > prev_entry->end) {
  199. prev = rb_next(prev);
  200. prev_entry = rb_entry(prev, struct tree_entry, rb_node);
  201. }
  202. *prev_ret = prev;
  203. return NULL;
  204. }
  205. static inline struct rb_node *tree_search(struct rb_root *root, u64 offset)
  206. {
  207. struct rb_node *prev;
  208. struct rb_node *ret;
  209. ret = __tree_search(root, offset, &prev);
  210. if (!ret)
  211. return prev;
  212. return ret;
  213. }
  214. static int tree_delete(struct rb_root *root, u64 offset)
  215. {
  216. struct rb_node *node;
  217. struct tree_entry *entry;
  218. node = __tree_search(root, offset, NULL);
  219. if (!node)
  220. return -ENOENT;
  221. entry = rb_entry(node, struct tree_entry, rb_node);
  222. entry->in_tree = 0;
  223. rb_erase(node, root);
  224. return 0;
  225. }
  226. /*
  227. * add_extent_mapping tries a simple backward merge with existing
  228. * mappings. The extent_map struct passed in will be inserted into
  229. * the tree directly (no copies made, just a reference taken).
  230. */
  231. int add_extent_mapping(struct extent_map_tree *tree,
  232. struct extent_map *em)
  233. {
  234. int ret = 0;
  235. struct extent_map *prev = NULL;
  236. struct rb_node *rb;
  237. write_lock_irq(&tree->lock);
  238. rb = tree_insert(&tree->map, em->end, &em->rb_node);
  239. if (rb) {
  240. prev = rb_entry(rb, struct extent_map, rb_node);
  241. printk("found extent map %Lu %Lu on insert of %Lu %Lu\n", prev->start, prev->end, em->start, em->end);
  242. ret = -EEXIST;
  243. goto out;
  244. }
  245. atomic_inc(&em->refs);
  246. if (em->start != 0) {
  247. rb = rb_prev(&em->rb_node);
  248. if (rb)
  249. prev = rb_entry(rb, struct extent_map, rb_node);
  250. if (prev && prev->end + 1 == em->start &&
  251. ((em->block_start == EXTENT_MAP_HOLE &&
  252. prev->block_start == EXTENT_MAP_HOLE) ||
  253. (em->block_start == EXTENT_MAP_INLINE &&
  254. prev->block_start == EXTENT_MAP_INLINE) ||
  255. (em->block_start == EXTENT_MAP_DELALLOC &&
  256. prev->block_start == EXTENT_MAP_DELALLOC) ||
  257. (em->block_start < EXTENT_MAP_DELALLOC - 1 &&
  258. em->block_start == prev->block_end + 1))) {
  259. em->start = prev->start;
  260. em->block_start = prev->block_start;
  261. rb_erase(&prev->rb_node, &tree->map);
  262. prev->in_tree = 0;
  263. free_extent_map(prev);
  264. }
  265. }
  266. out:
  267. write_unlock_irq(&tree->lock);
  268. return ret;
  269. }
  270. EXPORT_SYMBOL(add_extent_mapping);
  271. /*
  272. * lookup_extent_mapping returns the first extent_map struct in the
  273. * tree that intersects the [start, end] (inclusive) range. There may
  274. * be additional objects in the tree that intersect, so check the object
  275. * returned carefully to make sure you don't need additional lookups.
  276. */
  277. struct extent_map *lookup_extent_mapping(struct extent_map_tree *tree,
  278. u64 start, u64 end)
  279. {
  280. struct extent_map *em;
  281. struct rb_node *rb_node;
  282. read_lock_irq(&tree->lock);
  283. rb_node = tree_search(&tree->map, start);
  284. if (!rb_node) {
  285. em = NULL;
  286. goto out;
  287. }
  288. if (IS_ERR(rb_node)) {
  289. em = ERR_PTR(PTR_ERR(rb_node));
  290. goto out;
  291. }
  292. em = rb_entry(rb_node, struct extent_map, rb_node);
  293. if (em->end < start || em->start > end) {
  294. em = NULL;
  295. goto out;
  296. }
  297. atomic_inc(&em->refs);
  298. out:
  299. read_unlock_irq(&tree->lock);
  300. return em;
  301. }
  302. EXPORT_SYMBOL(lookup_extent_mapping);
  303. /*
  304. * removes an extent_map struct from the tree. No reference counts are
  305. * dropped, and no checks are done to see if the range is in use
  306. */
  307. int remove_extent_mapping(struct extent_map_tree *tree, struct extent_map *em)
  308. {
  309. int ret;
  310. write_lock_irq(&tree->lock);
  311. ret = tree_delete(&tree->map, em->end);
  312. write_unlock_irq(&tree->lock);
  313. return ret;
  314. }
  315. EXPORT_SYMBOL(remove_extent_mapping);
  316. /*
  317. * utility function to look for merge candidates inside a given range.
  318. * Any extents with matching state are merged together into a single
  319. * extent in the tree. Extents with EXTENT_IO in their state field
  320. * are not merged because the end_io handlers need to be able to do
  321. * operations on them without sleeping (or doing allocations/splits).
  322. *
  323. * This should be called with the tree lock held.
  324. */
  325. static int merge_state(struct extent_map_tree *tree,
  326. struct extent_state *state)
  327. {
  328. struct extent_state *other;
  329. struct rb_node *other_node;
  330. if (state->state & EXTENT_IOBITS)
  331. return 0;
  332. other_node = rb_prev(&state->rb_node);
  333. if (other_node) {
  334. other = rb_entry(other_node, struct extent_state, rb_node);
  335. if (other->end == state->start - 1 &&
  336. other->state == state->state) {
  337. state->start = other->start;
  338. other->in_tree = 0;
  339. rb_erase(&other->rb_node, &tree->state);
  340. free_extent_state(other);
  341. }
  342. }
  343. other_node = rb_next(&state->rb_node);
  344. if (other_node) {
  345. other = rb_entry(other_node, struct extent_state, rb_node);
  346. if (other->start == state->end + 1 &&
  347. other->state == state->state) {
  348. other->start = state->start;
  349. state->in_tree = 0;
  350. rb_erase(&state->rb_node, &tree->state);
  351. free_extent_state(state);
  352. }
  353. }
  354. return 0;
  355. }
  356. /*
  357. * insert an extent_state struct into the tree. 'bits' are set on the
  358. * struct before it is inserted.
  359. *
  360. * This may return -EEXIST if the extent is already there, in which case the
  361. * state struct is freed.
  362. *
  363. * The tree lock is not taken internally. This is a utility function and
  364. * probably isn't what you want to call (see set/clear_extent_bit).
  365. */
  366. static int insert_state(struct extent_map_tree *tree,
  367. struct extent_state *state, u64 start, u64 end,
  368. int bits)
  369. {
  370. struct rb_node *node;
  371. if (end < start) {
  372. printk("end < start %Lu %Lu\n", end, start);
  373. WARN_ON(1);
  374. }
  375. state->state |= bits;
  376. state->start = start;
  377. state->end = end;
  378. node = tree_insert(&tree->state, end, &state->rb_node);
  379. if (node) {
  380. struct extent_state *found;
  381. found = rb_entry(node, struct extent_state, rb_node);
  382. printk("found node %Lu %Lu on insert of %Lu %Lu\n", found->start, found->end, start, end);
  383. free_extent_state(state);
  384. return -EEXIST;
  385. }
  386. merge_state(tree, state);
  387. return 0;
  388. }
  389. /*
  390. * split a given extent state struct in two, inserting the preallocated
  391. * struct 'prealloc' as the newly created second half. 'split' indicates an
  392. * offset inside 'orig' where it should be split.
  393. *
  394. * Before calling,
  395. * the tree has 'orig' at [orig->start, orig->end]. After calling, there
  396. * are two extent state structs in the tree:
  397. * prealloc: [orig->start, split - 1]
  398. * orig: [ split, orig->end ]
  399. *
  400. * The tree locks are not taken by this function. They need to be held
  401. * by the caller.
  402. */
  403. static int split_state(struct extent_map_tree *tree, struct extent_state *orig,
  404. struct extent_state *prealloc, u64 split)
  405. {
  406. struct rb_node *node;
  407. prealloc->start = orig->start;
  408. prealloc->end = split - 1;
  409. prealloc->state = orig->state;
  410. orig->start = split;
  411. node = tree_insert(&tree->state, prealloc->end, &prealloc->rb_node);
  412. if (node) {
  413. struct extent_state *found;
  414. found = rb_entry(node, struct extent_state, rb_node);
  415. printk("found node %Lu %Lu on insert of %Lu %Lu\n", found->start, found->end, prealloc->start, prealloc->end);
  416. free_extent_state(prealloc);
  417. return -EEXIST;
  418. }
  419. return 0;
  420. }
  421. /*
  422. * utility function to clear some bits in an extent state struct.
  423. * it will optionally wake up any one waiting on this state (wake == 1), or
  424. * forcibly remove the state from the tree (delete == 1).
  425. *
  426. * If no bits are set on the state struct after clearing things, the
  427. * struct is freed and removed from the tree
  428. */
  429. static int clear_state_bit(struct extent_map_tree *tree,
  430. struct extent_state *state, int bits, int wake,
  431. int delete)
  432. {
  433. int ret = state->state & bits;
  434. state->state &= ~bits;
  435. if (wake)
  436. wake_up(&state->wq);
  437. if (delete || state->state == 0) {
  438. if (state->in_tree) {
  439. rb_erase(&state->rb_node, &tree->state);
  440. state->in_tree = 0;
  441. free_extent_state(state);
  442. } else {
  443. WARN_ON(1);
  444. }
  445. } else {
  446. merge_state(tree, state);
  447. }
  448. return ret;
  449. }
  450. /*
  451. * clear some bits on a range in the tree. This may require splitting
  452. * or inserting elements in the tree, so the gfp mask is used to
  453. * indicate which allocations or sleeping are allowed.
  454. *
  455. * pass 'wake' == 1 to kick any sleepers, and 'delete' == 1 to remove
  456. * the given range from the tree regardless of state (ie for truncate).
  457. *
  458. * the range [start, end] is inclusive.
  459. *
  460. * This takes the tree lock, and returns < 0 on error, > 0 if any of the
  461. * bits were already set, or zero if none of the bits were already set.
  462. */
  463. int clear_extent_bit(struct extent_map_tree *tree, u64 start, u64 end,
  464. int bits, int wake, int delete, gfp_t mask)
  465. {
  466. struct extent_state *state;
  467. struct extent_state *prealloc = NULL;
  468. struct rb_node *node;
  469. unsigned long flags;
  470. int err;
  471. int set = 0;
  472. again:
  473. if (!prealloc && (mask & __GFP_WAIT)) {
  474. prealloc = alloc_extent_state(mask);
  475. if (!prealloc)
  476. return -ENOMEM;
  477. }
  478. write_lock_irqsave(&tree->lock, flags);
  479. /*
  480. * this search will find the extents that end after
  481. * our range starts
  482. */
  483. node = tree_search(&tree->state, start);
  484. if (!node)
  485. goto out;
  486. state = rb_entry(node, struct extent_state, rb_node);
  487. if (state->start > end)
  488. goto out;
  489. WARN_ON(state->end < start);
  490. /*
  491. * | ---- desired range ---- |
  492. * | state | or
  493. * | ------------- state -------------- |
  494. *
  495. * We need to split the extent we found, and may flip
  496. * bits on second half.
  497. *
  498. * If the extent we found extends past our range, we
  499. * just split and search again. It'll get split again
  500. * the next time though.
  501. *
  502. * If the extent we found is inside our range, we clear
  503. * the desired bit on it.
  504. */
  505. if (state->start < start) {
  506. err = split_state(tree, state, prealloc, start);
  507. BUG_ON(err == -EEXIST);
  508. prealloc = NULL;
  509. if (err)
  510. goto out;
  511. if (state->end <= end) {
  512. start = state->end + 1;
  513. set |= clear_state_bit(tree, state, bits,
  514. wake, delete);
  515. } else {
  516. start = state->start;
  517. }
  518. goto search_again;
  519. }
  520. /*
  521. * | ---- desired range ---- |
  522. * | state |
  523. * We need to split the extent, and clear the bit
  524. * on the first half
  525. */
  526. if (state->start <= end && state->end > end) {
  527. err = split_state(tree, state, prealloc, end + 1);
  528. BUG_ON(err == -EEXIST);
  529. if (wake)
  530. wake_up(&state->wq);
  531. set |= clear_state_bit(tree, prealloc, bits,
  532. wake, delete);
  533. prealloc = NULL;
  534. goto out;
  535. }
  536. start = state->end + 1;
  537. set |= clear_state_bit(tree, state, bits, wake, delete);
  538. goto search_again;
  539. out:
  540. write_unlock_irqrestore(&tree->lock, flags);
  541. if (prealloc)
  542. free_extent_state(prealloc);
  543. return set;
  544. search_again:
  545. if (start > end)
  546. goto out;
  547. write_unlock_irqrestore(&tree->lock, flags);
  548. if (mask & __GFP_WAIT)
  549. cond_resched();
  550. goto again;
  551. }
  552. EXPORT_SYMBOL(clear_extent_bit);
  553. static int wait_on_state(struct extent_map_tree *tree,
  554. struct extent_state *state)
  555. {
  556. DEFINE_WAIT(wait);
  557. prepare_to_wait(&state->wq, &wait, TASK_UNINTERRUPTIBLE);
  558. read_unlock_irq(&tree->lock);
  559. schedule();
  560. read_lock_irq(&tree->lock);
  561. finish_wait(&state->wq, &wait);
  562. return 0;
  563. }
  564. /*
  565. * waits for one or more bits to clear on a range in the state tree.
  566. * The range [start, end] is inclusive.
  567. * The tree lock is taken by this function
  568. */
  569. int wait_extent_bit(struct extent_map_tree *tree, u64 start, u64 end, int bits)
  570. {
  571. struct extent_state *state;
  572. struct rb_node *node;
  573. read_lock_irq(&tree->lock);
  574. again:
  575. while (1) {
  576. /*
  577. * this search will find all the extents that end after
  578. * our range starts
  579. */
  580. node = tree_search(&tree->state, start);
  581. if (!node)
  582. break;
  583. state = rb_entry(node, struct extent_state, rb_node);
  584. if (state->start > end)
  585. goto out;
  586. if (state->state & bits) {
  587. start = state->start;
  588. atomic_inc(&state->refs);
  589. wait_on_state(tree, state);
  590. free_extent_state(state);
  591. goto again;
  592. }
  593. start = state->end + 1;
  594. if (start > end)
  595. break;
  596. if (need_resched()) {
  597. read_unlock_irq(&tree->lock);
  598. cond_resched();
  599. read_lock_irq(&tree->lock);
  600. }
  601. }
  602. out:
  603. read_unlock_irq(&tree->lock);
  604. return 0;
  605. }
  606. EXPORT_SYMBOL(wait_extent_bit);
  607. /*
  608. * set some bits on a range in the tree. This may require allocations
  609. * or sleeping, so the gfp mask is used to indicate what is allowed.
  610. *
  611. * If 'exclusive' == 1, this will fail with -EEXIST if some part of the
  612. * range already has the desired bits set. The start of the existing
  613. * range is returned in failed_start in this case.
  614. *
  615. * [start, end] is inclusive
  616. * This takes the tree lock.
  617. */
  618. int set_extent_bit(struct extent_map_tree *tree, u64 start, u64 end, int bits,
  619. int exclusive, u64 *failed_start, gfp_t mask)
  620. {
  621. struct extent_state *state;
  622. struct extent_state *prealloc = NULL;
  623. struct rb_node *node;
  624. unsigned long flags;
  625. int err = 0;
  626. int set;
  627. u64 last_start;
  628. u64 last_end;
  629. again:
  630. if (!prealloc && (mask & __GFP_WAIT)) {
  631. prealloc = alloc_extent_state(mask);
  632. if (!prealloc)
  633. return -ENOMEM;
  634. }
  635. write_lock_irqsave(&tree->lock, flags);
  636. /*
  637. * this search will find all the extents that end after
  638. * our range starts.
  639. */
  640. node = tree_search(&tree->state, start);
  641. if (!node) {
  642. err = insert_state(tree, prealloc, start, end, bits);
  643. prealloc = NULL;
  644. BUG_ON(err == -EEXIST);
  645. goto out;
  646. }
  647. state = rb_entry(node, struct extent_state, rb_node);
  648. last_start = state->start;
  649. last_end = state->end;
  650. /*
  651. * | ---- desired range ---- |
  652. * | state |
  653. *
  654. * Just lock what we found and keep going
  655. */
  656. if (state->start == start && state->end <= end) {
  657. set = state->state & bits;
  658. if (set && exclusive) {
  659. *failed_start = state->start;
  660. err = -EEXIST;
  661. goto out;
  662. }
  663. state->state |= bits;
  664. start = state->end + 1;
  665. merge_state(tree, state);
  666. goto search_again;
  667. }
  668. /*
  669. * | ---- desired range ---- |
  670. * | state |
  671. * or
  672. * | ------------- state -------------- |
  673. *
  674. * We need to split the extent we found, and may flip bits on
  675. * second half.
  676. *
  677. * If the extent we found extends past our
  678. * range, we just split and search again. It'll get split
  679. * again the next time though.
  680. *
  681. * If the extent we found is inside our range, we set the
  682. * desired bit on it.
  683. */
  684. if (state->start < start) {
  685. set = state->state & bits;
  686. if (exclusive && set) {
  687. *failed_start = start;
  688. err = -EEXIST;
  689. goto out;
  690. }
  691. err = split_state(tree, state, prealloc, start);
  692. BUG_ON(err == -EEXIST);
  693. prealloc = NULL;
  694. if (err)
  695. goto out;
  696. if (state->end <= end) {
  697. state->state |= bits;
  698. start = state->end + 1;
  699. merge_state(tree, state);
  700. } else {
  701. start = state->start;
  702. }
  703. goto search_again;
  704. }
  705. /*
  706. * | ---- desired range ---- |
  707. * | state | or | state |
  708. *
  709. * There's a hole, we need to insert something in it and
  710. * ignore the extent we found.
  711. */
  712. if (state->start > start) {
  713. u64 this_end;
  714. if (end < last_start)
  715. this_end = end;
  716. else
  717. this_end = last_start -1;
  718. err = insert_state(tree, prealloc, start, this_end,
  719. bits);
  720. prealloc = NULL;
  721. BUG_ON(err == -EEXIST);
  722. if (err)
  723. goto out;
  724. start = this_end + 1;
  725. goto search_again;
  726. }
  727. /*
  728. * | ---- desired range ---- |
  729. * | state |
  730. * We need to split the extent, and set the bit
  731. * on the first half
  732. */
  733. if (state->start <= end && state->end > end) {
  734. set = state->state & bits;
  735. if (exclusive && set) {
  736. *failed_start = start;
  737. err = -EEXIST;
  738. goto out;
  739. }
  740. err = split_state(tree, state, prealloc, end + 1);
  741. BUG_ON(err == -EEXIST);
  742. prealloc->state |= bits;
  743. merge_state(tree, prealloc);
  744. prealloc = NULL;
  745. goto out;
  746. }
  747. goto search_again;
  748. out:
  749. write_unlock_irqrestore(&tree->lock, flags);
  750. if (prealloc)
  751. free_extent_state(prealloc);
  752. return err;
  753. search_again:
  754. if (start > end)
  755. goto out;
  756. write_unlock_irqrestore(&tree->lock, flags);
  757. if (mask & __GFP_WAIT)
  758. cond_resched();
  759. goto again;
  760. }
  761. EXPORT_SYMBOL(set_extent_bit);
  762. /* wrappers around set/clear extent bit */
  763. int set_extent_dirty(struct extent_map_tree *tree, u64 start, u64 end,
  764. gfp_t mask)
  765. {
  766. return set_extent_bit(tree, start, end, EXTENT_DIRTY, 0, NULL,
  767. mask);
  768. }
  769. EXPORT_SYMBOL(set_extent_dirty);
  770. int set_extent_bits(struct extent_map_tree *tree, u64 start, u64 end,
  771. int bits, gfp_t mask)
  772. {
  773. return set_extent_bit(tree, start, end, bits, 0, NULL,
  774. mask);
  775. }
  776. EXPORT_SYMBOL(set_extent_bits);
  777. int clear_extent_bits(struct extent_map_tree *tree, u64 start, u64 end,
  778. int bits, gfp_t mask)
  779. {
  780. return clear_extent_bit(tree, start, end, bits, 0, 0, mask);
  781. }
  782. EXPORT_SYMBOL(clear_extent_bits);
  783. int set_extent_delalloc(struct extent_map_tree *tree, u64 start, u64 end,
  784. gfp_t mask)
  785. {
  786. return set_extent_bit(tree, start, end,
  787. EXTENT_DELALLOC | EXTENT_DIRTY, 0, NULL,
  788. mask);
  789. }
  790. EXPORT_SYMBOL(set_extent_delalloc);
  791. int clear_extent_dirty(struct extent_map_tree *tree, u64 start, u64 end,
  792. gfp_t mask)
  793. {
  794. return clear_extent_bit(tree, start, end,
  795. EXTENT_DIRTY | EXTENT_DELALLOC, 0, 0, mask);
  796. }
  797. EXPORT_SYMBOL(clear_extent_dirty);
  798. int set_extent_new(struct extent_map_tree *tree, u64 start, u64 end,
  799. gfp_t mask)
  800. {
  801. return set_extent_bit(tree, start, end, EXTENT_NEW, 0, NULL,
  802. mask);
  803. }
  804. EXPORT_SYMBOL(set_extent_new);
  805. int clear_extent_new(struct extent_map_tree *tree, u64 start, u64 end,
  806. gfp_t mask)
  807. {
  808. return clear_extent_bit(tree, start, end, EXTENT_NEW, 0, 0, mask);
  809. }
  810. EXPORT_SYMBOL(clear_extent_new);
  811. int set_extent_uptodate(struct extent_map_tree *tree, u64 start, u64 end,
  812. gfp_t mask)
  813. {
  814. return set_extent_bit(tree, start, end, EXTENT_UPTODATE, 0, NULL,
  815. mask);
  816. }
  817. EXPORT_SYMBOL(set_extent_uptodate);
  818. int clear_extent_uptodate(struct extent_map_tree *tree, u64 start, u64 end,
  819. gfp_t mask)
  820. {
  821. return clear_extent_bit(tree, start, end, EXTENT_UPTODATE, 0, 0, mask);
  822. }
  823. EXPORT_SYMBOL(clear_extent_uptodate);
  824. int set_extent_writeback(struct extent_map_tree *tree, u64 start, u64 end,
  825. gfp_t mask)
  826. {
  827. return set_extent_bit(tree, start, end, EXTENT_WRITEBACK,
  828. 0, NULL, mask);
  829. }
  830. EXPORT_SYMBOL(set_extent_writeback);
  831. int clear_extent_writeback(struct extent_map_tree *tree, u64 start, u64 end,
  832. gfp_t mask)
  833. {
  834. return clear_extent_bit(tree, start, end, EXTENT_WRITEBACK, 1, 0, mask);
  835. }
  836. EXPORT_SYMBOL(clear_extent_writeback);
  837. int wait_on_extent_writeback(struct extent_map_tree *tree, u64 start, u64 end)
  838. {
  839. return wait_extent_bit(tree, start, end, EXTENT_WRITEBACK);
  840. }
  841. EXPORT_SYMBOL(wait_on_extent_writeback);
  842. /*
  843. * locks a range in ascending order, waiting for any locked regions
  844. * it hits on the way. [start,end] are inclusive, and this will sleep.
  845. */
  846. int lock_extent(struct extent_map_tree *tree, u64 start, u64 end, gfp_t mask)
  847. {
  848. int err;
  849. u64 failed_start;
  850. while (1) {
  851. err = set_extent_bit(tree, start, end, EXTENT_LOCKED, 1,
  852. &failed_start, mask);
  853. if (err == -EEXIST && (mask & __GFP_WAIT)) {
  854. wait_extent_bit(tree, failed_start, end, EXTENT_LOCKED);
  855. start = failed_start;
  856. } else {
  857. break;
  858. }
  859. WARN_ON(start > end);
  860. }
  861. return err;
  862. }
  863. EXPORT_SYMBOL(lock_extent);
  864. int unlock_extent(struct extent_map_tree *tree, u64 start, u64 end,
  865. gfp_t mask)
  866. {
  867. return clear_extent_bit(tree, start, end, EXTENT_LOCKED, 1, 0, mask);
  868. }
  869. EXPORT_SYMBOL(unlock_extent);
  870. /*
  871. * helper function to set pages and extents in the tree dirty
  872. */
  873. int set_range_dirty(struct extent_map_tree *tree, u64 start, u64 end)
  874. {
  875. unsigned long index = start >> PAGE_CACHE_SHIFT;
  876. unsigned long end_index = end >> PAGE_CACHE_SHIFT;
  877. struct page *page;
  878. while (index <= end_index) {
  879. page = find_get_page(tree->mapping, index);
  880. BUG_ON(!page);
  881. __set_page_dirty_nobuffers(page);
  882. page_cache_release(page);
  883. index++;
  884. }
  885. set_extent_dirty(tree, start, end, GFP_NOFS);
  886. return 0;
  887. }
  888. EXPORT_SYMBOL(set_range_dirty);
  889. /*
  890. * helper function to set both pages and extents in the tree writeback
  891. */
  892. int set_range_writeback(struct extent_map_tree *tree, u64 start, u64 end)
  893. {
  894. unsigned long index = start >> PAGE_CACHE_SHIFT;
  895. unsigned long end_index = end >> PAGE_CACHE_SHIFT;
  896. struct page *page;
  897. while (index <= end_index) {
  898. page = find_get_page(tree->mapping, index);
  899. BUG_ON(!page);
  900. set_page_writeback(page);
  901. page_cache_release(page);
  902. index++;
  903. }
  904. set_extent_writeback(tree, start, end, GFP_NOFS);
  905. return 0;
  906. }
  907. EXPORT_SYMBOL(set_range_writeback);
  908. int find_first_extent_bit(struct extent_map_tree *tree, u64 start,
  909. u64 *start_ret, u64 *end_ret, int bits)
  910. {
  911. struct rb_node *node;
  912. struct extent_state *state;
  913. int ret = 1;
  914. read_lock_irq(&tree->lock);
  915. /*
  916. * this search will find all the extents that end after
  917. * our range starts.
  918. */
  919. node = tree_search(&tree->state, start);
  920. if (!node || IS_ERR(node)) {
  921. goto out;
  922. }
  923. while(1) {
  924. state = rb_entry(node, struct extent_state, rb_node);
  925. if (state->end >= start && (state->state & bits)) {
  926. *start_ret = state->start;
  927. *end_ret = state->end;
  928. ret = 0;
  929. break;
  930. }
  931. node = rb_next(node);
  932. if (!node)
  933. break;
  934. }
  935. out:
  936. read_unlock_irq(&tree->lock);
  937. return ret;
  938. }
  939. EXPORT_SYMBOL(find_first_extent_bit);
  940. u64 find_lock_delalloc_range(struct extent_map_tree *tree,
  941. u64 *start, u64 *end, u64 max_bytes)
  942. {
  943. struct rb_node *node;
  944. struct extent_state *state;
  945. u64 cur_start = *start;
  946. u64 found = 0;
  947. u64 total_bytes = 0;
  948. write_lock_irq(&tree->lock);
  949. /*
  950. * this search will find all the extents that end after
  951. * our range starts.
  952. */
  953. search_again:
  954. node = tree_search(&tree->state, cur_start);
  955. if (!node || IS_ERR(node)) {
  956. goto out;
  957. }
  958. while(1) {
  959. state = rb_entry(node, struct extent_state, rb_node);
  960. if (found && state->start != cur_start) {
  961. goto out;
  962. }
  963. if (!(state->state & EXTENT_DELALLOC)) {
  964. goto out;
  965. }
  966. if (!found) {
  967. struct extent_state *prev_state;
  968. struct rb_node *prev_node = node;
  969. while(1) {
  970. prev_node = rb_prev(prev_node);
  971. if (!prev_node)
  972. break;
  973. prev_state = rb_entry(prev_node,
  974. struct extent_state,
  975. rb_node);
  976. if (!(prev_state->state & EXTENT_DELALLOC))
  977. break;
  978. state = prev_state;
  979. node = prev_node;
  980. }
  981. }
  982. if (state->state & EXTENT_LOCKED) {
  983. DEFINE_WAIT(wait);
  984. atomic_inc(&state->refs);
  985. prepare_to_wait(&state->wq, &wait,
  986. TASK_UNINTERRUPTIBLE);
  987. write_unlock_irq(&tree->lock);
  988. schedule();
  989. write_lock_irq(&tree->lock);
  990. finish_wait(&state->wq, &wait);
  991. free_extent_state(state);
  992. goto search_again;
  993. }
  994. state->state |= EXTENT_LOCKED;
  995. if (!found)
  996. *start = state->start;
  997. found++;
  998. *end = state->end;
  999. cur_start = state->end + 1;
  1000. node = rb_next(node);
  1001. if (!node)
  1002. break;
  1003. total_bytes += state->end - state->start + 1;
  1004. if (total_bytes >= max_bytes)
  1005. break;
  1006. }
  1007. out:
  1008. write_unlock_irq(&tree->lock);
  1009. return found;
  1010. }
  1011. /*
  1012. * helper function to lock both pages and extents in the tree.
  1013. * pages must be locked first.
  1014. */
  1015. int lock_range(struct extent_map_tree *tree, u64 start, u64 end)
  1016. {
  1017. unsigned long index = start >> PAGE_CACHE_SHIFT;
  1018. unsigned long end_index = end >> PAGE_CACHE_SHIFT;
  1019. struct page *page;
  1020. int err;
  1021. while (index <= end_index) {
  1022. page = grab_cache_page(tree->mapping, index);
  1023. if (!page) {
  1024. err = -ENOMEM;
  1025. goto failed;
  1026. }
  1027. if (IS_ERR(page)) {
  1028. err = PTR_ERR(page);
  1029. goto failed;
  1030. }
  1031. index++;
  1032. }
  1033. lock_extent(tree, start, end, GFP_NOFS);
  1034. return 0;
  1035. failed:
  1036. /*
  1037. * we failed above in getting the page at 'index', so we undo here
  1038. * up to but not including the page at 'index'
  1039. */
  1040. end_index = index;
  1041. index = start >> PAGE_CACHE_SHIFT;
  1042. while (index < end_index) {
  1043. page = find_get_page(tree->mapping, index);
  1044. unlock_page(page);
  1045. page_cache_release(page);
  1046. index++;
  1047. }
  1048. return err;
  1049. }
  1050. EXPORT_SYMBOL(lock_range);
  1051. /*
  1052. * helper function to unlock both pages and extents in the tree.
  1053. */
  1054. int unlock_range(struct extent_map_tree *tree, u64 start, u64 end)
  1055. {
  1056. unsigned long index = start >> PAGE_CACHE_SHIFT;
  1057. unsigned long end_index = end >> PAGE_CACHE_SHIFT;
  1058. struct page *page;
  1059. while (index <= end_index) {
  1060. page = find_get_page(tree->mapping, index);
  1061. unlock_page(page);
  1062. page_cache_release(page);
  1063. index++;
  1064. }
  1065. unlock_extent(tree, start, end, GFP_NOFS);
  1066. return 0;
  1067. }
  1068. EXPORT_SYMBOL(unlock_range);
  1069. int set_state_private(struct extent_map_tree *tree, u64 start, u64 private)
  1070. {
  1071. struct rb_node *node;
  1072. struct extent_state *state;
  1073. int ret = 0;
  1074. write_lock_irq(&tree->lock);
  1075. /*
  1076. * this search will find all the extents that end after
  1077. * our range starts.
  1078. */
  1079. node = tree_search(&tree->state, start);
  1080. if (!node || IS_ERR(node)) {
  1081. ret = -ENOENT;
  1082. goto out;
  1083. }
  1084. state = rb_entry(node, struct extent_state, rb_node);
  1085. if (state->start != start) {
  1086. ret = -ENOENT;
  1087. goto out;
  1088. }
  1089. state->private = private;
  1090. out:
  1091. write_unlock_irq(&tree->lock);
  1092. return ret;
  1093. }
  1094. int get_state_private(struct extent_map_tree *tree, u64 start, u64 *private)
  1095. {
  1096. struct rb_node *node;
  1097. struct extent_state *state;
  1098. int ret = 0;
  1099. read_lock_irq(&tree->lock);
  1100. /*
  1101. * this search will find all the extents that end after
  1102. * our range starts.
  1103. */
  1104. node = tree_search(&tree->state, start);
  1105. if (!node || IS_ERR(node)) {
  1106. ret = -ENOENT;
  1107. goto out;
  1108. }
  1109. state = rb_entry(node, struct extent_state, rb_node);
  1110. if (state->start != start) {
  1111. ret = -ENOENT;
  1112. goto out;
  1113. }
  1114. *private = state->private;
  1115. out:
  1116. read_unlock_irq(&tree->lock);
  1117. return ret;
  1118. }
  1119. /*
  1120. * searches a range in the state tree for a given mask.
  1121. * If 'filled' == 1, this returns 1 only if ever extent in the tree
  1122. * has the bits set. Otherwise, 1 is returned if any bit in the
  1123. * range is found set.
  1124. */
  1125. int test_range_bit(struct extent_map_tree *tree, u64 start, u64 end,
  1126. int bits, int filled)
  1127. {
  1128. struct extent_state *state = NULL;
  1129. struct rb_node *node;
  1130. int bitset = 0;
  1131. read_lock_irq(&tree->lock);
  1132. node = tree_search(&tree->state, start);
  1133. while (node && start <= end) {
  1134. state = rb_entry(node, struct extent_state, rb_node);
  1135. if (filled && state->start > start) {
  1136. bitset = 0;
  1137. break;
  1138. }
  1139. if (state->start > end)
  1140. break;
  1141. if (state->state & bits) {
  1142. bitset = 1;
  1143. if (!filled)
  1144. break;
  1145. } else if (filled) {
  1146. bitset = 0;
  1147. break;
  1148. }
  1149. start = state->end + 1;
  1150. if (start > end)
  1151. break;
  1152. node = rb_next(node);
  1153. }
  1154. read_unlock_irq(&tree->lock);
  1155. return bitset;
  1156. }
  1157. EXPORT_SYMBOL(test_range_bit);
  1158. /*
  1159. * helper function to set a given page up to date if all the
  1160. * extents in the tree for that page are up to date
  1161. */
  1162. static int check_page_uptodate(struct extent_map_tree *tree,
  1163. struct page *page)
  1164. {
  1165. u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
  1166. u64 end = start + PAGE_CACHE_SIZE - 1;
  1167. if (test_range_bit(tree, start, end, EXTENT_UPTODATE, 1))
  1168. SetPageUptodate(page);
  1169. return 0;
  1170. }
  1171. /*
  1172. * helper function to unlock a page if all the extents in the tree
  1173. * for that page are unlocked
  1174. */
  1175. static int check_page_locked(struct extent_map_tree *tree,
  1176. struct page *page)
  1177. {
  1178. u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
  1179. u64 end = start + PAGE_CACHE_SIZE - 1;
  1180. if (!test_range_bit(tree, start, end, EXTENT_LOCKED, 0))
  1181. unlock_page(page);
  1182. return 0;
  1183. }
  1184. /*
  1185. * helper function to end page writeback if all the extents
  1186. * in the tree for that page are done with writeback
  1187. */
  1188. static int check_page_writeback(struct extent_map_tree *tree,
  1189. struct page *page)
  1190. {
  1191. u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
  1192. u64 end = start + PAGE_CACHE_SIZE - 1;
  1193. if (!test_range_bit(tree, start, end, EXTENT_WRITEBACK, 0))
  1194. end_page_writeback(page);
  1195. return 0;
  1196. }
  1197. /* lots and lots of room for performance fixes in the end_bio funcs */
  1198. /*
  1199. * after a writepage IO is done, we need to:
  1200. * clear the uptodate bits on error
  1201. * clear the writeback bits in the extent tree for this IO
  1202. * end_page_writeback if the page has no more pending IO
  1203. *
  1204. * Scheduling is not allowed, so the extent state tree is expected
  1205. * to have one and only one object corresponding to this IO.
  1206. */
  1207. #if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,23)
  1208. static void end_bio_extent_writepage(struct bio *bio, int err)
  1209. #else
  1210. static int end_bio_extent_writepage(struct bio *bio,
  1211. unsigned int bytes_done, int err)
  1212. #endif
  1213. {
  1214. const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
  1215. struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
  1216. struct extent_map_tree *tree = bio->bi_private;
  1217. u64 start;
  1218. u64 end;
  1219. int whole_page;
  1220. #if LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,23)
  1221. if (bio->bi_size)
  1222. return 1;
  1223. #endif
  1224. do {
  1225. struct page *page = bvec->bv_page;
  1226. start = ((u64)page->index << PAGE_CACHE_SHIFT) +
  1227. bvec->bv_offset;
  1228. end = start + bvec->bv_len - 1;
  1229. if (bvec->bv_offset == 0 && bvec->bv_len == PAGE_CACHE_SIZE)
  1230. whole_page = 1;
  1231. else
  1232. whole_page = 0;
  1233. if (--bvec >= bio->bi_io_vec)
  1234. prefetchw(&bvec->bv_page->flags);
  1235. if (!uptodate) {
  1236. clear_extent_uptodate(tree, start, end, GFP_ATOMIC);
  1237. ClearPageUptodate(page);
  1238. SetPageError(page);
  1239. }
  1240. clear_extent_writeback(tree, start, end, GFP_ATOMIC);
  1241. if (whole_page)
  1242. end_page_writeback(page);
  1243. else
  1244. check_page_writeback(tree, page);
  1245. if (tree->ops && tree->ops->writepage_end_io_hook)
  1246. tree->ops->writepage_end_io_hook(page, start, end);
  1247. } while (bvec >= bio->bi_io_vec);
  1248. bio_put(bio);
  1249. #if LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,23)
  1250. return 0;
  1251. #endif
  1252. }
  1253. /*
  1254. * after a readpage IO is done, we need to:
  1255. * clear the uptodate bits on error
  1256. * set the uptodate bits if things worked
  1257. * set the page up to date if all extents in the tree are uptodate
  1258. * clear the lock bit in the extent tree
  1259. * unlock the page if there are no other extents locked for it
  1260. *
  1261. * Scheduling is not allowed, so the extent state tree is expected
  1262. * to have one and only one object corresponding to this IO.
  1263. */
  1264. #if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,23)
  1265. static void end_bio_extent_readpage(struct bio *bio, int err)
  1266. #else
  1267. static int end_bio_extent_readpage(struct bio *bio,
  1268. unsigned int bytes_done, int err)
  1269. #endif
  1270. {
  1271. int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
  1272. struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
  1273. struct extent_map_tree *tree = bio->bi_private;
  1274. u64 start;
  1275. u64 end;
  1276. int whole_page;
  1277. int ret;
  1278. #if LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,23)
  1279. if (bio->bi_size)
  1280. return 1;
  1281. #endif
  1282. do {
  1283. struct page *page = bvec->bv_page;
  1284. start = ((u64)page->index << PAGE_CACHE_SHIFT) +
  1285. bvec->bv_offset;
  1286. end = start + bvec->bv_len - 1;
  1287. if (bvec->bv_offset == 0 && bvec->bv_len == PAGE_CACHE_SIZE)
  1288. whole_page = 1;
  1289. else
  1290. whole_page = 0;
  1291. if (--bvec >= bio->bi_io_vec)
  1292. prefetchw(&bvec->bv_page->flags);
  1293. if (uptodate && tree->ops && tree->ops->readpage_end_io_hook) {
  1294. ret = tree->ops->readpage_end_io_hook(page, start, end);
  1295. if (ret)
  1296. uptodate = 0;
  1297. }
  1298. if (uptodate) {
  1299. set_extent_uptodate(tree, start, end, GFP_ATOMIC);
  1300. if (whole_page)
  1301. SetPageUptodate(page);
  1302. else
  1303. check_page_uptodate(tree, page);
  1304. } else {
  1305. ClearPageUptodate(page);
  1306. SetPageError(page);
  1307. }
  1308. unlock_extent(tree, start, end, GFP_ATOMIC);
  1309. if (whole_page)
  1310. unlock_page(page);
  1311. else
  1312. check_page_locked(tree, page);
  1313. } while (bvec >= bio->bi_io_vec);
  1314. bio_put(bio);
  1315. #if LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,23)
  1316. return 0;
  1317. #endif
  1318. }
  1319. /*
  1320. * IO done from prepare_write is pretty simple, we just unlock
  1321. * the structs in the extent tree when done, and set the uptodate bits
  1322. * as appropriate.
  1323. */
  1324. #if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,23)
  1325. static void end_bio_extent_preparewrite(struct bio *bio, int err)
  1326. #else
  1327. static int end_bio_extent_preparewrite(struct bio *bio,
  1328. unsigned int bytes_done, int err)
  1329. #endif
  1330. {
  1331. const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
  1332. struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
  1333. struct extent_map_tree *tree = bio->bi_private;
  1334. u64 start;
  1335. u64 end;
  1336. #if LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,23)
  1337. if (bio->bi_size)
  1338. return 1;
  1339. #endif
  1340. do {
  1341. struct page *page = bvec->bv_page;
  1342. start = ((u64)page->index << PAGE_CACHE_SHIFT) +
  1343. bvec->bv_offset;
  1344. end = start + bvec->bv_len - 1;
  1345. if (--bvec >= bio->bi_io_vec)
  1346. prefetchw(&bvec->bv_page->flags);
  1347. if (uptodate) {
  1348. set_extent_uptodate(tree, start, end, GFP_ATOMIC);
  1349. } else {
  1350. ClearPageUptodate(page);
  1351. SetPageError(page);
  1352. }
  1353. unlock_extent(tree, start, end, GFP_ATOMIC);
  1354. } while (bvec >= bio->bi_io_vec);
  1355. bio_put(bio);
  1356. #if LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,23)
  1357. return 0;
  1358. #endif
  1359. }
  1360. static struct bio *
  1361. extent_bio_alloc(struct block_device *bdev, u64 first_sector, int nr_vecs,
  1362. gfp_t gfp_flags)
  1363. {
  1364. struct bio *bio;
  1365. bio = bio_alloc(gfp_flags, nr_vecs);
  1366. if (bio == NULL && (current->flags & PF_MEMALLOC)) {
  1367. while (!bio && (nr_vecs /= 2))
  1368. bio = bio_alloc(gfp_flags, nr_vecs);
  1369. }
  1370. if (bio) {
  1371. bio->bi_bdev = bdev;
  1372. bio->bi_sector = first_sector;
  1373. }
  1374. return bio;
  1375. }
  1376. static int submit_one_bio(int rw, struct bio *bio)
  1377. {
  1378. int ret = 0;
  1379. bio_get(bio);
  1380. submit_bio(rw, bio);
  1381. if (bio_flagged(bio, BIO_EOPNOTSUPP))
  1382. ret = -EOPNOTSUPP;
  1383. bio_put(bio);
  1384. return ret;
  1385. }
  1386. static int submit_extent_page(int rw, struct extent_map_tree *tree,
  1387. struct page *page, sector_t sector,
  1388. size_t size, unsigned long offset,
  1389. struct block_device *bdev,
  1390. struct bio **bio_ret,
  1391. unsigned long max_pages,
  1392. bio_end_io_t end_io_func)
  1393. {
  1394. int ret = 0;
  1395. struct bio *bio;
  1396. int nr;
  1397. if (bio_ret && *bio_ret) {
  1398. bio = *bio_ret;
  1399. if (bio->bi_sector + (bio->bi_size >> 9) != sector ||
  1400. bio_add_page(bio, page, size, offset) < size) {
  1401. ret = submit_one_bio(rw, bio);
  1402. bio = NULL;
  1403. } else {
  1404. return 0;
  1405. }
  1406. }
  1407. nr = min_t(int, max_pages, bio_get_nr_vecs(bdev));
  1408. bio = extent_bio_alloc(bdev, sector, nr, GFP_NOFS | __GFP_HIGH);
  1409. if (!bio) {
  1410. printk("failed to allocate bio nr %d\n", nr);
  1411. }
  1412. bio_add_page(bio, page, size, offset);
  1413. bio->bi_end_io = end_io_func;
  1414. bio->bi_private = tree;
  1415. if (bio_ret) {
  1416. *bio_ret = bio;
  1417. } else {
  1418. ret = submit_one_bio(rw, bio);
  1419. }
  1420. return ret;
  1421. }
  1422. void set_page_extent_mapped(struct page *page)
  1423. {
  1424. if (!PagePrivate(page)) {
  1425. SetPagePrivate(page);
  1426. WARN_ON(!page->mapping->a_ops->invalidatepage);
  1427. set_page_private(page, EXTENT_PAGE_PRIVATE);
  1428. page_cache_get(page);
  1429. }
  1430. }
  1431. /*
  1432. * basic readpage implementation. Locked extent state structs are inserted
  1433. * into the tree that are removed when the IO is done (by the end_io
  1434. * handlers)
  1435. */
  1436. static int __extent_read_full_page(struct extent_map_tree *tree,
  1437. struct page *page,
  1438. get_extent_t *get_extent,
  1439. struct bio **bio)
  1440. {
  1441. struct inode *inode = page->mapping->host;
  1442. u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
  1443. u64 page_end = start + PAGE_CACHE_SIZE - 1;
  1444. u64 end;
  1445. u64 cur = start;
  1446. u64 extent_offset;
  1447. u64 last_byte = i_size_read(inode);
  1448. u64 block_start;
  1449. u64 cur_end;
  1450. sector_t sector;
  1451. struct extent_map *em;
  1452. struct block_device *bdev;
  1453. int ret;
  1454. int nr = 0;
  1455. size_t page_offset = 0;
  1456. size_t iosize;
  1457. size_t blocksize = inode->i_sb->s_blocksize;
  1458. set_page_extent_mapped(page);
  1459. end = page_end;
  1460. lock_extent(tree, start, end, GFP_NOFS);
  1461. while (cur <= end) {
  1462. if (cur >= last_byte) {
  1463. iosize = PAGE_CACHE_SIZE - page_offset;
  1464. zero_user_page(page, page_offset, iosize, KM_USER0);
  1465. set_extent_uptodate(tree, cur, cur + iosize - 1,
  1466. GFP_NOFS);
  1467. unlock_extent(tree, cur, cur + iosize - 1, GFP_NOFS);
  1468. break;
  1469. }
  1470. em = get_extent(inode, page, page_offset, cur, end, 0);
  1471. if (IS_ERR(em) || !em) {
  1472. SetPageError(page);
  1473. unlock_extent(tree, cur, end, GFP_NOFS);
  1474. break;
  1475. }
  1476. extent_offset = cur - em->start;
  1477. BUG_ON(em->end < cur);
  1478. BUG_ON(end < cur);
  1479. iosize = min(em->end - cur, end - cur) + 1;
  1480. cur_end = min(em->end, end);
  1481. iosize = (iosize + blocksize - 1) & ~((u64)blocksize - 1);
  1482. sector = (em->block_start + extent_offset) >> 9;
  1483. bdev = em->bdev;
  1484. block_start = em->block_start;
  1485. free_extent_map(em);
  1486. em = NULL;
  1487. /* we've found a hole, just zero and go on */
  1488. if (block_start == EXTENT_MAP_HOLE) {
  1489. zero_user_page(page, page_offset, iosize, KM_USER0);
  1490. set_extent_uptodate(tree, cur, cur + iosize - 1,
  1491. GFP_NOFS);
  1492. unlock_extent(tree, cur, cur + iosize - 1, GFP_NOFS);
  1493. cur = cur + iosize;
  1494. page_offset += iosize;
  1495. continue;
  1496. }
  1497. /* the get_extent function already copied into the page */
  1498. if (test_range_bit(tree, cur, cur_end, EXTENT_UPTODATE, 1)) {
  1499. unlock_extent(tree, cur, cur + iosize - 1, GFP_NOFS);
  1500. cur = cur + iosize;
  1501. page_offset += iosize;
  1502. continue;
  1503. }
  1504. ret = 0;
  1505. if (tree->ops && tree->ops->readpage_io_hook) {
  1506. ret = tree->ops->readpage_io_hook(page, cur,
  1507. cur + iosize - 1);
  1508. }
  1509. if (!ret) {
  1510. unsigned long nr = (last_byte >> PAGE_CACHE_SHIFT) + 1;
  1511. nr -= page->index;
  1512. ret = submit_extent_page(READ, tree, page,
  1513. sector, iosize, page_offset,
  1514. bdev, bio, nr,
  1515. end_bio_extent_readpage);
  1516. }
  1517. if (ret)
  1518. SetPageError(page);
  1519. cur = cur + iosize;
  1520. page_offset += iosize;
  1521. nr++;
  1522. }
  1523. if (!nr) {
  1524. if (!PageError(page))
  1525. SetPageUptodate(page);
  1526. unlock_page(page);
  1527. }
  1528. return 0;
  1529. }
  1530. int extent_read_full_page(struct extent_map_tree *tree, struct page *page,
  1531. get_extent_t *get_extent)
  1532. {
  1533. struct bio *bio = NULL;
  1534. int ret;
  1535. ret = __extent_read_full_page(tree, page, get_extent, &bio);
  1536. if (bio)
  1537. submit_one_bio(READ, bio);
  1538. return ret;
  1539. }
  1540. EXPORT_SYMBOL(extent_read_full_page);
  1541. /*
  1542. * the writepage semantics are similar to regular writepage. extent
  1543. * records are inserted to lock ranges in the tree, and as dirty areas
  1544. * are found, they are marked writeback. Then the lock bits are removed
  1545. * and the end_io handler clears the writeback ranges
  1546. */
  1547. static int __extent_writepage(struct page *page, struct writeback_control *wbc,
  1548. void *data)
  1549. {
  1550. struct inode *inode = page->mapping->host;
  1551. struct extent_page_data *epd = data;
  1552. struct extent_map_tree *tree = epd->tree;
  1553. u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
  1554. u64 delalloc_start;
  1555. u64 page_end = start + PAGE_CACHE_SIZE - 1;
  1556. u64 end;
  1557. u64 cur = start;
  1558. u64 extent_offset;
  1559. u64 last_byte = i_size_read(inode);
  1560. u64 block_start;
  1561. u64 iosize;
  1562. sector_t sector;
  1563. struct extent_map *em;
  1564. struct block_device *bdev;
  1565. int ret;
  1566. int nr = 0;
  1567. size_t page_offset = 0;
  1568. size_t blocksize;
  1569. loff_t i_size = i_size_read(inode);
  1570. unsigned long end_index = i_size >> PAGE_CACHE_SHIFT;
  1571. u64 nr_delalloc;
  1572. u64 delalloc_end;
  1573. WARN_ON(!PageLocked(page));
  1574. if (page->index > end_index) {
  1575. clear_extent_dirty(tree, start, page_end, GFP_NOFS);
  1576. unlock_page(page);
  1577. return 0;
  1578. }
  1579. if (page->index == end_index) {
  1580. size_t offset = i_size & (PAGE_CACHE_SIZE - 1);
  1581. zero_user_page(page, offset,
  1582. PAGE_CACHE_SIZE - offset, KM_USER0);
  1583. }
  1584. set_page_extent_mapped(page);
  1585. delalloc_start = start;
  1586. delalloc_end = 0;
  1587. while(delalloc_end < page_end) {
  1588. nr_delalloc = find_lock_delalloc_range(tree, &delalloc_start,
  1589. &delalloc_end,
  1590. 128 * 1024 * 1024);
  1591. if (nr_delalloc <= 0)
  1592. break;
  1593. tree->ops->fill_delalloc(inode, delalloc_start,
  1594. delalloc_end);
  1595. clear_extent_bit(tree, delalloc_start,
  1596. delalloc_end,
  1597. EXTENT_LOCKED | EXTENT_DELALLOC,
  1598. 1, 0, GFP_NOFS);
  1599. delalloc_start = delalloc_end + 1;
  1600. }
  1601. lock_extent(tree, start, page_end, GFP_NOFS);
  1602. end = page_end;
  1603. if (test_range_bit(tree, start, page_end, EXTENT_DELALLOC, 0)) {
  1604. printk("found delalloc bits after lock_extent\n");
  1605. }
  1606. if (last_byte <= start) {
  1607. clear_extent_dirty(tree, start, page_end, GFP_NOFS);
  1608. goto done;
  1609. }
  1610. set_extent_uptodate(tree, start, page_end, GFP_NOFS);
  1611. blocksize = inode->i_sb->s_blocksize;
  1612. while (cur <= end) {
  1613. if (cur >= last_byte) {
  1614. clear_extent_dirty(tree, cur, page_end, GFP_NOFS);
  1615. break;
  1616. }
  1617. em = epd->get_extent(inode, page, page_offset, cur, end, 1);
  1618. if (IS_ERR(em) || !em) {
  1619. SetPageError(page);
  1620. break;
  1621. }
  1622. extent_offset = cur - em->start;
  1623. BUG_ON(em->end < cur);
  1624. BUG_ON(end < cur);
  1625. iosize = min(em->end - cur, end - cur) + 1;
  1626. iosize = (iosize + blocksize - 1) & ~((u64)blocksize - 1);
  1627. sector = (em->block_start + extent_offset) >> 9;
  1628. bdev = em->bdev;
  1629. block_start = em->block_start;
  1630. free_extent_map(em);
  1631. em = NULL;
  1632. if (block_start == EXTENT_MAP_HOLE ||
  1633. block_start == EXTENT_MAP_INLINE) {
  1634. clear_extent_dirty(tree, cur,
  1635. cur + iosize - 1, GFP_NOFS);
  1636. cur = cur + iosize;
  1637. page_offset += iosize;
  1638. continue;
  1639. }
  1640. /* leave this out until we have a page_mkwrite call */
  1641. if (0 && !test_range_bit(tree, cur, cur + iosize - 1,
  1642. EXTENT_DIRTY, 0)) {
  1643. cur = cur + iosize;
  1644. page_offset += iosize;
  1645. continue;
  1646. }
  1647. clear_extent_dirty(tree, cur, cur + iosize - 1, GFP_NOFS);
  1648. if (tree->ops && tree->ops->writepage_io_hook) {
  1649. ret = tree->ops->writepage_io_hook(page, cur,
  1650. cur + iosize - 1);
  1651. } else {
  1652. ret = 0;
  1653. }
  1654. if (ret)
  1655. SetPageError(page);
  1656. else {
  1657. unsigned long nr = end_index + 1;
  1658. set_range_writeback(tree, cur, cur + iosize - 1);
  1659. ret = submit_extent_page(WRITE, tree, page, sector,
  1660. iosize, page_offset, bdev,
  1661. &epd->bio, nr,
  1662. end_bio_extent_writepage);
  1663. if (ret)
  1664. SetPageError(page);
  1665. }
  1666. cur = cur + iosize;
  1667. page_offset += iosize;
  1668. nr++;
  1669. }
  1670. done:
  1671. unlock_extent(tree, start, page_end, GFP_NOFS);
  1672. unlock_page(page);
  1673. return 0;
  1674. }
  1675. int extent_write_full_page(struct extent_map_tree *tree, struct page *page,
  1676. get_extent_t *get_extent,
  1677. struct writeback_control *wbc)
  1678. {
  1679. int ret;
  1680. struct extent_page_data epd = {
  1681. .bio = NULL,
  1682. .tree = tree,
  1683. .get_extent = get_extent,
  1684. };
  1685. ret = __extent_writepage(page, wbc, &epd);
  1686. if (epd.bio)
  1687. submit_one_bio(WRITE, epd.bio);
  1688. return ret;
  1689. }
  1690. EXPORT_SYMBOL(extent_write_full_page);
  1691. int extent_writepages(struct extent_map_tree *tree,
  1692. struct address_space *mapping,
  1693. get_extent_t *get_extent,
  1694. struct writeback_control *wbc)
  1695. {
  1696. int ret;
  1697. struct extent_page_data epd = {
  1698. .bio = NULL,
  1699. .tree = tree,
  1700. .get_extent = get_extent,
  1701. };
  1702. ret = write_cache_pages(mapping, wbc, __extent_writepage, &epd);
  1703. if (epd.bio)
  1704. submit_one_bio(WRITE, epd.bio);
  1705. return ret;
  1706. }
  1707. EXPORT_SYMBOL(extent_writepages);
  1708. int extent_readpages(struct extent_map_tree *tree,
  1709. struct address_space *mapping,
  1710. struct list_head *pages, unsigned nr_pages,
  1711. get_extent_t get_extent)
  1712. {
  1713. struct bio *bio = NULL;
  1714. unsigned page_idx;
  1715. struct pagevec pvec;
  1716. pagevec_init(&pvec, 0);
  1717. for (page_idx = 0; page_idx < nr_pages; page_idx++) {
  1718. struct page *page = list_entry(pages->prev, struct page, lru);
  1719. prefetchw(&page->flags);
  1720. list_del(&page->lru);
  1721. /*
  1722. * what we want to do here is call add_to_page_cache_lru,
  1723. * but that isn't exported, so we reproduce it here
  1724. */
  1725. if (!add_to_page_cache(page, mapping,
  1726. page->index, GFP_KERNEL)) {
  1727. /* open coding of lru_cache_add, also not exported */
  1728. page_cache_get(page);
  1729. if (!pagevec_add(&pvec, page))
  1730. __pagevec_lru_add(&pvec);
  1731. __extent_read_full_page(tree, page, get_extent, &bio);
  1732. }
  1733. page_cache_release(page);
  1734. }
  1735. if (pagevec_count(&pvec))
  1736. __pagevec_lru_add(&pvec);
  1737. BUG_ON(!list_empty(pages));
  1738. if (bio)
  1739. submit_one_bio(READ, bio);
  1740. return 0;
  1741. }
  1742. EXPORT_SYMBOL(extent_readpages);
  1743. /*
  1744. * basic invalidatepage code, this waits on any locked or writeback
  1745. * ranges corresponding to the page, and then deletes any extent state
  1746. * records from the tree
  1747. */
  1748. int extent_invalidatepage(struct extent_map_tree *tree,
  1749. struct page *page, unsigned long offset)
  1750. {
  1751. u64 start = ((u64)page->index << PAGE_CACHE_SHIFT);
  1752. u64 end = start + PAGE_CACHE_SIZE - 1;
  1753. size_t blocksize = page->mapping->host->i_sb->s_blocksize;
  1754. start += (offset + blocksize -1) & ~(blocksize - 1);
  1755. if (start > end)
  1756. return 0;
  1757. lock_extent(tree, start, end, GFP_NOFS);
  1758. wait_on_extent_writeback(tree, start, end);
  1759. clear_extent_bit(tree, start, end,
  1760. EXTENT_LOCKED | EXTENT_DIRTY | EXTENT_DELALLOC,
  1761. 1, 1, GFP_NOFS);
  1762. return 0;
  1763. }
  1764. EXPORT_SYMBOL(extent_invalidatepage);
  1765. /*
  1766. * simple commit_write call, set_range_dirty is used to mark both
  1767. * the pages and the extent records as dirty
  1768. */
  1769. int extent_commit_write(struct extent_map_tree *tree,
  1770. struct inode *inode, struct page *page,
  1771. unsigned from, unsigned to)
  1772. {
  1773. loff_t pos = ((loff_t)page->index << PAGE_CACHE_SHIFT) + to;
  1774. set_page_extent_mapped(page);
  1775. set_page_dirty(page);
  1776. if (pos > inode->i_size) {
  1777. i_size_write(inode, pos);
  1778. mark_inode_dirty(inode);
  1779. }
  1780. return 0;
  1781. }
  1782. EXPORT_SYMBOL(extent_commit_write);
  1783. int extent_prepare_write(struct extent_map_tree *tree,
  1784. struct inode *inode, struct page *page,
  1785. unsigned from, unsigned to, get_extent_t *get_extent)
  1786. {
  1787. u64 page_start = (u64)page->index << PAGE_CACHE_SHIFT;
  1788. u64 page_end = page_start + PAGE_CACHE_SIZE - 1;
  1789. u64 block_start;
  1790. u64 orig_block_start;
  1791. u64 block_end;
  1792. u64 cur_end;
  1793. struct extent_map *em;
  1794. unsigned blocksize = 1 << inode->i_blkbits;
  1795. size_t page_offset = 0;
  1796. size_t block_off_start;
  1797. size_t block_off_end;
  1798. int err = 0;
  1799. int iocount = 0;
  1800. int ret = 0;
  1801. int isnew;
  1802. set_page_extent_mapped(page);
  1803. block_start = (page_start + from) & ~((u64)blocksize - 1);
  1804. block_end = (page_start + to - 1) | (blocksize - 1);
  1805. orig_block_start = block_start;
  1806. lock_extent(tree, page_start, page_end, GFP_NOFS);
  1807. while(block_start <= block_end) {
  1808. em = get_extent(inode, page, page_offset, block_start,
  1809. block_end, 1);
  1810. if (IS_ERR(em) || !em) {
  1811. goto err;
  1812. }
  1813. cur_end = min(block_end, em->end);
  1814. block_off_start = block_start & (PAGE_CACHE_SIZE - 1);
  1815. block_off_end = block_off_start + blocksize;
  1816. isnew = clear_extent_new(tree, block_start, cur_end, GFP_NOFS);
  1817. if (!PageUptodate(page) && isnew &&
  1818. (block_off_end > to || block_off_start < from)) {
  1819. void *kaddr;
  1820. kaddr = kmap_atomic(page, KM_USER0);
  1821. if (block_off_end > to)
  1822. memset(kaddr + to, 0, block_off_end - to);
  1823. if (block_off_start < from)
  1824. memset(kaddr + block_off_start, 0,
  1825. from - block_off_start);
  1826. flush_dcache_page(page);
  1827. kunmap_atomic(kaddr, KM_USER0);
  1828. }
  1829. if (!isnew && !PageUptodate(page) &&
  1830. (block_off_end > to || block_off_start < from) &&
  1831. !test_range_bit(tree, block_start, cur_end,
  1832. EXTENT_UPTODATE, 1)) {
  1833. u64 sector;
  1834. u64 extent_offset = block_start - em->start;
  1835. size_t iosize;
  1836. sector = (em->block_start + extent_offset) >> 9;
  1837. iosize = (cur_end - block_start + blocksize - 1) &
  1838. ~((u64)blocksize - 1);
  1839. /*
  1840. * we've already got the extent locked, but we
  1841. * need to split the state such that our end_bio
  1842. * handler can clear the lock.
  1843. */
  1844. set_extent_bit(tree, block_start,
  1845. block_start + iosize - 1,
  1846. EXTENT_LOCKED, 0, NULL, GFP_NOFS);
  1847. ret = submit_extent_page(READ, tree, page,
  1848. sector, iosize, page_offset, em->bdev,
  1849. NULL, 1,
  1850. end_bio_extent_preparewrite);
  1851. iocount++;
  1852. block_start = block_start + iosize;
  1853. } else {
  1854. set_extent_uptodate(tree, block_start, cur_end,
  1855. GFP_NOFS);
  1856. unlock_extent(tree, block_start, cur_end, GFP_NOFS);
  1857. block_start = cur_end + 1;
  1858. }
  1859. page_offset = block_start & (PAGE_CACHE_SIZE - 1);
  1860. free_extent_map(em);
  1861. }
  1862. if (iocount) {
  1863. wait_extent_bit(tree, orig_block_start,
  1864. block_end, EXTENT_LOCKED);
  1865. }
  1866. check_page_uptodate(tree, page);
  1867. err:
  1868. /* FIXME, zero out newly allocated blocks on error */
  1869. return err;
  1870. }
  1871. EXPORT_SYMBOL(extent_prepare_write);
  1872. /*
  1873. * a helper for releasepage. As long as there are no locked extents
  1874. * in the range corresponding to the page, both state records and extent
  1875. * map records are removed
  1876. */
  1877. int try_release_extent_mapping(struct extent_map_tree *tree, struct page *page)
  1878. {
  1879. struct extent_map *em;
  1880. u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
  1881. u64 end = start + PAGE_CACHE_SIZE - 1;
  1882. u64 orig_start = start;
  1883. int ret = 1;
  1884. while (start <= end) {
  1885. em = lookup_extent_mapping(tree, start, end);
  1886. if (!em || IS_ERR(em))
  1887. break;
  1888. if (!test_range_bit(tree, em->start, em->end,
  1889. EXTENT_LOCKED, 0)) {
  1890. remove_extent_mapping(tree, em);
  1891. /* once for the rb tree */
  1892. free_extent_map(em);
  1893. }
  1894. start = em->end + 1;
  1895. /* once for us */
  1896. free_extent_map(em);
  1897. }
  1898. if (test_range_bit(tree, orig_start, end, EXTENT_LOCKED, 0))
  1899. ret = 0;
  1900. else
  1901. clear_extent_bit(tree, orig_start, end, EXTENT_UPTODATE,
  1902. 1, 1, GFP_NOFS);
  1903. return ret;
  1904. }
  1905. EXPORT_SYMBOL(try_release_extent_mapping);
  1906. sector_t extent_bmap(struct address_space *mapping, sector_t iblock,
  1907. get_extent_t *get_extent)
  1908. {
  1909. struct inode *inode = mapping->host;
  1910. u64 start = iblock << inode->i_blkbits;
  1911. u64 end = start + (1 << inode->i_blkbits) - 1;
  1912. sector_t sector = 0;
  1913. struct extent_map *em;
  1914. em = get_extent(inode, NULL, 0, start, end, 0);
  1915. if (!em || IS_ERR(em))
  1916. return 0;
  1917. if (em->block_start == EXTENT_MAP_INLINE ||
  1918. em->block_start == EXTENT_MAP_HOLE)
  1919. goto out;
  1920. sector = (em->block_start + start - em->start) >> inode->i_blkbits;
  1921. out:
  1922. free_extent_map(em);
  1923. return sector;
  1924. }
  1925. static int add_lru(struct extent_map_tree *tree, struct extent_buffer *eb)
  1926. {
  1927. if (list_empty(&eb->lru)) {
  1928. extent_buffer_get(eb);
  1929. list_add(&eb->lru, &tree->buffer_lru);
  1930. tree->lru_size++;
  1931. if (tree->lru_size >= BUFFER_LRU_MAX) {
  1932. struct extent_buffer *rm;
  1933. rm = list_entry(tree->buffer_lru.prev,
  1934. struct extent_buffer, lru);
  1935. tree->lru_size--;
  1936. list_del_init(&rm->lru);
  1937. free_extent_buffer(rm);
  1938. }
  1939. } else
  1940. list_move(&eb->lru, &tree->buffer_lru);
  1941. return 0;
  1942. }
  1943. static struct extent_buffer *find_lru(struct extent_map_tree *tree,
  1944. u64 start, unsigned long len)
  1945. {
  1946. struct list_head *lru = &tree->buffer_lru;
  1947. struct list_head *cur = lru->next;
  1948. struct extent_buffer *eb;
  1949. if (list_empty(lru))
  1950. return NULL;
  1951. do {
  1952. eb = list_entry(cur, struct extent_buffer, lru);
  1953. if (eb->start == start && eb->len == len) {
  1954. extent_buffer_get(eb);
  1955. return eb;
  1956. }
  1957. cur = cur->next;
  1958. } while (cur != lru);
  1959. return NULL;
  1960. }
  1961. static inline unsigned long num_extent_pages(u64 start, u64 len)
  1962. {
  1963. return ((start + len + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT) -
  1964. (start >> PAGE_CACHE_SHIFT);
  1965. }
  1966. static inline struct page *extent_buffer_page(struct extent_buffer *eb,
  1967. unsigned long i)
  1968. {
  1969. struct page *p;
  1970. struct address_space *mapping;
  1971. if (i == 0)
  1972. return eb->first_page;
  1973. i += eb->start >> PAGE_CACHE_SHIFT;
  1974. mapping = eb->first_page->mapping;
  1975. read_lock_irq(&mapping->tree_lock);
  1976. p = radix_tree_lookup(&mapping->page_tree, i);
  1977. read_unlock_irq(&mapping->tree_lock);
  1978. return p;
  1979. }
  1980. static struct extent_buffer *__alloc_extent_buffer(struct extent_map_tree *tree,
  1981. u64 start,
  1982. unsigned long len,
  1983. gfp_t mask)
  1984. {
  1985. struct extent_buffer *eb = NULL;
  1986. spin_lock(&tree->lru_lock);
  1987. eb = find_lru(tree, start, len);
  1988. spin_unlock(&tree->lru_lock);
  1989. if (eb) {
  1990. return eb;
  1991. }
  1992. eb = kmem_cache_zalloc(extent_buffer_cache, mask);
  1993. INIT_LIST_HEAD(&eb->lru);
  1994. eb->start = start;
  1995. eb->len = len;
  1996. atomic_set(&eb->refs, 1);
  1997. return eb;
  1998. }
  1999. static void __free_extent_buffer(struct extent_buffer *eb)
  2000. {
  2001. kmem_cache_free(extent_buffer_cache, eb);
  2002. }
  2003. struct extent_buffer *alloc_extent_buffer(struct extent_map_tree *tree,
  2004. u64 start, unsigned long len,
  2005. struct page *page0,
  2006. gfp_t mask)
  2007. {
  2008. unsigned long num_pages = num_extent_pages(start, len);
  2009. unsigned long i;
  2010. unsigned long index = start >> PAGE_CACHE_SHIFT;
  2011. struct extent_buffer *eb;
  2012. struct page *p;
  2013. struct address_space *mapping = tree->mapping;
  2014. int uptodate = 1;
  2015. eb = __alloc_extent_buffer(tree, start, len, mask);
  2016. if (!eb || IS_ERR(eb))
  2017. return NULL;
  2018. if (eb->flags & EXTENT_BUFFER_FILLED)
  2019. goto lru_add;
  2020. if (page0) {
  2021. eb->first_page = page0;
  2022. i = 1;
  2023. index++;
  2024. page_cache_get(page0);
  2025. mark_page_accessed(page0);
  2026. set_page_extent_mapped(page0);
  2027. WARN_ON(!PageUptodate(page0));
  2028. set_page_private(page0, EXTENT_PAGE_PRIVATE_FIRST_PAGE |
  2029. len << 2);
  2030. } else {
  2031. i = 0;
  2032. }
  2033. for (; i < num_pages; i++, index++) {
  2034. p = find_or_create_page(mapping, index, mask | __GFP_HIGHMEM);
  2035. if (!p) {
  2036. WARN_ON(1);
  2037. goto fail;
  2038. }
  2039. set_page_extent_mapped(p);
  2040. mark_page_accessed(p);
  2041. if (i == 0) {
  2042. eb->first_page = p;
  2043. set_page_private(p, EXTENT_PAGE_PRIVATE_FIRST_PAGE |
  2044. len << 2);
  2045. } else {
  2046. set_page_private(p, EXTENT_PAGE_PRIVATE);
  2047. }
  2048. if (!PageUptodate(p))
  2049. uptodate = 0;
  2050. unlock_page(p);
  2051. }
  2052. if (uptodate)
  2053. eb->flags |= EXTENT_UPTODATE;
  2054. eb->flags |= EXTENT_BUFFER_FILLED;
  2055. lru_add:
  2056. spin_lock(&tree->lru_lock);
  2057. add_lru(tree, eb);
  2058. spin_unlock(&tree->lru_lock);
  2059. return eb;
  2060. fail:
  2061. spin_lock(&tree->lru_lock);
  2062. list_del_init(&eb->lru);
  2063. spin_unlock(&tree->lru_lock);
  2064. if (!atomic_dec_and_test(&eb->refs))
  2065. return NULL;
  2066. for (index = 1; index < i; index++) {
  2067. page_cache_release(extent_buffer_page(eb, index));
  2068. }
  2069. if (i > 0)
  2070. page_cache_release(extent_buffer_page(eb, 0));
  2071. __free_extent_buffer(eb);
  2072. return NULL;
  2073. }
  2074. EXPORT_SYMBOL(alloc_extent_buffer);
  2075. struct extent_buffer *find_extent_buffer(struct extent_map_tree *tree,
  2076. u64 start, unsigned long len,
  2077. gfp_t mask)
  2078. {
  2079. unsigned long num_pages = num_extent_pages(start, len);
  2080. unsigned long i;
  2081. unsigned long index = start >> PAGE_CACHE_SHIFT;
  2082. struct extent_buffer *eb;
  2083. struct page *p;
  2084. struct address_space *mapping = tree->mapping;
  2085. int uptodate = 1;
  2086. eb = __alloc_extent_buffer(tree, start, len, mask);
  2087. if (!eb || IS_ERR(eb))
  2088. return NULL;
  2089. if (eb->flags & EXTENT_BUFFER_FILLED)
  2090. goto lru_add;
  2091. for (i = 0; i < num_pages; i++, index++) {
  2092. p = find_lock_page(mapping, index);
  2093. if (!p) {
  2094. goto fail;
  2095. }
  2096. set_page_extent_mapped(p);
  2097. mark_page_accessed(p);
  2098. if (i == 0) {
  2099. eb->first_page = p;
  2100. set_page_private(p, EXTENT_PAGE_PRIVATE_FIRST_PAGE |
  2101. len << 2);
  2102. } else {
  2103. set_page_private(p, EXTENT_PAGE_PRIVATE);
  2104. }
  2105. if (!PageUptodate(p))
  2106. uptodate = 0;
  2107. unlock_page(p);
  2108. }
  2109. if (uptodate)
  2110. eb->flags |= EXTENT_UPTODATE;
  2111. eb->flags |= EXTENT_BUFFER_FILLED;
  2112. lru_add:
  2113. spin_lock(&tree->lru_lock);
  2114. add_lru(tree, eb);
  2115. spin_unlock(&tree->lru_lock);
  2116. return eb;
  2117. fail:
  2118. spin_lock(&tree->lru_lock);
  2119. list_del_init(&eb->lru);
  2120. spin_unlock(&tree->lru_lock);
  2121. if (!atomic_dec_and_test(&eb->refs))
  2122. return NULL;
  2123. for (index = 1; index < i; index++) {
  2124. page_cache_release(extent_buffer_page(eb, index));
  2125. }
  2126. if (i > 0)
  2127. page_cache_release(extent_buffer_page(eb, 0));
  2128. __free_extent_buffer(eb);
  2129. return NULL;
  2130. }
  2131. EXPORT_SYMBOL(find_extent_buffer);
  2132. void free_extent_buffer(struct extent_buffer *eb)
  2133. {
  2134. unsigned long i;
  2135. unsigned long num_pages;
  2136. if (!eb)
  2137. return;
  2138. if (!atomic_dec_and_test(&eb->refs))
  2139. return;
  2140. WARN_ON(!list_empty(&eb->lru));
  2141. num_pages = num_extent_pages(eb->start, eb->len);
  2142. for (i = 1; i < num_pages; i++) {
  2143. page_cache_release(extent_buffer_page(eb, i));
  2144. }
  2145. page_cache_release(extent_buffer_page(eb, 0));
  2146. __free_extent_buffer(eb);
  2147. }
  2148. EXPORT_SYMBOL(free_extent_buffer);
  2149. int clear_extent_buffer_dirty(struct extent_map_tree *tree,
  2150. struct extent_buffer *eb)
  2151. {
  2152. int set;
  2153. unsigned long i;
  2154. unsigned long num_pages;
  2155. struct page *page;
  2156. u64 start = eb->start;
  2157. u64 end = start + eb->len - 1;
  2158. set = clear_extent_dirty(tree, start, end, GFP_NOFS);
  2159. num_pages = num_extent_pages(eb->start, eb->len);
  2160. for (i = 0; i < num_pages; i++) {
  2161. page = extent_buffer_page(eb, i);
  2162. lock_page(page);
  2163. /*
  2164. * if we're on the last page or the first page and the
  2165. * block isn't aligned on a page boundary, do extra checks
  2166. * to make sure we don't clean page that is partially dirty
  2167. */
  2168. if ((i == 0 && (eb->start & (PAGE_CACHE_SIZE - 1))) ||
  2169. ((i == num_pages - 1) &&
  2170. ((eb->start + eb->len) & (PAGE_CACHE_SIZE - 1)))) {
  2171. start = (u64)page->index << PAGE_CACHE_SHIFT;
  2172. end = start + PAGE_CACHE_SIZE - 1;
  2173. if (test_range_bit(tree, start, end,
  2174. EXTENT_DIRTY, 0)) {
  2175. unlock_page(page);
  2176. continue;
  2177. }
  2178. }
  2179. clear_page_dirty_for_io(page);
  2180. unlock_page(page);
  2181. }
  2182. return 0;
  2183. }
  2184. EXPORT_SYMBOL(clear_extent_buffer_dirty);
  2185. int wait_on_extent_buffer_writeback(struct extent_map_tree *tree,
  2186. struct extent_buffer *eb)
  2187. {
  2188. return wait_on_extent_writeback(tree, eb->start,
  2189. eb->start + eb->len - 1);
  2190. }
  2191. EXPORT_SYMBOL(wait_on_extent_buffer_writeback);
  2192. int set_extent_buffer_dirty(struct extent_map_tree *tree,
  2193. struct extent_buffer *eb)
  2194. {
  2195. unsigned long i;
  2196. unsigned long num_pages;
  2197. num_pages = num_extent_pages(eb->start, eb->len);
  2198. for (i = 0; i < num_pages; i++) {
  2199. struct page *page = extent_buffer_page(eb, i);
  2200. /* writepage may need to do something special for the
  2201. * first page, we have to make sure page->private is
  2202. * properly set. releasepage may drop page->private
  2203. * on us if the page isn't already dirty.
  2204. */
  2205. if (i == 0) {
  2206. lock_page(page);
  2207. set_page_private(page,
  2208. EXTENT_PAGE_PRIVATE_FIRST_PAGE |
  2209. eb->len << 2);
  2210. }
  2211. __set_page_dirty_nobuffers(extent_buffer_page(eb, i));
  2212. if (i == 0)
  2213. unlock_page(page);
  2214. }
  2215. return set_extent_dirty(tree, eb->start,
  2216. eb->start + eb->len - 1, GFP_NOFS);
  2217. }
  2218. EXPORT_SYMBOL(set_extent_buffer_dirty);
  2219. int set_extent_buffer_uptodate(struct extent_map_tree *tree,
  2220. struct extent_buffer *eb)
  2221. {
  2222. unsigned long i;
  2223. struct page *page;
  2224. unsigned long num_pages;
  2225. num_pages = num_extent_pages(eb->start, eb->len);
  2226. set_extent_uptodate(tree, eb->start, eb->start + eb->len - 1,
  2227. GFP_NOFS);
  2228. for (i = 0; i < num_pages; i++) {
  2229. page = extent_buffer_page(eb, i);
  2230. if ((i == 0 && (eb->start & (PAGE_CACHE_SIZE - 1))) ||
  2231. ((i == num_pages - 1) &&
  2232. ((eb->start + eb->len) & (PAGE_CACHE_SIZE - 1)))) {
  2233. check_page_uptodate(tree, page);
  2234. continue;
  2235. }
  2236. SetPageUptodate(page);
  2237. }
  2238. return 0;
  2239. }
  2240. EXPORT_SYMBOL(set_extent_buffer_uptodate);
  2241. int extent_buffer_uptodate(struct extent_map_tree *tree,
  2242. struct extent_buffer *eb)
  2243. {
  2244. if (eb->flags & EXTENT_UPTODATE)
  2245. return 1;
  2246. return test_range_bit(tree, eb->start, eb->start + eb->len - 1,
  2247. EXTENT_UPTODATE, 1);
  2248. }
  2249. EXPORT_SYMBOL(extent_buffer_uptodate);
  2250. int read_extent_buffer_pages(struct extent_map_tree *tree,
  2251. struct extent_buffer *eb,
  2252. u64 start,
  2253. int wait)
  2254. {
  2255. unsigned long i;
  2256. unsigned long start_i;
  2257. struct page *page;
  2258. int err;
  2259. int ret = 0;
  2260. unsigned long num_pages;
  2261. if (eb->flags & EXTENT_UPTODATE)
  2262. return 0;
  2263. if (0 && test_range_bit(tree, eb->start, eb->start + eb->len - 1,
  2264. EXTENT_UPTODATE, 1)) {
  2265. return 0;
  2266. }
  2267. if (start) {
  2268. WARN_ON(start < eb->start);
  2269. start_i = (start >> PAGE_CACHE_SHIFT) -
  2270. (eb->start >> PAGE_CACHE_SHIFT);
  2271. } else {
  2272. start_i = 0;
  2273. }
  2274. num_pages = num_extent_pages(eb->start, eb->len);
  2275. for (i = start_i; i < num_pages; i++) {
  2276. page = extent_buffer_page(eb, i);
  2277. if (PageUptodate(page)) {
  2278. continue;
  2279. }
  2280. if (!wait) {
  2281. if (TestSetPageLocked(page)) {
  2282. continue;
  2283. }
  2284. } else {
  2285. lock_page(page);
  2286. }
  2287. if (!PageUptodate(page)) {
  2288. err = page->mapping->a_ops->readpage(NULL, page);
  2289. if (err) {
  2290. ret = err;
  2291. }
  2292. } else {
  2293. unlock_page(page);
  2294. }
  2295. }
  2296. if (ret || !wait) {
  2297. return ret;
  2298. }
  2299. for (i = start_i; i < num_pages; i++) {
  2300. page = extent_buffer_page(eb, i);
  2301. wait_on_page_locked(page);
  2302. if (!PageUptodate(page)) {
  2303. ret = -EIO;
  2304. }
  2305. }
  2306. if (!ret)
  2307. eb->flags |= EXTENT_UPTODATE;
  2308. return ret;
  2309. }
  2310. EXPORT_SYMBOL(read_extent_buffer_pages);
  2311. void read_extent_buffer(struct extent_buffer *eb, void *dstv,
  2312. unsigned long start,
  2313. unsigned long len)
  2314. {
  2315. size_t cur;
  2316. size_t offset;
  2317. struct page *page;
  2318. char *kaddr;
  2319. char *dst = (char *)dstv;
  2320. size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
  2321. unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
  2322. unsigned long num_pages = num_extent_pages(eb->start, eb->len);
  2323. WARN_ON(start > eb->len);
  2324. WARN_ON(start + len > eb->start + eb->len);
  2325. offset = (start_offset + start) & ((unsigned long)PAGE_CACHE_SIZE - 1);
  2326. while(len > 0) {
  2327. page = extent_buffer_page(eb, i);
  2328. if (!PageUptodate(page)) {
  2329. printk("page %lu not up to date i %lu, total %lu, len %lu\n", page->index, i, num_pages, eb->len);
  2330. WARN_ON(1);
  2331. }
  2332. WARN_ON(!PageUptodate(page));
  2333. cur = min(len, (PAGE_CACHE_SIZE - offset));
  2334. kaddr = kmap_atomic(page, KM_USER1);
  2335. memcpy(dst, kaddr + offset, cur);
  2336. kunmap_atomic(kaddr, KM_USER1);
  2337. dst += cur;
  2338. len -= cur;
  2339. offset = 0;
  2340. i++;
  2341. }
  2342. }
  2343. EXPORT_SYMBOL(read_extent_buffer);
  2344. int map_private_extent_buffer(struct extent_buffer *eb, unsigned long start,
  2345. unsigned long min_len, char **token, char **map,
  2346. unsigned long *map_start,
  2347. unsigned long *map_len, int km)
  2348. {
  2349. size_t offset = start & (PAGE_CACHE_SIZE - 1);
  2350. char *kaddr;
  2351. struct page *p;
  2352. size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
  2353. unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
  2354. unsigned long end_i = (start_offset + start + min_len - 1) >>
  2355. PAGE_CACHE_SHIFT;
  2356. if (i != end_i)
  2357. return -EINVAL;
  2358. if (i == 0) {
  2359. offset = start_offset;
  2360. *map_start = 0;
  2361. } else {
  2362. offset = 0;
  2363. *map_start = ((u64)i << PAGE_CACHE_SHIFT) - start_offset;
  2364. }
  2365. if (start + min_len > eb->len) {
  2366. printk("bad mapping eb start %Lu len %lu, wanted %lu %lu\n", eb->start, eb->len, start, min_len);
  2367. WARN_ON(1);
  2368. }
  2369. p = extent_buffer_page(eb, i);
  2370. WARN_ON(!PageUptodate(p));
  2371. kaddr = kmap_atomic(p, km);
  2372. *token = kaddr;
  2373. *map = kaddr + offset;
  2374. *map_len = PAGE_CACHE_SIZE - offset;
  2375. return 0;
  2376. }
  2377. EXPORT_SYMBOL(map_private_extent_buffer);
  2378. int map_extent_buffer(struct extent_buffer *eb, unsigned long start,
  2379. unsigned long min_len,
  2380. char **token, char **map,
  2381. unsigned long *map_start,
  2382. unsigned long *map_len, int km)
  2383. {
  2384. int err;
  2385. int save = 0;
  2386. if (eb->map_token) {
  2387. unmap_extent_buffer(eb, eb->map_token, km);
  2388. eb->map_token = NULL;
  2389. save = 1;
  2390. }
  2391. err = map_private_extent_buffer(eb, start, min_len, token, map,
  2392. map_start, map_len, km);
  2393. if (!err && save) {
  2394. eb->map_token = *token;
  2395. eb->kaddr = *map;
  2396. eb->map_start = *map_start;
  2397. eb->map_len = *map_len;
  2398. }
  2399. return err;
  2400. }
  2401. EXPORT_SYMBOL(map_extent_buffer);
  2402. void unmap_extent_buffer(struct extent_buffer *eb, char *token, int km)
  2403. {
  2404. kunmap_atomic(token, km);
  2405. }
  2406. EXPORT_SYMBOL(unmap_extent_buffer);
  2407. int memcmp_extent_buffer(struct extent_buffer *eb, const void *ptrv,
  2408. unsigned long start,
  2409. unsigned long len)
  2410. {
  2411. size_t cur;
  2412. size_t offset;
  2413. struct page *page;
  2414. char *kaddr;
  2415. char *ptr = (char *)ptrv;
  2416. size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
  2417. unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
  2418. int ret = 0;
  2419. WARN_ON(start > eb->len);
  2420. WARN_ON(start + len > eb->start + eb->len);
  2421. offset = (start_offset + start) & ((unsigned long)PAGE_CACHE_SIZE - 1);
  2422. while(len > 0) {
  2423. page = extent_buffer_page(eb, i);
  2424. WARN_ON(!PageUptodate(page));
  2425. cur = min(len, (PAGE_CACHE_SIZE - offset));
  2426. kaddr = kmap_atomic(page, KM_USER0);
  2427. ret = memcmp(ptr, kaddr + offset, cur);
  2428. kunmap_atomic(kaddr, KM_USER0);
  2429. if (ret)
  2430. break;
  2431. ptr += cur;
  2432. len -= cur;
  2433. offset = 0;
  2434. i++;
  2435. }
  2436. return ret;
  2437. }
  2438. EXPORT_SYMBOL(memcmp_extent_buffer);
  2439. void write_extent_buffer(struct extent_buffer *eb, const void *srcv,
  2440. unsigned long start, unsigned long len)
  2441. {
  2442. size_t cur;
  2443. size_t offset;
  2444. struct page *page;
  2445. char *kaddr;
  2446. char *src = (char *)srcv;
  2447. size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
  2448. unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
  2449. WARN_ON(start > eb->len);
  2450. WARN_ON(start + len > eb->start + eb->len);
  2451. offset = (start_offset + start) & ((unsigned long)PAGE_CACHE_SIZE - 1);
  2452. while(len > 0) {
  2453. page = extent_buffer_page(eb, i);
  2454. WARN_ON(!PageUptodate(page));
  2455. cur = min(len, PAGE_CACHE_SIZE - offset);
  2456. kaddr = kmap_atomic(page, KM_USER1);
  2457. memcpy(kaddr + offset, src, cur);
  2458. kunmap_atomic(kaddr, KM_USER1);
  2459. src += cur;
  2460. len -= cur;
  2461. offset = 0;
  2462. i++;
  2463. }
  2464. }
  2465. EXPORT_SYMBOL(write_extent_buffer);
  2466. void memset_extent_buffer(struct extent_buffer *eb, char c,
  2467. unsigned long start, unsigned long len)
  2468. {
  2469. size_t cur;
  2470. size_t offset;
  2471. struct page *page;
  2472. char *kaddr;
  2473. size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
  2474. unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
  2475. WARN_ON(start > eb->len);
  2476. WARN_ON(start + len > eb->start + eb->len);
  2477. offset = (start_offset + start) & ((unsigned long)PAGE_CACHE_SIZE - 1);
  2478. while(len > 0) {
  2479. page = extent_buffer_page(eb, i);
  2480. WARN_ON(!PageUptodate(page));
  2481. cur = min(len, PAGE_CACHE_SIZE - offset);
  2482. kaddr = kmap_atomic(page, KM_USER0);
  2483. memset(kaddr + offset, c, cur);
  2484. kunmap_atomic(kaddr, KM_USER0);
  2485. len -= cur;
  2486. offset = 0;
  2487. i++;
  2488. }
  2489. }
  2490. EXPORT_SYMBOL(memset_extent_buffer);
  2491. void copy_extent_buffer(struct extent_buffer *dst, struct extent_buffer *src,
  2492. unsigned long dst_offset, unsigned long src_offset,
  2493. unsigned long len)
  2494. {
  2495. u64 dst_len = dst->len;
  2496. size_t cur;
  2497. size_t offset;
  2498. struct page *page;
  2499. char *kaddr;
  2500. size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
  2501. unsigned long i = (start_offset + dst_offset) >> PAGE_CACHE_SHIFT;
  2502. WARN_ON(src->len != dst_len);
  2503. offset = (start_offset + dst_offset) &
  2504. ((unsigned long)PAGE_CACHE_SIZE - 1);
  2505. while(len > 0) {
  2506. page = extent_buffer_page(dst, i);
  2507. WARN_ON(!PageUptodate(page));
  2508. cur = min(len, (unsigned long)(PAGE_CACHE_SIZE - offset));
  2509. kaddr = kmap_atomic(page, KM_USER0);
  2510. read_extent_buffer(src, kaddr + offset, src_offset, cur);
  2511. kunmap_atomic(kaddr, KM_USER0);
  2512. src_offset += cur;
  2513. len -= cur;
  2514. offset = 0;
  2515. i++;
  2516. }
  2517. }
  2518. EXPORT_SYMBOL(copy_extent_buffer);
  2519. static void move_pages(struct page *dst_page, struct page *src_page,
  2520. unsigned long dst_off, unsigned long src_off,
  2521. unsigned long len)
  2522. {
  2523. char *dst_kaddr = kmap_atomic(dst_page, KM_USER0);
  2524. if (dst_page == src_page) {
  2525. memmove(dst_kaddr + dst_off, dst_kaddr + src_off, len);
  2526. } else {
  2527. char *src_kaddr = kmap_atomic(src_page, KM_USER1);
  2528. char *p = dst_kaddr + dst_off + len;
  2529. char *s = src_kaddr + src_off + len;
  2530. while (len--)
  2531. *--p = *--s;
  2532. kunmap_atomic(src_kaddr, KM_USER1);
  2533. }
  2534. kunmap_atomic(dst_kaddr, KM_USER0);
  2535. }
  2536. static void copy_pages(struct page *dst_page, struct page *src_page,
  2537. unsigned long dst_off, unsigned long src_off,
  2538. unsigned long len)
  2539. {
  2540. char *dst_kaddr = kmap_atomic(dst_page, KM_USER0);
  2541. char *src_kaddr;
  2542. if (dst_page != src_page)
  2543. src_kaddr = kmap_atomic(src_page, KM_USER1);
  2544. else
  2545. src_kaddr = dst_kaddr;
  2546. memcpy(dst_kaddr + dst_off, src_kaddr + src_off, len);
  2547. kunmap_atomic(dst_kaddr, KM_USER0);
  2548. if (dst_page != src_page)
  2549. kunmap_atomic(src_kaddr, KM_USER1);
  2550. }
  2551. void memcpy_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
  2552. unsigned long src_offset, unsigned long len)
  2553. {
  2554. size_t cur;
  2555. size_t dst_off_in_page;
  2556. size_t src_off_in_page;
  2557. size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
  2558. unsigned long dst_i;
  2559. unsigned long src_i;
  2560. if (src_offset + len > dst->len) {
  2561. printk("memmove bogus src_offset %lu move len %lu len %lu\n",
  2562. src_offset, len, dst->len);
  2563. BUG_ON(1);
  2564. }
  2565. if (dst_offset + len > dst->len) {
  2566. printk("memmove bogus dst_offset %lu move len %lu len %lu\n",
  2567. dst_offset, len, dst->len);
  2568. BUG_ON(1);
  2569. }
  2570. while(len > 0) {
  2571. dst_off_in_page = (start_offset + dst_offset) &
  2572. ((unsigned long)PAGE_CACHE_SIZE - 1);
  2573. src_off_in_page = (start_offset + src_offset) &
  2574. ((unsigned long)PAGE_CACHE_SIZE - 1);
  2575. dst_i = (start_offset + dst_offset) >> PAGE_CACHE_SHIFT;
  2576. src_i = (start_offset + src_offset) >> PAGE_CACHE_SHIFT;
  2577. cur = min(len, (unsigned long)(PAGE_CACHE_SIZE -
  2578. src_off_in_page));
  2579. cur = min_t(unsigned long, cur,
  2580. (unsigned long)(PAGE_CACHE_SIZE - dst_off_in_page));
  2581. copy_pages(extent_buffer_page(dst, dst_i),
  2582. extent_buffer_page(dst, src_i),
  2583. dst_off_in_page, src_off_in_page, cur);
  2584. src_offset += cur;
  2585. dst_offset += cur;
  2586. len -= cur;
  2587. }
  2588. }
  2589. EXPORT_SYMBOL(memcpy_extent_buffer);
  2590. void memmove_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
  2591. unsigned long src_offset, unsigned long len)
  2592. {
  2593. size_t cur;
  2594. size_t dst_off_in_page;
  2595. size_t src_off_in_page;
  2596. unsigned long dst_end = dst_offset + len - 1;
  2597. unsigned long src_end = src_offset + len - 1;
  2598. size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
  2599. unsigned long dst_i;
  2600. unsigned long src_i;
  2601. if (src_offset + len > dst->len) {
  2602. printk("memmove bogus src_offset %lu move len %lu len %lu\n",
  2603. src_offset, len, dst->len);
  2604. BUG_ON(1);
  2605. }
  2606. if (dst_offset + len > dst->len) {
  2607. printk("memmove bogus dst_offset %lu move len %lu len %lu\n",
  2608. dst_offset, len, dst->len);
  2609. BUG_ON(1);
  2610. }
  2611. if (dst_offset < src_offset) {
  2612. memcpy_extent_buffer(dst, dst_offset, src_offset, len);
  2613. return;
  2614. }
  2615. while(len > 0) {
  2616. dst_i = (start_offset + dst_end) >> PAGE_CACHE_SHIFT;
  2617. src_i = (start_offset + src_end) >> PAGE_CACHE_SHIFT;
  2618. dst_off_in_page = (start_offset + dst_end) &
  2619. ((unsigned long)PAGE_CACHE_SIZE - 1);
  2620. src_off_in_page = (start_offset + src_end) &
  2621. ((unsigned long)PAGE_CACHE_SIZE - 1);
  2622. cur = min_t(unsigned long, len, src_off_in_page + 1);
  2623. cur = min(cur, dst_off_in_page + 1);
  2624. move_pages(extent_buffer_page(dst, dst_i),
  2625. extent_buffer_page(dst, src_i),
  2626. dst_off_in_page - cur + 1,
  2627. src_off_in_page - cur + 1, cur);
  2628. dst_end -= cur;
  2629. src_end -= cur;
  2630. len -= cur;
  2631. }
  2632. }
  2633. EXPORT_SYMBOL(memmove_extent_buffer);