ordered-data.c 31 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119
  1. /*
  2. * Copyright (C) 2007 Oracle. All rights reserved.
  3. *
  4. * This program is free software; you can redistribute it and/or
  5. * modify it under the terms of the GNU General Public
  6. * License v2 as published by the Free Software Foundation.
  7. *
  8. * This program is distributed in the hope that it will be useful,
  9. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  11. * General Public License for more details.
  12. *
  13. * You should have received a copy of the GNU General Public
  14. * License along with this program; if not, write to the
  15. * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  16. * Boston, MA 021110-1307, USA.
  17. */
  18. #include <linux/slab.h>
  19. #include <linux/blkdev.h>
  20. #include <linux/writeback.h>
  21. #include <linux/pagevec.h>
  22. #include "ctree.h"
  23. #include "transaction.h"
  24. #include "btrfs_inode.h"
  25. #include "extent_io.h"
  26. #include "disk-io.h"
  27. static struct kmem_cache *btrfs_ordered_extent_cache;
  28. static u64 entry_end(struct btrfs_ordered_extent *entry)
  29. {
  30. if (entry->file_offset + entry->len < entry->file_offset)
  31. return (u64)-1;
  32. return entry->file_offset + entry->len;
  33. }
  34. /* returns NULL if the insertion worked, or it returns the node it did find
  35. * in the tree
  36. */
  37. static struct rb_node *tree_insert(struct rb_root *root, u64 file_offset,
  38. struct rb_node *node)
  39. {
  40. struct rb_node **p = &root->rb_node;
  41. struct rb_node *parent = NULL;
  42. struct btrfs_ordered_extent *entry;
  43. while (*p) {
  44. parent = *p;
  45. entry = rb_entry(parent, struct btrfs_ordered_extent, rb_node);
  46. if (file_offset < entry->file_offset)
  47. p = &(*p)->rb_left;
  48. else if (file_offset >= entry_end(entry))
  49. p = &(*p)->rb_right;
  50. else
  51. return parent;
  52. }
  53. rb_link_node(node, parent, p);
  54. rb_insert_color(node, root);
  55. return NULL;
  56. }
  57. static void ordered_data_tree_panic(struct inode *inode, int errno,
  58. u64 offset)
  59. {
  60. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  61. btrfs_panic(fs_info, errno, "Inconsistency in ordered tree at offset "
  62. "%llu\n", offset);
  63. }
  64. /*
  65. * look for a given offset in the tree, and if it can't be found return the
  66. * first lesser offset
  67. */
  68. static struct rb_node *__tree_search(struct rb_root *root, u64 file_offset,
  69. struct rb_node **prev_ret)
  70. {
  71. struct rb_node *n = root->rb_node;
  72. struct rb_node *prev = NULL;
  73. struct rb_node *test;
  74. struct btrfs_ordered_extent *entry;
  75. struct btrfs_ordered_extent *prev_entry = NULL;
  76. while (n) {
  77. entry = rb_entry(n, struct btrfs_ordered_extent, rb_node);
  78. prev = n;
  79. prev_entry = entry;
  80. if (file_offset < entry->file_offset)
  81. n = n->rb_left;
  82. else if (file_offset >= entry_end(entry))
  83. n = n->rb_right;
  84. else
  85. return n;
  86. }
  87. if (!prev_ret)
  88. return NULL;
  89. while (prev && file_offset >= entry_end(prev_entry)) {
  90. test = rb_next(prev);
  91. if (!test)
  92. break;
  93. prev_entry = rb_entry(test, struct btrfs_ordered_extent,
  94. rb_node);
  95. if (file_offset < entry_end(prev_entry))
  96. break;
  97. prev = test;
  98. }
  99. if (prev)
  100. prev_entry = rb_entry(prev, struct btrfs_ordered_extent,
  101. rb_node);
  102. while (prev && file_offset < entry_end(prev_entry)) {
  103. test = rb_prev(prev);
  104. if (!test)
  105. break;
  106. prev_entry = rb_entry(test, struct btrfs_ordered_extent,
  107. rb_node);
  108. prev = test;
  109. }
  110. *prev_ret = prev;
  111. return NULL;
  112. }
  113. /*
  114. * helper to check if a given offset is inside a given entry
  115. */
  116. static int offset_in_entry(struct btrfs_ordered_extent *entry, u64 file_offset)
  117. {
  118. if (file_offset < entry->file_offset ||
  119. entry->file_offset + entry->len <= file_offset)
  120. return 0;
  121. return 1;
  122. }
  123. static int range_overlaps(struct btrfs_ordered_extent *entry, u64 file_offset,
  124. u64 len)
  125. {
  126. if (file_offset + len <= entry->file_offset ||
  127. entry->file_offset + entry->len <= file_offset)
  128. return 0;
  129. return 1;
  130. }
  131. /*
  132. * look find the first ordered struct that has this offset, otherwise
  133. * the first one less than this offset
  134. */
  135. static inline struct rb_node *tree_search(struct btrfs_ordered_inode_tree *tree,
  136. u64 file_offset)
  137. {
  138. struct rb_root *root = &tree->tree;
  139. struct rb_node *prev = NULL;
  140. struct rb_node *ret;
  141. struct btrfs_ordered_extent *entry;
  142. if (tree->last) {
  143. entry = rb_entry(tree->last, struct btrfs_ordered_extent,
  144. rb_node);
  145. if (offset_in_entry(entry, file_offset))
  146. return tree->last;
  147. }
  148. ret = __tree_search(root, file_offset, &prev);
  149. if (!ret)
  150. ret = prev;
  151. if (ret)
  152. tree->last = ret;
  153. return ret;
  154. }
  155. /* allocate and add a new ordered_extent into the per-inode tree.
  156. * file_offset is the logical offset in the file
  157. *
  158. * start is the disk block number of an extent already reserved in the
  159. * extent allocation tree
  160. *
  161. * len is the length of the extent
  162. *
  163. * The tree is given a single reference on the ordered extent that was
  164. * inserted.
  165. */
  166. static int __btrfs_add_ordered_extent(struct inode *inode, u64 file_offset,
  167. u64 start, u64 len, u64 disk_len,
  168. int type, int dio, int compress_type)
  169. {
  170. struct btrfs_root *root = BTRFS_I(inode)->root;
  171. struct btrfs_ordered_inode_tree *tree;
  172. struct rb_node *node;
  173. struct btrfs_ordered_extent *entry;
  174. tree = &BTRFS_I(inode)->ordered_tree;
  175. entry = kmem_cache_zalloc(btrfs_ordered_extent_cache, GFP_NOFS);
  176. if (!entry)
  177. return -ENOMEM;
  178. entry->file_offset = file_offset;
  179. entry->start = start;
  180. entry->len = len;
  181. if (!(BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM) &&
  182. !(type == BTRFS_ORDERED_NOCOW))
  183. entry->csum_bytes_left = disk_len;
  184. entry->disk_len = disk_len;
  185. entry->bytes_left = len;
  186. entry->inode = igrab(inode);
  187. entry->compress_type = compress_type;
  188. if (type != BTRFS_ORDERED_IO_DONE && type != BTRFS_ORDERED_COMPLETE)
  189. set_bit(type, &entry->flags);
  190. if (dio)
  191. set_bit(BTRFS_ORDERED_DIRECT, &entry->flags);
  192. /* one ref for the tree */
  193. atomic_set(&entry->refs, 1);
  194. init_waitqueue_head(&entry->wait);
  195. INIT_LIST_HEAD(&entry->list);
  196. INIT_LIST_HEAD(&entry->root_extent_list);
  197. INIT_LIST_HEAD(&entry->work_list);
  198. init_completion(&entry->completion);
  199. INIT_LIST_HEAD(&entry->log_list);
  200. trace_btrfs_ordered_extent_add(inode, entry);
  201. spin_lock_irq(&tree->lock);
  202. node = tree_insert(&tree->tree, file_offset,
  203. &entry->rb_node);
  204. if (node)
  205. ordered_data_tree_panic(inode, -EEXIST, file_offset);
  206. spin_unlock_irq(&tree->lock);
  207. spin_lock(&root->ordered_extent_lock);
  208. list_add_tail(&entry->root_extent_list,
  209. &root->ordered_extents);
  210. root->nr_ordered_extents++;
  211. if (root->nr_ordered_extents == 1) {
  212. spin_lock(&root->fs_info->ordered_root_lock);
  213. BUG_ON(!list_empty(&root->ordered_root));
  214. list_add_tail(&root->ordered_root,
  215. &root->fs_info->ordered_roots);
  216. spin_unlock(&root->fs_info->ordered_root_lock);
  217. }
  218. spin_unlock(&root->ordered_extent_lock);
  219. return 0;
  220. }
  221. int btrfs_add_ordered_extent(struct inode *inode, u64 file_offset,
  222. u64 start, u64 len, u64 disk_len, int type)
  223. {
  224. return __btrfs_add_ordered_extent(inode, file_offset, start, len,
  225. disk_len, type, 0,
  226. BTRFS_COMPRESS_NONE);
  227. }
  228. int btrfs_add_ordered_extent_dio(struct inode *inode, u64 file_offset,
  229. u64 start, u64 len, u64 disk_len, int type)
  230. {
  231. return __btrfs_add_ordered_extent(inode, file_offset, start, len,
  232. disk_len, type, 1,
  233. BTRFS_COMPRESS_NONE);
  234. }
  235. int btrfs_add_ordered_extent_compress(struct inode *inode, u64 file_offset,
  236. u64 start, u64 len, u64 disk_len,
  237. int type, int compress_type)
  238. {
  239. return __btrfs_add_ordered_extent(inode, file_offset, start, len,
  240. disk_len, type, 0,
  241. compress_type);
  242. }
  243. /*
  244. * Add a struct btrfs_ordered_sum into the list of checksums to be inserted
  245. * when an ordered extent is finished. If the list covers more than one
  246. * ordered extent, it is split across multiples.
  247. */
  248. void btrfs_add_ordered_sum(struct inode *inode,
  249. struct btrfs_ordered_extent *entry,
  250. struct btrfs_ordered_sum *sum)
  251. {
  252. struct btrfs_ordered_inode_tree *tree;
  253. tree = &BTRFS_I(inode)->ordered_tree;
  254. spin_lock_irq(&tree->lock);
  255. list_add_tail(&sum->list, &entry->list);
  256. WARN_ON(entry->csum_bytes_left < sum->len);
  257. entry->csum_bytes_left -= sum->len;
  258. if (entry->csum_bytes_left == 0)
  259. wake_up(&entry->wait);
  260. spin_unlock_irq(&tree->lock);
  261. }
  262. /*
  263. * this is used to account for finished IO across a given range
  264. * of the file. The IO may span ordered extents. If
  265. * a given ordered_extent is completely done, 1 is returned, otherwise
  266. * 0.
  267. *
  268. * test_and_set_bit on a flag in the struct btrfs_ordered_extent is used
  269. * to make sure this function only returns 1 once for a given ordered extent.
  270. *
  271. * file_offset is updated to one byte past the range that is recorded as
  272. * complete. This allows you to walk forward in the file.
  273. */
  274. int btrfs_dec_test_first_ordered_pending(struct inode *inode,
  275. struct btrfs_ordered_extent **cached,
  276. u64 *file_offset, u64 io_size, int uptodate)
  277. {
  278. struct btrfs_ordered_inode_tree *tree;
  279. struct rb_node *node;
  280. struct btrfs_ordered_extent *entry = NULL;
  281. int ret;
  282. unsigned long flags;
  283. u64 dec_end;
  284. u64 dec_start;
  285. u64 to_dec;
  286. tree = &BTRFS_I(inode)->ordered_tree;
  287. spin_lock_irqsave(&tree->lock, flags);
  288. node = tree_search(tree, *file_offset);
  289. if (!node) {
  290. ret = 1;
  291. goto out;
  292. }
  293. entry = rb_entry(node, struct btrfs_ordered_extent, rb_node);
  294. if (!offset_in_entry(entry, *file_offset)) {
  295. ret = 1;
  296. goto out;
  297. }
  298. dec_start = max(*file_offset, entry->file_offset);
  299. dec_end = min(*file_offset + io_size, entry->file_offset +
  300. entry->len);
  301. *file_offset = dec_end;
  302. if (dec_start > dec_end) {
  303. printk(KERN_CRIT "bad ordering dec_start %llu end %llu\n",
  304. dec_start, dec_end);
  305. }
  306. to_dec = dec_end - dec_start;
  307. if (to_dec > entry->bytes_left) {
  308. printk(KERN_CRIT "bad ordered accounting left %llu size %llu\n",
  309. entry->bytes_left, to_dec);
  310. }
  311. entry->bytes_left -= to_dec;
  312. if (!uptodate)
  313. set_bit(BTRFS_ORDERED_IOERR, &entry->flags);
  314. if (entry->bytes_left == 0)
  315. ret = test_and_set_bit(BTRFS_ORDERED_IO_DONE, &entry->flags);
  316. else
  317. ret = 1;
  318. out:
  319. if (!ret && cached && entry) {
  320. *cached = entry;
  321. atomic_inc(&entry->refs);
  322. }
  323. spin_unlock_irqrestore(&tree->lock, flags);
  324. return ret == 0;
  325. }
  326. /*
  327. * this is used to account for finished IO across a given range
  328. * of the file. The IO should not span ordered extents. If
  329. * a given ordered_extent is completely done, 1 is returned, otherwise
  330. * 0.
  331. *
  332. * test_and_set_bit on a flag in the struct btrfs_ordered_extent is used
  333. * to make sure this function only returns 1 once for a given ordered extent.
  334. */
  335. int btrfs_dec_test_ordered_pending(struct inode *inode,
  336. struct btrfs_ordered_extent **cached,
  337. u64 file_offset, u64 io_size, int uptodate)
  338. {
  339. struct btrfs_ordered_inode_tree *tree;
  340. struct rb_node *node;
  341. struct btrfs_ordered_extent *entry = NULL;
  342. unsigned long flags;
  343. int ret;
  344. tree = &BTRFS_I(inode)->ordered_tree;
  345. spin_lock_irqsave(&tree->lock, flags);
  346. if (cached && *cached) {
  347. entry = *cached;
  348. goto have_entry;
  349. }
  350. node = tree_search(tree, file_offset);
  351. if (!node) {
  352. ret = 1;
  353. goto out;
  354. }
  355. entry = rb_entry(node, struct btrfs_ordered_extent, rb_node);
  356. have_entry:
  357. if (!offset_in_entry(entry, file_offset)) {
  358. ret = 1;
  359. goto out;
  360. }
  361. if (io_size > entry->bytes_left) {
  362. printk(KERN_CRIT "bad ordered accounting left %llu size %llu\n",
  363. entry->bytes_left, io_size);
  364. }
  365. entry->bytes_left -= io_size;
  366. if (!uptodate)
  367. set_bit(BTRFS_ORDERED_IOERR, &entry->flags);
  368. if (entry->bytes_left == 0)
  369. ret = test_and_set_bit(BTRFS_ORDERED_IO_DONE, &entry->flags);
  370. else
  371. ret = 1;
  372. out:
  373. if (!ret && cached && entry) {
  374. *cached = entry;
  375. atomic_inc(&entry->refs);
  376. }
  377. spin_unlock_irqrestore(&tree->lock, flags);
  378. return ret == 0;
  379. }
  380. /* Needs to either be called under a log transaction or the log_mutex */
  381. void btrfs_get_logged_extents(struct btrfs_root *log, struct inode *inode)
  382. {
  383. struct btrfs_ordered_inode_tree *tree;
  384. struct btrfs_ordered_extent *ordered;
  385. struct rb_node *n;
  386. int index = log->log_transid % 2;
  387. tree = &BTRFS_I(inode)->ordered_tree;
  388. spin_lock_irq(&tree->lock);
  389. for (n = rb_first(&tree->tree); n; n = rb_next(n)) {
  390. ordered = rb_entry(n, struct btrfs_ordered_extent, rb_node);
  391. spin_lock(&log->log_extents_lock[index]);
  392. if (list_empty(&ordered->log_list)) {
  393. list_add_tail(&ordered->log_list, &log->logged_list[index]);
  394. atomic_inc(&ordered->refs);
  395. }
  396. spin_unlock(&log->log_extents_lock[index]);
  397. }
  398. spin_unlock_irq(&tree->lock);
  399. }
  400. void btrfs_wait_logged_extents(struct btrfs_root *log, u64 transid)
  401. {
  402. struct btrfs_ordered_extent *ordered;
  403. int index = transid % 2;
  404. spin_lock_irq(&log->log_extents_lock[index]);
  405. while (!list_empty(&log->logged_list[index])) {
  406. ordered = list_first_entry(&log->logged_list[index],
  407. struct btrfs_ordered_extent,
  408. log_list);
  409. list_del_init(&ordered->log_list);
  410. spin_unlock_irq(&log->log_extents_lock[index]);
  411. wait_event(ordered->wait, test_bit(BTRFS_ORDERED_IO_DONE,
  412. &ordered->flags));
  413. btrfs_put_ordered_extent(ordered);
  414. spin_lock_irq(&log->log_extents_lock[index]);
  415. }
  416. spin_unlock_irq(&log->log_extents_lock[index]);
  417. }
  418. void btrfs_free_logged_extents(struct btrfs_root *log, u64 transid)
  419. {
  420. struct btrfs_ordered_extent *ordered;
  421. int index = transid % 2;
  422. spin_lock_irq(&log->log_extents_lock[index]);
  423. while (!list_empty(&log->logged_list[index])) {
  424. ordered = list_first_entry(&log->logged_list[index],
  425. struct btrfs_ordered_extent,
  426. log_list);
  427. list_del_init(&ordered->log_list);
  428. spin_unlock_irq(&log->log_extents_lock[index]);
  429. btrfs_put_ordered_extent(ordered);
  430. spin_lock_irq(&log->log_extents_lock[index]);
  431. }
  432. spin_unlock_irq(&log->log_extents_lock[index]);
  433. }
  434. /*
  435. * used to drop a reference on an ordered extent. This will free
  436. * the extent if the last reference is dropped
  437. */
  438. void btrfs_put_ordered_extent(struct btrfs_ordered_extent *entry)
  439. {
  440. struct list_head *cur;
  441. struct btrfs_ordered_sum *sum;
  442. trace_btrfs_ordered_extent_put(entry->inode, entry);
  443. if (atomic_dec_and_test(&entry->refs)) {
  444. if (entry->inode)
  445. btrfs_add_delayed_iput(entry->inode);
  446. while (!list_empty(&entry->list)) {
  447. cur = entry->list.next;
  448. sum = list_entry(cur, struct btrfs_ordered_sum, list);
  449. list_del(&sum->list);
  450. kfree(sum);
  451. }
  452. kmem_cache_free(btrfs_ordered_extent_cache, entry);
  453. }
  454. }
  455. /*
  456. * remove an ordered extent from the tree. No references are dropped
  457. * and waiters are woken up.
  458. */
  459. void btrfs_remove_ordered_extent(struct inode *inode,
  460. struct btrfs_ordered_extent *entry)
  461. {
  462. struct btrfs_ordered_inode_tree *tree;
  463. struct btrfs_root *root = BTRFS_I(inode)->root;
  464. struct rb_node *node;
  465. tree = &BTRFS_I(inode)->ordered_tree;
  466. spin_lock_irq(&tree->lock);
  467. node = &entry->rb_node;
  468. rb_erase(node, &tree->tree);
  469. tree->last = NULL;
  470. set_bit(BTRFS_ORDERED_COMPLETE, &entry->flags);
  471. spin_unlock_irq(&tree->lock);
  472. spin_lock(&root->ordered_extent_lock);
  473. list_del_init(&entry->root_extent_list);
  474. root->nr_ordered_extents--;
  475. trace_btrfs_ordered_extent_remove(inode, entry);
  476. /*
  477. * we have no more ordered extents for this inode and
  478. * no dirty pages. We can safely remove it from the
  479. * list of ordered extents
  480. */
  481. if (RB_EMPTY_ROOT(&tree->tree) &&
  482. !mapping_tagged(inode->i_mapping, PAGECACHE_TAG_DIRTY)) {
  483. list_del_init(&BTRFS_I(inode)->ordered_operations);
  484. }
  485. if (!root->nr_ordered_extents) {
  486. spin_lock(&root->fs_info->ordered_root_lock);
  487. BUG_ON(list_empty(&root->ordered_root));
  488. list_del_init(&root->ordered_root);
  489. spin_unlock(&root->fs_info->ordered_root_lock);
  490. }
  491. spin_unlock(&root->ordered_extent_lock);
  492. wake_up(&entry->wait);
  493. }
  494. static void btrfs_run_ordered_extent_work(struct btrfs_work *work)
  495. {
  496. struct btrfs_ordered_extent *ordered;
  497. ordered = container_of(work, struct btrfs_ordered_extent, flush_work);
  498. btrfs_start_ordered_extent(ordered->inode, ordered, 1);
  499. complete(&ordered->completion);
  500. }
  501. /*
  502. * wait for all the ordered extents in a root. This is done when balancing
  503. * space between drives.
  504. */
  505. void btrfs_wait_ordered_extents(struct btrfs_root *root, int delay_iput)
  506. {
  507. struct list_head splice, works;
  508. struct btrfs_ordered_extent *ordered, *next;
  509. struct inode *inode;
  510. INIT_LIST_HEAD(&splice);
  511. INIT_LIST_HEAD(&works);
  512. mutex_lock(&root->fs_info->ordered_operations_mutex);
  513. spin_lock(&root->ordered_extent_lock);
  514. list_splice_init(&root->ordered_extents, &splice);
  515. while (!list_empty(&splice)) {
  516. ordered = list_first_entry(&splice, struct btrfs_ordered_extent,
  517. root_extent_list);
  518. list_move_tail(&ordered->root_extent_list,
  519. &root->ordered_extents);
  520. /*
  521. * the inode may be getting freed (in sys_unlink path).
  522. */
  523. inode = igrab(ordered->inode);
  524. if (!inode) {
  525. cond_resched_lock(&root->ordered_extent_lock);
  526. continue;
  527. }
  528. atomic_inc(&ordered->refs);
  529. spin_unlock(&root->ordered_extent_lock);
  530. ordered->flush_work.func = btrfs_run_ordered_extent_work;
  531. list_add_tail(&ordered->work_list, &works);
  532. btrfs_queue_worker(&root->fs_info->flush_workers,
  533. &ordered->flush_work);
  534. cond_resched();
  535. spin_lock(&root->ordered_extent_lock);
  536. }
  537. spin_unlock(&root->ordered_extent_lock);
  538. list_for_each_entry_safe(ordered, next, &works, work_list) {
  539. list_del_init(&ordered->work_list);
  540. wait_for_completion(&ordered->completion);
  541. inode = ordered->inode;
  542. btrfs_put_ordered_extent(ordered);
  543. if (delay_iput)
  544. btrfs_add_delayed_iput(inode);
  545. else
  546. iput(inode);
  547. cond_resched();
  548. }
  549. mutex_unlock(&root->fs_info->ordered_operations_mutex);
  550. }
  551. void btrfs_wait_all_ordered_extents(struct btrfs_fs_info *fs_info,
  552. int delay_iput)
  553. {
  554. struct btrfs_root *root;
  555. struct list_head splice;
  556. INIT_LIST_HEAD(&splice);
  557. spin_lock(&fs_info->ordered_root_lock);
  558. list_splice_init(&fs_info->ordered_roots, &splice);
  559. while (!list_empty(&splice)) {
  560. root = list_first_entry(&splice, struct btrfs_root,
  561. ordered_root);
  562. root = btrfs_grab_fs_root(root);
  563. BUG_ON(!root);
  564. list_move_tail(&root->ordered_root,
  565. &fs_info->ordered_roots);
  566. spin_unlock(&fs_info->ordered_root_lock);
  567. btrfs_wait_ordered_extents(root, delay_iput);
  568. btrfs_put_fs_root(root);
  569. spin_lock(&fs_info->ordered_root_lock);
  570. }
  571. spin_unlock(&fs_info->ordered_root_lock);
  572. }
  573. /*
  574. * this is used during transaction commit to write all the inodes
  575. * added to the ordered operation list. These files must be fully on
  576. * disk before the transaction commits.
  577. *
  578. * we have two modes here, one is to just start the IO via filemap_flush
  579. * and the other is to wait for all the io. When we wait, we have an
  580. * extra check to make sure the ordered operation list really is empty
  581. * before we return
  582. */
  583. int btrfs_run_ordered_operations(struct btrfs_trans_handle *trans,
  584. struct btrfs_root *root, int wait)
  585. {
  586. struct btrfs_inode *btrfs_inode;
  587. struct inode *inode;
  588. struct btrfs_transaction *cur_trans = trans->transaction;
  589. struct list_head splice;
  590. struct list_head works;
  591. struct btrfs_delalloc_work *work, *next;
  592. int ret = 0;
  593. INIT_LIST_HEAD(&splice);
  594. INIT_LIST_HEAD(&works);
  595. mutex_lock(&root->fs_info->ordered_extent_flush_mutex);
  596. spin_lock(&root->fs_info->ordered_root_lock);
  597. list_splice_init(&cur_trans->ordered_operations, &splice);
  598. while (!list_empty(&splice)) {
  599. btrfs_inode = list_entry(splice.next, struct btrfs_inode,
  600. ordered_operations);
  601. inode = &btrfs_inode->vfs_inode;
  602. list_del_init(&btrfs_inode->ordered_operations);
  603. /*
  604. * the inode may be getting freed (in sys_unlink path).
  605. */
  606. inode = igrab(inode);
  607. if (!inode)
  608. continue;
  609. if (!wait)
  610. list_add_tail(&BTRFS_I(inode)->ordered_operations,
  611. &cur_trans->ordered_operations);
  612. spin_unlock(&root->fs_info->ordered_root_lock);
  613. work = btrfs_alloc_delalloc_work(inode, wait, 1);
  614. if (!work) {
  615. spin_lock(&root->fs_info->ordered_root_lock);
  616. if (list_empty(&BTRFS_I(inode)->ordered_operations))
  617. list_add_tail(&btrfs_inode->ordered_operations,
  618. &splice);
  619. list_splice_tail(&splice,
  620. &cur_trans->ordered_operations);
  621. spin_unlock(&root->fs_info->ordered_root_lock);
  622. ret = -ENOMEM;
  623. goto out;
  624. }
  625. list_add_tail(&work->list, &works);
  626. btrfs_queue_worker(&root->fs_info->flush_workers,
  627. &work->work);
  628. cond_resched();
  629. spin_lock(&root->fs_info->ordered_root_lock);
  630. }
  631. spin_unlock(&root->fs_info->ordered_root_lock);
  632. out:
  633. list_for_each_entry_safe(work, next, &works, list) {
  634. list_del_init(&work->list);
  635. btrfs_wait_and_free_delalloc_work(work);
  636. }
  637. mutex_unlock(&root->fs_info->ordered_extent_flush_mutex);
  638. return ret;
  639. }
  640. /*
  641. * Used to start IO or wait for a given ordered extent to finish.
  642. *
  643. * If wait is one, this effectively waits on page writeback for all the pages
  644. * in the extent, and it waits on the io completion code to insert
  645. * metadata into the btree corresponding to the extent
  646. */
  647. void btrfs_start_ordered_extent(struct inode *inode,
  648. struct btrfs_ordered_extent *entry,
  649. int wait)
  650. {
  651. u64 start = entry->file_offset;
  652. u64 end = start + entry->len - 1;
  653. trace_btrfs_ordered_extent_start(inode, entry);
  654. /*
  655. * pages in the range can be dirty, clean or writeback. We
  656. * start IO on any dirty ones so the wait doesn't stall waiting
  657. * for the flusher thread to find them
  658. */
  659. if (!test_bit(BTRFS_ORDERED_DIRECT, &entry->flags))
  660. filemap_fdatawrite_range(inode->i_mapping, start, end);
  661. if (wait) {
  662. wait_event(entry->wait, test_bit(BTRFS_ORDERED_COMPLETE,
  663. &entry->flags));
  664. }
  665. }
  666. /*
  667. * Used to wait on ordered extents across a large range of bytes.
  668. */
  669. void btrfs_wait_ordered_range(struct inode *inode, u64 start, u64 len)
  670. {
  671. u64 end;
  672. u64 orig_end;
  673. struct btrfs_ordered_extent *ordered;
  674. if (start + len < start) {
  675. orig_end = INT_LIMIT(loff_t);
  676. } else {
  677. orig_end = start + len - 1;
  678. if (orig_end > INT_LIMIT(loff_t))
  679. orig_end = INT_LIMIT(loff_t);
  680. }
  681. /* start IO across the range first to instantiate any delalloc
  682. * extents
  683. */
  684. filemap_fdatawrite_range(inode->i_mapping, start, orig_end);
  685. /*
  686. * So with compression we will find and lock a dirty page and clear the
  687. * first one as dirty, setup an async extent, and immediately return
  688. * with the entire range locked but with nobody actually marked with
  689. * writeback. So we can't just filemap_write_and_wait_range() and
  690. * expect it to work since it will just kick off a thread to do the
  691. * actual work. So we need to call filemap_fdatawrite_range _again_
  692. * since it will wait on the page lock, which won't be unlocked until
  693. * after the pages have been marked as writeback and so we're good to go
  694. * from there. We have to do this otherwise we'll miss the ordered
  695. * extents and that results in badness. Please Josef, do not think you
  696. * know better and pull this out at some point in the future, it is
  697. * right and you are wrong.
  698. */
  699. if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT,
  700. &BTRFS_I(inode)->runtime_flags))
  701. filemap_fdatawrite_range(inode->i_mapping, start, orig_end);
  702. filemap_fdatawait_range(inode->i_mapping, start, orig_end);
  703. end = orig_end;
  704. while (1) {
  705. ordered = btrfs_lookup_first_ordered_extent(inode, end);
  706. if (!ordered)
  707. break;
  708. if (ordered->file_offset > orig_end) {
  709. btrfs_put_ordered_extent(ordered);
  710. break;
  711. }
  712. if (ordered->file_offset + ordered->len < start) {
  713. btrfs_put_ordered_extent(ordered);
  714. break;
  715. }
  716. btrfs_start_ordered_extent(inode, ordered, 1);
  717. end = ordered->file_offset;
  718. btrfs_put_ordered_extent(ordered);
  719. if (end == 0 || end == start)
  720. break;
  721. end--;
  722. }
  723. }
  724. /*
  725. * find an ordered extent corresponding to file_offset. return NULL if
  726. * nothing is found, otherwise take a reference on the extent and return it
  727. */
  728. struct btrfs_ordered_extent *btrfs_lookup_ordered_extent(struct inode *inode,
  729. u64 file_offset)
  730. {
  731. struct btrfs_ordered_inode_tree *tree;
  732. struct rb_node *node;
  733. struct btrfs_ordered_extent *entry = NULL;
  734. tree = &BTRFS_I(inode)->ordered_tree;
  735. spin_lock_irq(&tree->lock);
  736. node = tree_search(tree, file_offset);
  737. if (!node)
  738. goto out;
  739. entry = rb_entry(node, struct btrfs_ordered_extent, rb_node);
  740. if (!offset_in_entry(entry, file_offset))
  741. entry = NULL;
  742. if (entry)
  743. atomic_inc(&entry->refs);
  744. out:
  745. spin_unlock_irq(&tree->lock);
  746. return entry;
  747. }
  748. /* Since the DIO code tries to lock a wide area we need to look for any ordered
  749. * extents that exist in the range, rather than just the start of the range.
  750. */
  751. struct btrfs_ordered_extent *btrfs_lookup_ordered_range(struct inode *inode,
  752. u64 file_offset,
  753. u64 len)
  754. {
  755. struct btrfs_ordered_inode_tree *tree;
  756. struct rb_node *node;
  757. struct btrfs_ordered_extent *entry = NULL;
  758. tree = &BTRFS_I(inode)->ordered_tree;
  759. spin_lock_irq(&tree->lock);
  760. node = tree_search(tree, file_offset);
  761. if (!node) {
  762. node = tree_search(tree, file_offset + len);
  763. if (!node)
  764. goto out;
  765. }
  766. while (1) {
  767. entry = rb_entry(node, struct btrfs_ordered_extent, rb_node);
  768. if (range_overlaps(entry, file_offset, len))
  769. break;
  770. if (entry->file_offset >= file_offset + len) {
  771. entry = NULL;
  772. break;
  773. }
  774. entry = NULL;
  775. node = rb_next(node);
  776. if (!node)
  777. break;
  778. }
  779. out:
  780. if (entry)
  781. atomic_inc(&entry->refs);
  782. spin_unlock_irq(&tree->lock);
  783. return entry;
  784. }
  785. /*
  786. * lookup and return any extent before 'file_offset'. NULL is returned
  787. * if none is found
  788. */
  789. struct btrfs_ordered_extent *
  790. btrfs_lookup_first_ordered_extent(struct inode *inode, u64 file_offset)
  791. {
  792. struct btrfs_ordered_inode_tree *tree;
  793. struct rb_node *node;
  794. struct btrfs_ordered_extent *entry = NULL;
  795. tree = &BTRFS_I(inode)->ordered_tree;
  796. spin_lock_irq(&tree->lock);
  797. node = tree_search(tree, file_offset);
  798. if (!node)
  799. goto out;
  800. entry = rb_entry(node, struct btrfs_ordered_extent, rb_node);
  801. atomic_inc(&entry->refs);
  802. out:
  803. spin_unlock_irq(&tree->lock);
  804. return entry;
  805. }
  806. /*
  807. * After an extent is done, call this to conditionally update the on disk
  808. * i_size. i_size is updated to cover any fully written part of the file.
  809. */
  810. int btrfs_ordered_update_i_size(struct inode *inode, u64 offset,
  811. struct btrfs_ordered_extent *ordered)
  812. {
  813. struct btrfs_ordered_inode_tree *tree = &BTRFS_I(inode)->ordered_tree;
  814. u64 disk_i_size;
  815. u64 new_i_size;
  816. u64 i_size = i_size_read(inode);
  817. struct rb_node *node;
  818. struct rb_node *prev = NULL;
  819. struct btrfs_ordered_extent *test;
  820. int ret = 1;
  821. if (ordered)
  822. offset = entry_end(ordered);
  823. else
  824. offset = ALIGN(offset, BTRFS_I(inode)->root->sectorsize);
  825. spin_lock_irq(&tree->lock);
  826. disk_i_size = BTRFS_I(inode)->disk_i_size;
  827. /* truncate file */
  828. if (disk_i_size > i_size) {
  829. BTRFS_I(inode)->disk_i_size = i_size;
  830. ret = 0;
  831. goto out;
  832. }
  833. /*
  834. * if the disk i_size is already at the inode->i_size, or
  835. * this ordered extent is inside the disk i_size, we're done
  836. */
  837. if (disk_i_size == i_size)
  838. goto out;
  839. /*
  840. * We still need to update disk_i_size if outstanding_isize is greater
  841. * than disk_i_size.
  842. */
  843. if (offset <= disk_i_size &&
  844. (!ordered || ordered->outstanding_isize <= disk_i_size))
  845. goto out;
  846. /*
  847. * walk backward from this ordered extent to disk_i_size.
  848. * if we find an ordered extent then we can't update disk i_size
  849. * yet
  850. */
  851. if (ordered) {
  852. node = rb_prev(&ordered->rb_node);
  853. } else {
  854. prev = tree_search(tree, offset);
  855. /*
  856. * we insert file extents without involving ordered struct,
  857. * so there should be no ordered struct cover this offset
  858. */
  859. if (prev) {
  860. test = rb_entry(prev, struct btrfs_ordered_extent,
  861. rb_node);
  862. BUG_ON(offset_in_entry(test, offset));
  863. }
  864. node = prev;
  865. }
  866. for (; node; node = rb_prev(node)) {
  867. test = rb_entry(node, struct btrfs_ordered_extent, rb_node);
  868. /* We treat this entry as if it doesnt exist */
  869. if (test_bit(BTRFS_ORDERED_UPDATED_ISIZE, &test->flags))
  870. continue;
  871. if (test->file_offset + test->len <= disk_i_size)
  872. break;
  873. if (test->file_offset >= i_size)
  874. break;
  875. if (entry_end(test) > disk_i_size) {
  876. /*
  877. * we don't update disk_i_size now, so record this
  878. * undealt i_size. Or we will not know the real
  879. * i_size.
  880. */
  881. if (test->outstanding_isize < offset)
  882. test->outstanding_isize = offset;
  883. if (ordered &&
  884. ordered->outstanding_isize >
  885. test->outstanding_isize)
  886. test->outstanding_isize =
  887. ordered->outstanding_isize;
  888. goto out;
  889. }
  890. }
  891. new_i_size = min_t(u64, offset, i_size);
  892. /*
  893. * Some ordered extents may completed before the current one, and
  894. * we hold the real i_size in ->outstanding_isize.
  895. */
  896. if (ordered && ordered->outstanding_isize > new_i_size)
  897. new_i_size = min_t(u64, ordered->outstanding_isize, i_size);
  898. BTRFS_I(inode)->disk_i_size = new_i_size;
  899. ret = 0;
  900. out:
  901. /*
  902. * We need to do this because we can't remove ordered extents until
  903. * after the i_disk_size has been updated and then the inode has been
  904. * updated to reflect the change, so we need to tell anybody who finds
  905. * this ordered extent that we've already done all the real work, we
  906. * just haven't completed all the other work.
  907. */
  908. if (ordered)
  909. set_bit(BTRFS_ORDERED_UPDATED_ISIZE, &ordered->flags);
  910. spin_unlock_irq(&tree->lock);
  911. return ret;
  912. }
  913. /*
  914. * search the ordered extents for one corresponding to 'offset' and
  915. * try to find a checksum. This is used because we allow pages to
  916. * be reclaimed before their checksum is actually put into the btree
  917. */
  918. int btrfs_find_ordered_sum(struct inode *inode, u64 offset, u64 disk_bytenr,
  919. u32 *sum, int len)
  920. {
  921. struct btrfs_ordered_sum *ordered_sum;
  922. struct btrfs_ordered_extent *ordered;
  923. struct btrfs_ordered_inode_tree *tree = &BTRFS_I(inode)->ordered_tree;
  924. unsigned long num_sectors;
  925. unsigned long i;
  926. u32 sectorsize = BTRFS_I(inode)->root->sectorsize;
  927. int index = 0;
  928. ordered = btrfs_lookup_ordered_extent(inode, offset);
  929. if (!ordered)
  930. return 0;
  931. spin_lock_irq(&tree->lock);
  932. list_for_each_entry_reverse(ordered_sum, &ordered->list, list) {
  933. if (disk_bytenr >= ordered_sum->bytenr &&
  934. disk_bytenr < ordered_sum->bytenr + ordered_sum->len) {
  935. i = (disk_bytenr - ordered_sum->bytenr) >>
  936. inode->i_sb->s_blocksize_bits;
  937. num_sectors = ordered_sum->len >>
  938. inode->i_sb->s_blocksize_bits;
  939. num_sectors = min_t(int, len - index, num_sectors - i);
  940. memcpy(sum + index, ordered_sum->sums + i,
  941. num_sectors);
  942. index += (int)num_sectors;
  943. if (index == len)
  944. goto out;
  945. disk_bytenr += num_sectors * sectorsize;
  946. }
  947. }
  948. out:
  949. spin_unlock_irq(&tree->lock);
  950. btrfs_put_ordered_extent(ordered);
  951. return index;
  952. }
  953. /*
  954. * add a given inode to the list of inodes that must be fully on
  955. * disk before a transaction commit finishes.
  956. *
  957. * This basically gives us the ext3 style data=ordered mode, and it is mostly
  958. * used to make sure renamed files are fully on disk.
  959. *
  960. * It is a noop if the inode is already fully on disk.
  961. *
  962. * If trans is not null, we'll do a friendly check for a transaction that
  963. * is already flushing things and force the IO down ourselves.
  964. */
  965. void btrfs_add_ordered_operation(struct btrfs_trans_handle *trans,
  966. struct btrfs_root *root, struct inode *inode)
  967. {
  968. struct btrfs_transaction *cur_trans = trans->transaction;
  969. u64 last_mod;
  970. last_mod = max(BTRFS_I(inode)->generation, BTRFS_I(inode)->last_trans);
  971. /*
  972. * if this file hasn't been changed since the last transaction
  973. * commit, we can safely return without doing anything
  974. */
  975. if (last_mod < root->fs_info->last_trans_committed)
  976. return;
  977. spin_lock(&root->fs_info->ordered_root_lock);
  978. if (list_empty(&BTRFS_I(inode)->ordered_operations)) {
  979. list_add_tail(&BTRFS_I(inode)->ordered_operations,
  980. &cur_trans->ordered_operations);
  981. }
  982. spin_unlock(&root->fs_info->ordered_root_lock);
  983. }
  984. int __init ordered_data_init(void)
  985. {
  986. btrfs_ordered_extent_cache = kmem_cache_create("btrfs_ordered_extent",
  987. sizeof(struct btrfs_ordered_extent), 0,
  988. SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD,
  989. NULL);
  990. if (!btrfs_ordered_extent_cache)
  991. return -ENOMEM;
  992. return 0;
  993. }
  994. void ordered_data_exit(void)
  995. {
  996. if (btrfs_ordered_extent_cache)
  997. kmem_cache_destroy(btrfs_ordered_extent_cache);
  998. }