1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16#include <linux/kernel.h>
17#include <linux/module.h>
18#include <linux/spinlock.h>
19#include <linux/sched.h>
20#include <linux/fs.h>
21#include <linux/mm.h>
22#include <linux/writeback.h>
23#include <linux/blkdev.h>
24#include <linux/backing-dev.h>
25#include <linux/buffer_head.h>
26#include "internal.h"
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41static int writeback_acquire(struct backing_dev_info *bdi)
42{
43 return !test_and_set_bit(BDI_pdflush, &bdi->state);
44}
45
46
47
48
49
50
51
52int writeback_in_progress(struct backing_dev_info *bdi)
53{
54 return test_bit(BDI_pdflush, &bdi->state);
55}
56
57
58
59
60
61static void writeback_release(struct backing_dev_info *bdi)
62{
63 BUG_ON(!writeback_in_progress(bdi));
64 clear_bit(BDI_pdflush, &bdi->state);
65}
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94void __mark_inode_dirty(struct inode *inode, int flags)
95{
96 struct super_block *sb = inode->i_sb;
97
98
99
100
101
102 if (flags & (I_DIRTY_SYNC | I_DIRTY_DATASYNC)) {
103 if (sb->s_op->dirty_inode)
104 sb->s_op->dirty_inode(inode);
105 }
106
107
108
109
110
111 smp_mb();
112
113
114 if ((inode->i_state & flags) == flags)
115 return;
116
117 if (unlikely(block_dump)) {
118 struct dentry *dentry = NULL;
119 const char *name = "?";
120
121 if (!list_empty(&inode->i_dentry)) {
122 dentry = list_entry(inode->i_dentry.next,
123 struct dentry, d_alias);
124 if (dentry && dentry->d_name.name)
125 name = (const char *) dentry->d_name.name;
126 }
127
128 if (inode->i_ino || strcmp(inode->i_sb->s_id, "bdev"))
129 printk(KERN_DEBUG
130 "%s(%d): dirtied inode %lu (%s) on %s\n",
131 current->comm, task_pid_nr(current), inode->i_ino,
132 name, inode->i_sb->s_id);
133 }
134
135 spin_lock(&inode_lock);
136 if ((inode->i_state & flags) != flags) {
137 const int was_dirty = inode->i_state & I_DIRTY;
138
139 inode->i_state |= flags;
140
141
142
143
144
145
146 if (inode->i_state & I_SYNC)
147 goto out;
148
149
150
151
152
153 if (!S_ISBLK(inode->i_mode)) {
154 if (hlist_unhashed(&inode->i_hash))
155 goto out;
156 }
157 if (inode->i_state & (I_FREEING|I_CLEAR))
158 goto out;
159
160
161
162
163
164 if (!was_dirty) {
165 inode->dirtied_when = jiffies;
166 list_move(&inode->i_list, &sb->s_dirty);
167 }
168 }
169out:
170 spin_unlock(&inode_lock);
171}
172
173EXPORT_SYMBOL(__mark_inode_dirty);
174
175static int write_inode(struct inode *inode, int sync)
176{
177 if (inode->i_sb->s_op->write_inode && !is_bad_inode(inode))
178 return inode->i_sb->s_op->write_inode(inode, sync);
179 return 0;
180}
181
182
183
184
185
186
187
188
189
190
191static void redirty_tail(struct inode *inode)
192{
193 struct super_block *sb = inode->i_sb;
194
195 if (!list_empty(&sb->s_dirty)) {
196 struct inode *tail_inode;
197
198 tail_inode = list_entry(sb->s_dirty.next, struct inode, i_list);
199 if (!time_after_eq(inode->dirtied_when,
200 tail_inode->dirtied_when))
201 inode->dirtied_when = jiffies;
202 }
203 list_move(&inode->i_list, &sb->s_dirty);
204}
205
206
207
208
209static void requeue_io(struct inode *inode)
210{
211 list_move(&inode->i_list, &inode->i_sb->s_more_io);
212}
213
214static void inode_sync_complete(struct inode *inode)
215{
216
217
218
219 smp_mb();
220 wake_up_bit(&inode->i_state, __I_SYNC);
221}
222
223
224
225
226static void move_expired_inodes(struct list_head *delaying_queue,
227 struct list_head *dispatch_queue,
228 unsigned long *older_than_this)
229{
230 while (!list_empty(delaying_queue)) {
231 struct inode *inode = list_entry(delaying_queue->prev,
232 struct inode, i_list);
233 if (older_than_this &&
234 time_after(inode->dirtied_when, *older_than_this))
235 break;
236 list_move(&inode->i_list, dispatch_queue);
237 }
238}
239
240
241
242
243static void queue_io(struct super_block *sb,
244 unsigned long *older_than_this)
245{
246 list_splice_init(&sb->s_more_io, sb->s_io.prev);
247 move_expired_inodes(&sb->s_dirty, &sb->s_io, older_than_this);
248}
249
250int sb_has_dirty_inodes(struct super_block *sb)
251{
252 return !list_empty(&sb->s_dirty) ||
253 !list_empty(&sb->s_io) ||
254 !list_empty(&sb->s_more_io);
255}
256EXPORT_SYMBOL(sb_has_dirty_inodes);
257
258
259
260
261
262
263
264
265
266
267
268static int
269__sync_single_inode(struct inode *inode, struct writeback_control *wbc)
270{
271 unsigned dirty;
272 struct address_space *mapping = inode->i_mapping;
273 int wait = wbc->sync_mode == WB_SYNC_ALL;
274 int ret;
275
276 BUG_ON(inode->i_state & I_SYNC);
277
278
279 dirty = inode->i_state & I_DIRTY;
280 inode->i_state |= I_SYNC;
281 inode->i_state &= ~I_DIRTY;
282
283 spin_unlock(&inode_lock);
284
285 ret = do_writepages(mapping, wbc);
286
287
288 if (dirty & (I_DIRTY_SYNC | I_DIRTY_DATASYNC)) {
289 int err = write_inode(inode, wait);
290 if (ret == 0)
291 ret = err;
292 }
293
294 if (wait) {
295 int err = filemap_fdatawait(mapping);
296 if (ret == 0)
297 ret = err;
298 }
299
300 spin_lock(&inode_lock);
301 inode->i_state &= ~I_SYNC;
302 if (!(inode->i_state & I_FREEING)) {
303 if (!(inode->i_state & I_DIRTY) &&
304 mapping_tagged(mapping, PAGECACHE_TAG_DIRTY)) {
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319 if (wbc->for_kupdate) {
320
321
322
323
324
325 inode->i_state |= I_DIRTY_PAGES;
326 if (wbc->nr_to_write <= 0) {
327
328
329
330 requeue_io(inode);
331 } else {
332
333
334
335 redirty_tail(inode);
336 }
337 } else {
338
339
340
341
342
343
344
345 inode->i_state |= I_DIRTY_PAGES;
346 redirty_tail(inode);
347 }
348 } else if (inode->i_state & I_DIRTY) {
349
350
351
352
353 redirty_tail(inode);
354 } else if (atomic_read(&inode->i_count)) {
355
356
357
358 list_move(&inode->i_list, &inode_in_use);
359 } else {
360
361
362
363 list_move(&inode->i_list, &inode_unused);
364 }
365 }
366 inode_sync_complete(inode);
367 return ret;
368}
369
370
371
372
373
374
375static int
376__writeback_single_inode(struct inode *inode, struct writeback_control *wbc)
377{
378 wait_queue_head_t *wqh;
379
380 if (!atomic_read(&inode->i_count))
381 WARN_ON(!(inode->i_state & (I_WILL_FREE|I_FREEING)));
382 else
383 WARN_ON(inode->i_state & I_WILL_FREE);
384
385 if ((wbc->sync_mode != WB_SYNC_ALL) && (inode->i_state & I_SYNC)) {
386
387
388
389
390
391
392
393 requeue_io(inode);
394 return 0;
395 }
396
397
398
399
400 if (inode->i_state & I_SYNC) {
401 DEFINE_WAIT_BIT(wq, &inode->i_state, __I_SYNC);
402
403 wqh = bit_waitqueue(&inode->i_state, __I_SYNC);
404 do {
405 spin_unlock(&inode_lock);
406 __wait_on_bit(wqh, &wq, inode_wait,
407 TASK_UNINTERRUPTIBLE);
408 spin_lock(&inode_lock);
409 } while (inode->i_state & I_SYNC);
410 }
411 return __sync_single_inode(inode, wbc);
412}
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442void generic_sync_sb_inodes(struct super_block *sb,
443 struct writeback_control *wbc)
444{
445 const unsigned long start = jiffies;
446
447 spin_lock(&inode_lock);
448 if (!wbc->for_kupdate || list_empty(&sb->s_io))
449 queue_io(sb, wbc->older_than_this);
450
451 while (!list_empty(&sb->s_io)) {
452 struct inode *inode = list_entry(sb->s_io.prev,
453 struct inode, i_list);
454 struct address_space *mapping = inode->i_mapping;
455 struct backing_dev_info *bdi = mapping->backing_dev_info;
456 long pages_skipped;
457
458 if (!bdi_cap_writeback_dirty(bdi)) {
459 redirty_tail(inode);
460 if (sb_is_blkdev_sb(sb)) {
461
462
463
464
465 continue;
466 }
467
468
469
470
471
472 break;
473 }
474
475 if (wbc->nonblocking && bdi_write_congested(bdi)) {
476 wbc->encountered_congestion = 1;
477 if (!sb_is_blkdev_sb(sb))
478 break;
479 requeue_io(inode);
480 continue;
481 }
482
483 if (wbc->bdi && bdi != wbc->bdi) {
484 if (!sb_is_blkdev_sb(sb))
485 break;
486 requeue_io(inode);
487 continue;
488 }
489
490
491 if (time_after(inode->dirtied_when, start))
492 break;
493
494
495 if (current_is_pdflush() && !writeback_acquire(bdi))
496 break;
497
498 BUG_ON(inode->i_state & I_FREEING);
499 __iget(inode);
500 pages_skipped = wbc->pages_skipped;
501 __writeback_single_inode(inode, wbc);
502 if (wbc->sync_mode == WB_SYNC_HOLD) {
503 inode->dirtied_when = jiffies;
504 list_move(&inode->i_list, &sb->s_dirty);
505 }
506 if (current_is_pdflush())
507 writeback_release(bdi);
508 if (wbc->pages_skipped != pages_skipped) {
509
510
511
512
513 redirty_tail(inode);
514 }
515 spin_unlock(&inode_lock);
516 iput(inode);
517 cond_resched();
518 spin_lock(&inode_lock);
519 if (wbc->nr_to_write <= 0) {
520 wbc->more_io = 1;
521 break;
522 }
523 if (!list_empty(&sb->s_more_io))
524 wbc->more_io = 1;
525 }
526 spin_unlock(&inode_lock);
527 return;
528}
529EXPORT_SYMBOL_GPL(generic_sync_sb_inodes);
530
531static void sync_sb_inodes(struct super_block *sb,
532 struct writeback_control *wbc)
533{
534 generic_sync_sb_inodes(sb, wbc);
535}
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556void
557writeback_inodes(struct writeback_control *wbc)
558{
559 struct super_block *sb;
560
561 might_sleep();
562 spin_lock(&sb_lock);
563restart:
564 list_for_each_entry_reverse(sb, &super_blocks, s_list) {
565 if (sb_has_dirty_inodes(sb)) {
566
567 sb->s_count++;
568 spin_unlock(&sb_lock);
569
570
571
572
573
574 if (down_read_trylock(&sb->s_umount)) {
575 if (sb->s_root)
576 sync_sb_inodes(sb, wbc);
577 up_read(&sb->s_umount);
578 }
579 spin_lock(&sb_lock);
580 if (__put_super_and_need_restart(sb))
581 goto restart;
582 }
583 if (wbc->nr_to_write <= 0)
584 break;
585 }
586 spin_unlock(&sb_lock);
587}
588
589
590
591
592
593
594
595
596
597
598
599
600void sync_inodes_sb(struct super_block *sb, int wait)
601{
602 struct writeback_control wbc = {
603 .sync_mode = wait ? WB_SYNC_ALL : WB_SYNC_HOLD,
604 .range_start = 0,
605 .range_end = LLONG_MAX,
606 };
607 unsigned long nr_dirty = global_page_state(NR_FILE_DIRTY);
608 unsigned long nr_unstable = global_page_state(NR_UNSTABLE_NFS);
609
610 wbc.nr_to_write = nr_dirty + nr_unstable +
611 (inodes_stat.nr_inodes - inodes_stat.nr_unused) +
612 nr_dirty + nr_unstable;
613 wbc.nr_to_write += wbc.nr_to_write / 2;
614 sync_sb_inodes(sb, &wbc);
615}
616
617
618
619
620static void set_sb_syncing(int val)
621{
622 struct super_block *sb;
623 spin_lock(&sb_lock);
624 list_for_each_entry_reverse(sb, &super_blocks, s_list)
625 sb->s_syncing = val;
626 spin_unlock(&sb_lock);
627}
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648static void __sync_inodes(int wait)
649{
650 struct super_block *sb;
651
652 spin_lock(&sb_lock);
653restart:
654 list_for_each_entry(sb, &super_blocks, s_list) {
655 if (sb->s_syncing)
656 continue;
657 sb->s_syncing = 1;
658 sb->s_count++;
659 spin_unlock(&sb_lock);
660 down_read(&sb->s_umount);
661 if (sb->s_root) {
662 sync_inodes_sb(sb, wait);
663 sync_blockdev(sb->s_bdev);
664 }
665 up_read(&sb->s_umount);
666 spin_lock(&sb_lock);
667 if (__put_super_and_need_restart(sb))
668 goto restart;
669 }
670 spin_unlock(&sb_lock);
671}
672
673void sync_inodes(int wait)
674{
675 set_sb_syncing(0);
676 __sync_inodes(0);
677
678 if (wait) {
679 set_sb_syncing(0);
680 __sync_inodes(1);
681 }
682}
683
684
685
686
687
688
689
690
691
692
693
694int write_inode_now(struct inode *inode, int sync)
695{
696 int ret;
697 struct writeback_control wbc = {
698 .nr_to_write = LONG_MAX,
699 .sync_mode = sync ? WB_SYNC_ALL : WB_SYNC_NONE,
700 .range_start = 0,
701 .range_end = LLONG_MAX,
702 };
703
704 if (!mapping_cap_writeback_dirty(inode->i_mapping))
705 wbc.nr_to_write = 0;
706
707 might_sleep();
708 spin_lock(&inode_lock);
709 ret = __writeback_single_inode(inode, &wbc);
710 spin_unlock(&inode_lock);
711 if (sync)
712 inode_sync_wait(inode);
713 return ret;
714}
715EXPORT_SYMBOL(write_inode_now);
716
717
718
719
720
721
722
723
724
725
726
727
728int sync_inode(struct inode *inode, struct writeback_control *wbc)
729{
730 int ret;
731
732 spin_lock(&inode_lock);
733 ret = __writeback_single_inode(inode, wbc);
734 spin_unlock(&inode_lock);
735 return ret;
736}
737EXPORT_SYMBOL(sync_inode);
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756int generic_osync_inode(struct inode *inode, struct address_space *mapping, int what)
757{
758 int err = 0;
759 int need_write_inode_now = 0;
760 int err2;
761
762 if (what & OSYNC_DATA)
763 err = filemap_fdatawrite(mapping);
764 if (what & (OSYNC_METADATA|OSYNC_DATA)) {
765 err2 = sync_mapping_buffers(mapping);
766 if (!err)
767 err = err2;
768 }
769 if (what & OSYNC_DATA) {
770 err2 = filemap_fdatawait(mapping);
771 if (!err)
772 err = err2;
773 }
774
775 spin_lock(&inode_lock);
776 if ((inode->i_state & I_DIRTY) &&
777 ((what & OSYNC_INODE) || (inode->i_state & I_DIRTY_DATASYNC)))
778 need_write_inode_now = 1;
779 spin_unlock(&inode_lock);
780
781 if (need_write_inode_now) {
782 err2 = write_inode_now(inode, 1);
783 if (!err)
784 err = err2;
785 }
786 else
787 inode_sync_wait(inode);
788
789 return err;
790}
791EXPORT_SYMBOL(generic_osync_inode);