1
2
3
4
5
6
7
8#include <linux/mm.h>
9#include <linux/hugetlb.h>
10#include <linux/mman.h>
11#include <linux/slab.h>
12#include <linux/kernel_stat.h>
13#include <linux/swap.h>
14#include <linux/vmalloc.h>
15#include <linux/pagemap.h>
16#include <linux/namei.h>
17#include <linux/shm.h>
18#include <linux/blkdev.h>
19#include <linux/writeback.h>
20#include <linux/proc_fs.h>
21#include <linux/seq_file.h>
22#include <linux/init.h>
23#include <linux/module.h>
24#include <linux/rmap.h>
25#include <linux/security.h>
26#include <linux/backing-dev.h>
27#include <linux/mutex.h>
28#include <linux/capability.h>
29#include <linux/syscalls.h>
30#include <linux/memcontrol.h>
31
32#include <asm/pgtable.h>
33#include <asm/tlbflush.h>
34#include <linux/swapops.h>
35
36static DEFINE_SPINLOCK(swap_lock);
37static unsigned int nr_swapfiles;
38long total_swap_pages;
39static int swap_overflow;
40static int least_priority;
41
42static const char Bad_file[] = "Bad swap file entry ";
43static const char Unused_file[] = "Unused swap file entry ";
44static const char Bad_offset[] = "Bad swap offset entry ";
45static const char Unused_offset[] = "Unused swap offset entry ";
46
47static struct swap_list_t swap_list = {-1, -1};
48
49static struct swap_info_struct swap_info[MAX_SWAPFILES];
50
51static DEFINE_MUTEX(swapon_mutex);
52
53
54
55
56
57
58static DECLARE_RWSEM(swap_unplug_sem);
59
60void swap_unplug_io_fn(struct backing_dev_info *unused_bdi, struct page *page)
61{
62 swp_entry_t entry;
63
64 down_read(&swap_unplug_sem);
65 entry.val = page_private(page);
66 if (PageSwapCache(page)) {
67 struct block_device *bdev = swap_info[swp_type(entry)].bdev;
68 struct backing_dev_info *bdi;
69
70
71
72
73
74
75
76
77
78 WARN_ON(page_count(page) <= 1);
79
80 bdi = bdev->bd_inode->i_mapping->backing_dev_info;
81 blk_run_backing_dev(bdi, page);
82 }
83 up_read(&swap_unplug_sem);
84}
85
86#define SWAPFILE_CLUSTER 256
87#define LATENCY_LIMIT 256
88
89static inline unsigned long scan_swap_map(struct swap_info_struct *si)
90{
91 unsigned long offset, last_in_cluster;
92 int latency_ration = LATENCY_LIMIT;
93
94
95
96
97
98
99
100
101
102
103
104 si->flags += SWP_SCANNING;
105 if (unlikely(!si->cluster_nr)) {
106 si->cluster_nr = SWAPFILE_CLUSTER - 1;
107 if (si->pages - si->inuse_pages < SWAPFILE_CLUSTER)
108 goto lowest;
109 spin_unlock(&swap_lock);
110
111 offset = si->lowest_bit;
112 last_in_cluster = offset + SWAPFILE_CLUSTER - 1;
113
114
115 for (; last_in_cluster <= si->highest_bit; offset++) {
116 if (si->swap_map[offset])
117 last_in_cluster = offset + SWAPFILE_CLUSTER;
118 else if (offset == last_in_cluster) {
119 spin_lock(&swap_lock);
120 si->cluster_next = offset-SWAPFILE_CLUSTER+1;
121 goto cluster;
122 }
123 if (unlikely(--latency_ration < 0)) {
124 cond_resched();
125 latency_ration = LATENCY_LIMIT;
126 }
127 }
128 spin_lock(&swap_lock);
129 goto lowest;
130 }
131
132 si->cluster_nr--;
133cluster:
134 offset = si->cluster_next;
135 if (offset > si->highest_bit)
136lowest: offset = si->lowest_bit;
137checks: if (!(si->flags & SWP_WRITEOK))
138 goto no_page;
139 if (!si->highest_bit)
140 goto no_page;
141 if (!si->swap_map[offset]) {
142 if (offset == si->lowest_bit)
143 si->lowest_bit++;
144 if (offset == si->highest_bit)
145 si->highest_bit--;
146 si->inuse_pages++;
147 if (si->inuse_pages == si->pages) {
148 si->lowest_bit = si->max;
149 si->highest_bit = 0;
150 }
151 si->swap_map[offset] = 1;
152 si->cluster_next = offset + 1;
153 si->flags -= SWP_SCANNING;
154 return offset;
155 }
156
157 spin_unlock(&swap_lock);
158 while (++offset <= si->highest_bit) {
159 if (!si->swap_map[offset]) {
160 spin_lock(&swap_lock);
161 goto checks;
162 }
163 if (unlikely(--latency_ration < 0)) {
164 cond_resched();
165 latency_ration = LATENCY_LIMIT;
166 }
167 }
168 spin_lock(&swap_lock);
169 goto lowest;
170
171no_page:
172 si->flags -= SWP_SCANNING;
173 return 0;
174}
175
176swp_entry_t get_swap_page(void)
177{
178 struct swap_info_struct *si;
179 pgoff_t offset;
180 int type, next;
181 int wrapped = 0;
182
183 spin_lock(&swap_lock);
184 if (nr_swap_pages <= 0)
185 goto noswap;
186 nr_swap_pages--;
187
188 for (type = swap_list.next; type >= 0 && wrapped < 2; type = next) {
189 si = swap_info + type;
190 next = si->next;
191 if (next < 0 ||
192 (!wrapped && si->prio != swap_info[next].prio)) {
193 next = swap_list.head;
194 wrapped++;
195 }
196
197 if (!si->highest_bit)
198 continue;
199 if (!(si->flags & SWP_WRITEOK))
200 continue;
201
202 swap_list.next = next;
203 offset = scan_swap_map(si);
204 if (offset) {
205 spin_unlock(&swap_lock);
206 return swp_entry(type, offset);
207 }
208 next = swap_list.next;
209 }
210
211 nr_swap_pages++;
212noswap:
213 spin_unlock(&swap_lock);
214 return (swp_entry_t) {0};
215}
216
217swp_entry_t get_swap_page_of_type(int type)
218{
219 struct swap_info_struct *si;
220 pgoff_t offset;
221
222 spin_lock(&swap_lock);
223 si = swap_info + type;
224 if (si->flags & SWP_WRITEOK) {
225 nr_swap_pages--;
226 offset = scan_swap_map(si);
227 if (offset) {
228 spin_unlock(&swap_lock);
229 return swp_entry(type, offset);
230 }
231 nr_swap_pages++;
232 }
233 spin_unlock(&swap_lock);
234 return (swp_entry_t) {0};
235}
236
237static struct swap_info_struct * swap_info_get(swp_entry_t entry)
238{
239 struct swap_info_struct * p;
240 unsigned long offset, type;
241
242 if (!entry.val)
243 goto out;
244 type = swp_type(entry);
245 if (type >= nr_swapfiles)
246 goto bad_nofile;
247 p = & swap_info[type];
248 if (!(p->flags & SWP_USED))
249 goto bad_device;
250 offset = swp_offset(entry);
251 if (offset >= p->max)
252 goto bad_offset;
253 if (!p->swap_map[offset])
254 goto bad_free;
255 spin_lock(&swap_lock);
256 return p;
257
258bad_free:
259 printk(KERN_ERR "swap_free: %s%08lx\n", Unused_offset, entry.val);
260 goto out;
261bad_offset:
262 printk(KERN_ERR "swap_free: %s%08lx\n", Bad_offset, entry.val);
263 goto out;
264bad_device:
265 printk(KERN_ERR "swap_free: %s%08lx\n", Unused_file, entry.val);
266 goto out;
267bad_nofile:
268 printk(KERN_ERR "swap_free: %s%08lx\n", Bad_file, entry.val);
269out:
270 return NULL;
271}
272
273static int swap_entry_free(struct swap_info_struct *p, unsigned long offset)
274{
275 int count = p->swap_map[offset];
276
277 if (count < SWAP_MAP_MAX) {
278 count--;
279 p->swap_map[offset] = count;
280 if (!count) {
281 if (offset < p->lowest_bit)
282 p->lowest_bit = offset;
283 if (offset > p->highest_bit)
284 p->highest_bit = offset;
285 if (p->prio > swap_info[swap_list.next].prio)
286 swap_list.next = p - swap_info;
287 nr_swap_pages++;
288 p->inuse_pages--;
289 }
290 }
291 return count;
292}
293
294
295
296
297
298void swap_free(swp_entry_t entry)
299{
300 struct swap_info_struct * p;
301
302 p = swap_info_get(entry);
303 if (p) {
304 swap_entry_free(p, swp_offset(entry));
305 spin_unlock(&swap_lock);
306 }
307}
308
309
310
311
312static inline int page_swapcount(struct page *page)
313{
314 int count = 0;
315 struct swap_info_struct *p;
316 swp_entry_t entry;
317
318 entry.val = page_private(page);
319 p = swap_info_get(entry);
320 if (p) {
321
322 count = p->swap_map[swp_offset(entry)] - 1;
323 spin_unlock(&swap_lock);
324 }
325 return count;
326}
327
328
329
330
331
332int can_share_swap_page(struct page *page)
333{
334 int count;
335
336 BUG_ON(!PageLocked(page));
337 count = page_mapcount(page);
338 if (count <= 1 && PageSwapCache(page))
339 count += page_swapcount(page);
340 return count == 1;
341}
342
343
344
345
346
347static int remove_exclusive_swap_page_count(struct page *page, int count)
348{
349 int retval;
350 struct swap_info_struct * p;
351 swp_entry_t entry;
352
353 BUG_ON(PagePrivate(page));
354 BUG_ON(!PageLocked(page));
355
356 if (!PageSwapCache(page))
357 return 0;
358 if (PageWriteback(page))
359 return 0;
360 if (page_count(page) != count)
361 return 0;
362
363 entry.val = page_private(page);
364 p = swap_info_get(entry);
365 if (!p)
366 return 0;
367
368
369 retval = 0;
370 if (p->swap_map[swp_offset(entry)] == 1) {
371
372 spin_lock_irq(&swapper_space.tree_lock);
373 if ((page_count(page) == count) && !PageWriteback(page)) {
374 __delete_from_swap_cache(page);
375 SetPageDirty(page);
376 retval = 1;
377 }
378 spin_unlock_irq(&swapper_space.tree_lock);
379 }
380 spin_unlock(&swap_lock);
381
382 if (retval) {
383 swap_free(entry);
384 page_cache_release(page);
385 }
386
387 return retval;
388}
389
390
391
392
393
394int remove_exclusive_swap_page(struct page *page)
395{
396 return remove_exclusive_swap_page_count(page, 2);
397}
398
399
400
401
402
403
404int remove_exclusive_swap_page_ref(struct page *page)
405{
406 return remove_exclusive_swap_page_count(page, 2 + page_mapcount(page));
407}
408
409
410
411
412
413void free_swap_and_cache(swp_entry_t entry)
414{
415 struct swap_info_struct * p;
416 struct page *page = NULL;
417
418 if (is_migration_entry(entry))
419 return;
420
421 p = swap_info_get(entry);
422 if (p) {
423 if (swap_entry_free(p, swp_offset(entry)) == 1) {
424 page = find_get_page(&swapper_space, entry.val);
425 if (page && !trylock_page(page)) {
426 page_cache_release(page);
427 page = NULL;
428 }
429 }
430 spin_unlock(&swap_lock);
431 }
432 if (page) {
433 int one_user;
434
435 BUG_ON(PagePrivate(page));
436 one_user = (page_count(page) == 2);
437
438
439 if (PageSwapCache(page) && !PageWriteback(page) &&
440 (one_user || vm_swap_full())) {
441 delete_from_swap_cache(page);
442 SetPageDirty(page);
443 }
444 unlock_page(page);
445 page_cache_release(page);
446 }
447}
448
449#ifdef CONFIG_HIBERNATION
450
451
452
453
454
455
456
457
458int swap_type_of(dev_t device, sector_t offset, struct block_device **bdev_p)
459{
460 struct block_device *bdev = NULL;
461 int i;
462
463 if (device)
464 bdev = bdget(device);
465
466 spin_lock(&swap_lock);
467 for (i = 0; i < nr_swapfiles; i++) {
468 struct swap_info_struct *sis = swap_info + i;
469
470 if (!(sis->flags & SWP_WRITEOK))
471 continue;
472
473 if (!bdev) {
474 if (bdev_p)
475 *bdev_p = sis->bdev;
476
477 spin_unlock(&swap_lock);
478 return i;
479 }
480 if (bdev == sis->bdev) {
481 struct swap_extent *se;
482
483 se = list_entry(sis->extent_list.next,
484 struct swap_extent, list);
485 if (se->start_block == offset) {
486 if (bdev_p)
487 *bdev_p = sis->bdev;
488
489 spin_unlock(&swap_lock);
490 bdput(bdev);
491 return i;
492 }
493 }
494 }
495 spin_unlock(&swap_lock);
496 if (bdev)
497 bdput(bdev);
498
499 return -ENODEV;
500}
501
502
503
504
505
506
507
508unsigned int count_swap_pages(int type, int free)
509{
510 unsigned int n = 0;
511
512 if (type < nr_swapfiles) {
513 spin_lock(&swap_lock);
514 if (swap_info[type].flags & SWP_WRITEOK) {
515 n = swap_info[type].pages;
516 if (free)
517 n -= swap_info[type].inuse_pages;
518 }
519 spin_unlock(&swap_lock);
520 }
521 return n;
522}
523#endif
524
525
526
527
528
529
530static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd,
531 unsigned long addr, swp_entry_t entry, struct page *page)
532{
533 spinlock_t *ptl;
534 pte_t *pte;
535 int ret = 1;
536
537 if (mem_cgroup_charge(page, vma->vm_mm, GFP_KERNEL))
538 ret = -ENOMEM;
539
540 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
541 if (unlikely(!pte_same(*pte, swp_entry_to_pte(entry)))) {
542 if (ret > 0)
543 mem_cgroup_uncharge_page(page);
544 ret = 0;
545 goto out;
546 }
547
548 inc_mm_counter(vma->vm_mm, anon_rss);
549 get_page(page);
550 set_pte_at(vma->vm_mm, addr, pte,
551 pte_mkold(mk_pte(page, vma->vm_page_prot)));
552 page_add_anon_rmap(page, vma, addr);
553 swap_free(entry);
554
555
556
557
558 activate_page(page);
559out:
560 pte_unmap_unlock(pte, ptl);
561 return ret;
562}
563
564static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
565 unsigned long addr, unsigned long end,
566 swp_entry_t entry, struct page *page)
567{
568 pte_t swp_pte = swp_entry_to_pte(entry);
569 pte_t *pte;
570 int ret = 0;
571
572
573
574
575
576
577
578
579
580
581 pte = pte_offset_map(pmd, addr);
582 do {
583
584
585
586
587 if (unlikely(pte_same(*pte, swp_pte))) {
588 pte_unmap(pte);
589 ret = unuse_pte(vma, pmd, addr, entry, page);
590 if (ret)
591 goto out;
592 pte = pte_offset_map(pmd, addr);
593 }
594 } while (pte++, addr += PAGE_SIZE, addr != end);
595 pte_unmap(pte - 1);
596out:
597 return ret;
598}
599
600static inline int unuse_pmd_range(struct vm_area_struct *vma, pud_t *pud,
601 unsigned long addr, unsigned long end,
602 swp_entry_t entry, struct page *page)
603{
604 pmd_t *pmd;
605 unsigned long next;
606 int ret;
607
608 pmd = pmd_offset(pud, addr);
609 do {
610 next = pmd_addr_end(addr, end);
611 if (pmd_none_or_clear_bad(pmd))
612 continue;
613 ret = unuse_pte_range(vma, pmd, addr, next, entry, page);
614 if (ret)
615 return ret;
616 } while (pmd++, addr = next, addr != end);
617 return 0;
618}
619
620static inline int unuse_pud_range(struct vm_area_struct *vma, pgd_t *pgd,
621 unsigned long addr, unsigned long end,
622 swp_entry_t entry, struct page *page)
623{
624 pud_t *pud;
625 unsigned long next;
626 int ret;
627
628 pud = pud_offset(pgd, addr);
629 do {
630 next = pud_addr_end(addr, end);
631 if (pud_none_or_clear_bad(pud))
632 continue;
633 ret = unuse_pmd_range(vma, pud, addr, next, entry, page);
634 if (ret)
635 return ret;
636 } while (pud++, addr = next, addr != end);
637 return 0;
638}
639
640static int unuse_vma(struct vm_area_struct *vma,
641 swp_entry_t entry, struct page *page)
642{
643 pgd_t *pgd;
644 unsigned long addr, end, next;
645 int ret;
646
647 if (page->mapping) {
648 addr = page_address_in_vma(page, vma);
649 if (addr == -EFAULT)
650 return 0;
651 else
652 end = addr + PAGE_SIZE;
653 } else {
654 addr = vma->vm_start;
655 end = vma->vm_end;
656 }
657
658 pgd = pgd_offset(vma->vm_mm, addr);
659 do {
660 next = pgd_addr_end(addr, end);
661 if (pgd_none_or_clear_bad(pgd))
662 continue;
663 ret = unuse_pud_range(vma, pgd, addr, next, entry, page);
664 if (ret)
665 return ret;
666 } while (pgd++, addr = next, addr != end);
667 return 0;
668}
669
670static int unuse_mm(struct mm_struct *mm,
671 swp_entry_t entry, struct page *page)
672{
673 struct vm_area_struct *vma;
674 int ret = 0;
675
676 if (!down_read_trylock(&mm->mmap_sem)) {
677
678
679
680
681 activate_page(page);
682 unlock_page(page);
683 down_read(&mm->mmap_sem);
684 lock_page(page);
685 }
686 for (vma = mm->mmap; vma; vma = vma->vm_next) {
687 if (vma->anon_vma && (ret = unuse_vma(vma, entry, page)))
688 break;
689 }
690 up_read(&mm->mmap_sem);
691 return (ret < 0)? ret: 0;
692}
693
694
695
696
697
698static unsigned int find_next_to_unuse(struct swap_info_struct *si,
699 unsigned int prev)
700{
701 unsigned int max = si->max;
702 unsigned int i = prev;
703 int count;
704
705
706
707
708
709
710
711 for (;;) {
712 if (++i >= max) {
713 if (!prev) {
714 i = 0;
715 break;
716 }
717
718
719
720
721 max = prev + 1;
722 prev = 0;
723 i = 1;
724 }
725 count = si->swap_map[i];
726 if (count && count != SWAP_MAP_BAD)
727 break;
728 }
729 return i;
730}
731
732
733
734
735
736
737static int try_to_unuse(unsigned int type)
738{
739 struct swap_info_struct * si = &swap_info[type];
740 struct mm_struct *start_mm;
741 unsigned short *swap_map;
742 unsigned short swcount;
743 struct page *page;
744 swp_entry_t entry;
745 unsigned int i = 0;
746 int retval = 0;
747 int reset_overflow = 0;
748 int shmem;
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765 start_mm = &init_mm;
766 atomic_inc(&init_mm.mm_users);
767
768
769
770
771
772
773 while ((i = find_next_to_unuse(si, i)) != 0) {
774 if (signal_pending(current)) {
775 retval = -EINTR;
776 break;
777 }
778
779
780
781
782
783
784 swap_map = &si->swap_map[i];
785 entry = swp_entry(type, i);
786 page = read_swap_cache_async(entry,
787 GFP_HIGHUSER_MOVABLE, NULL, 0);
788 if (!page) {
789
790
791
792
793
794
795 if (!*swap_map)
796 continue;
797 retval = -ENOMEM;
798 break;
799 }
800
801
802
803
804 if (atomic_read(&start_mm->mm_users) == 1) {
805 mmput(start_mm);
806 start_mm = &init_mm;
807 atomic_inc(&init_mm.mm_users);
808 }
809
810
811
812
813
814
815
816
817
818 wait_on_page_locked(page);
819 wait_on_page_writeback(page);
820 lock_page(page);
821 wait_on_page_writeback(page);
822
823
824
825
826
827
828 shmem = 0;
829 swcount = *swap_map;
830 if (swcount > 1) {
831 if (start_mm == &init_mm)
832 shmem = shmem_unuse(entry, page);
833 else
834 retval = unuse_mm(start_mm, entry, page);
835 }
836 if (*swap_map > 1) {
837 int set_start_mm = (*swap_map >= swcount);
838 struct list_head *p = &start_mm->mmlist;
839 struct mm_struct *new_start_mm = start_mm;
840 struct mm_struct *prev_mm = start_mm;
841 struct mm_struct *mm;
842
843 atomic_inc(&new_start_mm->mm_users);
844 atomic_inc(&prev_mm->mm_users);
845 spin_lock(&mmlist_lock);
846 while (*swap_map > 1 && !retval && !shmem &&
847 (p = p->next) != &start_mm->mmlist) {
848 mm = list_entry(p, struct mm_struct, mmlist);
849 if (!atomic_inc_not_zero(&mm->mm_users))
850 continue;
851 spin_unlock(&mmlist_lock);
852 mmput(prev_mm);
853 prev_mm = mm;
854
855 cond_resched();
856
857 swcount = *swap_map;
858 if (swcount <= 1)
859 ;
860 else if (mm == &init_mm) {
861 set_start_mm = 1;
862 shmem = shmem_unuse(entry, page);
863 } else
864 retval = unuse_mm(mm, entry, page);
865 if (set_start_mm && *swap_map < swcount) {
866 mmput(new_start_mm);
867 atomic_inc(&mm->mm_users);
868 new_start_mm = mm;
869 set_start_mm = 0;
870 }
871 spin_lock(&mmlist_lock);
872 }
873 spin_unlock(&mmlist_lock);
874 mmput(prev_mm);
875 mmput(start_mm);
876 start_mm = new_start_mm;
877 }
878 if (shmem) {
879
880 if (shmem > 0)
881 continue;
882 retval = shmem;
883 break;
884 }
885 if (retval) {
886 unlock_page(page);
887 page_cache_release(page);
888 break;
889 }
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904 if (*swap_map == SWAP_MAP_MAX) {
905 spin_lock(&swap_lock);
906 *swap_map = 1;
907 spin_unlock(&swap_lock);
908 reset_overflow = 1;
909 }
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924 if ((*swap_map > 1) && PageDirty(page) && PageSwapCache(page)) {
925 struct writeback_control wbc = {
926 .sync_mode = WB_SYNC_NONE,
927 };
928
929 swap_writepage(page, &wbc);
930 lock_page(page);
931 wait_on_page_writeback(page);
932 }
933 if (PageSwapCache(page))
934 delete_from_swap_cache(page);
935
936
937
938
939
940
941 SetPageDirty(page);
942 unlock_page(page);
943 page_cache_release(page);
944
945
946
947
948
949 cond_resched();
950 }
951
952 mmput(start_mm);
953 if (reset_overflow) {
954 printk(KERN_WARNING "swapoff: cleared swap entry overflow\n");
955 swap_overflow = 0;
956 }
957 return retval;
958}
959
960
961
962
963
964
965
966static void drain_mmlist(void)
967{
968 struct list_head *p, *next;
969 unsigned int i;
970
971 for (i = 0; i < nr_swapfiles; i++)
972 if (swap_info[i].inuse_pages)
973 return;
974 spin_lock(&mmlist_lock);
975 list_for_each_safe(p, next, &init_mm.mmlist)
976 list_del_init(p);
977 spin_unlock(&mmlist_lock);
978}
979
980
981
982
983
984sector_t map_swap_page(struct swap_info_struct *sis, pgoff_t offset)
985{
986 struct swap_extent *se = sis->curr_swap_extent;
987 struct swap_extent *start_se = se;
988
989 for ( ; ; ) {
990 struct list_head *lh;
991
992 if (se->start_page <= offset &&
993 offset < (se->start_page + se->nr_pages)) {
994 return se->start_block + (offset - se->start_page);
995 }
996 lh = se->list.next;
997 if (lh == &sis->extent_list)
998 lh = lh->next;
999 se = list_entry(lh, struct swap_extent, list);
1000 sis->curr_swap_extent = se;
1001 BUG_ON(se == start_se);
1002 }
1003}
1004
1005#ifdef CONFIG_HIBERNATION
1006
1007
1008
1009
1010sector_t swapdev_block(int swap_type, pgoff_t offset)
1011{
1012 struct swap_info_struct *sis;
1013
1014 if (swap_type >= nr_swapfiles)
1015 return 0;
1016
1017 sis = swap_info + swap_type;
1018 return (sis->flags & SWP_WRITEOK) ? map_swap_page(sis, offset) : 0;
1019}
1020#endif
1021
1022
1023
1024
1025static void destroy_swap_extents(struct swap_info_struct *sis)
1026{
1027 while (!list_empty(&sis->extent_list)) {
1028 struct swap_extent *se;
1029
1030 se = list_entry(sis->extent_list.next,
1031 struct swap_extent, list);
1032 list_del(&se->list);
1033 kfree(se);
1034 }
1035}
1036
1037
1038
1039
1040
1041
1042
1043static int
1044add_swap_extent(struct swap_info_struct *sis, unsigned long start_page,
1045 unsigned long nr_pages, sector_t start_block)
1046{
1047 struct swap_extent *se;
1048 struct swap_extent *new_se;
1049 struct list_head *lh;
1050
1051 lh = sis->extent_list.prev;
1052 if (lh != &sis->extent_list) {
1053 se = list_entry(lh, struct swap_extent, list);
1054 BUG_ON(se->start_page + se->nr_pages != start_page);
1055 if (se->start_block + se->nr_pages == start_block) {
1056
1057 se->nr_pages += nr_pages;
1058 return 0;
1059 }
1060 }
1061
1062
1063
1064
1065 new_se = kmalloc(sizeof(*se), GFP_KERNEL);
1066 if (new_se == NULL)
1067 return -ENOMEM;
1068 new_se->start_page = start_page;
1069 new_se->nr_pages = nr_pages;
1070 new_se->start_block = start_block;
1071
1072 list_add_tail(&new_se->list, &sis->extent_list);
1073 return 1;
1074}
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107static int setup_swap_extents(struct swap_info_struct *sis, sector_t *span)
1108{
1109 struct inode *inode;
1110 unsigned blocks_per_page;
1111 unsigned long page_no;
1112 unsigned blkbits;
1113 sector_t probe_block;
1114 sector_t last_block;
1115 sector_t lowest_block = -1;
1116 sector_t highest_block = 0;
1117 int nr_extents = 0;
1118 int ret;
1119
1120 inode = sis->swap_file->f_mapping->host;
1121 if (S_ISBLK(inode->i_mode)) {
1122 ret = add_swap_extent(sis, 0, sis->max, 0);
1123 *span = sis->pages;
1124 goto done;
1125 }
1126
1127 blkbits = inode->i_blkbits;
1128 blocks_per_page = PAGE_SIZE >> blkbits;
1129
1130
1131
1132
1133
1134 probe_block = 0;
1135 page_no = 0;
1136 last_block = i_size_read(inode) >> blkbits;
1137 while ((probe_block + blocks_per_page) <= last_block &&
1138 page_no < sis->max) {
1139 unsigned block_in_page;
1140 sector_t first_block;
1141
1142 first_block = bmap(inode, probe_block);
1143 if (first_block == 0)
1144 goto bad_bmap;
1145
1146
1147
1148
1149 if (first_block & (blocks_per_page - 1)) {
1150 probe_block++;
1151 goto reprobe;
1152 }
1153
1154 for (block_in_page = 1; block_in_page < blocks_per_page;
1155 block_in_page++) {
1156 sector_t block;
1157
1158 block = bmap(inode, probe_block + block_in_page);
1159 if (block == 0)
1160 goto bad_bmap;
1161 if (block != first_block + block_in_page) {
1162
1163 probe_block++;
1164 goto reprobe;
1165 }
1166 }
1167
1168 first_block >>= (PAGE_SHIFT - blkbits);
1169 if (page_no) {
1170 if (first_block < lowest_block)
1171 lowest_block = first_block;
1172 if (first_block > highest_block)
1173 highest_block = first_block;
1174 }
1175
1176
1177
1178
1179 ret = add_swap_extent(sis, page_no, 1, first_block);
1180 if (ret < 0)
1181 goto out;
1182 nr_extents += ret;
1183 page_no++;
1184 probe_block += blocks_per_page;
1185reprobe:
1186 continue;
1187 }
1188 ret = nr_extents;
1189 *span = 1 + highest_block - lowest_block;
1190 if (page_no == 0)
1191 page_no = 1;
1192 sis->max = page_no;
1193 sis->pages = page_no - 1;
1194 sis->highest_bit = page_no - 1;
1195done:
1196 sis->curr_swap_extent = list_entry(sis->extent_list.prev,
1197 struct swap_extent, list);
1198 goto out;
1199bad_bmap:
1200 printk(KERN_ERR "swapon: swapfile has holes\n");
1201 ret = -EINVAL;
1202out:
1203 return ret;
1204}
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226asmlinkage long sys_swapoff(const char __user * specialfile)
1227{
1228 struct swap_info_struct * p = NULL;
1229 unsigned short *swap_map;
1230 struct file *swap_file, *victim;
1231 struct address_space *mapping;
1232 struct inode *inode;
1233 char * pathname;
1234 int i, type, prev;
1235 int err;
1236
1237 if (!capable(CAP_SYS_ADMIN))
1238 return -EPERM;
1239
1240 pathname = getname(specialfile);
1241 err = PTR_ERR(pathname);
1242 if (IS_ERR(pathname))
1243 goto out;
1244
1245 victim = filp_open(pathname, O_RDWR|O_LARGEFILE, 0);
1246 putname(pathname);
1247 err = PTR_ERR(victim);
1248 if (IS_ERR(victim))
1249 goto out;
1250
1251 mapping = victim->f_mapping;
1252 prev = -1;
1253 spin_lock(&swap_lock);
1254 for (type = swap_list.head; type >= 0; type = swap_info[type].next) {
1255 p = swap_info + type;
1256 if ((p->flags & SWP_ACTIVE) == SWP_ACTIVE) {
1257 if (p->swap_file->f_mapping == mapping)
1258 break;
1259 }
1260 prev = type;
1261 }
1262 if (type < 0) {
1263 err = -EINVAL;
1264 spin_unlock(&swap_lock);
1265 goto out_dput;
1266 }
1267 if (!security_vm_enough_memory(p->pages))
1268 vm_unacct_memory(p->pages);
1269 else {
1270 err = -ENOMEM;
1271 spin_unlock(&swap_lock);
1272 goto out_dput;
1273 }
1274 if (prev < 0) {
1275 swap_list.head = p->next;
1276 } else {
1277 swap_info[prev].next = p->next;
1278 }
1279 if (type == swap_list.next) {
1280
1281 swap_list.next = swap_list.head;
1282 }
1283 if (p->prio < 0) {
1284 for (i = p->next; i >= 0; i = swap_info[i].next)
1285 swap_info[i].prio = p->prio--;
1286 least_priority++;
1287 }
1288 nr_swap_pages -= p->pages;
1289 total_swap_pages -= p->pages;
1290 p->flags &= ~SWP_WRITEOK;
1291 spin_unlock(&swap_lock);
1292
1293 current->flags |= PF_SWAPOFF;
1294 err = try_to_unuse(type);
1295 current->flags &= ~PF_SWAPOFF;
1296
1297 if (err) {
1298
1299 spin_lock(&swap_lock);
1300 if (p->prio < 0)
1301 p->prio = --least_priority;
1302 prev = -1;
1303 for (i = swap_list.head; i >= 0; i = swap_info[i].next) {
1304 if (p->prio >= swap_info[i].prio)
1305 break;
1306 prev = i;
1307 }
1308 p->next = i;
1309 if (prev < 0)
1310 swap_list.head = swap_list.next = p - swap_info;
1311 else
1312 swap_info[prev].next = p - swap_info;
1313 nr_swap_pages += p->pages;
1314 total_swap_pages += p->pages;
1315 p->flags |= SWP_WRITEOK;
1316 spin_unlock(&swap_lock);
1317 goto out_dput;
1318 }
1319
1320
1321 down_write(&swap_unplug_sem);
1322 up_write(&swap_unplug_sem);
1323
1324 destroy_swap_extents(p);
1325 mutex_lock(&swapon_mutex);
1326 spin_lock(&swap_lock);
1327 drain_mmlist();
1328
1329
1330 p->highest_bit = 0;
1331 while (p->flags >= SWP_SCANNING) {
1332 spin_unlock(&swap_lock);
1333 schedule_timeout_uninterruptible(1);
1334 spin_lock(&swap_lock);
1335 }
1336
1337 swap_file = p->swap_file;
1338 p->swap_file = NULL;
1339 p->max = 0;
1340 swap_map = p->swap_map;
1341 p->swap_map = NULL;
1342 p->flags = 0;
1343 spin_unlock(&swap_lock);
1344 mutex_unlock(&swapon_mutex);
1345 vfree(swap_map);
1346 inode = mapping->host;
1347 if (S_ISBLK(inode->i_mode)) {
1348 struct block_device *bdev = I_BDEV(inode);
1349 set_blocksize(bdev, p->old_block_size);
1350 bd_release(bdev);
1351 } else {
1352 mutex_lock(&inode->i_mutex);
1353 inode->i_flags &= ~S_SWAPFILE;
1354 mutex_unlock(&inode->i_mutex);
1355 }
1356 filp_close(swap_file, NULL);
1357 err = 0;
1358
1359out_dput:
1360 filp_close(victim, NULL);
1361out:
1362 return err;
1363}
1364
1365#ifdef CONFIG_PROC_FS
1366
1367static void *swap_start(struct seq_file *swap, loff_t *pos)
1368{
1369 struct swap_info_struct *ptr = swap_info;
1370 int i;
1371 loff_t l = *pos;
1372
1373 mutex_lock(&swapon_mutex);
1374
1375 if (!l)
1376 return SEQ_START_TOKEN;
1377
1378 for (i = 0; i < nr_swapfiles; i++, ptr++) {
1379 if (!(ptr->flags & SWP_USED) || !ptr->swap_map)
1380 continue;
1381 if (!--l)
1382 return ptr;
1383 }
1384
1385 return NULL;
1386}
1387
1388static void *swap_next(struct seq_file *swap, void *v, loff_t *pos)
1389{
1390 struct swap_info_struct *ptr;
1391 struct swap_info_struct *endptr = swap_info + nr_swapfiles;
1392
1393 if (v == SEQ_START_TOKEN)
1394 ptr = swap_info;
1395 else {
1396 ptr = v;
1397 ptr++;
1398 }
1399
1400 for (; ptr < endptr; ptr++) {
1401 if (!(ptr->flags & SWP_USED) || !ptr->swap_map)
1402 continue;
1403 ++*pos;
1404 return ptr;
1405 }
1406
1407 return NULL;
1408}
1409
1410static void swap_stop(struct seq_file *swap, void *v)
1411{
1412 mutex_unlock(&swapon_mutex);
1413}
1414
1415static int swap_show(struct seq_file *swap, void *v)
1416{
1417 struct swap_info_struct *ptr = v;
1418 struct file *file;
1419 int len;
1420
1421 if (ptr == SEQ_START_TOKEN) {
1422 seq_puts(swap,"Filename\t\t\t\tType\t\tSize\tUsed\tPriority\n");
1423 return 0;
1424 }
1425
1426 file = ptr->swap_file;
1427 len = seq_path(swap, &file->f_path, " \t\n\\");
1428 seq_printf(swap, "%*s%s\t%u\t%u\t%d\n",
1429 len < 40 ? 40 - len : 1, " ",
1430 S_ISBLK(file->f_path.dentry->d_inode->i_mode) ?
1431 "partition" : "file\t",
1432 ptr->pages << (PAGE_SHIFT - 10),
1433 ptr->inuse_pages << (PAGE_SHIFT - 10),
1434 ptr->prio);
1435 return 0;
1436}
1437
1438static const struct seq_operations swaps_op = {
1439 .start = swap_start,
1440 .next = swap_next,
1441 .stop = swap_stop,
1442 .show = swap_show
1443};
1444
1445static int swaps_open(struct inode *inode, struct file *file)
1446{
1447 return seq_open(file, &swaps_op);
1448}
1449
1450static const struct file_operations proc_swaps_operations = {
1451 .open = swaps_open,
1452 .read = seq_read,
1453 .llseek = seq_lseek,
1454 .release = seq_release,
1455};
1456
1457static int __init procswaps_init(void)
1458{
1459 proc_create("swaps", 0, NULL, &proc_swaps_operations);
1460 return 0;
1461}
1462__initcall(procswaps_init);
1463#endif
1464
1465
1466
1467
1468
1469
1470asmlinkage long sys_swapon(const char __user * specialfile, int swap_flags)
1471{
1472 struct swap_info_struct * p;
1473 char *name = NULL;
1474 struct block_device *bdev = NULL;
1475 struct file *swap_file = NULL;
1476 struct address_space *mapping;
1477 unsigned int type;
1478 int i, prev;
1479 int error;
1480 union swap_header *swap_header = NULL;
1481 int swap_header_version;
1482 unsigned int nr_good_pages = 0;
1483 int nr_extents = 0;
1484 sector_t span;
1485 unsigned long maxpages = 1;
1486 int swapfilesize;
1487 unsigned short *swap_map = NULL;
1488 struct page *page = NULL;
1489 struct inode *inode = NULL;
1490 int did_down = 0;
1491
1492 if (!capable(CAP_SYS_ADMIN))
1493 return -EPERM;
1494 spin_lock(&swap_lock);
1495 p = swap_info;
1496 for (type = 0 ; type < nr_swapfiles ; type++,p++)
1497 if (!(p->flags & SWP_USED))
1498 break;
1499 error = -EPERM;
1500 if (type >= MAX_SWAPFILES) {
1501 spin_unlock(&swap_lock);
1502 goto out;
1503 }
1504 if (type >= nr_swapfiles)
1505 nr_swapfiles = type+1;
1506 memset(p, 0, sizeof(*p));
1507 INIT_LIST_HEAD(&p->extent_list);
1508 p->flags = SWP_USED;
1509 p->next = -1;
1510 spin_unlock(&swap_lock);
1511 name = getname(specialfile);
1512 error = PTR_ERR(name);
1513 if (IS_ERR(name)) {
1514 name = NULL;
1515 goto bad_swap_2;
1516 }
1517 swap_file = filp_open(name, O_RDWR|O_LARGEFILE, 0);
1518 error = PTR_ERR(swap_file);
1519 if (IS_ERR(swap_file)) {
1520 swap_file = NULL;
1521 goto bad_swap_2;
1522 }
1523
1524 p->swap_file = swap_file;
1525 mapping = swap_file->f_mapping;
1526 inode = mapping->host;
1527
1528 error = -EBUSY;
1529 for (i = 0; i < nr_swapfiles; i++) {
1530 struct swap_info_struct *q = &swap_info[i];
1531
1532 if (i == type || !q->swap_file)
1533 continue;
1534 if (mapping == q->swap_file->f_mapping)
1535 goto bad_swap;
1536 }
1537
1538 error = -EINVAL;
1539 if (S_ISBLK(inode->i_mode)) {
1540 bdev = I_BDEV(inode);
1541 error = bd_claim(bdev, sys_swapon);
1542 if (error < 0) {
1543 bdev = NULL;
1544 error = -EINVAL;
1545 goto bad_swap;
1546 }
1547 p->old_block_size = block_size(bdev);
1548 error = set_blocksize(bdev, PAGE_SIZE);
1549 if (error < 0)
1550 goto bad_swap;
1551 p->bdev = bdev;
1552 } else if (S_ISREG(inode->i_mode)) {
1553 p->bdev = inode->i_sb->s_bdev;
1554 mutex_lock(&inode->i_mutex);
1555 did_down = 1;
1556 if (IS_SWAPFILE(inode)) {
1557 error = -EBUSY;
1558 goto bad_swap;
1559 }
1560 } else {
1561 goto bad_swap;
1562 }
1563
1564 swapfilesize = i_size_read(inode) >> PAGE_SHIFT;
1565
1566
1567
1568
1569 if (!mapping->a_ops->readpage) {
1570 error = -EINVAL;
1571 goto bad_swap;
1572 }
1573 page = read_mapping_page(mapping, 0, swap_file);
1574 if (IS_ERR(page)) {
1575 error = PTR_ERR(page);
1576 goto bad_swap;
1577 }
1578 kmap(page);
1579 swap_header = page_address(page);
1580
1581 if (!memcmp("SWAP-SPACE",swap_header->magic.magic,10))
1582 swap_header_version = 1;
1583 else if (!memcmp("SWAPSPACE2",swap_header->magic.magic,10))
1584 swap_header_version = 2;
1585 else {
1586 printk(KERN_ERR "Unable to find swap-space signature\n");
1587 error = -EINVAL;
1588 goto bad_swap;
1589 }
1590
1591 switch (swap_header_version) {
1592 case 1:
1593 printk(KERN_ERR "version 0 swap is no longer supported. "
1594 "Use mkswap -v1 %s\n", name);
1595 error = -EINVAL;
1596 goto bad_swap;
1597 case 2:
1598
1599 if (swab32(swap_header->info.version) == 1) {
1600 swab32s(&swap_header->info.version);
1601 swab32s(&swap_header->info.last_page);
1602 swab32s(&swap_header->info.nr_badpages);
1603 for (i = 0; i < swap_header->info.nr_badpages; i++)
1604 swab32s(&swap_header->info.badpages[i]);
1605 }
1606
1607
1608 if (swap_header->info.version != 1) {
1609 printk(KERN_WARNING
1610 "Unable to handle swap header version %d\n",
1611 swap_header->info.version);
1612 error = -EINVAL;
1613 goto bad_swap;
1614 }
1615
1616 p->lowest_bit = 1;
1617 p->cluster_next = 1;
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633 maxpages = swp_offset(pte_to_swp_entry(swp_entry_to_pte(swp_entry(0,~0UL)))) - 1;
1634 if (maxpages > swap_header->info.last_page)
1635 maxpages = swap_header->info.last_page;
1636 p->highest_bit = maxpages - 1;
1637
1638 error = -EINVAL;
1639 if (!maxpages)
1640 goto bad_swap;
1641 if (swapfilesize && maxpages > swapfilesize) {
1642 printk(KERN_WARNING
1643 "Swap area shorter than signature indicates\n");
1644 goto bad_swap;
1645 }
1646 if (swap_header->info.nr_badpages && S_ISREG(inode->i_mode))
1647 goto bad_swap;
1648 if (swap_header->info.nr_badpages > MAX_SWAP_BADPAGES)
1649 goto bad_swap;
1650
1651
1652 swap_map = vmalloc(maxpages * sizeof(short));
1653 if (!swap_map) {
1654 error = -ENOMEM;
1655 goto bad_swap;
1656 }
1657
1658 error = 0;
1659 memset(swap_map, 0, maxpages * sizeof(short));
1660 for (i = 0; i < swap_header->info.nr_badpages; i++) {
1661 int page_nr = swap_header->info.badpages[i];
1662 if (page_nr <= 0 || page_nr >= swap_header->info.last_page)
1663 error = -EINVAL;
1664 else
1665 swap_map[page_nr] = SWAP_MAP_BAD;
1666 }
1667 nr_good_pages = swap_header->info.last_page -
1668 swap_header->info.nr_badpages -
1669 1 ;
1670 if (error)
1671 goto bad_swap;
1672 }
1673
1674 if (nr_good_pages) {
1675 swap_map[0] = SWAP_MAP_BAD;
1676 p->max = maxpages;
1677 p->pages = nr_good_pages;
1678 nr_extents = setup_swap_extents(p, &span);
1679 if (nr_extents < 0) {
1680 error = nr_extents;
1681 goto bad_swap;
1682 }
1683 nr_good_pages = p->pages;
1684 }
1685 if (!nr_good_pages) {
1686 printk(KERN_WARNING "Empty swap-file\n");
1687 error = -EINVAL;
1688 goto bad_swap;
1689 }
1690
1691 mutex_lock(&swapon_mutex);
1692 spin_lock(&swap_lock);
1693 if (swap_flags & SWAP_FLAG_PREFER)
1694 p->prio =
1695 (swap_flags & SWAP_FLAG_PRIO_MASK) >> SWAP_FLAG_PRIO_SHIFT;
1696 else
1697 p->prio = --least_priority;
1698 p->swap_map = swap_map;
1699 p->flags = SWP_ACTIVE;
1700 nr_swap_pages += nr_good_pages;
1701 total_swap_pages += nr_good_pages;
1702
1703 printk(KERN_INFO "Adding %uk swap on %s. "
1704 "Priority:%d extents:%d across:%lluk\n",
1705 nr_good_pages<<(PAGE_SHIFT-10), name, p->prio,
1706 nr_extents, (unsigned long long)span<<(PAGE_SHIFT-10));
1707
1708
1709 prev = -1;
1710 for (i = swap_list.head; i >= 0; i = swap_info[i].next) {
1711 if (p->prio >= swap_info[i].prio) {
1712 break;
1713 }
1714 prev = i;
1715 }
1716 p->next = i;
1717 if (prev < 0) {
1718 swap_list.head = swap_list.next = p - swap_info;
1719 } else {
1720 swap_info[prev].next = p - swap_info;
1721 }
1722 spin_unlock(&swap_lock);
1723 mutex_unlock(&swapon_mutex);
1724 error = 0;
1725 goto out;
1726bad_swap:
1727 if (bdev) {
1728 set_blocksize(bdev, p->old_block_size);
1729 bd_release(bdev);
1730 }
1731 destroy_swap_extents(p);
1732bad_swap_2:
1733 spin_lock(&swap_lock);
1734 p->swap_file = NULL;
1735 p->flags = 0;
1736 spin_unlock(&swap_lock);
1737 vfree(swap_map);
1738 if (swap_file)
1739 filp_close(swap_file, NULL);
1740out:
1741 if (page && !IS_ERR(page)) {
1742 kunmap(page);
1743 page_cache_release(page);
1744 }
1745 if (name)
1746 putname(name);
1747 if (did_down) {
1748 if (!error)
1749 inode->i_flags |= S_SWAPFILE;
1750 mutex_unlock(&inode->i_mutex);
1751 }
1752 return error;
1753}
1754
1755void si_swapinfo(struct sysinfo *val)
1756{
1757 unsigned int i;
1758 unsigned long nr_to_be_unused = 0;
1759
1760 spin_lock(&swap_lock);
1761 for (i = 0; i < nr_swapfiles; i++) {
1762 if (!(swap_info[i].flags & SWP_USED) ||
1763 (swap_info[i].flags & SWP_WRITEOK))
1764 continue;
1765 nr_to_be_unused += swap_info[i].inuse_pages;
1766 }
1767 val->freeswap = nr_swap_pages + nr_to_be_unused;
1768 val->totalswap = total_swap_pages + nr_to_be_unused;
1769 spin_unlock(&swap_lock);
1770}
1771
1772
1773
1774
1775
1776
1777
1778int swap_duplicate(swp_entry_t entry)
1779{
1780 struct swap_info_struct * p;
1781 unsigned long offset, type;
1782 int result = 0;
1783
1784 if (is_migration_entry(entry))
1785 return 1;
1786
1787 type = swp_type(entry);
1788 if (type >= nr_swapfiles)
1789 goto bad_file;
1790 p = type + swap_info;
1791 offset = swp_offset(entry);
1792
1793 spin_lock(&swap_lock);
1794 if (offset < p->max && p->swap_map[offset]) {
1795 if (p->swap_map[offset] < SWAP_MAP_MAX - 1) {
1796 p->swap_map[offset]++;
1797 result = 1;
1798 } else if (p->swap_map[offset] <= SWAP_MAP_MAX) {
1799 if (swap_overflow++ < 5)
1800 printk(KERN_WARNING "swap_dup: swap entry overflow\n");
1801 p->swap_map[offset] = SWAP_MAP_MAX;
1802 result = 1;
1803 }
1804 }
1805 spin_unlock(&swap_lock);
1806out:
1807 return result;
1808
1809bad_file:
1810 printk(KERN_ERR "swap_dup: %s%08lx\n", Bad_file, entry.val);
1811 goto out;
1812}
1813
1814struct swap_info_struct *
1815get_swap_info_struct(unsigned type)
1816{
1817 return &swap_info[type];
1818}
1819
1820
1821
1822
1823
1824int valid_swaphandles(swp_entry_t entry, unsigned long *offset)
1825{
1826 struct swap_info_struct *si;
1827 int our_page_cluster = page_cluster;
1828 pgoff_t target, toff;
1829 pgoff_t base, end;
1830 int nr_pages = 0;
1831
1832 if (!our_page_cluster)
1833 return 0;
1834
1835 si = &swap_info[swp_type(entry)];
1836 target = swp_offset(entry);
1837 base = (target >> our_page_cluster) << our_page_cluster;
1838 end = base + (1 << our_page_cluster);
1839 if (!base)
1840 base++;
1841
1842 spin_lock(&swap_lock);
1843 if (end > si->max)
1844 end = si->max;
1845
1846
1847 for (toff = target; ++toff < end; nr_pages++) {
1848
1849 if (!si->swap_map[toff])
1850 break;
1851 if (si->swap_map[toff] == SWAP_MAP_BAD)
1852 break;
1853 }
1854
1855 for (toff = target; --toff >= base; nr_pages++) {
1856
1857 if (!si->swap_map[toff])
1858 break;
1859 if (si->swap_map[toff] == SWAP_MAP_BAD)
1860 break;
1861 }
1862 spin_unlock(&swap_lock);
1863
1864
1865
1866
1867
1868 *offset = ++toff;
1869 return nr_pages? ++nr_pages: 0;
1870}