1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26#include <linux/module.h>
27#include <linux/init.h>
28#include <linux/fs.h>
29#include <linux/xattr.h>
30#include <linux/exportfs.h>
31#include <linux/generic_acl.h>
32#include <linux/mm.h>
33#include <linux/mman.h>
34#include <linux/file.h>
35#include <linux/swap.h>
36#include <linux/pagemap.h>
37#include <linux/string.h>
38#include <linux/slab.h>
39#include <linux/backing-dev.h>
40#include <linux/shmem_fs.h>
41#include <linux/mount.h>
42#include <linux/writeback.h>
43#include <linux/vfs.h>
44#include <linux/blkdev.h>
45#include <linux/security.h>
46#include <linux/swapops.h>
47#include <linux/mempolicy.h>
48#include <linux/namei.h>
49#include <linux/ctype.h>
50#include <linux/migrate.h>
51#include <linux/highmem.h>
52#include <linux/seq_file.h>
53#include <linux/magic.h>
54
55#include <asm/uaccess.h>
56#include <asm/div64.h>
57#include <asm/pgtable.h>
58
59#define ENTRIES_PER_PAGE (PAGE_CACHE_SIZE/sizeof(unsigned long))
60#define ENTRIES_PER_PAGEPAGE (ENTRIES_PER_PAGE*ENTRIES_PER_PAGE)
61#define BLOCKS_PER_PAGE (PAGE_CACHE_SIZE/512)
62
63#define SHMEM_MAX_INDEX (SHMEM_NR_DIRECT + (ENTRIES_PER_PAGEPAGE/2) * (ENTRIES_PER_PAGE+1))
64#define SHMEM_MAX_BYTES ((unsigned long long)SHMEM_MAX_INDEX << PAGE_CACHE_SHIFT)
65
66#define VM_ACCT(size) (PAGE_CACHE_ALIGN(size) >> PAGE_SHIFT)
67
68
69#define SHMEM_PAGEIN VM_READ
70#define SHMEM_TRUNCATE VM_WRITE
71
72
73#define LATENCY_LIMIT 64
74
75
76#define BOGO_DIRENT_SIZE 20
77
78
79enum sgp_type {
80 SGP_READ,
81 SGP_CACHE,
82 SGP_DIRTY,
83 SGP_WRITE,
84};
85
86#ifdef CONFIG_TMPFS
87static unsigned long shmem_default_max_blocks(void)
88{
89 return totalram_pages / 2;
90}
91
92static unsigned long shmem_default_max_inodes(void)
93{
94 return min(totalram_pages - totalhigh_pages, totalram_pages / 2);
95}
96#endif
97
98static int shmem_getpage(struct inode *inode, unsigned long idx,
99 struct page **pagep, enum sgp_type sgp, int *type);
100
101static inline struct page *shmem_dir_alloc(gfp_t gfp_mask)
102{
103
104
105
106
107
108
109
110 return alloc_pages((gfp_mask & ~GFP_MOVABLE_MASK) | __GFP_ZERO,
111 PAGE_CACHE_SHIFT-PAGE_SHIFT);
112}
113
114static inline void shmem_dir_free(struct page *page)
115{
116 __free_pages(page, PAGE_CACHE_SHIFT-PAGE_SHIFT);
117}
118
119static struct page **shmem_dir_map(struct page *page)
120{
121 return (struct page **)kmap_atomic(page, KM_USER0);
122}
123
124static inline void shmem_dir_unmap(struct page **dir)
125{
126 kunmap_atomic(dir, KM_USER0);
127}
128
129static swp_entry_t *shmem_swp_map(struct page *page)
130{
131 return (swp_entry_t *)kmap_atomic(page, KM_USER1);
132}
133
134static inline void shmem_swp_balance_unmap(void)
135{
136
137
138
139
140
141
142
143 (void) kmap_atomic(ZERO_PAGE(0), KM_USER1);
144}
145
146static inline void shmem_swp_unmap(swp_entry_t *entry)
147{
148 kunmap_atomic(entry, KM_USER1);
149}
150
151static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb)
152{
153 return sb->s_fs_info;
154}
155
156
157
158
159
160
161
162static inline int shmem_acct_size(unsigned long flags, loff_t size)
163{
164 return (flags & VM_ACCOUNT) ?
165 security_vm_enough_memory_kern(VM_ACCT(size)) : 0;
166}
167
168static inline void shmem_unacct_size(unsigned long flags, loff_t size)
169{
170 if (flags & VM_ACCOUNT)
171 vm_unacct_memory(VM_ACCT(size));
172}
173
174
175
176
177
178
179
180static inline int shmem_acct_block(unsigned long flags)
181{
182 return (flags & VM_ACCOUNT) ?
183 0 : security_vm_enough_memory_kern(VM_ACCT(PAGE_CACHE_SIZE));
184}
185
186static inline void shmem_unacct_blocks(unsigned long flags, long pages)
187{
188 if (!(flags & VM_ACCOUNT))
189 vm_unacct_memory(pages * VM_ACCT(PAGE_CACHE_SIZE));
190}
191
192static const struct super_operations shmem_ops;
193static const struct address_space_operations shmem_aops;
194static const struct file_operations shmem_file_operations;
195static const struct inode_operations shmem_inode_operations;
196static const struct inode_operations shmem_dir_inode_operations;
197static const struct inode_operations shmem_special_inode_operations;
198static struct vm_operations_struct shmem_vm_ops;
199
200static struct backing_dev_info shmem_backing_dev_info __read_mostly = {
201 .ra_pages = 0,
202 .capabilities = BDI_CAP_NO_ACCT_AND_WRITEBACK | BDI_CAP_SWAP_BACKED,
203 .unplug_io_fn = default_unplug_io_fn,
204};
205
206static LIST_HEAD(shmem_swaplist);
207static DEFINE_MUTEX(shmem_swaplist_mutex);
208
209static void shmem_free_blocks(struct inode *inode, long pages)
210{
211 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
212 if (sbinfo->max_blocks) {
213 spin_lock(&sbinfo->stat_lock);
214 sbinfo->free_blocks += pages;
215 inode->i_blocks -= pages*BLOCKS_PER_PAGE;
216 spin_unlock(&sbinfo->stat_lock);
217 }
218}
219
220static int shmem_reserve_inode(struct super_block *sb)
221{
222 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
223 if (sbinfo->max_inodes) {
224 spin_lock(&sbinfo->stat_lock);
225 if (!sbinfo->free_inodes) {
226 spin_unlock(&sbinfo->stat_lock);
227 return -ENOSPC;
228 }
229 sbinfo->free_inodes--;
230 spin_unlock(&sbinfo->stat_lock);
231 }
232 return 0;
233}
234
235static void shmem_free_inode(struct super_block *sb)
236{
237 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
238 if (sbinfo->max_inodes) {
239 spin_lock(&sbinfo->stat_lock);
240 sbinfo->free_inodes++;
241 spin_unlock(&sbinfo->stat_lock);
242 }
243}
244
245
246
247
248
249
250
251
252
253
254
255
256
257static void shmem_recalc_inode(struct inode *inode)
258{
259 struct shmem_inode_info *info = SHMEM_I(inode);
260 long freed;
261
262 freed = info->alloced - info->swapped - inode->i_mapping->nrpages;
263 if (freed > 0) {
264 info->alloced -= freed;
265 shmem_unacct_blocks(info->flags, freed);
266 shmem_free_blocks(inode, freed);
267 }
268}
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307static swp_entry_t *shmem_swp_entry(struct shmem_inode_info *info, unsigned long index, struct page **page)
308{
309 unsigned long offset;
310 struct page **dir;
311 struct page *subdir;
312
313 if (index < SHMEM_NR_DIRECT) {
314 shmem_swp_balance_unmap();
315 return info->i_direct+index;
316 }
317 if (!info->i_indirect) {
318 if (page) {
319 info->i_indirect = *page;
320 *page = NULL;
321 }
322 return NULL;
323 }
324
325 index -= SHMEM_NR_DIRECT;
326 offset = index % ENTRIES_PER_PAGE;
327 index /= ENTRIES_PER_PAGE;
328 dir = shmem_dir_map(info->i_indirect);
329
330 if (index >= ENTRIES_PER_PAGE/2) {
331 index -= ENTRIES_PER_PAGE/2;
332 dir += ENTRIES_PER_PAGE/2 + index/ENTRIES_PER_PAGE;
333 index %= ENTRIES_PER_PAGE;
334 subdir = *dir;
335 if (!subdir) {
336 if (page) {
337 *dir = *page;
338 *page = NULL;
339 }
340 shmem_dir_unmap(dir);
341 return NULL;
342 }
343 shmem_dir_unmap(dir);
344 dir = shmem_dir_map(subdir);
345 }
346
347 dir += index;
348 subdir = *dir;
349 if (!subdir) {
350 if (!page || !(subdir = *page)) {
351 shmem_dir_unmap(dir);
352 return NULL;
353 }
354 *dir = subdir;
355 *page = NULL;
356 }
357 shmem_dir_unmap(dir);
358 return shmem_swp_map(subdir) + offset;
359}
360
361static void shmem_swp_set(struct shmem_inode_info *info, swp_entry_t *entry, unsigned long value)
362{
363 long incdec = value? 1: -1;
364
365 entry->val = value;
366 info->swapped += incdec;
367 if ((unsigned long)(entry - info->i_direct) >= SHMEM_NR_DIRECT) {
368 struct page *page = kmap_atomic_to_page(entry);
369 set_page_private(page, page_private(page) + incdec);
370 }
371}
372
373
374
375
376
377
378
379
380
381static swp_entry_t *shmem_swp_alloc(struct shmem_inode_info *info, unsigned long index, enum sgp_type sgp)
382{
383 struct inode *inode = &info->vfs_inode;
384 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
385 struct page *page = NULL;
386 swp_entry_t *entry;
387
388 if (sgp != SGP_WRITE &&
389 ((loff_t) index << PAGE_CACHE_SHIFT) >= i_size_read(inode))
390 return ERR_PTR(-EINVAL);
391
392 while (!(entry = shmem_swp_entry(info, index, &page))) {
393 if (sgp == SGP_READ)
394 return shmem_swp_map(ZERO_PAGE(0));
395
396
397
398
399
400 if (sbinfo->max_blocks) {
401 spin_lock(&sbinfo->stat_lock);
402 if (sbinfo->free_blocks <= 1) {
403 spin_unlock(&sbinfo->stat_lock);
404 return ERR_PTR(-ENOSPC);
405 }
406 sbinfo->free_blocks--;
407 inode->i_blocks += BLOCKS_PER_PAGE;
408 spin_unlock(&sbinfo->stat_lock);
409 }
410
411 spin_unlock(&info->lock);
412 page = shmem_dir_alloc(mapping_gfp_mask(inode->i_mapping));
413 if (page)
414 set_page_private(page, 0);
415 spin_lock(&info->lock);
416
417 if (!page) {
418 shmem_free_blocks(inode, 1);
419 return ERR_PTR(-ENOMEM);
420 }
421 if (sgp != SGP_WRITE &&
422 ((loff_t) index << PAGE_CACHE_SHIFT) >= i_size_read(inode)) {
423 entry = ERR_PTR(-EINVAL);
424 break;
425 }
426 if (info->next_index <= index)
427 info->next_index = index + 1;
428 }
429 if (page) {
430
431 shmem_free_blocks(inode, 1);
432 shmem_dir_free(page);
433 }
434 if (info->next_index <= index && !IS_ERR(entry))
435 info->next_index = index + 1;
436 return entry;
437}
438
439
440
441
442
443
444
445static int shmem_free_swp(swp_entry_t *dir, swp_entry_t *edir,
446 spinlock_t *punch_lock)
447{
448 spinlock_t *punch_unlock = NULL;
449 swp_entry_t *ptr;
450 int freed = 0;
451
452 for (ptr = dir; ptr < edir; ptr++) {
453 if (ptr->val) {
454 if (unlikely(punch_lock)) {
455 punch_unlock = punch_lock;
456 punch_lock = NULL;
457 spin_lock(punch_unlock);
458 if (!ptr->val)
459 continue;
460 }
461 free_swap_and_cache(*ptr);
462 *ptr = (swp_entry_t){0};
463 freed++;
464 }
465 }
466 if (punch_unlock)
467 spin_unlock(punch_unlock);
468 return freed;
469}
470
471static int shmem_map_and_free_swp(struct page *subdir, int offset,
472 int limit, struct page ***dir, spinlock_t *punch_lock)
473{
474 swp_entry_t *ptr;
475 int freed = 0;
476
477 ptr = shmem_swp_map(subdir);
478 for (; offset < limit; offset += LATENCY_LIMIT) {
479 int size = limit - offset;
480 if (size > LATENCY_LIMIT)
481 size = LATENCY_LIMIT;
482 freed += shmem_free_swp(ptr+offset, ptr+offset+size,
483 punch_lock);
484 if (need_resched()) {
485 shmem_swp_unmap(ptr);
486 if (*dir) {
487 shmem_dir_unmap(*dir);
488 *dir = NULL;
489 }
490 cond_resched();
491 ptr = shmem_swp_map(subdir);
492 }
493 }
494 shmem_swp_unmap(ptr);
495 return freed;
496}
497
498static void shmem_free_pages(struct list_head *next)
499{
500 struct page *page;
501 int freed = 0;
502
503 do {
504 page = container_of(next, struct page, lru);
505 next = next->next;
506 shmem_dir_free(page);
507 freed++;
508 if (freed >= LATENCY_LIMIT) {
509 cond_resched();
510 freed = 0;
511 }
512 } while (next);
513}
514
515static void shmem_truncate_range(struct inode *inode, loff_t start, loff_t end)
516{
517 struct shmem_inode_info *info = SHMEM_I(inode);
518 unsigned long idx;
519 unsigned long size;
520 unsigned long limit;
521 unsigned long stage;
522 unsigned long diroff;
523 struct page **dir;
524 struct page *topdir;
525 struct page *middir;
526 struct page *subdir;
527 swp_entry_t *ptr;
528 LIST_HEAD(pages_to_free);
529 long nr_pages_to_free = 0;
530 long nr_swaps_freed = 0;
531 int offset;
532 int freed;
533 int punch_hole;
534 spinlock_t *needs_lock;
535 spinlock_t *punch_lock;
536 unsigned long upper_limit;
537
538 inode->i_ctime = inode->i_mtime = CURRENT_TIME;
539 idx = (start + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
540 if (idx >= info->next_index)
541 return;
542
543 spin_lock(&info->lock);
544 info->flags |= SHMEM_TRUNCATE;
545 if (likely(end == (loff_t) -1)) {
546 limit = info->next_index;
547 upper_limit = SHMEM_MAX_INDEX;
548 info->next_index = idx;
549 needs_lock = NULL;
550 punch_hole = 0;
551 } else {
552 if (end + 1 >= inode->i_size) {
553 limit = (inode->i_size + PAGE_CACHE_SIZE - 1) >>
554 PAGE_CACHE_SHIFT;
555 upper_limit = SHMEM_MAX_INDEX;
556 } else {
557 limit = (end + 1) >> PAGE_CACHE_SHIFT;
558 upper_limit = limit;
559 }
560 needs_lock = &info->lock;
561 punch_hole = 1;
562 }
563
564 topdir = info->i_indirect;
565 if (topdir && idx <= SHMEM_NR_DIRECT && !punch_hole) {
566 info->i_indirect = NULL;
567 nr_pages_to_free++;
568 list_add(&topdir->lru, &pages_to_free);
569 }
570 spin_unlock(&info->lock);
571
572 if (info->swapped && idx < SHMEM_NR_DIRECT) {
573 ptr = info->i_direct;
574 size = limit;
575 if (size > SHMEM_NR_DIRECT)
576 size = SHMEM_NR_DIRECT;
577 nr_swaps_freed = shmem_free_swp(ptr+idx, ptr+size, needs_lock);
578 }
579
580
581
582
583
584 if (!topdir || limit <= SHMEM_NR_DIRECT)
585 goto done2;
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600 upper_limit -= SHMEM_NR_DIRECT;
601 limit -= SHMEM_NR_DIRECT;
602 idx = (idx > SHMEM_NR_DIRECT)? (idx - SHMEM_NR_DIRECT): 0;
603 offset = idx % ENTRIES_PER_PAGE;
604 idx -= offset;
605
606 dir = shmem_dir_map(topdir);
607 stage = ENTRIES_PER_PAGEPAGE/2;
608 if (idx < ENTRIES_PER_PAGEPAGE/2) {
609 middir = topdir;
610 diroff = idx/ENTRIES_PER_PAGE;
611 } else {
612 dir += ENTRIES_PER_PAGE/2;
613 dir += (idx - ENTRIES_PER_PAGEPAGE/2)/ENTRIES_PER_PAGEPAGE;
614 while (stage <= idx)
615 stage += ENTRIES_PER_PAGEPAGE;
616 middir = *dir;
617 if (*dir) {
618 diroff = ((idx - ENTRIES_PER_PAGEPAGE/2) %
619 ENTRIES_PER_PAGEPAGE) / ENTRIES_PER_PAGE;
620 if (!diroff && !offset && upper_limit >= stage) {
621 if (needs_lock) {
622 spin_lock(needs_lock);
623 *dir = NULL;
624 spin_unlock(needs_lock);
625 needs_lock = NULL;
626 } else
627 *dir = NULL;
628 nr_pages_to_free++;
629 list_add(&middir->lru, &pages_to_free);
630 }
631 shmem_dir_unmap(dir);
632 dir = shmem_dir_map(middir);
633 } else {
634 diroff = 0;
635 offset = 0;
636 idx = stage;
637 }
638 }
639
640 for (; idx < limit; idx += ENTRIES_PER_PAGE, diroff++) {
641 if (unlikely(idx == stage)) {
642 shmem_dir_unmap(dir);
643 dir = shmem_dir_map(topdir) +
644 ENTRIES_PER_PAGE/2 + idx/ENTRIES_PER_PAGEPAGE;
645 while (!*dir) {
646 dir++;
647 idx += ENTRIES_PER_PAGEPAGE;
648 if (idx >= limit)
649 goto done1;
650 }
651 stage = idx + ENTRIES_PER_PAGEPAGE;
652 middir = *dir;
653 if (punch_hole)
654 needs_lock = &info->lock;
655 if (upper_limit >= stage) {
656 if (needs_lock) {
657 spin_lock(needs_lock);
658 *dir = NULL;
659 spin_unlock(needs_lock);
660 needs_lock = NULL;
661 } else
662 *dir = NULL;
663 nr_pages_to_free++;
664 list_add(&middir->lru, &pages_to_free);
665 }
666 shmem_dir_unmap(dir);
667 cond_resched();
668 dir = shmem_dir_map(middir);
669 diroff = 0;
670 }
671 punch_lock = needs_lock;
672 subdir = dir[diroff];
673 if (subdir && !offset && upper_limit-idx >= ENTRIES_PER_PAGE) {
674 if (needs_lock) {
675 spin_lock(needs_lock);
676 dir[diroff] = NULL;
677 spin_unlock(needs_lock);
678 punch_lock = NULL;
679 } else
680 dir[diroff] = NULL;
681 nr_pages_to_free++;
682 list_add(&subdir->lru, &pages_to_free);
683 }
684 if (subdir && page_private(subdir) ) {
685 size = limit - idx;
686 if (size > ENTRIES_PER_PAGE)
687 size = ENTRIES_PER_PAGE;
688 freed = shmem_map_and_free_swp(subdir,
689 offset, size, &dir, punch_lock);
690 if (!dir)
691 dir = shmem_dir_map(middir);
692 nr_swaps_freed += freed;
693 if (offset || punch_lock) {
694 spin_lock(&info->lock);
695 set_page_private(subdir,
696 page_private(subdir) - freed);
697 spin_unlock(&info->lock);
698 } else
699 BUG_ON(page_private(subdir) != freed);
700 }
701 offset = 0;
702 }
703done1:
704 shmem_dir_unmap(dir);
705done2:
706 if (inode->i_mapping->nrpages && (info->flags & SHMEM_PAGEIN)) {
707
708
709
710
711
712
713
714
715
716
717
718
719 truncate_inode_pages_range(inode->i_mapping, start, end);
720 if (punch_hole)
721 unmap_mapping_range(inode->i_mapping, start,
722 end - start, 1);
723 }
724
725 spin_lock(&info->lock);
726 info->flags &= ~SHMEM_TRUNCATE;
727 info->swapped -= nr_swaps_freed;
728 if (nr_pages_to_free)
729 shmem_free_blocks(inode, nr_pages_to_free);
730 shmem_recalc_inode(inode);
731 spin_unlock(&info->lock);
732
733
734
735
736 if (!list_empty(&pages_to_free)) {
737 pages_to_free.prev->next = NULL;
738 shmem_free_pages(pages_to_free.next);
739 }
740}
741
742static void shmem_truncate(struct inode *inode)
743{
744 shmem_truncate_range(inode, inode->i_size, (loff_t)-1);
745}
746
747static int shmem_notify_change(struct dentry *dentry, struct iattr *attr)
748{
749 struct inode *inode = dentry->d_inode;
750 struct page *page = NULL;
751 int error;
752
753 if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) {
754 if (attr->ia_size < inode->i_size) {
755
756
757
758
759
760
761
762 if (attr->ia_size & (PAGE_CACHE_SIZE-1)) {
763 (void) shmem_getpage(inode,
764 attr->ia_size>>PAGE_CACHE_SHIFT,
765 &page, SGP_READ, NULL);
766 if (page)
767 unlock_page(page);
768 }
769
770
771
772
773
774
775
776 if (attr->ia_size) {
777 struct shmem_inode_info *info = SHMEM_I(inode);
778 spin_lock(&info->lock);
779 info->flags &= ~SHMEM_PAGEIN;
780 spin_unlock(&info->lock);
781 }
782 }
783 }
784
785 error = inode_change_ok(inode, attr);
786 if (!error)
787 error = inode_setattr(inode, attr);
788#ifdef CONFIG_TMPFS_POSIX_ACL
789 if (!error && (attr->ia_valid & ATTR_MODE))
790 error = generic_acl_chmod(inode, &shmem_acl_ops);
791#endif
792 if (page)
793 page_cache_release(page);
794 return error;
795}
796
797static void shmem_delete_inode(struct inode *inode)
798{
799 struct shmem_inode_info *info = SHMEM_I(inode);
800
801 if (inode->i_op->truncate == shmem_truncate) {
802 truncate_inode_pages(inode->i_mapping, 0);
803 shmem_unacct_size(info->flags, inode->i_size);
804 inode->i_size = 0;
805 shmem_truncate(inode);
806 if (!list_empty(&info->swaplist)) {
807 mutex_lock(&shmem_swaplist_mutex);
808 list_del_init(&info->swaplist);
809 mutex_unlock(&shmem_swaplist_mutex);
810 }
811 }
812 BUG_ON(inode->i_blocks);
813 shmem_free_inode(inode->i_sb);
814 clear_inode(inode);
815}
816
817static inline int shmem_find_swp(swp_entry_t entry, swp_entry_t *dir, swp_entry_t *edir)
818{
819 swp_entry_t *ptr;
820
821 for (ptr = dir; ptr < edir; ptr++) {
822 if (ptr->val == entry.val)
823 return ptr - dir;
824 }
825 return -1;
826}
827
828static int shmem_unuse_inode(struct shmem_inode_info *info, swp_entry_t entry, struct page *page)
829{
830 struct inode *inode;
831 unsigned long idx;
832 unsigned long size;
833 unsigned long limit;
834 unsigned long stage;
835 struct page **dir;
836 struct page *subdir;
837 swp_entry_t *ptr;
838 int offset;
839 int error;
840
841 idx = 0;
842 ptr = info->i_direct;
843 spin_lock(&info->lock);
844 if (!info->swapped) {
845 list_del_init(&info->swaplist);
846 goto lost2;
847 }
848 limit = info->next_index;
849 size = limit;
850 if (size > SHMEM_NR_DIRECT)
851 size = SHMEM_NR_DIRECT;
852 offset = shmem_find_swp(entry, ptr, ptr+size);
853 if (offset >= 0)
854 goto found;
855 if (!info->i_indirect)
856 goto lost2;
857
858 dir = shmem_dir_map(info->i_indirect);
859 stage = SHMEM_NR_DIRECT + ENTRIES_PER_PAGEPAGE/2;
860
861 for (idx = SHMEM_NR_DIRECT; idx < limit; idx += ENTRIES_PER_PAGE, dir++) {
862 if (unlikely(idx == stage)) {
863 shmem_dir_unmap(dir-1);
864 if (cond_resched_lock(&info->lock)) {
865
866 if (limit > info->next_index) {
867 limit = info->next_index;
868 if (idx >= limit)
869 goto lost2;
870 }
871 }
872 dir = shmem_dir_map(info->i_indirect) +
873 ENTRIES_PER_PAGE/2 + idx/ENTRIES_PER_PAGEPAGE;
874 while (!*dir) {
875 dir++;
876 idx += ENTRIES_PER_PAGEPAGE;
877 if (idx >= limit)
878 goto lost1;
879 }
880 stage = idx + ENTRIES_PER_PAGEPAGE;
881 subdir = *dir;
882 shmem_dir_unmap(dir);
883 dir = shmem_dir_map(subdir);
884 }
885 subdir = *dir;
886 if (subdir && page_private(subdir)) {
887 ptr = shmem_swp_map(subdir);
888 size = limit - idx;
889 if (size > ENTRIES_PER_PAGE)
890 size = ENTRIES_PER_PAGE;
891 offset = shmem_find_swp(entry, ptr, ptr+size);
892 shmem_swp_unmap(ptr);
893 if (offset >= 0) {
894 shmem_dir_unmap(dir);
895 goto found;
896 }
897 }
898 }
899lost1:
900 shmem_dir_unmap(dir-1);
901lost2:
902 spin_unlock(&info->lock);
903 return 0;
904found:
905 idx += offset;
906 inode = igrab(&info->vfs_inode);
907 spin_unlock(&info->lock);
908
909
910
911
912
913
914
915
916 if (shmem_swaplist.next != &info->swaplist)
917 list_move_tail(&shmem_swaplist, &info->swaplist);
918 mutex_unlock(&shmem_swaplist_mutex);
919
920 error = 1;
921 if (!inode)
922 goto out;
923
924 error = mem_cgroup_cache_charge(page, current->mm, GFP_KERNEL);
925 if (error)
926 goto out;
927 error = radix_tree_preload(GFP_KERNEL);
928 if (error) {
929 mem_cgroup_uncharge_cache_page(page);
930 goto out;
931 }
932 error = 1;
933
934 spin_lock(&info->lock);
935 ptr = shmem_swp_entry(info, idx, NULL);
936 if (ptr && ptr->val == entry.val) {
937 error = add_to_page_cache_locked(page, inode->i_mapping,
938 idx, GFP_NOWAIT);
939
940 } else
941 mem_cgroup_uncharge_cache_page(page);
942
943 if (error == -EEXIST) {
944 struct page *filepage = find_get_page(inode->i_mapping, idx);
945 error = 1;
946 if (filepage) {
947
948
949
950
951 if (PageUptodate(filepage))
952 error = 0;
953 page_cache_release(filepage);
954 }
955 }
956 if (!error) {
957 delete_from_swap_cache(page);
958 set_page_dirty(page);
959 info->flags |= SHMEM_PAGEIN;
960 shmem_swp_set(info, ptr, 0);
961 swap_free(entry);
962 error = 1;
963 }
964 if (ptr)
965 shmem_swp_unmap(ptr);
966 spin_unlock(&info->lock);
967 radix_tree_preload_end();
968out:
969 unlock_page(page);
970 page_cache_release(page);
971 iput(inode);
972 return error;
973}
974
975
976
977
978int shmem_unuse(swp_entry_t entry, struct page *page)
979{
980 struct list_head *p, *next;
981 struct shmem_inode_info *info;
982 int found = 0;
983
984 mutex_lock(&shmem_swaplist_mutex);
985 list_for_each_safe(p, next, &shmem_swaplist) {
986 info = list_entry(p, struct shmem_inode_info, swaplist);
987 found = shmem_unuse_inode(info, entry, page);
988 cond_resched();
989 if (found)
990 goto out;
991 }
992 mutex_unlock(&shmem_swaplist_mutex);
993out: return found;
994}
995
996
997
998
999static int shmem_writepage(struct page *page, struct writeback_control *wbc)
1000{
1001 struct shmem_inode_info *info;
1002 swp_entry_t *entry, swap;
1003 struct address_space *mapping;
1004 unsigned long index;
1005 struct inode *inode;
1006
1007 BUG_ON(!PageLocked(page));
1008 mapping = page->mapping;
1009 index = page->index;
1010 inode = mapping->host;
1011 info = SHMEM_I(inode);
1012 if (info->flags & VM_LOCKED)
1013 goto redirty;
1014 if (!total_swap_pages)
1015 goto redirty;
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025 if (wbc->for_reclaim)
1026 swap = get_swap_page();
1027 else
1028 swap.val = 0;
1029
1030 spin_lock(&info->lock);
1031 if (index >= info->next_index) {
1032 BUG_ON(!(info->flags & SHMEM_TRUNCATE));
1033 goto unlock;
1034 }
1035 entry = shmem_swp_entry(info, index, NULL);
1036 if (entry->val) {
1037
1038
1039
1040
1041 free_swap_and_cache(*entry);
1042 shmem_swp_set(info, entry, 0);
1043 }
1044 shmem_recalc_inode(inode);
1045
1046 if (swap.val && add_to_swap_cache(page, swap, GFP_ATOMIC) == 0) {
1047 remove_from_page_cache(page);
1048 shmem_swp_set(info, entry, swap.val);
1049 shmem_swp_unmap(entry);
1050 if (list_empty(&info->swaplist))
1051 inode = igrab(inode);
1052 else
1053 inode = NULL;
1054 spin_unlock(&info->lock);
1055 swap_duplicate(swap);
1056 BUG_ON(page_mapped(page));
1057 page_cache_release(page);
1058 set_page_dirty(page);
1059 unlock_page(page);
1060 if (inode) {
1061 mutex_lock(&shmem_swaplist_mutex);
1062
1063 list_move_tail(&info->swaplist, &shmem_swaplist);
1064 mutex_unlock(&shmem_swaplist_mutex);
1065 iput(inode);
1066 }
1067 return 0;
1068 }
1069
1070 shmem_swp_unmap(entry);
1071unlock:
1072 spin_unlock(&info->lock);
1073 swap_free(swap);
1074redirty:
1075 set_page_dirty(page);
1076 if (wbc->for_reclaim)
1077 return AOP_WRITEPAGE_ACTIVATE;
1078 unlock_page(page);
1079 return 0;
1080}
1081
1082#ifdef CONFIG_NUMA
1083#ifdef CONFIG_TMPFS
1084static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
1085{
1086 char buffer[64];
1087
1088 if (!mpol || mpol->mode == MPOL_DEFAULT)
1089 return;
1090
1091 mpol_to_str(buffer, sizeof(buffer), mpol, 1);
1092
1093 seq_printf(seq, ",mpol=%s", buffer);
1094}
1095
1096static struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
1097{
1098 struct mempolicy *mpol = NULL;
1099 if (sbinfo->mpol) {
1100 spin_lock(&sbinfo->stat_lock);
1101 mpol = sbinfo->mpol;
1102 mpol_get(mpol);
1103 spin_unlock(&sbinfo->stat_lock);
1104 }
1105 return mpol;
1106}
1107#endif
1108
1109static struct page *shmem_swapin(swp_entry_t entry, gfp_t gfp,
1110 struct shmem_inode_info *info, unsigned long idx)
1111{
1112 struct mempolicy mpol, *spol;
1113 struct vm_area_struct pvma;
1114 struct page *page;
1115
1116 spol = mpol_cond_copy(&mpol,
1117 mpol_shared_policy_lookup(&info->policy, idx));
1118
1119
1120 pvma.vm_start = 0;
1121 pvma.vm_pgoff = idx;
1122 pvma.vm_ops = NULL;
1123 pvma.vm_policy = spol;
1124 page = swapin_readahead(entry, gfp, &pvma, 0);
1125 return page;
1126}
1127
1128static struct page *shmem_alloc_page(gfp_t gfp,
1129 struct shmem_inode_info *info, unsigned long idx)
1130{
1131 struct vm_area_struct pvma;
1132
1133
1134 pvma.vm_start = 0;
1135 pvma.vm_pgoff = idx;
1136 pvma.vm_ops = NULL;
1137 pvma.vm_policy = mpol_shared_policy_lookup(&info->policy, idx);
1138
1139
1140
1141
1142 return alloc_page_vma(gfp, &pvma, 0);
1143}
1144#else
1145#ifdef CONFIG_TMPFS
1146static inline void shmem_show_mpol(struct seq_file *seq, struct mempolicy *p)
1147{
1148}
1149#endif
1150
1151static inline struct page *shmem_swapin(swp_entry_t entry, gfp_t gfp,
1152 struct shmem_inode_info *info, unsigned long idx)
1153{
1154 return swapin_readahead(entry, gfp, NULL, 0);
1155}
1156
1157static inline struct page *shmem_alloc_page(gfp_t gfp,
1158 struct shmem_inode_info *info, unsigned long idx)
1159{
1160 return alloc_page(gfp);
1161}
1162#endif
1163
1164#if !defined(CONFIG_NUMA) || !defined(CONFIG_TMPFS)
1165static inline struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
1166{
1167 return NULL;
1168}
1169#endif
1170
1171
1172
1173
1174
1175
1176
1177
1178static int shmem_getpage(struct inode *inode, unsigned long idx,
1179 struct page **pagep, enum sgp_type sgp, int *type)
1180{
1181 struct address_space *mapping = inode->i_mapping;
1182 struct shmem_inode_info *info = SHMEM_I(inode);
1183 struct shmem_sb_info *sbinfo;
1184 struct page *filepage = *pagep;
1185 struct page *swappage;
1186 swp_entry_t *entry;
1187 swp_entry_t swap;
1188 gfp_t gfp;
1189 int error;
1190
1191 if (idx >= SHMEM_MAX_INDEX)
1192 return -EFBIG;
1193
1194 if (type)
1195 *type = 0;
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205repeat:
1206 if (!filepage)
1207 filepage = find_lock_page(mapping, idx);
1208 if (filepage && PageUptodate(filepage))
1209 goto done;
1210 error = 0;
1211 gfp = mapping_gfp_mask(mapping);
1212 if (!filepage) {
1213
1214
1215
1216
1217 error = radix_tree_preload(gfp & ~__GFP_HIGHMEM);
1218 if (error)
1219 goto failed;
1220 radix_tree_preload_end();
1221 }
1222
1223 spin_lock(&info->lock);
1224 shmem_recalc_inode(inode);
1225 entry = shmem_swp_alloc(info, idx, sgp);
1226 if (IS_ERR(entry)) {
1227 spin_unlock(&info->lock);
1228 error = PTR_ERR(entry);
1229 goto failed;
1230 }
1231 swap = *entry;
1232
1233 if (swap.val) {
1234
1235 swappage = lookup_swap_cache(swap);
1236 if (!swappage) {
1237 shmem_swp_unmap(entry);
1238
1239 if (type && !(*type & VM_FAULT_MAJOR)) {
1240 __count_vm_event(PGMAJFAULT);
1241 *type |= VM_FAULT_MAJOR;
1242 }
1243 spin_unlock(&info->lock);
1244 swappage = shmem_swapin(swap, gfp, info, idx);
1245 if (!swappage) {
1246 spin_lock(&info->lock);
1247 entry = shmem_swp_alloc(info, idx, sgp);
1248 if (IS_ERR(entry))
1249 error = PTR_ERR(entry);
1250 else {
1251 if (entry->val == swap.val)
1252 error = -ENOMEM;
1253 shmem_swp_unmap(entry);
1254 }
1255 spin_unlock(&info->lock);
1256 if (error)
1257 goto failed;
1258 goto repeat;
1259 }
1260 wait_on_page_locked(swappage);
1261 page_cache_release(swappage);
1262 goto repeat;
1263 }
1264
1265
1266 if (!trylock_page(swappage)) {
1267 shmem_swp_unmap(entry);
1268 spin_unlock(&info->lock);
1269 wait_on_page_locked(swappage);
1270 page_cache_release(swappage);
1271 goto repeat;
1272 }
1273 if (PageWriteback(swappage)) {
1274 shmem_swp_unmap(entry);
1275 spin_unlock(&info->lock);
1276 wait_on_page_writeback(swappage);
1277 unlock_page(swappage);
1278 page_cache_release(swappage);
1279 goto repeat;
1280 }
1281 if (!PageUptodate(swappage)) {
1282 shmem_swp_unmap(entry);
1283 spin_unlock(&info->lock);
1284 unlock_page(swappage);
1285 page_cache_release(swappage);
1286 error = -EIO;
1287 goto failed;
1288 }
1289
1290 if (filepage) {
1291 shmem_swp_set(info, entry, 0);
1292 shmem_swp_unmap(entry);
1293 delete_from_swap_cache(swappage);
1294 spin_unlock(&info->lock);
1295 copy_highpage(filepage, swappage);
1296 unlock_page(swappage);
1297 page_cache_release(swappage);
1298 flush_dcache_page(filepage);
1299 SetPageUptodate(filepage);
1300 set_page_dirty(filepage);
1301 swap_free(swap);
1302 } else if (!(error = add_to_page_cache_locked(swappage, mapping,
1303 idx, GFP_NOWAIT))) {
1304 info->flags |= SHMEM_PAGEIN;
1305 shmem_swp_set(info, entry, 0);
1306 shmem_swp_unmap(entry);
1307 delete_from_swap_cache(swappage);
1308 spin_unlock(&info->lock);
1309 filepage = swappage;
1310 set_page_dirty(filepage);
1311 swap_free(swap);
1312 } else {
1313 shmem_swp_unmap(entry);
1314 spin_unlock(&info->lock);
1315 unlock_page(swappage);
1316 page_cache_release(swappage);
1317 if (error == -ENOMEM) {
1318
1319 error = mem_cgroup_shrink_usage(current->mm,
1320 gfp);
1321 if (error)
1322 goto failed;
1323 }
1324 goto repeat;
1325 }
1326 } else if (sgp == SGP_READ && !filepage) {
1327 shmem_swp_unmap(entry);
1328 filepage = find_get_page(mapping, idx);
1329 if (filepage &&
1330 (!PageUptodate(filepage) || !trylock_page(filepage))) {
1331 spin_unlock(&info->lock);
1332 wait_on_page_locked(filepage);
1333 page_cache_release(filepage);
1334 filepage = NULL;
1335 goto repeat;
1336 }
1337 spin_unlock(&info->lock);
1338 } else {
1339 shmem_swp_unmap(entry);
1340 sbinfo = SHMEM_SB(inode->i_sb);
1341 if (sbinfo->max_blocks) {
1342 spin_lock(&sbinfo->stat_lock);
1343 if (sbinfo->free_blocks == 0 ||
1344 shmem_acct_block(info->flags)) {
1345 spin_unlock(&sbinfo->stat_lock);
1346 spin_unlock(&info->lock);
1347 error = -ENOSPC;
1348 goto failed;
1349 }
1350 sbinfo->free_blocks--;
1351 inode->i_blocks += BLOCKS_PER_PAGE;
1352 spin_unlock(&sbinfo->stat_lock);
1353 } else if (shmem_acct_block(info->flags)) {
1354 spin_unlock(&info->lock);
1355 error = -ENOSPC;
1356 goto failed;
1357 }
1358
1359 if (!filepage) {
1360 int ret;
1361
1362 spin_unlock(&info->lock);
1363 filepage = shmem_alloc_page(gfp, info, idx);
1364 if (!filepage) {
1365 shmem_unacct_blocks(info->flags, 1);
1366 shmem_free_blocks(inode, 1);
1367 error = -ENOMEM;
1368 goto failed;
1369 }
1370 SetPageSwapBacked(filepage);
1371
1372
1373 error = mem_cgroup_cache_charge(filepage, current->mm,
1374 gfp & ~__GFP_HIGHMEM);
1375 if (error) {
1376 page_cache_release(filepage);
1377 shmem_unacct_blocks(info->flags, 1);
1378 shmem_free_blocks(inode, 1);
1379 filepage = NULL;
1380 goto failed;
1381 }
1382
1383 spin_lock(&info->lock);
1384 entry = shmem_swp_alloc(info, idx, sgp);
1385 if (IS_ERR(entry))
1386 error = PTR_ERR(entry);
1387 else {
1388 swap = *entry;
1389 shmem_swp_unmap(entry);
1390 }
1391 ret = error || swap.val;
1392 if (ret)
1393 mem_cgroup_uncharge_cache_page(filepage);
1394 else
1395 ret = add_to_page_cache_lru(filepage, mapping,
1396 idx, GFP_NOWAIT);
1397
1398
1399
1400
1401 if (ret) {
1402 spin_unlock(&info->lock);
1403 page_cache_release(filepage);
1404 shmem_unacct_blocks(info->flags, 1);
1405 shmem_free_blocks(inode, 1);
1406 filepage = NULL;
1407 if (error)
1408 goto failed;
1409 goto repeat;
1410 }
1411 info->flags |= SHMEM_PAGEIN;
1412 }
1413
1414 info->alloced++;
1415 spin_unlock(&info->lock);
1416 clear_highpage(filepage);
1417 flush_dcache_page(filepage);
1418 SetPageUptodate(filepage);
1419 if (sgp == SGP_DIRTY)
1420 set_page_dirty(filepage);
1421 }
1422done:
1423 *pagep = filepage;
1424 return 0;
1425
1426failed:
1427 if (*pagep != filepage) {
1428 unlock_page(filepage);
1429 page_cache_release(filepage);
1430 }
1431 return error;
1432}
1433
1434static int shmem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1435{
1436 struct inode *inode = vma->vm_file->f_path.dentry->d_inode;
1437 int error;
1438 int ret;
1439
1440 if (((loff_t)vmf->pgoff << PAGE_CACHE_SHIFT) >= i_size_read(inode))
1441 return VM_FAULT_SIGBUS;
1442
1443 error = shmem_getpage(inode, vmf->pgoff, &vmf->page, SGP_CACHE, &ret);
1444 if (error)
1445 return ((error == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS);
1446
1447 mark_page_accessed(vmf->page);
1448 return ret | VM_FAULT_LOCKED;
1449}
1450
1451#ifdef CONFIG_NUMA
1452static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *new)
1453{
1454 struct inode *i = vma->vm_file->f_path.dentry->d_inode;
1455 return mpol_set_shared_policy(&SHMEM_I(i)->policy, vma, new);
1456}
1457
1458static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma,
1459 unsigned long addr)
1460{
1461 struct inode *i = vma->vm_file->f_path.dentry->d_inode;
1462 unsigned long idx;
1463
1464 idx = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
1465 return mpol_shared_policy_lookup(&SHMEM_I(i)->policy, idx);
1466}
1467#endif
1468
1469int shmem_lock(struct file *file, int lock, struct user_struct *user)
1470{
1471 struct inode *inode = file->f_path.dentry->d_inode;
1472 struct shmem_inode_info *info = SHMEM_I(inode);
1473 int retval = -ENOMEM;
1474
1475 spin_lock(&info->lock);
1476 if (lock && !(info->flags & VM_LOCKED)) {
1477 if (!user_shm_lock(inode->i_size, user))
1478 goto out_nomem;
1479 info->flags |= VM_LOCKED;
1480 mapping_set_unevictable(file->f_mapping);
1481 }
1482 if (!lock && (info->flags & VM_LOCKED) && user) {
1483 user_shm_unlock(inode->i_size, user);
1484 info->flags &= ~VM_LOCKED;
1485 mapping_clear_unevictable(file->f_mapping);
1486 scan_mapping_unevictable_pages(file->f_mapping);
1487 }
1488 retval = 0;
1489
1490out_nomem:
1491 spin_unlock(&info->lock);
1492 return retval;
1493}
1494
1495static int shmem_mmap(struct file *file, struct vm_area_struct *vma)
1496{
1497 file_accessed(file);
1498 vma->vm_ops = &shmem_vm_ops;
1499 vma->vm_flags |= VM_CAN_NONLINEAR;
1500 return 0;
1501}
1502
1503static struct inode *
1504shmem_get_inode(struct super_block *sb, int mode, dev_t dev)
1505{
1506 struct inode *inode;
1507 struct shmem_inode_info *info;
1508 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
1509
1510 if (shmem_reserve_inode(sb))
1511 return NULL;
1512
1513 inode = new_inode(sb);
1514 if (inode) {
1515 inode->i_mode = mode;
1516 inode->i_uid = current->fsuid;
1517 inode->i_gid = current->fsgid;
1518 inode->i_blocks = 0;
1519 inode->i_mapping->backing_dev_info = &shmem_backing_dev_info;
1520 inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
1521 inode->i_generation = get_seconds();
1522 info = SHMEM_I(inode);
1523 memset(info, 0, (char *)inode - (char *)info);
1524 spin_lock_init(&info->lock);
1525 INIT_LIST_HEAD(&info->swaplist);
1526
1527 switch (mode & S_IFMT) {
1528 default:
1529 inode->i_op = &shmem_special_inode_operations;
1530 init_special_inode(inode, mode, dev);
1531 break;
1532 case S_IFREG:
1533 inode->i_mapping->a_ops = &shmem_aops;
1534 inode->i_op = &shmem_inode_operations;
1535 inode->i_fop = &shmem_file_operations;
1536 mpol_shared_policy_init(&info->policy,
1537 shmem_get_sbmpol(sbinfo));
1538 break;
1539 case S_IFDIR:
1540 inc_nlink(inode);
1541
1542 inode->i_size = 2 * BOGO_DIRENT_SIZE;
1543 inode->i_op = &shmem_dir_inode_operations;
1544 inode->i_fop = &simple_dir_operations;
1545 break;
1546 case S_IFLNK:
1547
1548
1549
1550
1551 mpol_shared_policy_init(&info->policy, NULL);
1552 break;
1553 }
1554 } else
1555 shmem_free_inode(sb);
1556 return inode;
1557}
1558
1559#ifdef CONFIG_TMPFS
1560static const struct inode_operations shmem_symlink_inode_operations;
1561static const struct inode_operations shmem_symlink_inline_operations;
1562
1563
1564
1565
1566
1567
1568static int shmem_readpage(struct file *file, struct page *page)
1569{
1570 struct inode *inode = page->mapping->host;
1571 int error = shmem_getpage(inode, page->index, &page, SGP_CACHE, NULL);
1572 unlock_page(page);
1573 return error;
1574}
1575
1576static int
1577shmem_write_begin(struct file *file, struct address_space *mapping,
1578 loff_t pos, unsigned len, unsigned flags,
1579 struct page **pagep, void **fsdata)
1580{
1581 struct inode *inode = mapping->host;
1582 pgoff_t index = pos >> PAGE_CACHE_SHIFT;
1583 *pagep = NULL;
1584 return shmem_getpage(inode, index, pagep, SGP_WRITE, NULL);
1585}
1586
1587static int
1588shmem_write_end(struct file *file, struct address_space *mapping,
1589 loff_t pos, unsigned len, unsigned copied,
1590 struct page *page, void *fsdata)
1591{
1592 struct inode *inode = mapping->host;
1593
1594 if (pos + copied > inode->i_size)
1595 i_size_write(inode, pos + copied);
1596
1597 unlock_page(page);
1598 set_page_dirty(page);
1599 page_cache_release(page);
1600
1601 return copied;
1602}
1603
1604static void do_shmem_file_read(struct file *filp, loff_t *ppos, read_descriptor_t *desc, read_actor_t actor)
1605{
1606 struct inode *inode = filp->f_path.dentry->d_inode;
1607 struct address_space *mapping = inode->i_mapping;
1608 unsigned long index, offset;
1609 enum sgp_type sgp = SGP_READ;
1610
1611
1612
1613
1614
1615
1616 if (segment_eq(get_fs(), KERNEL_DS))
1617 sgp = SGP_DIRTY;
1618
1619 index = *ppos >> PAGE_CACHE_SHIFT;
1620 offset = *ppos & ~PAGE_CACHE_MASK;
1621
1622 for (;;) {
1623 struct page *page = NULL;
1624 unsigned long end_index, nr, ret;
1625 loff_t i_size = i_size_read(inode);
1626
1627 end_index = i_size >> PAGE_CACHE_SHIFT;
1628 if (index > end_index)
1629 break;
1630 if (index == end_index) {
1631 nr = i_size & ~PAGE_CACHE_MASK;
1632 if (nr <= offset)
1633 break;
1634 }
1635
1636 desc->error = shmem_getpage(inode, index, &page, sgp, NULL);
1637 if (desc->error) {
1638 if (desc->error == -EINVAL)
1639 desc->error = 0;
1640 break;
1641 }
1642 if (page)
1643 unlock_page(page);
1644
1645
1646
1647
1648
1649 nr = PAGE_CACHE_SIZE;
1650 i_size = i_size_read(inode);
1651 end_index = i_size >> PAGE_CACHE_SHIFT;
1652 if (index == end_index) {
1653 nr = i_size & ~PAGE_CACHE_MASK;
1654 if (nr <= offset) {
1655 if (page)
1656 page_cache_release(page);
1657 break;
1658 }
1659 }
1660 nr -= offset;
1661
1662 if (page) {
1663
1664
1665
1666
1667
1668 if (mapping_writably_mapped(mapping))
1669 flush_dcache_page(page);
1670
1671
1672
1673 if (!offset)
1674 mark_page_accessed(page);
1675 } else {
1676 page = ZERO_PAGE(0);
1677 page_cache_get(page);
1678 }
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690 ret = actor(desc, page, offset, nr);
1691 offset += ret;
1692 index += offset >> PAGE_CACHE_SHIFT;
1693 offset &= ~PAGE_CACHE_MASK;
1694
1695 page_cache_release(page);
1696 if (ret != nr || !desc->count)
1697 break;
1698
1699 cond_resched();
1700 }
1701
1702 *ppos = ((loff_t) index << PAGE_CACHE_SHIFT) + offset;
1703 file_accessed(filp);
1704}
1705
1706static ssize_t shmem_file_aio_read(struct kiocb *iocb,
1707 const struct iovec *iov, unsigned long nr_segs, loff_t pos)
1708{
1709 struct file *filp = iocb->ki_filp;
1710 ssize_t retval;
1711 unsigned long seg;
1712 size_t count;
1713 loff_t *ppos = &iocb->ki_pos;
1714
1715 retval = generic_segment_checks(iov, &nr_segs, &count, VERIFY_WRITE);
1716 if (retval)
1717 return retval;
1718
1719 for (seg = 0; seg < nr_segs; seg++) {
1720 read_descriptor_t desc;
1721
1722 desc.written = 0;
1723 desc.arg.buf = iov[seg].iov_base;
1724 desc.count = iov[seg].iov_len;
1725 if (desc.count == 0)
1726 continue;
1727 desc.error = 0;
1728 do_shmem_file_read(filp, ppos, &desc, file_read_actor);
1729 retval += desc.written;
1730 if (desc.error) {
1731 retval = retval ?: desc.error;
1732 break;
1733 }
1734 if (desc.count > 0)
1735 break;
1736 }
1737 return retval;
1738}
1739
1740static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf)
1741{
1742 struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb);
1743
1744 buf->f_type = TMPFS_MAGIC;
1745 buf->f_bsize = PAGE_CACHE_SIZE;
1746 buf->f_namelen = NAME_MAX;
1747 spin_lock(&sbinfo->stat_lock);
1748 if (sbinfo->max_blocks) {
1749 buf->f_blocks = sbinfo->max_blocks;
1750 buf->f_bavail = buf->f_bfree = sbinfo->free_blocks;
1751 }
1752 if (sbinfo->max_inodes) {
1753 buf->f_files = sbinfo->max_inodes;
1754 buf->f_ffree = sbinfo->free_inodes;
1755 }
1756
1757 spin_unlock(&sbinfo->stat_lock);
1758 return 0;
1759}
1760
1761
1762
1763
1764static int
1765shmem_mknod(struct inode *dir, struct dentry *dentry, int mode, dev_t dev)
1766{
1767 struct inode *inode = shmem_get_inode(dir->i_sb, mode, dev);
1768 int error = -ENOSPC;
1769
1770 if (inode) {
1771 error = security_inode_init_security(inode, dir, NULL, NULL,
1772 NULL);
1773 if (error) {
1774 if (error != -EOPNOTSUPP) {
1775 iput(inode);
1776 return error;
1777 }
1778 }
1779 error = shmem_acl_init(inode, dir);
1780 if (error) {
1781 iput(inode);
1782 return error;
1783 }
1784 if (dir->i_mode & S_ISGID) {
1785 inode->i_gid = dir->i_gid;
1786 if (S_ISDIR(mode))
1787 inode->i_mode |= S_ISGID;
1788 }
1789 dir->i_size += BOGO_DIRENT_SIZE;
1790 dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1791 d_instantiate(dentry, inode);
1792 dget(dentry);
1793 }
1794 return error;
1795}
1796
1797static int shmem_mkdir(struct inode *dir, struct dentry *dentry, int mode)
1798{
1799 int error;
1800
1801 if ((error = shmem_mknod(dir, dentry, mode | S_IFDIR, 0)))
1802 return error;
1803 inc_nlink(dir);
1804 return 0;
1805}
1806
1807static int shmem_create(struct inode *dir, struct dentry *dentry, int mode,
1808 struct nameidata *nd)
1809{
1810 return shmem_mknod(dir, dentry, mode | S_IFREG, 0);
1811}
1812
1813
1814
1815
1816static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry)
1817{
1818 struct inode *inode = old_dentry->d_inode;
1819 int ret;
1820
1821
1822
1823
1824
1825
1826 ret = shmem_reserve_inode(inode->i_sb);
1827 if (ret)
1828 goto out;
1829
1830 dir->i_size += BOGO_DIRENT_SIZE;
1831 inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1832 inc_nlink(inode);
1833 atomic_inc(&inode->i_count);
1834 dget(dentry);
1835 d_instantiate(dentry, inode);
1836out:
1837 return ret;
1838}
1839
1840static int shmem_unlink(struct inode *dir, struct dentry *dentry)
1841{
1842 struct inode *inode = dentry->d_inode;
1843
1844 if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode))
1845 shmem_free_inode(inode->i_sb);
1846
1847 dir->i_size -= BOGO_DIRENT_SIZE;
1848 inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1849 drop_nlink(inode);
1850 dput(dentry);
1851 return 0;
1852}
1853
1854static int shmem_rmdir(struct inode *dir, struct dentry *dentry)
1855{
1856 if (!simple_empty(dentry))
1857 return -ENOTEMPTY;
1858
1859 drop_nlink(dentry->d_inode);
1860 drop_nlink(dir);
1861 return shmem_unlink(dir, dentry);
1862}
1863
1864
1865
1866
1867
1868
1869
1870static int shmem_rename(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry)
1871{
1872 struct inode *inode = old_dentry->d_inode;
1873 int they_are_dirs = S_ISDIR(inode->i_mode);
1874
1875 if (!simple_empty(new_dentry))
1876 return -ENOTEMPTY;
1877
1878 if (new_dentry->d_inode) {
1879 (void) shmem_unlink(new_dir, new_dentry);
1880 if (they_are_dirs)
1881 drop_nlink(old_dir);
1882 } else if (they_are_dirs) {
1883 drop_nlink(old_dir);
1884 inc_nlink(new_dir);
1885 }
1886
1887 old_dir->i_size -= BOGO_DIRENT_SIZE;
1888 new_dir->i_size += BOGO_DIRENT_SIZE;
1889 old_dir->i_ctime = old_dir->i_mtime =
1890 new_dir->i_ctime = new_dir->i_mtime =
1891 inode->i_ctime = CURRENT_TIME;
1892 return 0;
1893}
1894
1895static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *symname)
1896{
1897 int error;
1898 int len;
1899 struct inode *inode;
1900 struct page *page = NULL;
1901 char *kaddr;
1902 struct shmem_inode_info *info;
1903
1904 len = strlen(symname) + 1;
1905 if (len > PAGE_CACHE_SIZE)
1906 return -ENAMETOOLONG;
1907
1908 inode = shmem_get_inode(dir->i_sb, S_IFLNK|S_IRWXUGO, 0);
1909 if (!inode)
1910 return -ENOSPC;
1911
1912 error = security_inode_init_security(inode, dir, NULL, NULL,
1913 NULL);
1914 if (error) {
1915 if (error != -EOPNOTSUPP) {
1916 iput(inode);
1917 return error;
1918 }
1919 error = 0;
1920 }
1921
1922 info = SHMEM_I(inode);
1923 inode->i_size = len-1;
1924 if (len <= (char *)inode - (char *)info) {
1925
1926 memcpy(info, symname, len);
1927 inode->i_op = &shmem_symlink_inline_operations;
1928 } else {
1929 error = shmem_getpage(inode, 0, &page, SGP_WRITE, NULL);
1930 if (error) {
1931 iput(inode);
1932 return error;
1933 }
1934 unlock_page(page);
1935 inode->i_mapping->a_ops = &shmem_aops;
1936 inode->i_op = &shmem_symlink_inode_operations;
1937 kaddr = kmap_atomic(page, KM_USER0);
1938 memcpy(kaddr, symname, len);
1939 kunmap_atomic(kaddr, KM_USER0);
1940 set_page_dirty(page);
1941 page_cache_release(page);
1942 }
1943 if (dir->i_mode & S_ISGID)
1944 inode->i_gid = dir->i_gid;
1945 dir->i_size += BOGO_DIRENT_SIZE;
1946 dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1947 d_instantiate(dentry, inode);
1948 dget(dentry);
1949 return 0;
1950}
1951
1952static void *shmem_follow_link_inline(struct dentry *dentry, struct nameidata *nd)
1953{
1954 nd_set_link(nd, (char *)SHMEM_I(dentry->d_inode));
1955 return NULL;
1956}
1957
1958static void *shmem_follow_link(struct dentry *dentry, struct nameidata *nd)
1959{
1960 struct page *page = NULL;
1961 int res = shmem_getpage(dentry->d_inode, 0, &page, SGP_READ, NULL);
1962 nd_set_link(nd, res ? ERR_PTR(res) : kmap(page));
1963 if (page)
1964 unlock_page(page);
1965 return page;
1966}
1967
1968static void shmem_put_link(struct dentry *dentry, struct nameidata *nd, void *cookie)
1969{
1970 if (!IS_ERR(nd_get_link(nd))) {
1971 struct page *page = cookie;
1972 kunmap(page);
1973 mark_page_accessed(page);
1974 page_cache_release(page);
1975 }
1976}
1977
1978static const struct inode_operations shmem_symlink_inline_operations = {
1979 .readlink = generic_readlink,
1980 .follow_link = shmem_follow_link_inline,
1981};
1982
1983static const struct inode_operations shmem_symlink_inode_operations = {
1984 .truncate = shmem_truncate,
1985 .readlink = generic_readlink,
1986 .follow_link = shmem_follow_link,
1987 .put_link = shmem_put_link,
1988};
1989
1990#ifdef CONFIG_TMPFS_POSIX_ACL
1991
1992
1993
1994
1995
1996
1997
1998static size_t shmem_xattr_security_list(struct inode *inode, char *list,
1999 size_t list_len, const char *name,
2000 size_t name_len)
2001{
2002 return security_inode_listsecurity(inode, list, list_len);
2003}
2004
2005static int shmem_xattr_security_get(struct inode *inode, const char *name,
2006 void *buffer, size_t size)
2007{
2008 if (strcmp(name, "") == 0)
2009 return -EINVAL;
2010 return xattr_getsecurity(inode, name, buffer, size);
2011}
2012
2013static int shmem_xattr_security_set(struct inode *inode, const char *name,
2014 const void *value, size_t size, int flags)
2015{
2016 if (strcmp(name, "") == 0)
2017 return -EINVAL;
2018 return security_inode_setsecurity(inode, name, value, size, flags);
2019}
2020
2021static struct xattr_handler shmem_xattr_security_handler = {
2022 .prefix = XATTR_SECURITY_PREFIX,
2023 .list = shmem_xattr_security_list,
2024 .get = shmem_xattr_security_get,
2025 .set = shmem_xattr_security_set,
2026};
2027
2028static struct xattr_handler *shmem_xattr_handlers[] = {
2029 &shmem_xattr_acl_access_handler,
2030 &shmem_xattr_acl_default_handler,
2031 &shmem_xattr_security_handler,
2032 NULL
2033};
2034#endif
2035
2036static struct dentry *shmem_get_parent(struct dentry *child)
2037{
2038 return ERR_PTR(-ESTALE);
2039}
2040
2041static int shmem_match(struct inode *ino, void *vfh)
2042{
2043 __u32 *fh = vfh;
2044 __u64 inum = fh[2];
2045 inum = (inum << 32) | fh[1];
2046 return ino->i_ino == inum && fh[0] == ino->i_generation;
2047}
2048
2049static struct dentry *shmem_fh_to_dentry(struct super_block *sb,
2050 struct fid *fid, int fh_len, int fh_type)
2051{
2052 struct inode *inode;
2053 struct dentry *dentry = NULL;
2054 u64 inum = fid->raw[2];
2055 inum = (inum << 32) | fid->raw[1];
2056
2057 if (fh_len < 3)
2058 return NULL;
2059
2060 inode = ilookup5(sb, (unsigned long)(inum + fid->raw[0]),
2061 shmem_match, fid->raw);
2062 if (inode) {
2063 dentry = d_find_alias(inode);
2064 iput(inode);
2065 }
2066
2067 return dentry;
2068}
2069
2070static int shmem_encode_fh(struct dentry *dentry, __u32 *fh, int *len,
2071 int connectable)
2072{
2073 struct inode *inode = dentry->d_inode;
2074
2075 if (*len < 3)
2076 return 255;
2077
2078 if (hlist_unhashed(&inode->i_hash)) {
2079
2080
2081
2082
2083
2084 static DEFINE_SPINLOCK(lock);
2085 spin_lock(&lock);
2086 if (hlist_unhashed(&inode->i_hash))
2087 __insert_inode_hash(inode,
2088 inode->i_ino + inode->i_generation);
2089 spin_unlock(&lock);
2090 }
2091
2092 fh[0] = inode->i_generation;
2093 fh[1] = inode->i_ino;
2094 fh[2] = ((__u64)inode->i_ino) >> 32;
2095
2096 *len = 3;
2097 return 1;
2098}
2099
2100static const struct export_operations shmem_export_ops = {
2101 .get_parent = shmem_get_parent,
2102 .encode_fh = shmem_encode_fh,
2103 .fh_to_dentry = shmem_fh_to_dentry,
2104};
2105
2106static int shmem_parse_options(char *options, struct shmem_sb_info *sbinfo,
2107 bool remount)
2108{
2109 char *this_char, *value, *rest;
2110
2111 while (options != NULL) {
2112 this_char = options;
2113 for (;;) {
2114
2115
2116
2117
2118
2119 options = strchr(options, ',');
2120 if (options == NULL)
2121 break;
2122 options++;
2123 if (!isdigit(*options)) {
2124 options[-1] = '\0';
2125 break;
2126 }
2127 }
2128 if (!*this_char)
2129 continue;
2130 if ((value = strchr(this_char,'=')) != NULL) {
2131 *value++ = 0;
2132 } else {
2133 printk(KERN_ERR
2134 "tmpfs: No value for mount option '%s'\n",
2135 this_char);
2136 return 1;
2137 }
2138
2139 if (!strcmp(this_char,"size")) {
2140 unsigned long long size;
2141 size = memparse(value,&rest);
2142 if (*rest == '%') {
2143 size <<= PAGE_SHIFT;
2144 size *= totalram_pages;
2145 do_div(size, 100);
2146 rest++;
2147 }
2148 if (*rest)
2149 goto bad_val;
2150 sbinfo->max_blocks =
2151 DIV_ROUND_UP(size, PAGE_CACHE_SIZE);
2152 } else if (!strcmp(this_char,"nr_blocks")) {
2153 sbinfo->max_blocks = memparse(value, &rest);
2154 if (*rest)
2155 goto bad_val;
2156 } else if (!strcmp(this_char,"nr_inodes")) {
2157 sbinfo->max_inodes = memparse(value, &rest);
2158 if (*rest)
2159 goto bad_val;
2160 } else if (!strcmp(this_char,"mode")) {
2161 if (remount)
2162 continue;
2163 sbinfo->mode = simple_strtoul(value, &rest, 8) & 07777;
2164 if (*rest)
2165 goto bad_val;
2166 } else if (!strcmp(this_char,"uid")) {
2167 if (remount)
2168 continue;
2169 sbinfo->uid = simple_strtoul(value, &rest, 0);
2170 if (*rest)
2171 goto bad_val;
2172 } else if (!strcmp(this_char,"gid")) {
2173 if (remount)
2174 continue;
2175 sbinfo->gid = simple_strtoul(value, &rest, 0);
2176 if (*rest)
2177 goto bad_val;
2178 } else if (!strcmp(this_char,"mpol")) {
2179 if (mpol_parse_str(value, &sbinfo->mpol, 1))
2180 goto bad_val;
2181 } else {
2182 printk(KERN_ERR "tmpfs: Bad mount option %s\n",
2183 this_char);
2184 return 1;
2185 }
2186 }
2187 return 0;
2188
2189bad_val:
2190 printk(KERN_ERR "tmpfs: Bad value '%s' for mount option '%s'\n",
2191 value, this_char);
2192 return 1;
2193
2194}
2195
2196static int shmem_remount_fs(struct super_block *sb, int *flags, char *data)
2197{
2198 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
2199 struct shmem_sb_info config = *sbinfo;
2200 unsigned long blocks;
2201 unsigned long inodes;
2202 int error = -EINVAL;
2203
2204 if (shmem_parse_options(data, &config, true))
2205 return error;
2206
2207 spin_lock(&sbinfo->stat_lock);
2208 blocks = sbinfo->max_blocks - sbinfo->free_blocks;
2209 inodes = sbinfo->max_inodes - sbinfo->free_inodes;
2210 if (config.max_blocks < blocks)
2211 goto out;
2212 if (config.max_inodes < inodes)
2213 goto out;
2214
2215
2216
2217
2218
2219
2220 if (config.max_blocks && !sbinfo->max_blocks)
2221 goto out;
2222 if (config.max_inodes && !sbinfo->max_inodes)
2223 goto out;
2224
2225 error = 0;
2226 sbinfo->max_blocks = config.max_blocks;
2227 sbinfo->free_blocks = config.max_blocks - blocks;
2228 sbinfo->max_inodes = config.max_inodes;
2229 sbinfo->free_inodes = config.max_inodes - inodes;
2230
2231 mpol_put(sbinfo->mpol);
2232 sbinfo->mpol = config.mpol;
2233out:
2234 spin_unlock(&sbinfo->stat_lock);
2235 return error;
2236}
2237
2238static int shmem_show_options(struct seq_file *seq, struct vfsmount *vfs)
2239{
2240 struct shmem_sb_info *sbinfo = SHMEM_SB(vfs->mnt_sb);
2241
2242 if (sbinfo->max_blocks != shmem_default_max_blocks())
2243 seq_printf(seq, ",size=%luk",
2244 sbinfo->max_blocks << (PAGE_CACHE_SHIFT - 10));
2245 if (sbinfo->max_inodes != shmem_default_max_inodes())
2246 seq_printf(seq, ",nr_inodes=%lu", sbinfo->max_inodes);
2247 if (sbinfo->mode != (S_IRWXUGO | S_ISVTX))
2248 seq_printf(seq, ",mode=%03o", sbinfo->mode);
2249 if (sbinfo->uid != 0)
2250 seq_printf(seq, ",uid=%u", sbinfo->uid);
2251 if (sbinfo->gid != 0)
2252 seq_printf(seq, ",gid=%u", sbinfo->gid);
2253 shmem_show_mpol(seq, sbinfo->mpol);
2254 return 0;
2255}
2256#endif
2257
2258static void shmem_put_super(struct super_block *sb)
2259{
2260 kfree(sb->s_fs_info);
2261 sb->s_fs_info = NULL;
2262}
2263
2264static int shmem_fill_super(struct super_block *sb,
2265 void *data, int silent)
2266{
2267 struct inode *inode;
2268 struct dentry *root;
2269 struct shmem_sb_info *sbinfo;
2270 int err = -ENOMEM;
2271
2272
2273 sbinfo = kmalloc(max((int)sizeof(struct shmem_sb_info),
2274 L1_CACHE_BYTES), GFP_KERNEL);
2275 if (!sbinfo)
2276 return -ENOMEM;
2277
2278 sbinfo->max_blocks = 0;
2279 sbinfo->max_inodes = 0;
2280 sbinfo->mode = S_IRWXUGO | S_ISVTX;
2281 sbinfo->uid = current->fsuid;
2282 sbinfo->gid = current->fsgid;
2283 sbinfo->mpol = NULL;
2284 sb->s_fs_info = sbinfo;
2285
2286#ifdef CONFIG_TMPFS
2287
2288
2289
2290
2291
2292 if (!(sb->s_flags & MS_NOUSER)) {
2293 sbinfo->max_blocks = shmem_default_max_blocks();
2294 sbinfo->max_inodes = shmem_default_max_inodes();
2295 if (shmem_parse_options(data, sbinfo, false)) {
2296 err = -EINVAL;
2297 goto failed;
2298 }
2299 }
2300 sb->s_export_op = &shmem_export_ops;
2301#else
2302 sb->s_flags |= MS_NOUSER;
2303#endif
2304
2305 spin_lock_init(&sbinfo->stat_lock);
2306 sbinfo->free_blocks = sbinfo->max_blocks;
2307 sbinfo->free_inodes = sbinfo->max_inodes;
2308
2309 sb->s_maxbytes = SHMEM_MAX_BYTES;
2310 sb->s_blocksize = PAGE_CACHE_SIZE;
2311 sb->s_blocksize_bits = PAGE_CACHE_SHIFT;
2312 sb->s_magic = TMPFS_MAGIC;
2313 sb->s_op = &shmem_ops;
2314 sb->s_time_gran = 1;
2315#ifdef CONFIG_TMPFS_POSIX_ACL
2316 sb->s_xattr = shmem_xattr_handlers;
2317 sb->s_flags |= MS_POSIXACL;
2318#endif
2319
2320 inode = shmem_get_inode(sb, S_IFDIR | sbinfo->mode, 0);
2321 if (!inode)
2322 goto failed;
2323 inode->i_uid = sbinfo->uid;
2324 inode->i_gid = sbinfo->gid;
2325 root = d_alloc_root(inode);
2326 if (!root)
2327 goto failed_iput;
2328 sb->s_root = root;
2329 return 0;
2330
2331failed_iput:
2332 iput(inode);
2333failed:
2334 shmem_put_super(sb);
2335 return err;
2336}
2337
2338static struct kmem_cache *shmem_inode_cachep;
2339
2340static struct inode *shmem_alloc_inode(struct super_block *sb)
2341{
2342 struct shmem_inode_info *p;
2343 p = (struct shmem_inode_info *)kmem_cache_alloc(shmem_inode_cachep, GFP_KERNEL);
2344 if (!p)
2345 return NULL;
2346 return &p->vfs_inode;
2347}
2348
2349static void shmem_destroy_inode(struct inode *inode)
2350{
2351 if ((inode->i_mode & S_IFMT) == S_IFREG) {
2352
2353 mpol_free_shared_policy(&SHMEM_I(inode)->policy);
2354 }
2355 shmem_acl_destroy_inode(inode);
2356 kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode));
2357}
2358
2359static void init_once(void *foo)
2360{
2361 struct shmem_inode_info *p = (struct shmem_inode_info *) foo;
2362
2363 inode_init_once(&p->vfs_inode);
2364#ifdef CONFIG_TMPFS_POSIX_ACL
2365 p->i_acl = NULL;
2366 p->i_default_acl = NULL;
2367#endif
2368}
2369
2370static int init_inodecache(void)
2371{
2372 shmem_inode_cachep = kmem_cache_create("shmem_inode_cache",
2373 sizeof(struct shmem_inode_info),
2374 0, SLAB_PANIC, init_once);
2375 return 0;
2376}
2377
2378static void destroy_inodecache(void)
2379{
2380 kmem_cache_destroy(shmem_inode_cachep);
2381}
2382
2383static const struct address_space_operations shmem_aops = {
2384 .writepage = shmem_writepage,
2385 .set_page_dirty = __set_page_dirty_no_writeback,
2386#ifdef CONFIG_TMPFS
2387 .readpage = shmem_readpage,
2388 .write_begin = shmem_write_begin,
2389 .write_end = shmem_write_end,
2390#endif
2391 .migratepage = migrate_page,
2392};
2393
2394static const struct file_operations shmem_file_operations = {
2395 .mmap = shmem_mmap,
2396#ifdef CONFIG_TMPFS
2397 .llseek = generic_file_llseek,
2398 .read = do_sync_read,
2399 .write = do_sync_write,
2400 .aio_read = shmem_file_aio_read,
2401 .aio_write = generic_file_aio_write,
2402 .fsync = simple_sync_file,
2403 .splice_read = generic_file_splice_read,
2404 .splice_write = generic_file_splice_write,
2405#endif
2406};
2407
2408static const struct inode_operations shmem_inode_operations = {
2409 .truncate = shmem_truncate,
2410 .setattr = shmem_notify_change,
2411 .truncate_range = shmem_truncate_range,
2412#ifdef CONFIG_TMPFS_POSIX_ACL
2413 .setxattr = generic_setxattr,
2414 .getxattr = generic_getxattr,
2415 .listxattr = generic_listxattr,
2416 .removexattr = generic_removexattr,
2417 .permission = shmem_permission,
2418#endif
2419
2420};
2421
2422static const struct inode_operations shmem_dir_inode_operations = {
2423#ifdef CONFIG_TMPFS
2424 .create = shmem_create,
2425 .lookup = simple_lookup,
2426 .link = shmem_link,
2427 .unlink = shmem_unlink,
2428 .symlink = shmem_symlink,
2429 .mkdir = shmem_mkdir,
2430 .rmdir = shmem_rmdir,
2431 .mknod = shmem_mknod,
2432 .rename = shmem_rename,
2433#endif
2434#ifdef CONFIG_TMPFS_POSIX_ACL
2435 .setattr = shmem_notify_change,
2436 .setxattr = generic_setxattr,
2437 .getxattr = generic_getxattr,
2438 .listxattr = generic_listxattr,
2439 .removexattr = generic_removexattr,
2440 .permission = shmem_permission,
2441#endif
2442};
2443
2444static const struct inode_operations shmem_special_inode_operations = {
2445#ifdef CONFIG_TMPFS_POSIX_ACL
2446 .setattr = shmem_notify_change,
2447 .setxattr = generic_setxattr,
2448 .getxattr = generic_getxattr,
2449 .listxattr = generic_listxattr,
2450 .removexattr = generic_removexattr,
2451 .permission = shmem_permission,
2452#endif
2453};
2454
2455static const struct super_operations shmem_ops = {
2456 .alloc_inode = shmem_alloc_inode,
2457 .destroy_inode = shmem_destroy_inode,
2458#ifdef CONFIG_TMPFS
2459 .statfs = shmem_statfs,
2460 .remount_fs = shmem_remount_fs,
2461 .show_options = shmem_show_options,
2462#endif
2463 .delete_inode = shmem_delete_inode,
2464 .drop_inode = generic_delete_inode,
2465 .put_super = shmem_put_super,
2466};
2467
2468static struct vm_operations_struct shmem_vm_ops = {
2469 .fault = shmem_fault,
2470#ifdef CONFIG_NUMA
2471 .set_policy = shmem_set_policy,
2472 .get_policy = shmem_get_policy,
2473#endif
2474};
2475
2476
2477static int shmem_get_sb(struct file_system_type *fs_type,
2478 int flags, const char *dev_name, void *data, struct vfsmount *mnt)
2479{
2480 return get_sb_nodev(fs_type, flags, data, shmem_fill_super, mnt);
2481}
2482
2483static struct file_system_type tmpfs_fs_type = {
2484 .owner = THIS_MODULE,
2485 .name = "tmpfs",
2486 .get_sb = shmem_get_sb,
2487 .kill_sb = kill_litter_super,
2488};
2489static struct vfsmount *shm_mnt;
2490
2491static int __init init_tmpfs(void)
2492{
2493 int error;
2494
2495 error = bdi_init(&shmem_backing_dev_info);
2496 if (error)
2497 goto out4;
2498
2499 error = init_inodecache();
2500 if (error)
2501 goto out3;
2502
2503 error = register_filesystem(&tmpfs_fs_type);
2504 if (error) {
2505 printk(KERN_ERR "Could not register tmpfs\n");
2506 goto out2;
2507 }
2508
2509 shm_mnt = vfs_kern_mount(&tmpfs_fs_type, MS_NOUSER,
2510 tmpfs_fs_type.name, NULL);
2511 if (IS_ERR(shm_mnt)) {
2512 error = PTR_ERR(shm_mnt);
2513 printk(KERN_ERR "Could not kern_mount tmpfs\n");
2514 goto out1;
2515 }
2516 return 0;
2517
2518out1:
2519 unregister_filesystem(&tmpfs_fs_type);
2520out2:
2521 destroy_inodecache();
2522out3:
2523 bdi_destroy(&shmem_backing_dev_info);
2524out4:
2525 shm_mnt = ERR_PTR(error);
2526 return error;
2527}
2528module_init(init_tmpfs)
2529
2530
2531
2532
2533
2534
2535
2536struct file *shmem_file_setup(char *name, loff_t size, unsigned long flags)
2537{
2538 int error;
2539 struct file *file;
2540 struct inode *inode;
2541 struct dentry *dentry, *root;
2542 struct qstr this;
2543
2544 if (IS_ERR(shm_mnt))
2545 return (void *)shm_mnt;
2546
2547 if (size < 0 || size > SHMEM_MAX_BYTES)
2548 return ERR_PTR(-EINVAL);
2549
2550 if (shmem_acct_size(flags, size))
2551 return ERR_PTR(-ENOMEM);
2552
2553 error = -ENOMEM;
2554 this.name = name;
2555 this.len = strlen(name);
2556 this.hash = 0;
2557 root = shm_mnt->mnt_root;
2558 dentry = d_alloc(root, &this);
2559 if (!dentry)
2560 goto put_memory;
2561
2562 error = -ENFILE;
2563 file = get_empty_filp();
2564 if (!file)
2565 goto put_dentry;
2566
2567 error = -ENOSPC;
2568 inode = shmem_get_inode(root->d_sb, S_IFREG | S_IRWXUGO, 0);
2569 if (!inode)
2570 goto close_file;
2571
2572 SHMEM_I(inode)->flags = flags & VM_ACCOUNT;
2573 d_instantiate(dentry, inode);
2574 inode->i_size = size;
2575 inode->i_nlink = 0;
2576 init_file(file, shm_mnt, dentry, FMODE_WRITE | FMODE_READ,
2577 &shmem_file_operations);
2578 return file;
2579
2580close_file:
2581 put_filp(file);
2582put_dentry:
2583 dput(dentry);
2584put_memory:
2585 shmem_unacct_size(flags, size);
2586 return ERR_PTR(error);
2587}
2588EXPORT_SYMBOL_GPL(shmem_file_setup);
2589
2590
2591
2592
2593
2594int shmem_zero_setup(struct vm_area_struct *vma)
2595{
2596 struct file *file;
2597 loff_t size = vma->vm_end - vma->vm_start;
2598
2599 file = shmem_file_setup("dev/zero", size, vma->vm_flags);
2600 if (IS_ERR(file))
2601 return PTR_ERR(file);
2602
2603 if (vma->vm_file)
2604 fput(vma->vm_file);
2605 vma->vm_file = file;
2606 vma->vm_ops = &shmem_vm_ops;
2607 return 0;
2608}