1
2
3
4
5#include <linux/highmem.h>
6#include <linux/bootmem.h>
7#include <linux/module.h>
8#include <linux/sched.h>
9#include <linux/slab.h>
10#include <linux/mm.h>
11#include <linux/interrupt.h>
12#include <linux/seq_file.h>
13#include <linux/debugfs.h>
14
15#include <asm/e820.h>
16#include <asm/processor.h>
17#include <asm/tlbflush.h>
18#include <asm/sections.h>
19#include <asm/uaccess.h>
20#include <asm/pgalloc.h>
21#include <asm/proto.h>
22#include <asm/pat.h>
23
24
25
26
27struct cpa_data {
28 unsigned long *vaddr;
29 pgprot_t mask_set;
30 pgprot_t mask_clr;
31 int numpages;
32 int flags;
33 unsigned long pfn;
34 unsigned force_split : 1;
35 int curpage;
36};
37
38
39
40
41
42
43
44static DEFINE_SPINLOCK(cpa_lock);
45
46#define CPA_FLUSHTLB 1
47#define CPA_ARRAY 2
48
49#ifdef CONFIG_PROC_FS
50static unsigned long direct_pages_count[PG_LEVEL_NUM];
51
52void update_page_count(int level, unsigned long pages)
53{
54 unsigned long flags;
55
56
57 spin_lock_irqsave(&pgd_lock, flags);
58 direct_pages_count[level] += pages;
59 spin_unlock_irqrestore(&pgd_lock, flags);
60}
61
62static void split_page_count(int level)
63{
64 direct_pages_count[level]--;
65 direct_pages_count[level - 1] += PTRS_PER_PTE;
66}
67
68void arch_report_meminfo(struct seq_file *m)
69{
70 seq_printf(m, "DirectMap4k: %8lu kB\n",
71 direct_pages_count[PG_LEVEL_4K] << 2);
72#if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
73 seq_printf(m, "DirectMap2M: %8lu kB\n",
74 direct_pages_count[PG_LEVEL_2M] << 11);
75#else
76 seq_printf(m, "DirectMap4M: %8lu kB\n",
77 direct_pages_count[PG_LEVEL_2M] << 12);
78#endif
79#ifdef CONFIG_X86_64
80 if (direct_gbpages)
81 seq_printf(m, "DirectMap1G: %8lu kB\n",
82 direct_pages_count[PG_LEVEL_1G] << 20);
83#endif
84}
85#else
86static inline void split_page_count(int level) { }
87#endif
88
89#ifdef CONFIG_X86_64
90
91static inline unsigned long highmap_start_pfn(void)
92{
93 return __pa(_text) >> PAGE_SHIFT;
94}
95
96static inline unsigned long highmap_end_pfn(void)
97{
98 return __pa(roundup((unsigned long)_end, PMD_SIZE)) >> PAGE_SHIFT;
99}
100
101#endif
102
103#ifdef CONFIG_DEBUG_PAGEALLOC
104# define debug_pagealloc 1
105#else
106# define debug_pagealloc 0
107#endif
108
109static inline int
110within(unsigned long addr, unsigned long start, unsigned long end)
111{
112 return addr >= start && addr < end;
113}
114
115
116
117
118
119
120
121
122
123
124
125
126
127void clflush_cache_range(void *vaddr, unsigned int size)
128{
129 void *vend = vaddr + size - 1;
130
131 mb();
132
133 for (; vaddr < vend; vaddr += boot_cpu_data.x86_clflush_size)
134 clflush(vaddr);
135
136
137
138 clflush(vend);
139
140 mb();
141}
142
143static void __cpa_flush_all(void *arg)
144{
145 unsigned long cache = (unsigned long)arg;
146
147
148
149
150
151 __flush_tlb_all();
152
153 if (cache && boot_cpu_data.x86_model >= 4)
154 wbinvd();
155}
156
157static void cpa_flush_all(unsigned long cache)
158{
159 BUG_ON(irqs_disabled());
160
161 on_each_cpu(__cpa_flush_all, (void *) cache, 1);
162}
163
164static void __cpa_flush_range(void *arg)
165{
166
167
168
169
170
171 __flush_tlb_all();
172}
173
174static void cpa_flush_range(unsigned long start, int numpages, int cache)
175{
176 unsigned int i, level;
177 unsigned long addr;
178
179 BUG_ON(irqs_disabled());
180 WARN_ON(PAGE_ALIGN(start) != start);
181
182 on_each_cpu(__cpa_flush_range, NULL, 1);
183
184 if (!cache)
185 return;
186
187
188
189
190
191
192
193 for (i = 0, addr = start; i < numpages; i++, addr += PAGE_SIZE) {
194 pte_t *pte = lookup_address(addr, &level);
195
196
197
198
199 if (pte && (pte_val(*pte) & _PAGE_PRESENT))
200 clflush_cache_range((void *) addr, PAGE_SIZE);
201 }
202}
203
204static void cpa_flush_array(unsigned long *start, int numpages, int cache)
205{
206 unsigned int i, level;
207 unsigned long *addr;
208
209 BUG_ON(irqs_disabled());
210
211 on_each_cpu(__cpa_flush_range, NULL, 1);
212
213 if (!cache)
214 return;
215
216
217 if (numpages >= 1024) {
218 if (boot_cpu_data.x86_model >= 4)
219 wbinvd();
220 return;
221 }
222
223
224
225
226
227
228 for (i = 0, addr = start; i < numpages; i++, addr++) {
229 pte_t *pte = lookup_address(*addr, &level);
230
231
232
233
234 if (pte && (pte_val(*pte) & _PAGE_PRESENT))
235 clflush_cache_range((void *) *addr, PAGE_SIZE);
236 }
237}
238
239
240
241
242
243
244
245static inline pgprot_t static_protections(pgprot_t prot, unsigned long address,
246 unsigned long pfn)
247{
248 pgprot_t forbidden = __pgprot(0);
249
250
251
252
253
254 if (within(pfn, BIOS_BEGIN >> PAGE_SHIFT, BIOS_END >> PAGE_SHIFT))
255 pgprot_val(forbidden) |= _PAGE_NX;
256
257
258
259
260
261
262 if (within(address, (unsigned long)_text, (unsigned long)_etext))
263 pgprot_val(forbidden) |= _PAGE_NX;
264
265
266
267
268
269 if (within(pfn, __pa((unsigned long)__start_rodata) >> PAGE_SHIFT,
270 __pa((unsigned long)__end_rodata) >> PAGE_SHIFT))
271 pgprot_val(forbidden) |= _PAGE_RW;
272
273 prot = __pgprot(pgprot_val(prot) & ~pgprot_val(forbidden));
274
275 return prot;
276}
277
278
279
280
281
282
283
284
285
286pte_t *lookup_address(unsigned long address, unsigned int *level)
287{
288 pgd_t *pgd = pgd_offset_k(address);
289 pud_t *pud;
290 pmd_t *pmd;
291
292 *level = PG_LEVEL_NONE;
293
294 if (pgd_none(*pgd))
295 return NULL;
296
297 pud = pud_offset(pgd, address);
298 if (pud_none(*pud))
299 return NULL;
300
301 *level = PG_LEVEL_1G;
302 if (pud_large(*pud) || !pud_present(*pud))
303 return (pte_t *)pud;
304
305 pmd = pmd_offset(pud, address);
306 if (pmd_none(*pmd))
307 return NULL;
308
309 *level = PG_LEVEL_2M;
310 if (pmd_large(*pmd) || !pmd_present(*pmd))
311 return (pte_t *)pmd;
312
313 *level = PG_LEVEL_4K;
314
315 return pte_offset_kernel(pmd, address);
316}
317EXPORT_SYMBOL_GPL(lookup_address);
318
319
320
321
322static void __set_pmd_pte(pte_t *kpte, unsigned long address, pte_t pte)
323{
324
325 set_pte_atomic(kpte, pte);
326#ifdef CONFIG_X86_32
327 if (!SHARED_KERNEL_PMD) {
328 struct page *page;
329
330 list_for_each_entry(page, &pgd_list, lru) {
331 pgd_t *pgd;
332 pud_t *pud;
333 pmd_t *pmd;
334
335 pgd = (pgd_t *)page_address(page) + pgd_index(address);
336 pud = pud_offset(pgd, address);
337 pmd = pmd_offset(pud, address);
338 set_pte_atomic((pte_t *)pmd, pte);
339 }
340 }
341#endif
342}
343
344static int
345try_preserve_large_page(pte_t *kpte, unsigned long address,
346 struct cpa_data *cpa)
347{
348 unsigned long nextpage_addr, numpages, pmask, psize, flags, addr, pfn;
349 pte_t new_pte, old_pte, *tmp;
350 pgprot_t old_prot, new_prot;
351 int i, do_split = 1;
352 unsigned int level;
353
354 if (cpa->force_split)
355 return 1;
356
357 spin_lock_irqsave(&pgd_lock, flags);
358
359
360
361
362 tmp = lookup_address(address, &level);
363 if (tmp != kpte)
364 goto out_unlock;
365
366 switch (level) {
367 case PG_LEVEL_2M:
368 psize = PMD_PAGE_SIZE;
369 pmask = PMD_PAGE_MASK;
370 break;
371#ifdef CONFIG_X86_64
372 case PG_LEVEL_1G:
373 psize = PUD_PAGE_SIZE;
374 pmask = PUD_PAGE_MASK;
375 break;
376#endif
377 default:
378 do_split = -EINVAL;
379 goto out_unlock;
380 }
381
382
383
384
385
386 nextpage_addr = (address + psize) & pmask;
387 numpages = (nextpage_addr - address) >> PAGE_SHIFT;
388 if (numpages < cpa->numpages)
389 cpa->numpages = numpages;
390
391
392
393
394 old_pte = *kpte;
395 old_prot = new_prot = pte_pgprot(old_pte);
396
397 pgprot_val(new_prot) &= ~pgprot_val(cpa->mask_clr);
398 pgprot_val(new_prot) |= pgprot_val(cpa->mask_set);
399
400
401
402
403
404 pfn = pte_pfn(old_pte) + ((address & (psize - 1)) >> PAGE_SHIFT);
405 cpa->pfn = pfn;
406
407 new_prot = static_protections(new_prot, address, pfn);
408
409
410
411
412
413
414 addr = address + PAGE_SIZE;
415 pfn++;
416 for (i = 1; i < cpa->numpages; i++, addr += PAGE_SIZE, pfn++) {
417 pgprot_t chk_prot = static_protections(new_prot, addr, pfn);
418
419 if (pgprot_val(chk_prot) != pgprot_val(new_prot))
420 goto out_unlock;
421 }
422
423
424
425
426
427 if (pgprot_val(new_prot) == pgprot_val(old_prot)) {
428 do_split = 0;
429 goto out_unlock;
430 }
431
432
433
434
435
436
437
438
439
440 if (address == (nextpage_addr - psize) && cpa->numpages == numpages) {
441
442
443
444
445 new_pte = pfn_pte(pte_pfn(old_pte), canon_pgprot(new_prot));
446 __set_pmd_pte(kpte, address, new_pte);
447 cpa->flags |= CPA_FLUSHTLB;
448 do_split = 0;
449 }
450
451out_unlock:
452 spin_unlock_irqrestore(&pgd_lock, flags);
453
454 return do_split;
455}
456
457static int split_large_page(pte_t *kpte, unsigned long address)
458{
459 unsigned long flags, pfn, pfninc = 1;
460 unsigned int i, level;
461 pte_t *pbase, *tmp;
462 pgprot_t ref_prot;
463 struct page *base;
464
465 if (!debug_pagealloc)
466 spin_unlock(&cpa_lock);
467 base = alloc_pages(GFP_KERNEL, 0);
468 if (!debug_pagealloc)
469 spin_lock(&cpa_lock);
470 if (!base)
471 return -ENOMEM;
472
473 spin_lock_irqsave(&pgd_lock, flags);
474
475
476
477
478 tmp = lookup_address(address, &level);
479 if (tmp != kpte)
480 goto out_unlock;
481
482 pbase = (pte_t *)page_address(base);
483 paravirt_alloc_pte(&init_mm, page_to_pfn(base));
484 ref_prot = pte_pgprot(pte_clrhuge(*kpte));
485
486#ifdef CONFIG_X86_64
487 if (level == PG_LEVEL_1G) {
488 pfninc = PMD_PAGE_SIZE >> PAGE_SHIFT;
489 pgprot_val(ref_prot) |= _PAGE_PSE;
490 }
491#endif
492
493
494
495
496 pfn = pte_pfn(*kpte);
497 for (i = 0; i < PTRS_PER_PTE; i++, pfn += pfninc)
498 set_pte(&pbase[i], pfn_pte(pfn, ref_prot));
499
500 if (address >= (unsigned long)__va(0) &&
501 address < (unsigned long)__va(max_low_pfn_mapped << PAGE_SHIFT))
502 split_page_count(level);
503
504#ifdef CONFIG_X86_64
505 if (address >= (unsigned long)__va(1UL<<32) &&
506 address < (unsigned long)__va(max_pfn_mapped << PAGE_SHIFT))
507 split_page_count(level);
508#endif
509
510
511
512
513
514
515
516
517
518
519
520 ref_prot = pte_pgprot(pte_mkexec(pte_clrhuge(*kpte)));
521 pgprot_val(ref_prot) |= _PAGE_PRESENT;
522 __set_pmd_pte(kpte, address, mk_pte(base, ref_prot));
523 base = NULL;
524
525out_unlock:
526
527
528
529
530 if (base)
531 __free_page(base);
532 spin_unlock_irqrestore(&pgd_lock, flags);
533
534 return 0;
535}
536
537static int __change_page_attr(struct cpa_data *cpa, int primary)
538{
539 unsigned long address;
540 int do_split, err;
541 unsigned int level;
542 pte_t *kpte, old_pte;
543
544 if (cpa->flags & CPA_ARRAY)
545 address = cpa->vaddr[cpa->curpage];
546 else
547 address = *cpa->vaddr;
548
549repeat:
550 kpte = lookup_address(address, &level);
551 if (!kpte)
552 return 0;
553
554 old_pte = *kpte;
555 if (!pte_val(old_pte)) {
556 if (!primary)
557 return 0;
558 WARN(1, KERN_WARNING "CPA: called for zero pte. "
559 "vaddr = %lx cpa->vaddr = %lx\n", address,
560 *cpa->vaddr);
561 return -EINVAL;
562 }
563
564 if (level == PG_LEVEL_4K) {
565 pte_t new_pte;
566 pgprot_t new_prot = pte_pgprot(old_pte);
567 unsigned long pfn = pte_pfn(old_pte);
568
569 pgprot_val(new_prot) &= ~pgprot_val(cpa->mask_clr);
570 pgprot_val(new_prot) |= pgprot_val(cpa->mask_set);
571
572 new_prot = static_protections(new_prot, address, pfn);
573
574
575
576
577
578
579 new_pte = pfn_pte(pfn, canon_pgprot(new_prot));
580 cpa->pfn = pfn;
581
582
583
584 if (pte_val(old_pte) != pte_val(new_pte)) {
585 set_pte_atomic(kpte, new_pte);
586 cpa->flags |= CPA_FLUSHTLB;
587 }
588 cpa->numpages = 1;
589 return 0;
590 }
591
592
593
594
595
596 do_split = try_preserve_large_page(kpte, address, cpa);
597
598
599
600
601
602 if (do_split <= 0)
603 return do_split;
604
605
606
607
608 err = split_large_page(kpte, address);
609 if (!err) {
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628 flush_tlb_all();
629 goto repeat;
630 }
631
632 return err;
633}
634
635static int __change_page_attr_set_clr(struct cpa_data *cpa, int checkalias);
636
637static int cpa_process_alias(struct cpa_data *cpa)
638{
639 struct cpa_data alias_cpa;
640 int ret = 0;
641 unsigned long temp_cpa_vaddr, vaddr;
642
643 if (cpa->pfn >= max_pfn_mapped)
644 return 0;
645
646#ifdef CONFIG_X86_64
647 if (cpa->pfn >= max_low_pfn_mapped && cpa->pfn < (1UL<<(32-PAGE_SHIFT)))
648 return 0;
649#endif
650
651
652
653
654 if (cpa->flags & CPA_ARRAY)
655 vaddr = cpa->vaddr[cpa->curpage];
656 else
657 vaddr = *cpa->vaddr;
658
659 if (!(within(vaddr, PAGE_OFFSET,
660 PAGE_OFFSET + (max_low_pfn_mapped << PAGE_SHIFT))
661#ifdef CONFIG_X86_64
662 || within(vaddr, PAGE_OFFSET + (1UL<<32),
663 PAGE_OFFSET + (max_pfn_mapped << PAGE_SHIFT))
664#endif
665 )) {
666
667 alias_cpa = *cpa;
668 temp_cpa_vaddr = (unsigned long) __va(cpa->pfn << PAGE_SHIFT);
669 alias_cpa.vaddr = &temp_cpa_vaddr;
670 alias_cpa.flags &= ~CPA_ARRAY;
671
672
673 ret = __change_page_attr_set_clr(&alias_cpa, 0);
674 }
675
676#ifdef CONFIG_X86_64
677 if (ret)
678 return ret;
679
680
681
682
683 if (within(vaddr, (unsigned long) _text, (unsigned long) _end))
684 return 0;
685
686
687
688
689
690 if (!within(cpa->pfn, highmap_start_pfn(), highmap_end_pfn()))
691 return 0;
692
693 alias_cpa = *cpa;
694 temp_cpa_vaddr = (cpa->pfn << PAGE_SHIFT) + __START_KERNEL_map - phys_base;
695 alias_cpa.vaddr = &temp_cpa_vaddr;
696 alias_cpa.flags &= ~CPA_ARRAY;
697
698
699
700
701 __change_page_attr_set_clr(&alias_cpa, 0);
702#endif
703 return ret;
704}
705
706static int __change_page_attr_set_clr(struct cpa_data *cpa, int checkalias)
707{
708 int ret, numpages = cpa->numpages;
709
710 while (numpages) {
711
712
713
714
715 cpa->numpages = numpages;
716
717 if (cpa->flags & CPA_ARRAY)
718 cpa->numpages = 1;
719
720 if (!debug_pagealloc)
721 spin_lock(&cpa_lock);
722 ret = __change_page_attr(cpa, checkalias);
723 if (!debug_pagealloc)
724 spin_unlock(&cpa_lock);
725 if (ret)
726 return ret;
727
728 if (checkalias) {
729 ret = cpa_process_alias(cpa);
730 if (ret)
731 return ret;
732 }
733
734
735
736
737
738
739 BUG_ON(cpa->numpages > numpages);
740 numpages -= cpa->numpages;
741 if (cpa->flags & CPA_ARRAY)
742 cpa->curpage++;
743 else
744 *cpa->vaddr += cpa->numpages * PAGE_SIZE;
745
746 }
747 return 0;
748}
749
750static inline int cache_attr(pgprot_t attr)
751{
752 return pgprot_val(attr) &
753 (_PAGE_PAT | _PAGE_PAT_LARGE | _PAGE_PWT | _PAGE_PCD);
754}
755
756static int change_page_attr_set_clr(unsigned long *addr, int numpages,
757 pgprot_t mask_set, pgprot_t mask_clr,
758 int force_split, int array)
759{
760 struct cpa_data cpa;
761 int ret, cache, checkalias;
762
763
764
765
766
767 mask_set = canon_pgprot(mask_set);
768 mask_clr = canon_pgprot(mask_clr);
769 if (!pgprot_val(mask_set) && !pgprot_val(mask_clr) && !force_split)
770 return 0;
771
772
773 if (!array) {
774 if (*addr & ~PAGE_MASK) {
775 *addr &= PAGE_MASK;
776
777
778
779 WARN_ON_ONCE(1);
780 }
781 } else {
782 int i;
783 for (i = 0; i < numpages; i++) {
784 if (addr[i] & ~PAGE_MASK) {
785 addr[i] &= PAGE_MASK;
786 WARN_ON_ONCE(1);
787 }
788 }
789 }
790
791
792 kmap_flush_unused();
793
794 vm_unmap_aliases();
795
796 cpa.vaddr = addr;
797 cpa.numpages = numpages;
798 cpa.mask_set = mask_set;
799 cpa.mask_clr = mask_clr;
800 cpa.flags = 0;
801 cpa.curpage = 0;
802 cpa.force_split = force_split;
803
804 if (array)
805 cpa.flags |= CPA_ARRAY;
806
807
808 checkalias = (pgprot_val(mask_set) | pgprot_val(mask_clr)) != _PAGE_NX;
809
810 ret = __change_page_attr_set_clr(&cpa, checkalias);
811
812
813
814
815 if (!(cpa.flags & CPA_FLUSHTLB))
816 goto out;
817
818
819
820
821
822 cache = cache_attr(mask_set);
823
824
825
826
827
828
829
830 if (!ret && cpu_has_clflush) {
831 if (cpa.flags & CPA_ARRAY)
832 cpa_flush_array(addr, numpages, cache);
833 else
834 cpa_flush_range(*addr, numpages, cache);
835 } else
836 cpa_flush_all(cache);
837
838out:
839 return ret;
840}
841
842static inline int change_page_attr_set(unsigned long *addr, int numpages,
843 pgprot_t mask, int array)
844{
845 return change_page_attr_set_clr(addr, numpages, mask, __pgprot(0), 0,
846 array);
847}
848
849static inline int change_page_attr_clear(unsigned long *addr, int numpages,
850 pgprot_t mask, int array)
851{
852 return change_page_attr_set_clr(addr, numpages, __pgprot(0), mask, 0,
853 array);
854}
855
856int _set_memory_uc(unsigned long addr, int numpages)
857{
858
859
860
861 return change_page_attr_set(&addr, numpages,
862 __pgprot(_PAGE_CACHE_UC_MINUS), 0);
863}
864
865int set_memory_uc(unsigned long addr, int numpages)
866{
867
868
869
870 if (reserve_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE,
871 _PAGE_CACHE_UC_MINUS, NULL))
872 return -EINVAL;
873
874 return _set_memory_uc(addr, numpages);
875}
876EXPORT_SYMBOL(set_memory_uc);
877
878int set_memory_array_uc(unsigned long *addr, int addrinarray)
879{
880 unsigned long start;
881 unsigned long end;
882 int i;
883
884
885
886 for (i = 0; i < addrinarray; i++) {
887 start = __pa(addr[i]);
888 for (end = start + PAGE_SIZE; i < addrinarray - 1; end += PAGE_SIZE) {
889 if (end != __pa(addr[i + 1]))
890 break;
891 i++;
892 }
893 if (reserve_memtype(start, end, _PAGE_CACHE_UC_MINUS, NULL))
894 goto out;
895 }
896
897 return change_page_attr_set(addr, addrinarray,
898 __pgprot(_PAGE_CACHE_UC_MINUS), 1);
899out:
900 for (i = 0; i < addrinarray; i++) {
901 unsigned long tmp = __pa(addr[i]);
902
903 if (tmp == start)
904 break;
905 for (end = tmp + PAGE_SIZE; i < addrinarray - 1; end += PAGE_SIZE) {
906 if (end != __pa(addr[i + 1]))
907 break;
908 i++;
909 }
910 free_memtype(tmp, end);
911 }
912 return -EINVAL;
913}
914EXPORT_SYMBOL(set_memory_array_uc);
915
916int _set_memory_wc(unsigned long addr, int numpages)
917{
918 return change_page_attr_set(&addr, numpages,
919 __pgprot(_PAGE_CACHE_WC), 0);
920}
921
922int set_memory_wc(unsigned long addr, int numpages)
923{
924 if (!pat_enabled)
925 return set_memory_uc(addr, numpages);
926
927 if (reserve_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE,
928 _PAGE_CACHE_WC, NULL))
929 return -EINVAL;
930
931 return _set_memory_wc(addr, numpages);
932}
933EXPORT_SYMBOL(set_memory_wc);
934
935int _set_memory_wb(unsigned long addr, int numpages)
936{
937 return change_page_attr_clear(&addr, numpages,
938 __pgprot(_PAGE_CACHE_MASK), 0);
939}
940
941int set_memory_wb(unsigned long addr, int numpages)
942{
943 free_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE);
944
945 return _set_memory_wb(addr, numpages);
946}
947EXPORT_SYMBOL(set_memory_wb);
948
949int set_memory_array_wb(unsigned long *addr, int addrinarray)
950{
951 int i;
952
953 for (i = 0; i < addrinarray; i++) {
954 unsigned long start = __pa(addr[i]);
955 unsigned long end;
956
957 for (end = start + PAGE_SIZE; i < addrinarray - 1; end += PAGE_SIZE) {
958 if (end != __pa(addr[i + 1]))
959 break;
960 i++;
961 }
962 free_memtype(start, end);
963 }
964 return change_page_attr_clear(addr, addrinarray,
965 __pgprot(_PAGE_CACHE_MASK), 1);
966}
967EXPORT_SYMBOL(set_memory_array_wb);
968
969int set_memory_x(unsigned long addr, int numpages)
970{
971 return change_page_attr_clear(&addr, numpages, __pgprot(_PAGE_NX), 0);
972}
973EXPORT_SYMBOL(set_memory_x);
974
975int set_memory_nx(unsigned long addr, int numpages)
976{
977 return change_page_attr_set(&addr, numpages, __pgprot(_PAGE_NX), 0);
978}
979EXPORT_SYMBOL(set_memory_nx);
980
981int set_memory_ro(unsigned long addr, int numpages)
982{
983 return change_page_attr_clear(&addr, numpages, __pgprot(_PAGE_RW), 0);
984}
985EXPORT_SYMBOL_GPL(set_memory_ro);
986
987int set_memory_rw(unsigned long addr, int numpages)
988{
989 return change_page_attr_set(&addr, numpages, __pgprot(_PAGE_RW), 0);
990}
991EXPORT_SYMBOL_GPL(set_memory_rw);
992
993int set_memory_np(unsigned long addr, int numpages)
994{
995 return change_page_attr_clear(&addr, numpages, __pgprot(_PAGE_PRESENT), 0);
996}
997
998int set_memory_4k(unsigned long addr, int numpages)
999{
1000 return change_page_attr_set_clr(&addr, numpages, __pgprot(0),
1001 __pgprot(0), 1, 0);
1002}
1003
1004int set_pages_uc(struct page *page, int numpages)
1005{
1006 unsigned long addr = (unsigned long)page_address(page);
1007
1008 return set_memory_uc(addr, numpages);
1009}
1010EXPORT_SYMBOL(set_pages_uc);
1011
1012int set_pages_wb(struct page *page, int numpages)
1013{
1014 unsigned long addr = (unsigned long)page_address(page);
1015
1016 return set_memory_wb(addr, numpages);
1017}
1018EXPORT_SYMBOL(set_pages_wb);
1019
1020int set_pages_x(struct page *page, int numpages)
1021{
1022 unsigned long addr = (unsigned long)page_address(page);
1023
1024 return set_memory_x(addr, numpages);
1025}
1026EXPORT_SYMBOL(set_pages_x);
1027
1028int set_pages_nx(struct page *page, int numpages)
1029{
1030 unsigned long addr = (unsigned long)page_address(page);
1031
1032 return set_memory_nx(addr, numpages);
1033}
1034EXPORT_SYMBOL(set_pages_nx);
1035
1036int set_pages_ro(struct page *page, int numpages)
1037{
1038 unsigned long addr = (unsigned long)page_address(page);
1039
1040 return set_memory_ro(addr, numpages);
1041}
1042
1043int set_pages_rw(struct page *page, int numpages)
1044{
1045 unsigned long addr = (unsigned long)page_address(page);
1046
1047 return set_memory_rw(addr, numpages);
1048}
1049
1050#ifdef CONFIG_DEBUG_PAGEALLOC
1051
1052static int __set_pages_p(struct page *page, int numpages)
1053{
1054 unsigned long tempaddr = (unsigned long) page_address(page);
1055 struct cpa_data cpa = { .vaddr = &tempaddr,
1056 .numpages = numpages,
1057 .mask_set = __pgprot(_PAGE_PRESENT | _PAGE_RW),
1058 .mask_clr = __pgprot(0),
1059 .flags = 0};
1060
1061
1062
1063
1064
1065
1066
1067 return __change_page_attr_set_clr(&cpa, 0);
1068}
1069
1070static int __set_pages_np(struct page *page, int numpages)
1071{
1072 unsigned long tempaddr = (unsigned long) page_address(page);
1073 struct cpa_data cpa = { .vaddr = &tempaddr,
1074 .numpages = numpages,
1075 .mask_set = __pgprot(0),
1076 .mask_clr = __pgprot(_PAGE_PRESENT | _PAGE_RW),
1077 .flags = 0};
1078
1079
1080
1081
1082
1083
1084
1085 return __change_page_attr_set_clr(&cpa, 0);
1086}
1087
1088void kernel_map_pages(struct page *page, int numpages, int enable)
1089{
1090 if (PageHighMem(page))
1091 return;
1092 if (!enable) {
1093 debug_check_no_locks_freed(page_address(page),
1094 numpages * PAGE_SIZE);
1095 }
1096
1097
1098
1099
1100 if (!debug_pagealloc_enabled)
1101 return;
1102
1103
1104
1105
1106
1107
1108 if (enable)
1109 __set_pages_p(page, numpages);
1110 else
1111 __set_pages_np(page, numpages);
1112
1113
1114
1115
1116
1117 __flush_tlb_all();
1118}
1119
1120#ifdef CONFIG_HIBERNATION
1121
1122bool kernel_page_present(struct page *page)
1123{
1124 unsigned int level;
1125 pte_t *pte;
1126
1127 if (PageHighMem(page))
1128 return false;
1129
1130 pte = lookup_address((unsigned long)page_address(page), &level);
1131 return (pte_val(*pte) & _PAGE_PRESENT);
1132}
1133
1134#endif
1135
1136#endif
1137
1138
1139
1140
1141
1142#ifdef CONFIG_CPA_DEBUG
1143#include "pageattr-test.c"
1144#endif