init.c 30 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068
  1. /*
  2. * Copyright (C) 1995 Linus Torvalds
  3. * Copyright 2010 Tilera Corporation. All Rights Reserved.
  4. *
  5. * This program is free software; you can redistribute it and/or
  6. * modify it under the terms of the GNU General Public License
  7. * as published by the Free Software Foundation, version 2.
  8. *
  9. * This program is distributed in the hope that it will be useful, but
  10. * WITHOUT ANY WARRANTY; without even the implied warranty of
  11. * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
  12. * NON INFRINGEMENT. See the GNU General Public License for
  13. * more details.
  14. */
  15. #include <linux/module.h>
  16. #include <linux/signal.h>
  17. #include <linux/sched.h>
  18. #include <linux/kernel.h>
  19. #include <linux/errno.h>
  20. #include <linux/string.h>
  21. #include <linux/types.h>
  22. #include <linux/ptrace.h>
  23. #include <linux/mman.h>
  24. #include <linux/mm.h>
  25. #include <linux/hugetlb.h>
  26. #include <linux/swap.h>
  27. #include <linux/smp.h>
  28. #include <linux/init.h>
  29. #include <linux/highmem.h>
  30. #include <linux/pagemap.h>
  31. #include <linux/poison.h>
  32. #include <linux/bootmem.h>
  33. #include <linux/slab.h>
  34. #include <linux/proc_fs.h>
  35. #include <linux/efi.h>
  36. #include <linux/memory_hotplug.h>
  37. #include <linux/uaccess.h>
  38. #include <asm/mmu_context.h>
  39. #include <asm/processor.h>
  40. #include <asm/pgtable.h>
  41. #include <asm/pgalloc.h>
  42. #include <asm/dma.h>
  43. #include <asm/fixmap.h>
  44. #include <asm/tlb.h>
  45. #include <asm/tlbflush.h>
  46. #include <asm/sections.h>
  47. #include <asm/setup.h>
  48. #include <asm/homecache.h>
  49. #include <hv/hypervisor.h>
  50. #include <arch/chip.h>
  51. #include "migrate.h"
  52. #define clear_pgd(pmdptr) (*(pmdptr) = hv_pte(0))
  53. #ifndef __tilegx__
  54. unsigned long VMALLOC_RESERVE = CONFIG_VMALLOC_RESERVE;
  55. EXPORT_SYMBOL(VMALLOC_RESERVE);
  56. #endif
  57. /* Create an L2 page table */
  58. static pte_t * __init alloc_pte(void)
  59. {
  60. return __alloc_bootmem(L2_KERNEL_PGTABLE_SIZE, HV_PAGE_TABLE_ALIGN, 0);
  61. }
  62. /*
  63. * L2 page tables per controller. We allocate these all at once from
  64. * the bootmem allocator and store them here. This saves on kernel L2
  65. * page table memory, compared to allocating a full 64K page per L2
  66. * page table, and also means that in cases where we use huge pages,
  67. * we are guaranteed to later be able to shatter those huge pages and
  68. * switch to using these page tables instead, without requiring
  69. * further allocation. Each l2_ptes[] entry points to the first page
  70. * table for the first hugepage-size piece of memory on the
  71. * controller; other page tables are just indexed directly, i.e. the
  72. * L2 page tables are contiguous in memory for each controller.
  73. */
  74. static pte_t *l2_ptes[MAX_NUMNODES];
  75. static int num_l2_ptes[MAX_NUMNODES];
  76. static void init_prealloc_ptes(int node, int pages)
  77. {
  78. BUG_ON(pages & (PTRS_PER_PTE - 1));
  79. if (pages) {
  80. num_l2_ptes[node] = pages;
  81. l2_ptes[node] = __alloc_bootmem(pages * sizeof(pte_t),
  82. HV_PAGE_TABLE_ALIGN, 0);
  83. }
  84. }
  85. pte_t *get_prealloc_pte(unsigned long pfn)
  86. {
  87. int node = pfn_to_nid(pfn);
  88. pfn &= ~(-1UL << (NR_PA_HIGHBIT_SHIFT - PAGE_SHIFT));
  89. BUG_ON(node >= MAX_NUMNODES);
  90. BUG_ON(pfn >= num_l2_ptes[node]);
  91. return &l2_ptes[node][pfn];
  92. }
  93. /*
  94. * What caching do we expect pages from the heap to have when
  95. * they are allocated during bootup? (Once we've installed the
  96. * "real" swapper_pg_dir.)
  97. */
  98. static int initial_heap_home(void)
  99. {
  100. #if CHIP_HAS_CBOX_HOME_MAP()
  101. if (hash_default)
  102. return PAGE_HOME_HASH;
  103. #endif
  104. return smp_processor_id();
  105. }
  106. /*
  107. * Place a pointer to an L2 page table in a middle page
  108. * directory entry.
  109. */
  110. static void __init assign_pte(pmd_t *pmd, pte_t *page_table)
  111. {
  112. phys_addr_t pa = __pa(page_table);
  113. unsigned long l2_ptfn = pa >> HV_LOG2_PAGE_TABLE_ALIGN;
  114. pte_t pteval = hv_pte_set_ptfn(__pgprot(_PAGE_TABLE), l2_ptfn);
  115. BUG_ON((pa & (HV_PAGE_TABLE_ALIGN-1)) != 0);
  116. pteval = pte_set_home(pteval, initial_heap_home());
  117. *(pte_t *)pmd = pteval;
  118. if (page_table != (pte_t *)pmd_page_vaddr(*pmd))
  119. BUG();
  120. }
  121. #ifdef __tilegx__
  122. static inline pmd_t *alloc_pmd(void)
  123. {
  124. return __alloc_bootmem(L1_KERNEL_PGTABLE_SIZE, HV_PAGE_TABLE_ALIGN, 0);
  125. }
  126. static inline void assign_pmd(pud_t *pud, pmd_t *pmd)
  127. {
  128. assign_pte((pmd_t *)pud, (pte_t *)pmd);
  129. }
  130. #endif /* __tilegx__ */
  131. /* Replace the given pmd with a full PTE table. */
  132. void __init shatter_pmd(pmd_t *pmd)
  133. {
  134. pte_t *pte = get_prealloc_pte(pte_pfn(*(pte_t *)pmd));
  135. assign_pte(pmd, pte);
  136. }
  137. #ifdef __tilegx__
  138. static pmd_t *__init get_pmd(pgd_t pgtables[], unsigned long va)
  139. {
  140. pud_t *pud = pud_offset(&pgtables[pgd_index(va)], va);
  141. if (pud_none(*pud))
  142. assign_pmd(pud, alloc_pmd());
  143. return pmd_offset(pud, va);
  144. }
  145. #else
  146. static pmd_t *__init get_pmd(pgd_t pgtables[], unsigned long va)
  147. {
  148. return pmd_offset(pud_offset(&pgtables[pgd_index(va)], va), va);
  149. }
  150. #endif
  151. /*
  152. * This function initializes a certain range of kernel virtual memory
  153. * with new bootmem page tables, everywhere page tables are missing in
  154. * the given range.
  155. */
  156. /*
  157. * NOTE: The pagetables are allocated contiguous on the physical space
  158. * so we can cache the place of the first one and move around without
  159. * checking the pgd every time.
  160. */
  161. static void __init page_table_range_init(unsigned long start,
  162. unsigned long end, pgd_t *pgd)
  163. {
  164. unsigned long vaddr;
  165. start = round_down(start, PMD_SIZE);
  166. end = round_up(end, PMD_SIZE);
  167. for (vaddr = start; vaddr < end; vaddr += PMD_SIZE) {
  168. pmd_t *pmd = get_pmd(pgd, vaddr);
  169. if (pmd_none(*pmd))
  170. assign_pte(pmd, alloc_pte());
  171. }
  172. }
  173. #if CHIP_HAS_CBOX_HOME_MAP()
  174. static int __initdata ktext_hash = 1; /* .text pages */
  175. static int __initdata kdata_hash = 1; /* .data and .bss pages */
  176. int __write_once hash_default = 1; /* kernel allocator pages */
  177. EXPORT_SYMBOL(hash_default);
  178. int __write_once kstack_hash = 1; /* if no homecaching, use h4h */
  179. #endif /* CHIP_HAS_CBOX_HOME_MAP */
  180. /*
  181. * CPUs to use to for striping the pages of kernel data. If hash-for-home
  182. * is available, this is only relevant if kcache_hash sets up the
  183. * .data and .bss to be page-homed, and we don't want the default mode
  184. * of using the full set of kernel cpus for the striping.
  185. */
  186. static __initdata struct cpumask kdata_mask;
  187. static __initdata int kdata_arg_seen;
  188. int __write_once kdata_huge; /* if no homecaching, small pages */
  189. /* Combine a generic pgprot_t with cache home to get a cache-aware pgprot. */
  190. static pgprot_t __init construct_pgprot(pgprot_t prot, int home)
  191. {
  192. prot = pte_set_home(prot, home);
  193. #if CHIP_HAS_CBOX_HOME_MAP()
  194. if (home == PAGE_HOME_IMMUTABLE) {
  195. if (ktext_hash)
  196. prot = hv_pte_set_mode(prot, HV_PTE_MODE_CACHE_HASH_L3);
  197. else
  198. prot = hv_pte_set_mode(prot, HV_PTE_MODE_CACHE_NO_L3);
  199. }
  200. #endif
  201. return prot;
  202. }
  203. /*
  204. * For a given kernel data VA, how should it be cached?
  205. * We return the complete pgprot_t with caching bits set.
  206. */
  207. static pgprot_t __init init_pgprot(ulong address)
  208. {
  209. int cpu;
  210. unsigned long page;
  211. enum { CODE_DELTA = MEM_SV_INTRPT - PAGE_OFFSET };
  212. #if CHIP_HAS_CBOX_HOME_MAP()
  213. /* For kdata=huge, everything is just hash-for-home. */
  214. if (kdata_huge)
  215. return construct_pgprot(PAGE_KERNEL, PAGE_HOME_HASH);
  216. #endif
  217. /* We map the aliased pages of permanent text inaccessible. */
  218. if (address < (ulong) _sinittext - CODE_DELTA)
  219. return PAGE_NONE;
  220. /*
  221. * We map read-only data non-coherent for performance. We could
  222. * use neighborhood caching on TILE64, but it's not clear it's a win.
  223. */
  224. if ((address >= (ulong) __start_rodata &&
  225. address < (ulong) __end_rodata) ||
  226. address == (ulong) empty_zero_page) {
  227. return construct_pgprot(PAGE_KERNEL_RO, PAGE_HOME_IMMUTABLE);
  228. }
  229. #ifndef __tilegx__
  230. #if !ATOMIC_LOCKS_FOUND_VIA_TABLE()
  231. /* Force the atomic_locks[] array page to be hash-for-home. */
  232. if (address == (ulong) atomic_locks)
  233. return construct_pgprot(PAGE_KERNEL, PAGE_HOME_HASH);
  234. #endif
  235. #endif
  236. /*
  237. * Everything else that isn't data or bss is heap, so mark it
  238. * with the initial heap home (hash-for-home, or this cpu). This
  239. * includes any addresses after the loaded image and any address before
  240. * _einitdata, since we already captured the case of text before
  241. * _sinittext, and __pa(einittext) is approximately __pa(sinitdata).
  242. *
  243. * All the LOWMEM pages that we mark this way will get their
  244. * struct page homecache properly marked later, in set_page_homes().
  245. * The HIGHMEM pages we leave with a default zero for their
  246. * homes, but with a zero free_time we don't have to actually
  247. * do a flush action the first time we use them, either.
  248. */
  249. if (address >= (ulong) _end || address < (ulong) _einitdata)
  250. return construct_pgprot(PAGE_KERNEL, initial_heap_home());
  251. #if CHIP_HAS_CBOX_HOME_MAP()
  252. /* Use hash-for-home if requested for data/bss. */
  253. if (kdata_hash)
  254. return construct_pgprot(PAGE_KERNEL, PAGE_HOME_HASH);
  255. #endif
  256. /*
  257. * Make the w1data homed like heap to start with, to avoid
  258. * making it part of the page-striped data area when we're just
  259. * going to convert it to read-only soon anyway.
  260. */
  261. if (address >= (ulong)__w1data_begin && address < (ulong)__w1data_end)
  262. return construct_pgprot(PAGE_KERNEL, initial_heap_home());
  263. /*
  264. * Otherwise we just hand out consecutive cpus. To avoid
  265. * requiring this function to hold state, we just walk forward from
  266. * _sdata by PAGE_SIZE, skipping the readonly and init data, to reach
  267. * the requested address, while walking cpu home around kdata_mask.
  268. * This is typically no more than a dozen or so iterations.
  269. */
  270. page = (((ulong)__w1data_end) + PAGE_SIZE - 1) & PAGE_MASK;
  271. BUG_ON(address < page || address >= (ulong)_end);
  272. cpu = cpumask_first(&kdata_mask);
  273. for (; page < address; page += PAGE_SIZE) {
  274. if (page >= (ulong)&init_thread_union &&
  275. page < (ulong)&init_thread_union + THREAD_SIZE)
  276. continue;
  277. if (page == (ulong)empty_zero_page)
  278. continue;
  279. #ifndef __tilegx__
  280. #if !ATOMIC_LOCKS_FOUND_VIA_TABLE()
  281. if (page == (ulong)atomic_locks)
  282. continue;
  283. #endif
  284. #endif
  285. cpu = cpumask_next(cpu, &kdata_mask);
  286. if (cpu == NR_CPUS)
  287. cpu = cpumask_first(&kdata_mask);
  288. }
  289. return construct_pgprot(PAGE_KERNEL, cpu);
  290. }
  291. /*
  292. * This function sets up how we cache the kernel text. If we have
  293. * hash-for-home support, normally that is used instead (see the
  294. * kcache_hash boot flag for more information). But if we end up
  295. * using a page-based caching technique, this option sets up the
  296. * details of that. In addition, the "ktext=nocache" option may
  297. * always be used to disable local caching of text pages, if desired.
  298. */
  299. static int __initdata ktext_arg_seen;
  300. static int __initdata ktext_small;
  301. static int __initdata ktext_local;
  302. static int __initdata ktext_all;
  303. static int __initdata ktext_nondataplane;
  304. static int __initdata ktext_nocache;
  305. static struct cpumask __initdata ktext_mask;
  306. static int __init setup_ktext(char *str)
  307. {
  308. if (str == NULL)
  309. return -EINVAL;
  310. /* If you have a leading "nocache", turn off ktext caching */
  311. if (strncmp(str, "nocache", 7) == 0) {
  312. ktext_nocache = 1;
  313. pr_info("ktext: disabling local caching of kernel text\n");
  314. str += 7;
  315. if (*str == ',')
  316. ++str;
  317. if (*str == '\0')
  318. return 0;
  319. }
  320. ktext_arg_seen = 1;
  321. /* Default setting on Tile64: use a huge page */
  322. if (strcmp(str, "huge") == 0)
  323. pr_info("ktext: using one huge locally cached page\n");
  324. /* Pay TLB cost but get no cache benefit: cache small pages locally */
  325. else if (strcmp(str, "local") == 0) {
  326. ktext_small = 1;
  327. ktext_local = 1;
  328. pr_info("ktext: using small pages with local caching\n");
  329. }
  330. /* Neighborhood cache ktext pages on all cpus. */
  331. else if (strcmp(str, "all") == 0) {
  332. ktext_small = 1;
  333. ktext_all = 1;
  334. pr_info("ktext: using maximal caching neighborhood\n");
  335. }
  336. /* Neighborhood ktext pages on specified mask */
  337. else if (cpulist_parse(str, &ktext_mask) == 0) {
  338. char buf[NR_CPUS * 5];
  339. cpulist_scnprintf(buf, sizeof(buf), &ktext_mask);
  340. if (cpumask_weight(&ktext_mask) > 1) {
  341. ktext_small = 1;
  342. pr_info("ktext: using caching neighborhood %s "
  343. "with small pages\n", buf);
  344. } else {
  345. pr_info("ktext: caching on cpu %s with one huge page\n",
  346. buf);
  347. }
  348. }
  349. else if (*str)
  350. return -EINVAL;
  351. return 0;
  352. }
  353. early_param("ktext", setup_ktext);
  354. static inline pgprot_t ktext_set_nocache(pgprot_t prot)
  355. {
  356. if (!ktext_nocache)
  357. prot = hv_pte_set_nc(prot);
  358. #if CHIP_HAS_NC_AND_NOALLOC_BITS()
  359. else
  360. prot = hv_pte_set_no_alloc_l2(prot);
  361. #endif
  362. return prot;
  363. }
  364. /* Temporary page table we use for staging. */
  365. static pgd_t pgtables[PTRS_PER_PGD]
  366. __attribute__((aligned(HV_PAGE_TABLE_ALIGN)));
  367. /*
  368. * This maps the physical memory to kernel virtual address space, a total
  369. * of max_low_pfn pages, by creating page tables starting from address
  370. * PAGE_OFFSET.
  371. *
  372. * This routine transitions us from using a set of compiled-in large
  373. * pages to using some more precise caching, including removing access
  374. * to code pages mapped at PAGE_OFFSET (executed only at MEM_SV_START)
  375. * marking read-only data as locally cacheable, striping the remaining
  376. * .data and .bss across all the available tiles, and removing access
  377. * to pages above the top of RAM (thus ensuring a page fault from a bad
  378. * virtual address rather than a hypervisor shoot down for accessing
  379. * memory outside the assigned limits).
  380. */
  381. static void __init kernel_physical_mapping_init(pgd_t *pgd_base)
  382. {
  383. unsigned long long irqmask;
  384. unsigned long address, pfn;
  385. pmd_t *pmd;
  386. pte_t *pte;
  387. int pte_ofs;
  388. const struct cpumask *my_cpu_mask = cpumask_of(smp_processor_id());
  389. struct cpumask kstripe_mask;
  390. int rc, i;
  391. #if CHIP_HAS_CBOX_HOME_MAP()
  392. if (ktext_arg_seen && ktext_hash) {
  393. pr_warning("warning: \"ktext\" boot argument ignored"
  394. " if \"kcache_hash\" sets up text hash-for-home\n");
  395. ktext_small = 0;
  396. }
  397. if (kdata_arg_seen && kdata_hash) {
  398. pr_warning("warning: \"kdata\" boot argument ignored"
  399. " if \"kcache_hash\" sets up data hash-for-home\n");
  400. }
  401. if (kdata_huge && !hash_default) {
  402. pr_warning("warning: disabling \"kdata=huge\"; requires"
  403. " kcache_hash=all or =allbutstack\n");
  404. kdata_huge = 0;
  405. }
  406. #endif
  407. /*
  408. * Set up a mask for cpus to use for kernel striping.
  409. * This is normally all cpus, but minus dataplane cpus if any.
  410. * If the dataplane covers the whole chip, we stripe over
  411. * the whole chip too.
  412. */
  413. cpumask_copy(&kstripe_mask, cpu_possible_mask);
  414. if (!kdata_arg_seen)
  415. kdata_mask = kstripe_mask;
  416. /* Allocate and fill in L2 page tables */
  417. for (i = 0; i < MAX_NUMNODES; ++i) {
  418. #ifdef CONFIG_HIGHMEM
  419. unsigned long end_pfn = node_lowmem_end_pfn[i];
  420. #else
  421. unsigned long end_pfn = node_end_pfn[i];
  422. #endif
  423. unsigned long end_huge_pfn = 0;
  424. /* Pre-shatter the last huge page to allow per-cpu pages. */
  425. if (kdata_huge)
  426. end_huge_pfn = end_pfn - (HPAGE_SIZE >> PAGE_SHIFT);
  427. pfn = node_start_pfn[i];
  428. /* Allocate enough memory to hold L2 page tables for node. */
  429. init_prealloc_ptes(i, end_pfn - pfn);
  430. address = (unsigned long) pfn_to_kaddr(pfn);
  431. while (pfn < end_pfn) {
  432. BUG_ON(address & (HPAGE_SIZE-1));
  433. pmd = get_pmd(pgtables, address);
  434. pte = get_prealloc_pte(pfn);
  435. if (pfn < end_huge_pfn) {
  436. pgprot_t prot = init_pgprot(address);
  437. *(pte_t *)pmd = pte_mkhuge(pfn_pte(pfn, prot));
  438. for (pte_ofs = 0; pte_ofs < PTRS_PER_PTE;
  439. pfn++, pte_ofs++, address += PAGE_SIZE)
  440. pte[pte_ofs] = pfn_pte(pfn, prot);
  441. } else {
  442. if (kdata_huge)
  443. printk(KERN_DEBUG "pre-shattered huge"
  444. " page at %#lx\n", address);
  445. for (pte_ofs = 0; pte_ofs < PTRS_PER_PTE;
  446. pfn++, pte_ofs++, address += PAGE_SIZE) {
  447. pgprot_t prot = init_pgprot(address);
  448. pte[pte_ofs] = pfn_pte(pfn, prot);
  449. }
  450. assign_pte(pmd, pte);
  451. }
  452. }
  453. }
  454. /*
  455. * Set or check ktext_map now that we have cpu_possible_mask
  456. * and kstripe_mask to work with.
  457. */
  458. if (ktext_all)
  459. cpumask_copy(&ktext_mask, cpu_possible_mask);
  460. else if (ktext_nondataplane)
  461. ktext_mask = kstripe_mask;
  462. else if (!cpumask_empty(&ktext_mask)) {
  463. /* Sanity-check any mask that was requested */
  464. struct cpumask bad;
  465. cpumask_andnot(&bad, &ktext_mask, cpu_possible_mask);
  466. cpumask_and(&ktext_mask, &ktext_mask, cpu_possible_mask);
  467. if (!cpumask_empty(&bad)) {
  468. char buf[NR_CPUS * 5];
  469. cpulist_scnprintf(buf, sizeof(buf), &bad);
  470. pr_info("ktext: not using unavailable cpus %s\n", buf);
  471. }
  472. if (cpumask_empty(&ktext_mask)) {
  473. pr_warning("ktext: no valid cpus; caching on %d.\n",
  474. smp_processor_id());
  475. cpumask_copy(&ktext_mask,
  476. cpumask_of(smp_processor_id()));
  477. }
  478. }
  479. address = MEM_SV_INTRPT;
  480. pmd = get_pmd(pgtables, address);
  481. pfn = 0; /* code starts at PA 0 */
  482. if (ktext_small) {
  483. /* Allocate an L2 PTE for the kernel text */
  484. int cpu = 0;
  485. pgprot_t prot = construct_pgprot(PAGE_KERNEL_EXEC,
  486. PAGE_HOME_IMMUTABLE);
  487. if (ktext_local) {
  488. if (ktext_nocache)
  489. prot = hv_pte_set_mode(prot,
  490. HV_PTE_MODE_UNCACHED);
  491. else
  492. prot = hv_pte_set_mode(prot,
  493. HV_PTE_MODE_CACHE_NO_L3);
  494. } else {
  495. prot = hv_pte_set_mode(prot,
  496. HV_PTE_MODE_CACHE_TILE_L3);
  497. cpu = cpumask_first(&ktext_mask);
  498. prot = ktext_set_nocache(prot);
  499. }
  500. BUG_ON(address != (unsigned long)_text);
  501. pte = NULL;
  502. for (; address < (unsigned long)_einittext;
  503. pfn++, address += PAGE_SIZE) {
  504. pte_ofs = pte_index(address);
  505. if (pte_ofs == 0) {
  506. if (pte)
  507. assign_pte(pmd++, pte);
  508. pte = alloc_pte();
  509. }
  510. if (!ktext_local) {
  511. prot = set_remote_cache_cpu(prot, cpu);
  512. cpu = cpumask_next(cpu, &ktext_mask);
  513. if (cpu == NR_CPUS)
  514. cpu = cpumask_first(&ktext_mask);
  515. }
  516. pte[pte_ofs] = pfn_pte(pfn, prot);
  517. }
  518. if (pte)
  519. assign_pte(pmd, pte);
  520. } else {
  521. pte_t pteval = pfn_pte(0, PAGE_KERNEL_EXEC);
  522. pteval = pte_mkhuge(pteval);
  523. #if CHIP_HAS_CBOX_HOME_MAP()
  524. if (ktext_hash) {
  525. pteval = hv_pte_set_mode(pteval,
  526. HV_PTE_MODE_CACHE_HASH_L3);
  527. pteval = ktext_set_nocache(pteval);
  528. } else
  529. #endif /* CHIP_HAS_CBOX_HOME_MAP() */
  530. if (cpumask_weight(&ktext_mask) == 1) {
  531. pteval = set_remote_cache_cpu(pteval,
  532. cpumask_first(&ktext_mask));
  533. pteval = hv_pte_set_mode(pteval,
  534. HV_PTE_MODE_CACHE_TILE_L3);
  535. pteval = ktext_set_nocache(pteval);
  536. } else if (ktext_nocache)
  537. pteval = hv_pte_set_mode(pteval,
  538. HV_PTE_MODE_UNCACHED);
  539. else
  540. pteval = hv_pte_set_mode(pteval,
  541. HV_PTE_MODE_CACHE_NO_L3);
  542. for (; address < (unsigned long)_einittext;
  543. pfn += PFN_DOWN(HPAGE_SIZE), address += HPAGE_SIZE)
  544. *(pte_t *)(pmd++) = pfn_pte(pfn, pteval);
  545. }
  546. /* Set swapper_pgprot here so it is flushed to memory right away. */
  547. swapper_pgprot = init_pgprot((unsigned long)swapper_pg_dir);
  548. /*
  549. * Since we may be changing the caching of the stack and page
  550. * table itself, we invoke an assembly helper to do the
  551. * following steps:
  552. *
  553. * - flush the cache so we start with an empty slate
  554. * - install pgtables[] as the real page table
  555. * - flush the TLB so the new page table takes effect
  556. */
  557. irqmask = interrupt_mask_save_mask();
  558. interrupt_mask_set_mask(-1ULL);
  559. rc = flush_and_install_context(__pa(pgtables),
  560. init_pgprot((unsigned long)pgtables),
  561. __get_cpu_var(current_asid),
  562. cpumask_bits(my_cpu_mask));
  563. interrupt_mask_restore_mask(irqmask);
  564. BUG_ON(rc != 0);
  565. /* Copy the page table back to the normal swapper_pg_dir. */
  566. memcpy(pgd_base, pgtables, sizeof(pgtables));
  567. __install_page_table(pgd_base, __get_cpu_var(current_asid),
  568. swapper_pgprot);
  569. /*
  570. * We just read swapper_pgprot and thus brought it into the cache,
  571. * with its new home & caching mode. When we start the other CPUs,
  572. * they're going to reference swapper_pgprot via their initial fake
  573. * VA-is-PA mappings, which cache everything locally. At that
  574. * time, if it's in our cache with a conflicting home, the
  575. * simulator's coherence checker will complain. So, flush it out
  576. * of our cache; we're not going to ever use it again anyway.
  577. */
  578. __insn_finv(&swapper_pgprot);
  579. }
  580. /*
  581. * devmem_is_allowed() checks to see if /dev/mem access to a certain address
  582. * is valid. The argument is a physical page number.
  583. *
  584. * On Tile, the only valid things for which we can just hand out unchecked
  585. * PTEs are the kernel code and data. Anything else might change its
  586. * homing with time, and we wouldn't know to adjust the /dev/mem PTEs.
  587. * Note that init_thread_union is released to heap soon after boot,
  588. * so we include it in the init data.
  589. *
  590. * For TILE-Gx, we might want to consider allowing access to PA
  591. * regions corresponding to PCI space, etc.
  592. */
  593. int devmem_is_allowed(unsigned long pagenr)
  594. {
  595. return pagenr < kaddr_to_pfn(_end) &&
  596. !(pagenr >= kaddr_to_pfn(&init_thread_union) ||
  597. pagenr < kaddr_to_pfn(_einitdata)) &&
  598. !(pagenr >= kaddr_to_pfn(_sinittext) ||
  599. pagenr <= kaddr_to_pfn(_einittext-1));
  600. }
  601. #ifdef CONFIG_HIGHMEM
  602. static void __init permanent_kmaps_init(pgd_t *pgd_base)
  603. {
  604. pgd_t *pgd;
  605. pud_t *pud;
  606. pmd_t *pmd;
  607. pte_t *pte;
  608. unsigned long vaddr;
  609. vaddr = PKMAP_BASE;
  610. page_table_range_init(vaddr, vaddr + PAGE_SIZE*LAST_PKMAP, pgd_base);
  611. pgd = swapper_pg_dir + pgd_index(vaddr);
  612. pud = pud_offset(pgd, vaddr);
  613. pmd = pmd_offset(pud, vaddr);
  614. pte = pte_offset_kernel(pmd, vaddr);
  615. pkmap_page_table = pte;
  616. }
  617. #endif /* CONFIG_HIGHMEM */
  618. #ifndef CONFIG_64BIT
  619. static void __init init_free_pfn_range(unsigned long start, unsigned long end)
  620. {
  621. unsigned long pfn;
  622. struct page *page = pfn_to_page(start);
  623. for (pfn = start; pfn < end; ) {
  624. /* Optimize by freeing pages in large batches */
  625. int order = __ffs(pfn);
  626. int count, i;
  627. struct page *p;
  628. if (order >= MAX_ORDER)
  629. order = MAX_ORDER-1;
  630. count = 1 << order;
  631. while (pfn + count > end) {
  632. count >>= 1;
  633. --order;
  634. }
  635. for (p = page, i = 0; i < count; ++i, ++p) {
  636. __ClearPageReserved(p);
  637. /*
  638. * Hacky direct set to avoid unnecessary
  639. * lock take/release for EVERY page here.
  640. */
  641. p->_count.counter = 0;
  642. p->_mapcount.counter = -1;
  643. }
  644. init_page_count(page);
  645. __free_pages(page, order);
  646. adjust_managed_page_count(page, count);
  647. page += count;
  648. pfn += count;
  649. }
  650. }
  651. static void __init set_non_bootmem_pages_init(void)
  652. {
  653. struct zone *z;
  654. for_each_zone(z) {
  655. unsigned long start, end;
  656. int nid = z->zone_pgdat->node_id;
  657. #ifdef CONFIG_HIGHMEM
  658. int idx = zone_idx(z);
  659. #endif
  660. start = z->zone_start_pfn;
  661. end = start + z->spanned_pages;
  662. start = max(start, node_free_pfn[nid]);
  663. start = max(start, max_low_pfn);
  664. #ifdef CONFIG_HIGHMEM
  665. if (idx == ZONE_HIGHMEM)
  666. totalhigh_pages += z->spanned_pages;
  667. #endif
  668. if (kdata_huge) {
  669. unsigned long percpu_pfn = node_percpu_pfn[nid];
  670. if (start < percpu_pfn && end > percpu_pfn)
  671. end = percpu_pfn;
  672. }
  673. #ifdef CONFIG_PCI
  674. if (start <= pci_reserve_start_pfn &&
  675. end > pci_reserve_start_pfn) {
  676. if (end > pci_reserve_end_pfn)
  677. init_free_pfn_range(pci_reserve_end_pfn, end);
  678. end = pci_reserve_start_pfn;
  679. }
  680. #endif
  681. init_free_pfn_range(start, end);
  682. }
  683. }
  684. #endif
  685. /*
  686. * paging_init() sets up the page tables - note that all of lowmem is
  687. * already mapped by head.S.
  688. */
  689. void __init paging_init(void)
  690. {
  691. #ifdef __tilegx__
  692. pud_t *pud;
  693. #endif
  694. pgd_t *pgd_base = swapper_pg_dir;
  695. kernel_physical_mapping_init(pgd_base);
  696. /*
  697. * Fixed mappings, only the page table structure has to be
  698. * created - mappings will be set by set_fixmap():
  699. */
  700. page_table_range_init(fix_to_virt(__end_of_fixed_addresses - 1),
  701. FIXADDR_TOP, pgd_base);
  702. #ifdef CONFIG_HIGHMEM
  703. permanent_kmaps_init(pgd_base);
  704. #endif
  705. #ifdef __tilegx__
  706. /*
  707. * Since GX allocates just one pmd_t array worth of vmalloc space,
  708. * we go ahead and allocate it statically here, then share it
  709. * globally. As a result we don't have to worry about any task
  710. * changing init_mm once we get up and running, and there's no
  711. * need for e.g. vmalloc_sync_all().
  712. */
  713. BUILD_BUG_ON(pgd_index(VMALLOC_START) != pgd_index(VMALLOC_END - 1));
  714. pud = pud_offset(pgd_base + pgd_index(VMALLOC_START), VMALLOC_START);
  715. assign_pmd(pud, alloc_pmd());
  716. #endif
  717. }
  718. /*
  719. * Walk the kernel page tables and derive the page_home() from
  720. * the PTEs, so that set_pte() can properly validate the caching
  721. * of all PTEs it sees.
  722. */
  723. void __init set_page_homes(void)
  724. {
  725. }
  726. static void __init set_max_mapnr_init(void)
  727. {
  728. #ifdef CONFIG_FLATMEM
  729. max_mapnr = max_low_pfn;
  730. #endif
  731. }
  732. void __init mem_init(void)
  733. {
  734. int i;
  735. #ifndef __tilegx__
  736. void *last;
  737. #endif
  738. #ifdef CONFIG_FLATMEM
  739. BUG_ON(!mem_map);
  740. #endif
  741. #ifdef CONFIG_HIGHMEM
  742. /* check that fixmap and pkmap do not overlap */
  743. if (PKMAP_ADDR(LAST_PKMAP-1) >= FIXADDR_START) {
  744. pr_err("fixmap and kmap areas overlap"
  745. " - this will crash\n");
  746. pr_err("pkstart: %lxh pkend: %lxh fixstart %lxh\n",
  747. PKMAP_BASE, PKMAP_ADDR(LAST_PKMAP-1),
  748. FIXADDR_START);
  749. BUG();
  750. }
  751. #endif
  752. set_max_mapnr_init();
  753. /* this will put all bootmem onto the freelists */
  754. free_all_bootmem();
  755. #ifndef CONFIG_64BIT
  756. /* count all remaining LOWMEM and give all HIGHMEM to page allocator */
  757. set_non_bootmem_pages_init();
  758. #endif
  759. mem_init_print_info(NULL);
  760. /*
  761. * In debug mode, dump some interesting memory mappings.
  762. */
  763. #ifdef CONFIG_HIGHMEM
  764. printk(KERN_DEBUG " KMAP %#lx - %#lx\n",
  765. FIXADDR_START, FIXADDR_TOP + PAGE_SIZE - 1);
  766. printk(KERN_DEBUG " PKMAP %#lx - %#lx\n",
  767. PKMAP_BASE, PKMAP_ADDR(LAST_PKMAP) - 1);
  768. #endif
  769. #ifdef CONFIG_HUGEVMAP
  770. printk(KERN_DEBUG " HUGEMAP %#lx - %#lx\n",
  771. HUGE_VMAP_BASE, HUGE_VMAP_END - 1);
  772. #endif
  773. printk(KERN_DEBUG " VMALLOC %#lx - %#lx\n",
  774. _VMALLOC_START, _VMALLOC_END - 1);
  775. #ifdef __tilegx__
  776. for (i = MAX_NUMNODES-1; i >= 0; --i) {
  777. struct pglist_data *node = &node_data[i];
  778. if (node->node_present_pages) {
  779. unsigned long start = (unsigned long)
  780. pfn_to_kaddr(node->node_start_pfn);
  781. unsigned long end = start +
  782. (node->node_present_pages << PAGE_SHIFT);
  783. printk(KERN_DEBUG " MEM%d %#lx - %#lx\n",
  784. i, start, end - 1);
  785. }
  786. }
  787. #else
  788. last = high_memory;
  789. for (i = MAX_NUMNODES-1; i >= 0; --i) {
  790. if ((unsigned long)vbase_map[i] != -1UL) {
  791. printk(KERN_DEBUG " LOWMEM%d %#lx - %#lx\n",
  792. i, (unsigned long) (vbase_map[i]),
  793. (unsigned long) (last-1));
  794. last = vbase_map[i];
  795. }
  796. }
  797. #endif
  798. #ifndef __tilegx__
  799. /*
  800. * Convert from using one lock for all atomic operations to
  801. * one per cpu.
  802. */
  803. __init_atomic_per_cpu();
  804. #endif
  805. }
  806. /*
  807. * this is for the non-NUMA, single node SMP system case.
  808. * Specifically, in the case of x86, we will always add
  809. * memory to the highmem for now.
  810. */
  811. #ifndef CONFIG_NEED_MULTIPLE_NODES
  812. int arch_add_memory(u64 start, u64 size)
  813. {
  814. struct pglist_data *pgdata = &contig_page_data;
  815. struct zone *zone = pgdata->node_zones + MAX_NR_ZONES-1;
  816. unsigned long start_pfn = start >> PAGE_SHIFT;
  817. unsigned long nr_pages = size >> PAGE_SHIFT;
  818. return __add_pages(zone, start_pfn, nr_pages);
  819. }
  820. int remove_memory(u64 start, u64 size)
  821. {
  822. return -EINVAL;
  823. }
  824. #ifdef CONFIG_MEMORY_HOTREMOVE
  825. int arch_remove_memory(u64 start, u64 size)
  826. {
  827. /* TODO */
  828. return -EBUSY;
  829. }
  830. #endif
  831. #endif
  832. struct kmem_cache *pgd_cache;
  833. void __init pgtable_cache_init(void)
  834. {
  835. pgd_cache = kmem_cache_create("pgd", SIZEOF_PGD, SIZEOF_PGD, 0, NULL);
  836. if (!pgd_cache)
  837. panic("pgtable_cache_init(): Cannot create pgd cache");
  838. }
  839. #if !CHIP_HAS_COHERENT_LOCAL_CACHE()
  840. /*
  841. * The __w1data area holds data that is only written during initialization,
  842. * and is read-only and thus freely cacheable thereafter. Fix the page
  843. * table entries that cover that region accordingly.
  844. */
  845. static void mark_w1data_ro(void)
  846. {
  847. /* Loop over page table entries */
  848. unsigned long addr = (unsigned long)__w1data_begin;
  849. BUG_ON((addr & (PAGE_SIZE-1)) != 0);
  850. for (; addr <= (unsigned long)__w1data_end - 1; addr += PAGE_SIZE) {
  851. unsigned long pfn = kaddr_to_pfn((void *)addr);
  852. pte_t *ptep = virt_to_pte(NULL, addr);
  853. BUG_ON(pte_huge(*ptep)); /* not relevant for kdata_huge */
  854. set_pte_at(&init_mm, addr, ptep, pfn_pte(pfn, PAGE_KERNEL_RO));
  855. }
  856. }
  857. #endif
  858. #ifdef CONFIG_DEBUG_PAGEALLOC
  859. static long __write_once initfree;
  860. #else
  861. static long __write_once initfree = 1;
  862. #endif
  863. /* Select whether to free (1) or mark unusable (0) the __init pages. */
  864. static int __init set_initfree(char *str)
  865. {
  866. long val;
  867. if (strict_strtol(str, 0, &val) == 0) {
  868. initfree = val;
  869. pr_info("initfree: %s free init pages\n",
  870. initfree ? "will" : "won't");
  871. }
  872. return 1;
  873. }
  874. __setup("initfree=", set_initfree);
  875. static void free_init_pages(char *what, unsigned long begin, unsigned long end)
  876. {
  877. unsigned long addr = (unsigned long) begin;
  878. if (kdata_huge && !initfree) {
  879. pr_warning("Warning: ignoring initfree=0:"
  880. " incompatible with kdata=huge\n");
  881. initfree = 1;
  882. }
  883. end = (end + PAGE_SIZE - 1) & PAGE_MASK;
  884. local_flush_tlb_pages(NULL, begin, PAGE_SIZE, end - begin);
  885. for (addr = begin; addr < end; addr += PAGE_SIZE) {
  886. /*
  887. * Note we just reset the home here directly in the
  888. * page table. We know this is safe because our caller
  889. * just flushed the caches on all the other cpus,
  890. * and they won't be touching any of these pages.
  891. */
  892. int pfn = kaddr_to_pfn((void *)addr);
  893. struct page *page = pfn_to_page(pfn);
  894. pte_t *ptep = virt_to_pte(NULL, addr);
  895. if (!initfree) {
  896. /*
  897. * If debugging page accesses then do not free
  898. * this memory but mark them not present - any
  899. * buggy init-section access will create a
  900. * kernel page fault:
  901. */
  902. pte_clear(&init_mm, addr, ptep);
  903. continue;
  904. }
  905. if (pte_huge(*ptep))
  906. BUG_ON(!kdata_huge);
  907. else
  908. set_pte_at(&init_mm, addr, ptep,
  909. pfn_pte(pfn, PAGE_KERNEL));
  910. memset((void *)addr, POISON_FREE_INITMEM, PAGE_SIZE);
  911. free_reserved_page(page);
  912. }
  913. pr_info("Freeing %s: %ldk freed\n", what, (end - begin) >> 10);
  914. }
  915. void free_initmem(void)
  916. {
  917. const unsigned long text_delta = MEM_SV_INTRPT - PAGE_OFFSET;
  918. /*
  919. * Evict the dirty initdata on the boot cpu, evict the w1data
  920. * wherever it's homed, and evict all the init code everywhere.
  921. * We are guaranteed that no one will touch the init pages any
  922. * more, and although other cpus may be touching the w1data,
  923. * we only actually change the caching on tile64, which won't
  924. * be keeping local copies in the other tiles' caches anyway.
  925. */
  926. homecache_evict(&cpu_cacheable_map);
  927. /* Free the data pages that we won't use again after init. */
  928. free_init_pages("unused kernel data",
  929. (unsigned long)_sinitdata,
  930. (unsigned long)_einitdata);
  931. /*
  932. * Free the pages mapped from 0xc0000000 that correspond to code
  933. * pages from MEM_SV_INTRPT that we won't use again after init.
  934. */
  935. free_init_pages("unused kernel text",
  936. (unsigned long)_sinittext - text_delta,
  937. (unsigned long)_einittext - text_delta);
  938. #if !CHIP_HAS_COHERENT_LOCAL_CACHE()
  939. /*
  940. * Upgrade the .w1data section to globally cached.
  941. * We don't do this on tilepro, since the cache architecture
  942. * pretty much makes it irrelevant, and in any case we end
  943. * up having racing issues with other tiles that may touch
  944. * the data after we flush the cache but before we update
  945. * the PTEs and flush the TLBs, causing sharer shootdowns
  946. * later. Even though this is to clean data, it seems like
  947. * an unnecessary complication.
  948. */
  949. mark_w1data_ro();
  950. #endif
  951. /* Do a global TLB flush so everyone sees the changes. */
  952. flush_tlb_all();
  953. }