init.c 49 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786
  1. /* $Id: init.c,v 1.209 2002/02/09 19:49:31 davem Exp $
  2. * arch/sparc64/mm/init.c
  3. *
  4. * Copyright (C) 1996-1999 David S. Miller (davem@caip.rutgers.edu)
  5. * Copyright (C) 1997-1999 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
  6. */
  7. #include <linux/config.h>
  8. #include <linux/kernel.h>
  9. #include <linux/sched.h>
  10. #include <linux/string.h>
  11. #include <linux/init.h>
  12. #include <linux/bootmem.h>
  13. #include <linux/mm.h>
  14. #include <linux/hugetlb.h>
  15. #include <linux/slab.h>
  16. #include <linux/initrd.h>
  17. #include <linux/swap.h>
  18. #include <linux/pagemap.h>
  19. #include <linux/fs.h>
  20. #include <linux/seq_file.h>
  21. #include <linux/kprobes.h>
  22. #include <linux/cache.h>
  23. #include <asm/head.h>
  24. #include <asm/system.h>
  25. #include <asm/page.h>
  26. #include <asm/pgalloc.h>
  27. #include <asm/pgtable.h>
  28. #include <asm/oplib.h>
  29. #include <asm/iommu.h>
  30. #include <asm/io.h>
  31. #include <asm/uaccess.h>
  32. #include <asm/mmu_context.h>
  33. #include <asm/tlbflush.h>
  34. #include <asm/dma.h>
  35. #include <asm/starfire.h>
  36. #include <asm/tlb.h>
  37. #include <asm/spitfire.h>
  38. #include <asm/sections.h>
  39. extern void device_scan(void);
  40. struct sparc_phys_banks sp_banks[SPARC_PHYS_BANKS];
  41. unsigned long *sparc64_valid_addr_bitmap;
  42. /* Ugly, but necessary... -DaveM */
  43. unsigned long phys_base __read_mostly;
  44. unsigned long kern_base __read_mostly;
  45. unsigned long kern_size __read_mostly;
  46. unsigned long pfn_base __read_mostly;
  47. /* This is even uglier. We have a problem where the kernel may not be
  48. * located at phys_base. However, initial __alloc_bootmem() calls need to
  49. * be adjusted to be within the 4-8Megs that the kernel is mapped to, else
  50. * those page mappings wont work. Things are ok after inherit_prom_mappings
  51. * is called though. Dave says he'll clean this up some other time.
  52. * -- BenC
  53. */
  54. static unsigned long bootmap_base;
  55. /* get_new_mmu_context() uses "cache + 1". */
  56. DEFINE_SPINLOCK(ctx_alloc_lock);
  57. unsigned long tlb_context_cache = CTX_FIRST_VERSION - 1;
  58. #define CTX_BMAP_SLOTS (1UL << (CTX_NR_BITS - 6))
  59. unsigned long mmu_context_bmap[CTX_BMAP_SLOTS];
  60. /* References to special section boundaries */
  61. extern char _start[], _end[];
  62. /* Initial ramdisk setup */
  63. extern unsigned long sparc_ramdisk_image64;
  64. extern unsigned int sparc_ramdisk_image;
  65. extern unsigned int sparc_ramdisk_size;
  66. struct page *mem_map_zero __read_mostly;
  67. int bigkernel = 0;
  68. /* XXX Tune this... */
  69. #define PGT_CACHE_LOW 25
  70. #define PGT_CACHE_HIGH 50
  71. void check_pgt_cache(void)
  72. {
  73. preempt_disable();
  74. if (pgtable_cache_size > PGT_CACHE_HIGH) {
  75. do {
  76. if (pgd_quicklist)
  77. free_pgd_slow(get_pgd_fast());
  78. if (pte_quicklist[0])
  79. free_pte_slow(pte_alloc_one_fast(NULL, 0));
  80. if (pte_quicklist[1])
  81. free_pte_slow(pte_alloc_one_fast(NULL, 1 << (PAGE_SHIFT + 10)));
  82. } while (pgtable_cache_size > PGT_CACHE_LOW);
  83. }
  84. preempt_enable();
  85. }
  86. #ifdef CONFIG_DEBUG_DCFLUSH
  87. atomic_t dcpage_flushes = ATOMIC_INIT(0);
  88. #ifdef CONFIG_SMP
  89. atomic_t dcpage_flushes_xcall = ATOMIC_INIT(0);
  90. #endif
  91. #endif
  92. __inline__ void flush_dcache_page_impl(struct page *page)
  93. {
  94. #ifdef CONFIG_DEBUG_DCFLUSH
  95. atomic_inc(&dcpage_flushes);
  96. #endif
  97. #ifdef DCACHE_ALIASING_POSSIBLE
  98. __flush_dcache_page(page_address(page),
  99. ((tlb_type == spitfire) &&
  100. page_mapping(page) != NULL));
  101. #else
  102. if (page_mapping(page) != NULL &&
  103. tlb_type == spitfire)
  104. __flush_icache_page(__pa(page_address(page)));
  105. #endif
  106. }
  107. #define PG_dcache_dirty PG_arch_1
  108. #define PG_dcache_cpu_shift 24
  109. #define PG_dcache_cpu_mask (256 - 1)
  110. #if NR_CPUS > 256
  111. #error D-cache dirty tracking and thread_info->cpu need fixing for > 256 cpus
  112. #endif
  113. #define dcache_dirty_cpu(page) \
  114. (((page)->flags >> PG_dcache_cpu_shift) & PG_dcache_cpu_mask)
  115. static __inline__ void set_dcache_dirty(struct page *page, int this_cpu)
  116. {
  117. unsigned long mask = this_cpu;
  118. unsigned long non_cpu_bits;
  119. non_cpu_bits = ~(PG_dcache_cpu_mask << PG_dcache_cpu_shift);
  120. mask = (mask << PG_dcache_cpu_shift) | (1UL << PG_dcache_dirty);
  121. __asm__ __volatile__("1:\n\t"
  122. "ldx [%2], %%g7\n\t"
  123. "and %%g7, %1, %%g1\n\t"
  124. "or %%g1, %0, %%g1\n\t"
  125. "casx [%2], %%g7, %%g1\n\t"
  126. "cmp %%g7, %%g1\n\t"
  127. "membar #StoreLoad | #StoreStore\n\t"
  128. "bne,pn %%xcc, 1b\n\t"
  129. " nop"
  130. : /* no outputs */
  131. : "r" (mask), "r" (non_cpu_bits), "r" (&page->flags)
  132. : "g1", "g7");
  133. }
  134. static __inline__ void clear_dcache_dirty_cpu(struct page *page, unsigned long cpu)
  135. {
  136. unsigned long mask = (1UL << PG_dcache_dirty);
  137. __asm__ __volatile__("! test_and_clear_dcache_dirty\n"
  138. "1:\n\t"
  139. "ldx [%2], %%g7\n\t"
  140. "srlx %%g7, %4, %%g1\n\t"
  141. "and %%g1, %3, %%g1\n\t"
  142. "cmp %%g1, %0\n\t"
  143. "bne,pn %%icc, 2f\n\t"
  144. " andn %%g7, %1, %%g1\n\t"
  145. "casx [%2], %%g7, %%g1\n\t"
  146. "cmp %%g7, %%g1\n\t"
  147. "membar #StoreLoad | #StoreStore\n\t"
  148. "bne,pn %%xcc, 1b\n\t"
  149. " nop\n"
  150. "2:"
  151. : /* no outputs */
  152. : "r" (cpu), "r" (mask), "r" (&page->flags),
  153. "i" (PG_dcache_cpu_mask),
  154. "i" (PG_dcache_cpu_shift)
  155. : "g1", "g7");
  156. }
  157. extern void __update_mmu_cache(unsigned long mmu_context_hw, unsigned long address, pte_t pte, int code);
  158. void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t pte)
  159. {
  160. struct page *page;
  161. unsigned long pfn;
  162. unsigned long pg_flags;
  163. pfn = pte_pfn(pte);
  164. if (pfn_valid(pfn) &&
  165. (page = pfn_to_page(pfn), page_mapping(page)) &&
  166. ((pg_flags = page->flags) & (1UL << PG_dcache_dirty))) {
  167. int cpu = ((pg_flags >> PG_dcache_cpu_shift) &
  168. PG_dcache_cpu_mask);
  169. int this_cpu = get_cpu();
  170. /* This is just to optimize away some function calls
  171. * in the SMP case.
  172. */
  173. if (cpu == this_cpu)
  174. flush_dcache_page_impl(page);
  175. else
  176. smp_flush_dcache_page_impl(page, cpu);
  177. clear_dcache_dirty_cpu(page, cpu);
  178. put_cpu();
  179. }
  180. if (get_thread_fault_code())
  181. __update_mmu_cache(CTX_NRBITS(vma->vm_mm->context),
  182. address, pte, get_thread_fault_code());
  183. }
  184. void flush_dcache_page(struct page *page)
  185. {
  186. struct address_space *mapping;
  187. int this_cpu;
  188. /* Do not bother with the expensive D-cache flush if it
  189. * is merely the zero page. The 'bigcore' testcase in GDB
  190. * causes this case to run millions of times.
  191. */
  192. if (page == ZERO_PAGE(0))
  193. return;
  194. this_cpu = get_cpu();
  195. mapping = page_mapping(page);
  196. if (mapping && !mapping_mapped(mapping)) {
  197. int dirty = test_bit(PG_dcache_dirty, &page->flags);
  198. if (dirty) {
  199. int dirty_cpu = dcache_dirty_cpu(page);
  200. if (dirty_cpu == this_cpu)
  201. goto out;
  202. smp_flush_dcache_page_impl(page, dirty_cpu);
  203. }
  204. set_dcache_dirty(page, this_cpu);
  205. } else {
  206. /* We could delay the flush for the !page_mapping
  207. * case too. But that case is for exec env/arg
  208. * pages and those are %99 certainly going to get
  209. * faulted into the tlb (and thus flushed) anyways.
  210. */
  211. flush_dcache_page_impl(page);
  212. }
  213. out:
  214. put_cpu();
  215. }
  216. void __kprobes flush_icache_range(unsigned long start, unsigned long end)
  217. {
  218. /* Cheetah has coherent I-cache. */
  219. if (tlb_type == spitfire) {
  220. unsigned long kaddr;
  221. for (kaddr = start; kaddr < end; kaddr += PAGE_SIZE)
  222. __flush_icache_page(__get_phys(kaddr));
  223. }
  224. }
  225. unsigned long page_to_pfn(struct page *page)
  226. {
  227. return (unsigned long) ((page - mem_map) + pfn_base);
  228. }
  229. struct page *pfn_to_page(unsigned long pfn)
  230. {
  231. return (mem_map + (pfn - pfn_base));
  232. }
  233. void show_mem(void)
  234. {
  235. printk("Mem-info:\n");
  236. show_free_areas();
  237. printk("Free swap: %6ldkB\n",
  238. nr_swap_pages << (PAGE_SHIFT-10));
  239. printk("%ld pages of RAM\n", num_physpages);
  240. printk("%d free pages\n", nr_free_pages());
  241. printk("%d pages in page table cache\n",pgtable_cache_size);
  242. }
  243. void mmu_info(struct seq_file *m)
  244. {
  245. if (tlb_type == cheetah)
  246. seq_printf(m, "MMU Type\t: Cheetah\n");
  247. else if (tlb_type == cheetah_plus)
  248. seq_printf(m, "MMU Type\t: Cheetah+\n");
  249. else if (tlb_type == spitfire)
  250. seq_printf(m, "MMU Type\t: Spitfire\n");
  251. else
  252. seq_printf(m, "MMU Type\t: ???\n");
  253. #ifdef CONFIG_DEBUG_DCFLUSH
  254. seq_printf(m, "DCPageFlushes\t: %d\n",
  255. atomic_read(&dcpage_flushes));
  256. #ifdef CONFIG_SMP
  257. seq_printf(m, "DCPageFlushesXC\t: %d\n",
  258. atomic_read(&dcpage_flushes_xcall));
  259. #endif /* CONFIG_SMP */
  260. #endif /* CONFIG_DEBUG_DCFLUSH */
  261. }
  262. struct linux_prom_translation {
  263. unsigned long virt;
  264. unsigned long size;
  265. unsigned long data;
  266. };
  267. extern unsigned long prom_boot_page;
  268. extern void prom_remap(unsigned long physpage, unsigned long virtpage, int mmu_ihandle);
  269. extern int prom_get_mmu_ihandle(void);
  270. extern void register_prom_callbacks(void);
  271. /* Exported for SMP bootup purposes. */
  272. unsigned long kern_locked_tte_data;
  273. /* Exported for kernel TLB miss handling in ktlb.S */
  274. unsigned long prom_pmd_phys __read_mostly;
  275. unsigned int swapper_pgd_zero __read_mostly;
  276. void __init early_pgtable_allocfail(char *type)
  277. {
  278. prom_printf("inherit_prom_mappings: Cannot alloc kernel %s.\n", type);
  279. prom_halt();
  280. }
  281. #define BASE_PAGE_SIZE 8192
  282. static pmd_t *prompmd;
  283. /*
  284. * Translate PROM's mapping we capture at boot time into physical address.
  285. * The second parameter is only set from prom_callback() invocations.
  286. */
  287. unsigned long prom_virt_to_phys(unsigned long promva, int *error)
  288. {
  289. pmd_t *pmdp = prompmd + ((promva >> 23) & 0x7ff);
  290. pte_t *ptep;
  291. unsigned long base;
  292. if (pmd_none(*pmdp)) {
  293. if (error)
  294. *error = 1;
  295. return(0);
  296. }
  297. ptep = (pte_t *)__pmd_page(*pmdp) + ((promva >> 13) & 0x3ff);
  298. if (!pte_present(*ptep)) {
  299. if (error)
  300. *error = 1;
  301. return(0);
  302. }
  303. if (error) {
  304. *error = 0;
  305. return(pte_val(*ptep));
  306. }
  307. base = pte_val(*ptep) & _PAGE_PADDR;
  308. return(base + (promva & (BASE_PAGE_SIZE - 1)));
  309. }
  310. static void inherit_prom_mappings(void)
  311. {
  312. struct linux_prom_translation *trans;
  313. unsigned long phys_page, tte_vaddr, tte_data;
  314. void (*remap_func)(unsigned long, unsigned long, int);
  315. pmd_t *pmdp;
  316. pte_t *ptep;
  317. int node, n, i, tsz;
  318. node = prom_finddevice("/virtual-memory");
  319. n = prom_getproplen(node, "translations");
  320. if (n == 0 || n == -1) {
  321. prom_printf("Couldn't get translation property\n");
  322. prom_halt();
  323. }
  324. n += 5 * sizeof(struct linux_prom_translation);
  325. for (tsz = 1; tsz < n; tsz <<= 1)
  326. /* empty */;
  327. trans = __alloc_bootmem(tsz, SMP_CACHE_BYTES, bootmap_base);
  328. if (trans == NULL) {
  329. prom_printf("inherit_prom_mappings: Cannot alloc translations.\n");
  330. prom_halt();
  331. }
  332. memset(trans, 0, tsz);
  333. if ((n = prom_getproperty(node, "translations", (char *)trans, tsz)) == -1) {
  334. prom_printf("Couldn't get translation property\n");
  335. prom_halt();
  336. }
  337. n = n / sizeof(*trans);
  338. /*
  339. * The obp translations are saved based on 8k pagesize, since obp can
  340. * use a mixture of pagesizes. Misses to the 0xf0000000 - 0x100000000,
  341. * ie obp range, are handled in entry.S and do not use the vpte scheme
  342. * (see rant in inherit_locked_prom_mappings()).
  343. */
  344. #define OBP_PMD_SIZE 2048
  345. prompmd = __alloc_bootmem(OBP_PMD_SIZE, OBP_PMD_SIZE, bootmap_base);
  346. if (prompmd == NULL)
  347. early_pgtable_allocfail("pmd");
  348. memset(prompmd, 0, OBP_PMD_SIZE);
  349. for (i = 0; i < n; i++) {
  350. unsigned long vaddr;
  351. if (trans[i].virt >= LOW_OBP_ADDRESS && trans[i].virt < HI_OBP_ADDRESS) {
  352. for (vaddr = trans[i].virt;
  353. ((vaddr < trans[i].virt + trans[i].size) &&
  354. (vaddr < HI_OBP_ADDRESS));
  355. vaddr += BASE_PAGE_SIZE) {
  356. unsigned long val;
  357. pmdp = prompmd + ((vaddr >> 23) & 0x7ff);
  358. if (pmd_none(*pmdp)) {
  359. ptep = __alloc_bootmem(BASE_PAGE_SIZE,
  360. BASE_PAGE_SIZE,
  361. bootmap_base);
  362. if (ptep == NULL)
  363. early_pgtable_allocfail("pte");
  364. memset(ptep, 0, BASE_PAGE_SIZE);
  365. pmd_set(pmdp, ptep);
  366. }
  367. ptep = (pte_t *)__pmd_page(*pmdp) +
  368. ((vaddr >> 13) & 0x3ff);
  369. val = trans[i].data;
  370. /* Clear diag TTE bits. */
  371. if (tlb_type == spitfire)
  372. val &= ~0x0003fe0000000000UL;
  373. set_pte_at(&init_mm, vaddr,
  374. ptep, __pte(val | _PAGE_MODIFIED));
  375. trans[i].data += BASE_PAGE_SIZE;
  376. }
  377. }
  378. }
  379. prom_pmd_phys = __pa(prompmd);
  380. /* Now fixup OBP's idea about where we really are mapped. */
  381. prom_printf("Remapping the kernel... ");
  382. /* Spitfire Errata #32 workaround */
  383. /* NOTE: Using plain zero for the context value is
  384. * correct here, we are not using the Linux trap
  385. * tables yet so we should not use the special
  386. * UltraSPARC-III+ page size encodings yet.
  387. */
  388. __asm__ __volatile__("stxa %0, [%1] %2\n\t"
  389. "flush %%g6"
  390. : /* No outputs */
  391. : "r" (0), "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
  392. switch (tlb_type) {
  393. default:
  394. case spitfire:
  395. phys_page = spitfire_get_dtlb_data(sparc64_highest_locked_tlbent());
  396. break;
  397. case cheetah:
  398. case cheetah_plus:
  399. phys_page = cheetah_get_litlb_data(sparc64_highest_locked_tlbent());
  400. break;
  401. };
  402. phys_page &= _PAGE_PADDR;
  403. phys_page += ((unsigned long)&prom_boot_page -
  404. (unsigned long)KERNBASE);
  405. if (tlb_type == spitfire) {
  406. /* Lock this into i/d tlb entry 59 */
  407. __asm__ __volatile__(
  408. "stxa %%g0, [%2] %3\n\t"
  409. "stxa %0, [%1] %4\n\t"
  410. "membar #Sync\n\t"
  411. "flush %%g6\n\t"
  412. "stxa %%g0, [%2] %5\n\t"
  413. "stxa %0, [%1] %6\n\t"
  414. "membar #Sync\n\t"
  415. "flush %%g6"
  416. : : "r" (phys_page | _PAGE_VALID | _PAGE_SZ8K | _PAGE_CP |
  417. _PAGE_CV | _PAGE_P | _PAGE_L | _PAGE_W),
  418. "r" (59 << 3), "r" (TLB_TAG_ACCESS),
  419. "i" (ASI_DMMU), "i" (ASI_DTLB_DATA_ACCESS),
  420. "i" (ASI_IMMU), "i" (ASI_ITLB_DATA_ACCESS)
  421. : "memory");
  422. } else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
  423. /* Lock this into i/d tlb-0 entry 11 */
  424. __asm__ __volatile__(
  425. "stxa %%g0, [%2] %3\n\t"
  426. "stxa %0, [%1] %4\n\t"
  427. "membar #Sync\n\t"
  428. "flush %%g6\n\t"
  429. "stxa %%g0, [%2] %5\n\t"
  430. "stxa %0, [%1] %6\n\t"
  431. "membar #Sync\n\t"
  432. "flush %%g6"
  433. : : "r" (phys_page | _PAGE_VALID | _PAGE_SZ8K | _PAGE_CP |
  434. _PAGE_CV | _PAGE_P | _PAGE_L | _PAGE_W),
  435. "r" ((0 << 16) | (11 << 3)), "r" (TLB_TAG_ACCESS),
  436. "i" (ASI_DMMU), "i" (ASI_DTLB_DATA_ACCESS),
  437. "i" (ASI_IMMU), "i" (ASI_ITLB_DATA_ACCESS)
  438. : "memory");
  439. } else {
  440. /* Implement me :-) */
  441. BUG();
  442. }
  443. tte_vaddr = (unsigned long) KERNBASE;
  444. /* Spitfire Errata #32 workaround */
  445. /* NOTE: Using plain zero for the context value is
  446. * correct here, we are not using the Linux trap
  447. * tables yet so we should not use the special
  448. * UltraSPARC-III+ page size encodings yet.
  449. */
  450. __asm__ __volatile__("stxa %0, [%1] %2\n\t"
  451. "flush %%g6"
  452. : /* No outputs */
  453. : "r" (0),
  454. "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
  455. if (tlb_type == spitfire)
  456. tte_data = spitfire_get_dtlb_data(sparc64_highest_locked_tlbent());
  457. else
  458. tte_data = cheetah_get_ldtlb_data(sparc64_highest_locked_tlbent());
  459. kern_locked_tte_data = tte_data;
  460. remap_func = (void *) ((unsigned long) &prom_remap -
  461. (unsigned long) &prom_boot_page);
  462. /* Spitfire Errata #32 workaround */
  463. /* NOTE: Using plain zero for the context value is
  464. * correct here, we are not using the Linux trap
  465. * tables yet so we should not use the special
  466. * UltraSPARC-III+ page size encodings yet.
  467. */
  468. __asm__ __volatile__("stxa %0, [%1] %2\n\t"
  469. "flush %%g6"
  470. : /* No outputs */
  471. : "r" (0),
  472. "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
  473. remap_func((tlb_type == spitfire ?
  474. (spitfire_get_dtlb_data(sparc64_highest_locked_tlbent()) & _PAGE_PADDR) :
  475. (cheetah_get_litlb_data(sparc64_highest_locked_tlbent()) & _PAGE_PADDR)),
  476. (unsigned long) KERNBASE,
  477. prom_get_mmu_ihandle());
  478. if (bigkernel)
  479. remap_func(((tte_data + 0x400000) & _PAGE_PADDR),
  480. (unsigned long) KERNBASE + 0x400000, prom_get_mmu_ihandle());
  481. /* Flush out that temporary mapping. */
  482. spitfire_flush_dtlb_nucleus_page(0x0);
  483. spitfire_flush_itlb_nucleus_page(0x0);
  484. /* Now lock us back into the TLBs via OBP. */
  485. prom_dtlb_load(sparc64_highest_locked_tlbent(), tte_data, tte_vaddr);
  486. prom_itlb_load(sparc64_highest_locked_tlbent(), tte_data, tte_vaddr);
  487. if (bigkernel) {
  488. prom_dtlb_load(sparc64_highest_locked_tlbent()-1, tte_data + 0x400000,
  489. tte_vaddr + 0x400000);
  490. prom_itlb_load(sparc64_highest_locked_tlbent()-1, tte_data + 0x400000,
  491. tte_vaddr + 0x400000);
  492. }
  493. /* Re-read translations property. */
  494. if ((n = prom_getproperty(node, "translations", (char *)trans, tsz)) == -1) {
  495. prom_printf("Couldn't get translation property\n");
  496. prom_halt();
  497. }
  498. n = n / sizeof(*trans);
  499. for (i = 0; i < n; i++) {
  500. unsigned long vaddr = trans[i].virt;
  501. unsigned long size = trans[i].size;
  502. if (vaddr < 0xf0000000UL) {
  503. unsigned long avoid_start = (unsigned long) KERNBASE;
  504. unsigned long avoid_end = avoid_start + (4 * 1024 * 1024);
  505. if (bigkernel)
  506. avoid_end += (4 * 1024 * 1024);
  507. if (vaddr < avoid_start) {
  508. unsigned long top = vaddr + size;
  509. if (top > avoid_start)
  510. top = avoid_start;
  511. prom_unmap(top - vaddr, vaddr);
  512. }
  513. if ((vaddr + size) > avoid_end) {
  514. unsigned long bottom = vaddr;
  515. if (bottom < avoid_end)
  516. bottom = avoid_end;
  517. prom_unmap((vaddr + size) - bottom, bottom);
  518. }
  519. }
  520. }
  521. prom_printf("done.\n");
  522. register_prom_callbacks();
  523. }
  524. /* The OBP specifications for sun4u mark 0xfffffffc00000000 and
  525. * upwards as reserved for use by the firmware (I wonder if this
  526. * will be the same on Cheetah...). We use this virtual address
  527. * range for the VPTE table mappings of the nucleus so we need
  528. * to zap them when we enter the PROM. -DaveM
  529. */
  530. static void __flush_nucleus_vptes(void)
  531. {
  532. unsigned long prom_reserved_base = 0xfffffffc00000000UL;
  533. int i;
  534. /* Only DTLB must be checked for VPTE entries. */
  535. if (tlb_type == spitfire) {
  536. for (i = 0; i < 63; i++) {
  537. unsigned long tag;
  538. /* Spitfire Errata #32 workaround */
  539. /* NOTE: Always runs on spitfire, so no cheetah+
  540. * page size encodings.
  541. */
  542. __asm__ __volatile__("stxa %0, [%1] %2\n\t"
  543. "flush %%g6"
  544. : /* No outputs */
  545. : "r" (0),
  546. "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
  547. tag = spitfire_get_dtlb_tag(i);
  548. if (((tag & ~(PAGE_MASK)) == 0) &&
  549. ((tag & (PAGE_MASK)) >= prom_reserved_base)) {
  550. __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
  551. "membar #Sync"
  552. : /* no outputs */
  553. : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
  554. spitfire_put_dtlb_data(i, 0x0UL);
  555. }
  556. }
  557. } else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
  558. for (i = 0; i < 512; i++) {
  559. unsigned long tag = cheetah_get_dtlb_tag(i, 2);
  560. if ((tag & ~PAGE_MASK) == 0 &&
  561. (tag & PAGE_MASK) >= prom_reserved_base) {
  562. __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
  563. "membar #Sync"
  564. : /* no outputs */
  565. : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
  566. cheetah_put_dtlb_data(i, 0x0UL, 2);
  567. }
  568. if (tlb_type != cheetah_plus)
  569. continue;
  570. tag = cheetah_get_dtlb_tag(i, 3);
  571. if ((tag & ~PAGE_MASK) == 0 &&
  572. (tag & PAGE_MASK) >= prom_reserved_base) {
  573. __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
  574. "membar #Sync"
  575. : /* no outputs */
  576. : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
  577. cheetah_put_dtlb_data(i, 0x0UL, 3);
  578. }
  579. }
  580. } else {
  581. /* Implement me :-) */
  582. BUG();
  583. }
  584. }
  585. static int prom_ditlb_set;
  586. struct prom_tlb_entry {
  587. int tlb_ent;
  588. unsigned long tlb_tag;
  589. unsigned long tlb_data;
  590. };
  591. struct prom_tlb_entry prom_itlb[16], prom_dtlb[16];
  592. void prom_world(int enter)
  593. {
  594. unsigned long pstate;
  595. int i;
  596. if (!enter)
  597. set_fs((mm_segment_t) { get_thread_current_ds() });
  598. if (!prom_ditlb_set)
  599. return;
  600. /* Make sure the following runs atomically. */
  601. __asm__ __volatile__("flushw\n\t"
  602. "rdpr %%pstate, %0\n\t"
  603. "wrpr %0, %1, %%pstate"
  604. : "=r" (pstate)
  605. : "i" (PSTATE_IE));
  606. if (enter) {
  607. /* Kick out nucleus VPTEs. */
  608. __flush_nucleus_vptes();
  609. /* Install PROM world. */
  610. for (i = 0; i < 16; i++) {
  611. if (prom_dtlb[i].tlb_ent != -1) {
  612. __asm__ __volatile__("stxa %0, [%1] %2\n\t"
  613. "membar #Sync"
  614. : : "r" (prom_dtlb[i].tlb_tag), "r" (TLB_TAG_ACCESS),
  615. "i" (ASI_DMMU));
  616. if (tlb_type == spitfire)
  617. spitfire_put_dtlb_data(prom_dtlb[i].tlb_ent,
  618. prom_dtlb[i].tlb_data);
  619. else if (tlb_type == cheetah || tlb_type == cheetah_plus)
  620. cheetah_put_ldtlb_data(prom_dtlb[i].tlb_ent,
  621. prom_dtlb[i].tlb_data);
  622. }
  623. if (prom_itlb[i].tlb_ent != -1) {
  624. __asm__ __volatile__("stxa %0, [%1] %2\n\t"
  625. "membar #Sync"
  626. : : "r" (prom_itlb[i].tlb_tag),
  627. "r" (TLB_TAG_ACCESS),
  628. "i" (ASI_IMMU));
  629. if (tlb_type == spitfire)
  630. spitfire_put_itlb_data(prom_itlb[i].tlb_ent,
  631. prom_itlb[i].tlb_data);
  632. else if (tlb_type == cheetah || tlb_type == cheetah_plus)
  633. cheetah_put_litlb_data(prom_itlb[i].tlb_ent,
  634. prom_itlb[i].tlb_data);
  635. }
  636. }
  637. } else {
  638. for (i = 0; i < 16; i++) {
  639. if (prom_dtlb[i].tlb_ent != -1) {
  640. __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
  641. "membar #Sync"
  642. : : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
  643. if (tlb_type == spitfire)
  644. spitfire_put_dtlb_data(prom_dtlb[i].tlb_ent, 0x0UL);
  645. else
  646. cheetah_put_ldtlb_data(prom_dtlb[i].tlb_ent, 0x0UL);
  647. }
  648. if (prom_itlb[i].tlb_ent != -1) {
  649. __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
  650. "membar #Sync"
  651. : : "r" (TLB_TAG_ACCESS),
  652. "i" (ASI_IMMU));
  653. if (tlb_type == spitfire)
  654. spitfire_put_itlb_data(prom_itlb[i].tlb_ent, 0x0UL);
  655. else
  656. cheetah_put_litlb_data(prom_itlb[i].tlb_ent, 0x0UL);
  657. }
  658. }
  659. }
  660. __asm__ __volatile__("wrpr %0, 0, %%pstate"
  661. : : "r" (pstate));
  662. }
  663. void inherit_locked_prom_mappings(int save_p)
  664. {
  665. int i;
  666. int dtlb_seen = 0;
  667. int itlb_seen = 0;
  668. /* Fucking losing PROM has more mappings in the TLB, but
  669. * it (conveniently) fails to mention any of these in the
  670. * translations property. The only ones that matter are
  671. * the locked PROM tlb entries, so we impose the following
  672. * irrecovable rule on the PROM, it is allowed 8 locked
  673. * entries in the ITLB and 8 in the DTLB.
  674. *
  675. * Supposedly the upper 16GB of the address space is
  676. * reserved for OBP, BUT I WISH THIS WAS DOCUMENTED
  677. * SOMEWHERE!!!!!!!!!!!!!!!!! Furthermore the entire interface
  678. * used between the client program and the firmware on sun5
  679. * systems to coordinate mmu mappings is also COMPLETELY
  680. * UNDOCUMENTED!!!!!! Thanks S(t)un!
  681. */
  682. if (save_p) {
  683. for (i = 0; i < 16; i++) {
  684. prom_itlb[i].tlb_ent = -1;
  685. prom_dtlb[i].tlb_ent = -1;
  686. }
  687. }
  688. if (tlb_type == spitfire) {
  689. int high = SPITFIRE_HIGHEST_LOCKED_TLBENT - bigkernel;
  690. for (i = 0; i < high; i++) {
  691. unsigned long data;
  692. /* Spitfire Errata #32 workaround */
  693. /* NOTE: Always runs on spitfire, so no cheetah+
  694. * page size encodings.
  695. */
  696. __asm__ __volatile__("stxa %0, [%1] %2\n\t"
  697. "flush %%g6"
  698. : /* No outputs */
  699. : "r" (0),
  700. "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
  701. data = spitfire_get_dtlb_data(i);
  702. if ((data & (_PAGE_L|_PAGE_VALID)) == (_PAGE_L|_PAGE_VALID)) {
  703. unsigned long tag;
  704. /* Spitfire Errata #32 workaround */
  705. /* NOTE: Always runs on spitfire, so no
  706. * cheetah+ page size encodings.
  707. */
  708. __asm__ __volatile__("stxa %0, [%1] %2\n\t"
  709. "flush %%g6"
  710. : /* No outputs */
  711. : "r" (0),
  712. "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
  713. tag = spitfire_get_dtlb_tag(i);
  714. if (save_p) {
  715. prom_dtlb[dtlb_seen].tlb_ent = i;
  716. prom_dtlb[dtlb_seen].tlb_tag = tag;
  717. prom_dtlb[dtlb_seen].tlb_data = data;
  718. }
  719. __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
  720. "membar #Sync"
  721. : : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
  722. spitfire_put_dtlb_data(i, 0x0UL);
  723. dtlb_seen++;
  724. if (dtlb_seen > 15)
  725. break;
  726. }
  727. }
  728. for (i = 0; i < high; i++) {
  729. unsigned long data;
  730. /* Spitfire Errata #32 workaround */
  731. /* NOTE: Always runs on spitfire, so no
  732. * cheetah+ page size encodings.
  733. */
  734. __asm__ __volatile__("stxa %0, [%1] %2\n\t"
  735. "flush %%g6"
  736. : /* No outputs */
  737. : "r" (0),
  738. "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
  739. data = spitfire_get_itlb_data(i);
  740. if ((data & (_PAGE_L|_PAGE_VALID)) == (_PAGE_L|_PAGE_VALID)) {
  741. unsigned long tag;
  742. /* Spitfire Errata #32 workaround */
  743. /* NOTE: Always runs on spitfire, so no
  744. * cheetah+ page size encodings.
  745. */
  746. __asm__ __volatile__("stxa %0, [%1] %2\n\t"
  747. "flush %%g6"
  748. : /* No outputs */
  749. : "r" (0),
  750. "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
  751. tag = spitfire_get_itlb_tag(i);
  752. if (save_p) {
  753. prom_itlb[itlb_seen].tlb_ent = i;
  754. prom_itlb[itlb_seen].tlb_tag = tag;
  755. prom_itlb[itlb_seen].tlb_data = data;
  756. }
  757. __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
  758. "membar #Sync"
  759. : : "r" (TLB_TAG_ACCESS), "i" (ASI_IMMU));
  760. spitfire_put_itlb_data(i, 0x0UL);
  761. itlb_seen++;
  762. if (itlb_seen > 15)
  763. break;
  764. }
  765. }
  766. } else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
  767. int high = CHEETAH_HIGHEST_LOCKED_TLBENT - bigkernel;
  768. for (i = 0; i < high; i++) {
  769. unsigned long data;
  770. data = cheetah_get_ldtlb_data(i);
  771. if ((data & (_PAGE_L|_PAGE_VALID)) == (_PAGE_L|_PAGE_VALID)) {
  772. unsigned long tag;
  773. tag = cheetah_get_ldtlb_tag(i);
  774. if (save_p) {
  775. prom_dtlb[dtlb_seen].tlb_ent = i;
  776. prom_dtlb[dtlb_seen].tlb_tag = tag;
  777. prom_dtlb[dtlb_seen].tlb_data = data;
  778. }
  779. __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
  780. "membar #Sync"
  781. : : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
  782. cheetah_put_ldtlb_data(i, 0x0UL);
  783. dtlb_seen++;
  784. if (dtlb_seen > 15)
  785. break;
  786. }
  787. }
  788. for (i = 0; i < high; i++) {
  789. unsigned long data;
  790. data = cheetah_get_litlb_data(i);
  791. if ((data & (_PAGE_L|_PAGE_VALID)) == (_PAGE_L|_PAGE_VALID)) {
  792. unsigned long tag;
  793. tag = cheetah_get_litlb_tag(i);
  794. if (save_p) {
  795. prom_itlb[itlb_seen].tlb_ent = i;
  796. prom_itlb[itlb_seen].tlb_tag = tag;
  797. prom_itlb[itlb_seen].tlb_data = data;
  798. }
  799. __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
  800. "membar #Sync"
  801. : : "r" (TLB_TAG_ACCESS), "i" (ASI_IMMU));
  802. cheetah_put_litlb_data(i, 0x0UL);
  803. itlb_seen++;
  804. if (itlb_seen > 15)
  805. break;
  806. }
  807. }
  808. } else {
  809. /* Implement me :-) */
  810. BUG();
  811. }
  812. if (save_p)
  813. prom_ditlb_set = 1;
  814. }
  815. /* Give PROM back his world, done during reboots... */
  816. void prom_reload_locked(void)
  817. {
  818. int i;
  819. for (i = 0; i < 16; i++) {
  820. if (prom_dtlb[i].tlb_ent != -1) {
  821. __asm__ __volatile__("stxa %0, [%1] %2\n\t"
  822. "membar #Sync"
  823. : : "r" (prom_dtlb[i].tlb_tag), "r" (TLB_TAG_ACCESS),
  824. "i" (ASI_DMMU));
  825. if (tlb_type == spitfire)
  826. spitfire_put_dtlb_data(prom_dtlb[i].tlb_ent,
  827. prom_dtlb[i].tlb_data);
  828. else if (tlb_type == cheetah || tlb_type == cheetah_plus)
  829. cheetah_put_ldtlb_data(prom_dtlb[i].tlb_ent,
  830. prom_dtlb[i].tlb_data);
  831. }
  832. if (prom_itlb[i].tlb_ent != -1) {
  833. __asm__ __volatile__("stxa %0, [%1] %2\n\t"
  834. "membar #Sync"
  835. : : "r" (prom_itlb[i].tlb_tag),
  836. "r" (TLB_TAG_ACCESS),
  837. "i" (ASI_IMMU));
  838. if (tlb_type == spitfire)
  839. spitfire_put_itlb_data(prom_itlb[i].tlb_ent,
  840. prom_itlb[i].tlb_data);
  841. else
  842. cheetah_put_litlb_data(prom_itlb[i].tlb_ent,
  843. prom_itlb[i].tlb_data);
  844. }
  845. }
  846. }
  847. #ifdef DCACHE_ALIASING_POSSIBLE
  848. void __flush_dcache_range(unsigned long start, unsigned long end)
  849. {
  850. unsigned long va;
  851. if (tlb_type == spitfire) {
  852. int n = 0;
  853. for (va = start; va < end; va += 32) {
  854. spitfire_put_dcache_tag(va & 0x3fe0, 0x0);
  855. if (++n >= 512)
  856. break;
  857. }
  858. } else {
  859. start = __pa(start);
  860. end = __pa(end);
  861. for (va = start; va < end; va += 32)
  862. __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
  863. "membar #Sync"
  864. : /* no outputs */
  865. : "r" (va),
  866. "i" (ASI_DCACHE_INVALIDATE));
  867. }
  868. }
  869. #endif /* DCACHE_ALIASING_POSSIBLE */
  870. /* If not locked, zap it. */
  871. void __flush_tlb_all(void)
  872. {
  873. unsigned long pstate;
  874. int i;
  875. __asm__ __volatile__("flushw\n\t"
  876. "rdpr %%pstate, %0\n\t"
  877. "wrpr %0, %1, %%pstate"
  878. : "=r" (pstate)
  879. : "i" (PSTATE_IE));
  880. if (tlb_type == spitfire) {
  881. for (i = 0; i < 64; i++) {
  882. /* Spitfire Errata #32 workaround */
  883. /* NOTE: Always runs on spitfire, so no
  884. * cheetah+ page size encodings.
  885. */
  886. __asm__ __volatile__("stxa %0, [%1] %2\n\t"
  887. "flush %%g6"
  888. : /* No outputs */
  889. : "r" (0),
  890. "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
  891. if (!(spitfire_get_dtlb_data(i) & _PAGE_L)) {
  892. __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
  893. "membar #Sync"
  894. : /* no outputs */
  895. : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
  896. spitfire_put_dtlb_data(i, 0x0UL);
  897. }
  898. /* Spitfire Errata #32 workaround */
  899. /* NOTE: Always runs on spitfire, so no
  900. * cheetah+ page size encodings.
  901. */
  902. __asm__ __volatile__("stxa %0, [%1] %2\n\t"
  903. "flush %%g6"
  904. : /* No outputs */
  905. : "r" (0),
  906. "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
  907. if (!(spitfire_get_itlb_data(i) & _PAGE_L)) {
  908. __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
  909. "membar #Sync"
  910. : /* no outputs */
  911. : "r" (TLB_TAG_ACCESS), "i" (ASI_IMMU));
  912. spitfire_put_itlb_data(i, 0x0UL);
  913. }
  914. }
  915. } else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
  916. cheetah_flush_dtlb_all();
  917. cheetah_flush_itlb_all();
  918. }
  919. __asm__ __volatile__("wrpr %0, 0, %%pstate"
  920. : : "r" (pstate));
  921. }
  922. /* Caller does TLB context flushing on local CPU if necessary.
  923. * The caller also ensures that CTX_VALID(mm->context) is false.
  924. *
  925. * We must be careful about boundary cases so that we never
  926. * let the user have CTX 0 (nucleus) or we ever use a CTX
  927. * version of zero (and thus NO_CONTEXT would not be caught
  928. * by version mis-match tests in mmu_context.h).
  929. */
  930. void get_new_mmu_context(struct mm_struct *mm)
  931. {
  932. unsigned long ctx, new_ctx;
  933. unsigned long orig_pgsz_bits;
  934. spin_lock(&ctx_alloc_lock);
  935. orig_pgsz_bits = (mm->context.sparc64_ctx_val & CTX_PGSZ_MASK);
  936. ctx = (tlb_context_cache + 1) & CTX_NR_MASK;
  937. new_ctx = find_next_zero_bit(mmu_context_bmap, 1 << CTX_NR_BITS, ctx);
  938. if (new_ctx >= (1 << CTX_NR_BITS)) {
  939. new_ctx = find_next_zero_bit(mmu_context_bmap, ctx, 1);
  940. if (new_ctx >= ctx) {
  941. int i;
  942. new_ctx = (tlb_context_cache & CTX_VERSION_MASK) +
  943. CTX_FIRST_VERSION;
  944. if (new_ctx == 1)
  945. new_ctx = CTX_FIRST_VERSION;
  946. /* Don't call memset, for 16 entries that's just
  947. * plain silly...
  948. */
  949. mmu_context_bmap[0] = 3;
  950. mmu_context_bmap[1] = 0;
  951. mmu_context_bmap[2] = 0;
  952. mmu_context_bmap[3] = 0;
  953. for (i = 4; i < CTX_BMAP_SLOTS; i += 4) {
  954. mmu_context_bmap[i + 0] = 0;
  955. mmu_context_bmap[i + 1] = 0;
  956. mmu_context_bmap[i + 2] = 0;
  957. mmu_context_bmap[i + 3] = 0;
  958. }
  959. goto out;
  960. }
  961. }
  962. mmu_context_bmap[new_ctx>>6] |= (1UL << (new_ctx & 63));
  963. new_ctx |= (tlb_context_cache & CTX_VERSION_MASK);
  964. out:
  965. tlb_context_cache = new_ctx;
  966. mm->context.sparc64_ctx_val = new_ctx | orig_pgsz_bits;
  967. spin_unlock(&ctx_alloc_lock);
  968. }
  969. #ifndef CONFIG_SMP
  970. struct pgtable_cache_struct pgt_quicklists;
  971. #endif
  972. /* OK, we have to color these pages. The page tables are accessed
  973. * by non-Dcache enabled mapping in the VPTE area by the dtlb_backend.S
  974. * code, as well as by PAGE_OFFSET range direct-mapped addresses by
  975. * other parts of the kernel. By coloring, we make sure that the tlbmiss
  976. * fast handlers do not get data from old/garbage dcache lines that
  977. * correspond to an old/stale virtual address (user/kernel) that
  978. * previously mapped the pagetable page while accessing vpte range
  979. * addresses. The idea is that if the vpte color and PAGE_OFFSET range
  980. * color is the same, then when the kernel initializes the pagetable
  981. * using the later address range, accesses with the first address
  982. * range will see the newly initialized data rather than the garbage.
  983. */
  984. #ifdef DCACHE_ALIASING_POSSIBLE
  985. #define DC_ALIAS_SHIFT 1
  986. #else
  987. #define DC_ALIAS_SHIFT 0
  988. #endif
  989. pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address)
  990. {
  991. struct page *page;
  992. unsigned long color;
  993. {
  994. pte_t *ptep = pte_alloc_one_fast(mm, address);
  995. if (ptep)
  996. return ptep;
  997. }
  998. color = VPTE_COLOR(address);
  999. page = alloc_pages(GFP_KERNEL|__GFP_REPEAT, DC_ALIAS_SHIFT);
  1000. if (page) {
  1001. unsigned long *to_free;
  1002. unsigned long paddr;
  1003. pte_t *pte;
  1004. #ifdef DCACHE_ALIASING_POSSIBLE
  1005. set_page_count(page, 1);
  1006. ClearPageCompound(page);
  1007. set_page_count((page + 1), 1);
  1008. ClearPageCompound(page + 1);
  1009. #endif
  1010. paddr = (unsigned long) page_address(page);
  1011. memset((char *)paddr, 0, (PAGE_SIZE << DC_ALIAS_SHIFT));
  1012. if (!color) {
  1013. pte = (pte_t *) paddr;
  1014. to_free = (unsigned long *) (paddr + PAGE_SIZE);
  1015. } else {
  1016. pte = (pte_t *) (paddr + PAGE_SIZE);
  1017. to_free = (unsigned long *) paddr;
  1018. }
  1019. #ifdef DCACHE_ALIASING_POSSIBLE
  1020. /* Now free the other one up, adjust cache size. */
  1021. preempt_disable();
  1022. *to_free = (unsigned long) pte_quicklist[color ^ 0x1];
  1023. pte_quicklist[color ^ 0x1] = to_free;
  1024. pgtable_cache_size++;
  1025. preempt_enable();
  1026. #endif
  1027. return pte;
  1028. }
  1029. return NULL;
  1030. }
  1031. void sparc_ultra_dump_itlb(void)
  1032. {
  1033. int slot;
  1034. if (tlb_type == spitfire) {
  1035. printk ("Contents of itlb: ");
  1036. for (slot = 0; slot < 14; slot++) printk (" ");
  1037. printk ("%2x:%016lx,%016lx\n",
  1038. 0,
  1039. spitfire_get_itlb_tag(0), spitfire_get_itlb_data(0));
  1040. for (slot = 1; slot < 64; slot+=3) {
  1041. printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx %2x:%016lx,%016lx\n",
  1042. slot,
  1043. spitfire_get_itlb_tag(slot), spitfire_get_itlb_data(slot),
  1044. slot+1,
  1045. spitfire_get_itlb_tag(slot+1), spitfire_get_itlb_data(slot+1),
  1046. slot+2,
  1047. spitfire_get_itlb_tag(slot+2), spitfire_get_itlb_data(slot+2));
  1048. }
  1049. } else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
  1050. printk ("Contents of itlb0:\n");
  1051. for (slot = 0; slot < 16; slot+=2) {
  1052. printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n",
  1053. slot,
  1054. cheetah_get_litlb_tag(slot), cheetah_get_litlb_data(slot),
  1055. slot+1,
  1056. cheetah_get_litlb_tag(slot+1), cheetah_get_litlb_data(slot+1));
  1057. }
  1058. printk ("Contents of itlb2:\n");
  1059. for (slot = 0; slot < 128; slot+=2) {
  1060. printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n",
  1061. slot,
  1062. cheetah_get_itlb_tag(slot), cheetah_get_itlb_data(slot),
  1063. slot+1,
  1064. cheetah_get_itlb_tag(slot+1), cheetah_get_itlb_data(slot+1));
  1065. }
  1066. }
  1067. }
  1068. void sparc_ultra_dump_dtlb(void)
  1069. {
  1070. int slot;
  1071. if (tlb_type == spitfire) {
  1072. printk ("Contents of dtlb: ");
  1073. for (slot = 0; slot < 14; slot++) printk (" ");
  1074. printk ("%2x:%016lx,%016lx\n", 0,
  1075. spitfire_get_dtlb_tag(0), spitfire_get_dtlb_data(0));
  1076. for (slot = 1; slot < 64; slot+=3) {
  1077. printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx %2x:%016lx,%016lx\n",
  1078. slot,
  1079. spitfire_get_dtlb_tag(slot), spitfire_get_dtlb_data(slot),
  1080. slot+1,
  1081. spitfire_get_dtlb_tag(slot+1), spitfire_get_dtlb_data(slot+1),
  1082. slot+2,
  1083. spitfire_get_dtlb_tag(slot+2), spitfire_get_dtlb_data(slot+2));
  1084. }
  1085. } else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
  1086. printk ("Contents of dtlb0:\n");
  1087. for (slot = 0; slot < 16; slot+=2) {
  1088. printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n",
  1089. slot,
  1090. cheetah_get_ldtlb_tag(slot), cheetah_get_ldtlb_data(slot),
  1091. slot+1,
  1092. cheetah_get_ldtlb_tag(slot+1), cheetah_get_ldtlb_data(slot+1));
  1093. }
  1094. printk ("Contents of dtlb2:\n");
  1095. for (slot = 0; slot < 512; slot+=2) {
  1096. printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n",
  1097. slot,
  1098. cheetah_get_dtlb_tag(slot, 2), cheetah_get_dtlb_data(slot, 2),
  1099. slot+1,
  1100. cheetah_get_dtlb_tag(slot+1, 2), cheetah_get_dtlb_data(slot+1, 2));
  1101. }
  1102. if (tlb_type == cheetah_plus) {
  1103. printk ("Contents of dtlb3:\n");
  1104. for (slot = 0; slot < 512; slot+=2) {
  1105. printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n",
  1106. slot,
  1107. cheetah_get_dtlb_tag(slot, 3), cheetah_get_dtlb_data(slot, 3),
  1108. slot+1,
  1109. cheetah_get_dtlb_tag(slot+1, 3), cheetah_get_dtlb_data(slot+1, 3));
  1110. }
  1111. }
  1112. }
  1113. }
  1114. extern unsigned long cmdline_memory_size;
  1115. unsigned long __init bootmem_init(unsigned long *pages_avail)
  1116. {
  1117. unsigned long bootmap_size, start_pfn, end_pfn;
  1118. unsigned long end_of_phys_memory = 0UL;
  1119. unsigned long bootmap_pfn, bytes_avail, size;
  1120. int i;
  1121. #ifdef CONFIG_DEBUG_BOOTMEM
  1122. prom_printf("bootmem_init: Scan sp_banks, ");
  1123. #endif
  1124. bytes_avail = 0UL;
  1125. for (i = 0; sp_banks[i].num_bytes != 0; i++) {
  1126. end_of_phys_memory = sp_banks[i].base_addr +
  1127. sp_banks[i].num_bytes;
  1128. bytes_avail += sp_banks[i].num_bytes;
  1129. if (cmdline_memory_size) {
  1130. if (bytes_avail > cmdline_memory_size) {
  1131. unsigned long slack = bytes_avail - cmdline_memory_size;
  1132. bytes_avail -= slack;
  1133. end_of_phys_memory -= slack;
  1134. sp_banks[i].num_bytes -= slack;
  1135. if (sp_banks[i].num_bytes == 0) {
  1136. sp_banks[i].base_addr = 0xdeadbeef;
  1137. } else {
  1138. sp_banks[i+1].num_bytes = 0;
  1139. sp_banks[i+1].base_addr = 0xdeadbeef;
  1140. }
  1141. break;
  1142. }
  1143. }
  1144. }
  1145. *pages_avail = bytes_avail >> PAGE_SHIFT;
  1146. /* Start with page aligned address of last symbol in kernel
  1147. * image. The kernel is hard mapped below PAGE_OFFSET in a
  1148. * 4MB locked TLB translation.
  1149. */
  1150. start_pfn = PAGE_ALIGN(kern_base + kern_size) >> PAGE_SHIFT;
  1151. bootmap_pfn = start_pfn;
  1152. end_pfn = end_of_phys_memory >> PAGE_SHIFT;
  1153. #ifdef CONFIG_BLK_DEV_INITRD
  1154. /* Now have to check initial ramdisk, so that bootmap does not overwrite it */
  1155. if (sparc_ramdisk_image || sparc_ramdisk_image64) {
  1156. unsigned long ramdisk_image = sparc_ramdisk_image ?
  1157. sparc_ramdisk_image : sparc_ramdisk_image64;
  1158. if (ramdisk_image >= (unsigned long)_end - 2 * PAGE_SIZE)
  1159. ramdisk_image -= KERNBASE;
  1160. initrd_start = ramdisk_image + phys_base;
  1161. initrd_end = initrd_start + sparc_ramdisk_size;
  1162. if (initrd_end > end_of_phys_memory) {
  1163. printk(KERN_CRIT "initrd extends beyond end of memory "
  1164. "(0x%016lx > 0x%016lx)\ndisabling initrd\n",
  1165. initrd_end, end_of_phys_memory);
  1166. initrd_start = 0;
  1167. }
  1168. if (initrd_start) {
  1169. if (initrd_start >= (start_pfn << PAGE_SHIFT) &&
  1170. initrd_start < (start_pfn << PAGE_SHIFT) + 2 * PAGE_SIZE)
  1171. bootmap_pfn = PAGE_ALIGN (initrd_end) >> PAGE_SHIFT;
  1172. }
  1173. }
  1174. #endif
  1175. /* Initialize the boot-time allocator. */
  1176. max_pfn = max_low_pfn = end_pfn;
  1177. min_low_pfn = pfn_base;
  1178. #ifdef CONFIG_DEBUG_BOOTMEM
  1179. prom_printf("init_bootmem(min[%lx], bootmap[%lx], max[%lx])\n",
  1180. min_low_pfn, bootmap_pfn, max_low_pfn);
  1181. #endif
  1182. bootmap_size = init_bootmem_node(NODE_DATA(0), bootmap_pfn, pfn_base, end_pfn);
  1183. bootmap_base = bootmap_pfn << PAGE_SHIFT;
  1184. /* Now register the available physical memory with the
  1185. * allocator.
  1186. */
  1187. for (i = 0; sp_banks[i].num_bytes != 0; i++) {
  1188. #ifdef CONFIG_DEBUG_BOOTMEM
  1189. prom_printf("free_bootmem(sp_banks:%d): base[%lx] size[%lx]\n",
  1190. i, sp_banks[i].base_addr, sp_banks[i].num_bytes);
  1191. #endif
  1192. free_bootmem(sp_banks[i].base_addr, sp_banks[i].num_bytes);
  1193. }
  1194. #ifdef CONFIG_BLK_DEV_INITRD
  1195. if (initrd_start) {
  1196. size = initrd_end - initrd_start;
  1197. /* Resert the initrd image area. */
  1198. #ifdef CONFIG_DEBUG_BOOTMEM
  1199. prom_printf("reserve_bootmem(initrd): base[%llx] size[%lx]\n",
  1200. initrd_start, initrd_end);
  1201. #endif
  1202. reserve_bootmem(initrd_start, size);
  1203. *pages_avail -= PAGE_ALIGN(size) >> PAGE_SHIFT;
  1204. initrd_start += PAGE_OFFSET;
  1205. initrd_end += PAGE_OFFSET;
  1206. }
  1207. #endif
  1208. /* Reserve the kernel text/data/bss. */
  1209. #ifdef CONFIG_DEBUG_BOOTMEM
  1210. prom_printf("reserve_bootmem(kernel): base[%lx] size[%lx]\n", kern_base, kern_size);
  1211. #endif
  1212. reserve_bootmem(kern_base, kern_size);
  1213. *pages_avail -= PAGE_ALIGN(kern_size) >> PAGE_SHIFT;
  1214. /* Reserve the bootmem map. We do not account for it
  1215. * in pages_avail because we will release that memory
  1216. * in free_all_bootmem.
  1217. */
  1218. size = bootmap_size;
  1219. #ifdef CONFIG_DEBUG_BOOTMEM
  1220. prom_printf("reserve_bootmem(bootmap): base[%lx] size[%lx]\n",
  1221. (bootmap_pfn << PAGE_SHIFT), size);
  1222. #endif
  1223. reserve_bootmem((bootmap_pfn << PAGE_SHIFT), size);
  1224. *pages_avail -= PAGE_ALIGN(size) >> PAGE_SHIFT;
  1225. return end_pfn;
  1226. }
  1227. /* paging_init() sets up the page tables */
  1228. extern void cheetah_ecache_flush_init(void);
  1229. static unsigned long last_valid_pfn;
  1230. void __init paging_init(void)
  1231. {
  1232. extern pmd_t swapper_pmd_dir[1024];
  1233. unsigned long alias_base = kern_base + PAGE_OFFSET;
  1234. unsigned long second_alias_page = 0;
  1235. unsigned long pt, flags, end_pfn, pages_avail;
  1236. unsigned long shift = alias_base - ((unsigned long)KERNBASE);
  1237. unsigned long real_end;
  1238. set_bit(0, mmu_context_bmap);
  1239. real_end = (unsigned long)_end;
  1240. if ((real_end > ((unsigned long)KERNBASE + 0x400000)))
  1241. bigkernel = 1;
  1242. #ifdef CONFIG_BLK_DEV_INITRD
  1243. if (sparc_ramdisk_image || sparc_ramdisk_image64)
  1244. real_end = (PAGE_ALIGN(real_end) + PAGE_ALIGN(sparc_ramdisk_size));
  1245. #endif
  1246. /* We assume physical memory starts at some 4mb multiple,
  1247. * if this were not true we wouldn't boot up to this point
  1248. * anyways.
  1249. */
  1250. pt = kern_base | _PAGE_VALID | _PAGE_SZ4MB;
  1251. pt |= _PAGE_CP | _PAGE_CV | _PAGE_P | _PAGE_L | _PAGE_W;
  1252. local_irq_save(flags);
  1253. if (tlb_type == spitfire) {
  1254. __asm__ __volatile__(
  1255. " stxa %1, [%0] %3\n"
  1256. " stxa %2, [%5] %4\n"
  1257. " membar #Sync\n"
  1258. " flush %%g6\n"
  1259. " nop\n"
  1260. " nop\n"
  1261. " nop\n"
  1262. : /* No outputs */
  1263. : "r" (TLB_TAG_ACCESS), "r" (alias_base), "r" (pt),
  1264. "i" (ASI_DMMU), "i" (ASI_DTLB_DATA_ACCESS), "r" (61 << 3)
  1265. : "memory");
  1266. if (real_end >= KERNBASE + 0x340000) {
  1267. second_alias_page = alias_base + 0x400000;
  1268. __asm__ __volatile__(
  1269. " stxa %1, [%0] %3\n"
  1270. " stxa %2, [%5] %4\n"
  1271. " membar #Sync\n"
  1272. " flush %%g6\n"
  1273. " nop\n"
  1274. " nop\n"
  1275. " nop\n"
  1276. : /* No outputs */
  1277. : "r" (TLB_TAG_ACCESS), "r" (second_alias_page), "r" (pt + 0x400000),
  1278. "i" (ASI_DMMU), "i" (ASI_DTLB_DATA_ACCESS), "r" (60 << 3)
  1279. : "memory");
  1280. }
  1281. } else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
  1282. __asm__ __volatile__(
  1283. " stxa %1, [%0] %3\n"
  1284. " stxa %2, [%5] %4\n"
  1285. " membar #Sync\n"
  1286. " flush %%g6\n"
  1287. " nop\n"
  1288. " nop\n"
  1289. " nop\n"
  1290. : /* No outputs */
  1291. : "r" (TLB_TAG_ACCESS), "r" (alias_base), "r" (pt),
  1292. "i" (ASI_DMMU), "i" (ASI_DTLB_DATA_ACCESS), "r" ((0<<16) | (13<<3))
  1293. : "memory");
  1294. if (real_end >= KERNBASE + 0x340000) {
  1295. second_alias_page = alias_base + 0x400000;
  1296. __asm__ __volatile__(
  1297. " stxa %1, [%0] %3\n"
  1298. " stxa %2, [%5] %4\n"
  1299. " membar #Sync\n"
  1300. " flush %%g6\n"
  1301. " nop\n"
  1302. " nop\n"
  1303. " nop\n"
  1304. : /* No outputs */
  1305. : "r" (TLB_TAG_ACCESS), "r" (second_alias_page), "r" (pt + 0x400000),
  1306. "i" (ASI_DMMU), "i" (ASI_DTLB_DATA_ACCESS), "r" ((0<<16) | (12<<3))
  1307. : "memory");
  1308. }
  1309. }
  1310. local_irq_restore(flags);
  1311. /* Now set kernel pgd to upper alias so physical page computations
  1312. * work.
  1313. */
  1314. init_mm.pgd += ((shift) / (sizeof(pgd_t)));
  1315. memset(swapper_pmd_dir, 0, sizeof(swapper_pmd_dir));
  1316. /* Now can init the kernel/bad page tables. */
  1317. pud_set(pud_offset(&swapper_pg_dir[0], 0),
  1318. swapper_pmd_dir + (shift / sizeof(pgd_t)));
  1319. swapper_pgd_zero = pgd_val(init_mm.pgd[0]);
  1320. /* Setup bootmem... */
  1321. pages_avail = 0;
  1322. last_valid_pfn = end_pfn = bootmem_init(&pages_avail);
  1323. /* Inherit non-locked OBP mappings. */
  1324. inherit_prom_mappings();
  1325. /* Ok, we can use our TLB miss and window trap handlers safely.
  1326. * We need to do a quick peek here to see if we are on StarFire
  1327. * or not, so setup_tba can setup the IRQ globals correctly (it
  1328. * needs to get the hard smp processor id correctly).
  1329. */
  1330. {
  1331. extern void setup_tba(int);
  1332. setup_tba(this_is_starfire);
  1333. }
  1334. inherit_locked_prom_mappings(1);
  1335. /* We only created DTLB mapping of this stuff. */
  1336. spitfire_flush_dtlb_nucleus_page(alias_base);
  1337. if (second_alias_page)
  1338. spitfire_flush_dtlb_nucleus_page(second_alias_page);
  1339. __flush_tlb_all();
  1340. {
  1341. unsigned long zones_size[MAX_NR_ZONES];
  1342. unsigned long zholes_size[MAX_NR_ZONES];
  1343. unsigned long npages;
  1344. int znum;
  1345. for (znum = 0; znum < MAX_NR_ZONES; znum++)
  1346. zones_size[znum] = zholes_size[znum] = 0;
  1347. npages = end_pfn - pfn_base;
  1348. zones_size[ZONE_DMA] = npages;
  1349. zholes_size[ZONE_DMA] = npages - pages_avail;
  1350. free_area_init_node(0, &contig_page_data, zones_size,
  1351. phys_base >> PAGE_SHIFT, zholes_size);
  1352. }
  1353. device_scan();
  1354. }
  1355. /* Ok, it seems that the prom can allocate some more memory chunks
  1356. * as a side effect of some prom calls we perform during the
  1357. * boot sequence. My most likely theory is that it is from the
  1358. * prom_set_traptable() call, and OBP is allocating a scratchpad
  1359. * for saving client program register state etc.
  1360. */
  1361. static void __init sort_memlist(struct linux_mlist_p1275 *thislist)
  1362. {
  1363. int swapi = 0;
  1364. int i, mitr;
  1365. unsigned long tmpaddr, tmpsize;
  1366. unsigned long lowest;
  1367. for (i = 0; thislist[i].theres_more != 0; i++) {
  1368. lowest = thislist[i].start_adr;
  1369. for (mitr = i+1; thislist[mitr-1].theres_more != 0; mitr++)
  1370. if (thislist[mitr].start_adr < lowest) {
  1371. lowest = thislist[mitr].start_adr;
  1372. swapi = mitr;
  1373. }
  1374. if (lowest == thislist[i].start_adr)
  1375. continue;
  1376. tmpaddr = thislist[swapi].start_adr;
  1377. tmpsize = thislist[swapi].num_bytes;
  1378. for (mitr = swapi; mitr > i; mitr--) {
  1379. thislist[mitr].start_adr = thislist[mitr-1].start_adr;
  1380. thislist[mitr].num_bytes = thislist[mitr-1].num_bytes;
  1381. }
  1382. thislist[i].start_adr = tmpaddr;
  1383. thislist[i].num_bytes = tmpsize;
  1384. }
  1385. }
  1386. void __init rescan_sp_banks(void)
  1387. {
  1388. struct linux_prom64_registers memlist[64];
  1389. struct linux_mlist_p1275 avail[64], *mlist;
  1390. unsigned long bytes, base_paddr;
  1391. int num_regs, node = prom_finddevice("/memory");
  1392. int i;
  1393. num_regs = prom_getproperty(node, "available",
  1394. (char *) memlist, sizeof(memlist));
  1395. num_regs = (num_regs / sizeof(struct linux_prom64_registers));
  1396. for (i = 0; i < num_regs; i++) {
  1397. avail[i].start_adr = memlist[i].phys_addr;
  1398. avail[i].num_bytes = memlist[i].reg_size;
  1399. avail[i].theres_more = &avail[i + 1];
  1400. }
  1401. avail[i - 1].theres_more = NULL;
  1402. sort_memlist(avail);
  1403. mlist = &avail[0];
  1404. i = 0;
  1405. bytes = mlist->num_bytes;
  1406. base_paddr = mlist->start_adr;
  1407. sp_banks[0].base_addr = base_paddr;
  1408. sp_banks[0].num_bytes = bytes;
  1409. while (mlist->theres_more != NULL){
  1410. i++;
  1411. mlist = mlist->theres_more;
  1412. bytes = mlist->num_bytes;
  1413. if (i >= SPARC_PHYS_BANKS-1) {
  1414. printk ("The machine has more banks than "
  1415. "this kernel can support\n"
  1416. "Increase the SPARC_PHYS_BANKS "
  1417. "setting (currently %d)\n",
  1418. SPARC_PHYS_BANKS);
  1419. i = SPARC_PHYS_BANKS-1;
  1420. break;
  1421. }
  1422. sp_banks[i].base_addr = mlist->start_adr;
  1423. sp_banks[i].num_bytes = mlist->num_bytes;
  1424. }
  1425. i++;
  1426. sp_banks[i].base_addr = 0xdeadbeefbeefdeadUL;
  1427. sp_banks[i].num_bytes = 0;
  1428. for (i = 0; sp_banks[i].num_bytes != 0; i++)
  1429. sp_banks[i].num_bytes &= PAGE_MASK;
  1430. }
  1431. static void __init taint_real_pages(void)
  1432. {
  1433. struct sparc_phys_banks saved_sp_banks[SPARC_PHYS_BANKS];
  1434. int i;
  1435. for (i = 0; i < SPARC_PHYS_BANKS; i++) {
  1436. saved_sp_banks[i].base_addr =
  1437. sp_banks[i].base_addr;
  1438. saved_sp_banks[i].num_bytes =
  1439. sp_banks[i].num_bytes;
  1440. }
  1441. rescan_sp_banks();
  1442. /* Find changes discovered in the sp_bank rescan and
  1443. * reserve the lost portions in the bootmem maps.
  1444. */
  1445. for (i = 0; saved_sp_banks[i].num_bytes; i++) {
  1446. unsigned long old_start, old_end;
  1447. old_start = saved_sp_banks[i].base_addr;
  1448. old_end = old_start +
  1449. saved_sp_banks[i].num_bytes;
  1450. while (old_start < old_end) {
  1451. int n;
  1452. for (n = 0; sp_banks[n].num_bytes; n++) {
  1453. unsigned long new_start, new_end;
  1454. new_start = sp_banks[n].base_addr;
  1455. new_end = new_start + sp_banks[n].num_bytes;
  1456. if (new_start <= old_start &&
  1457. new_end >= (old_start + PAGE_SIZE)) {
  1458. set_bit (old_start >> 22,
  1459. sparc64_valid_addr_bitmap);
  1460. goto do_next_page;
  1461. }
  1462. }
  1463. reserve_bootmem(old_start, PAGE_SIZE);
  1464. do_next_page:
  1465. old_start += PAGE_SIZE;
  1466. }
  1467. }
  1468. }
  1469. void __init mem_init(void)
  1470. {
  1471. unsigned long codepages, datapages, initpages;
  1472. unsigned long addr, last;
  1473. int i;
  1474. i = last_valid_pfn >> ((22 - PAGE_SHIFT) + 6);
  1475. i += 1;
  1476. sparc64_valid_addr_bitmap = (unsigned long *)
  1477. __alloc_bootmem(i << 3, SMP_CACHE_BYTES, bootmap_base);
  1478. if (sparc64_valid_addr_bitmap == NULL) {
  1479. prom_printf("mem_init: Cannot alloc valid_addr_bitmap.\n");
  1480. prom_halt();
  1481. }
  1482. memset(sparc64_valid_addr_bitmap, 0, i << 3);
  1483. addr = PAGE_OFFSET + kern_base;
  1484. last = PAGE_ALIGN(kern_size) + addr;
  1485. while (addr < last) {
  1486. set_bit(__pa(addr) >> 22, sparc64_valid_addr_bitmap);
  1487. addr += PAGE_SIZE;
  1488. }
  1489. taint_real_pages();
  1490. max_mapnr = last_valid_pfn - pfn_base;
  1491. high_memory = __va(last_valid_pfn << PAGE_SHIFT);
  1492. #ifdef CONFIG_DEBUG_BOOTMEM
  1493. prom_printf("mem_init: Calling free_all_bootmem().\n");
  1494. #endif
  1495. totalram_pages = num_physpages = free_all_bootmem() - 1;
  1496. /*
  1497. * Set up the zero page, mark it reserved, so that page count
  1498. * is not manipulated when freeing the page from user ptes.
  1499. */
  1500. mem_map_zero = alloc_pages(GFP_KERNEL|__GFP_ZERO, 0);
  1501. if (mem_map_zero == NULL) {
  1502. prom_printf("paging_init: Cannot alloc zero page.\n");
  1503. prom_halt();
  1504. }
  1505. SetPageReserved(mem_map_zero);
  1506. codepages = (((unsigned long) _etext) - ((unsigned long) _start));
  1507. codepages = PAGE_ALIGN(codepages) >> PAGE_SHIFT;
  1508. datapages = (((unsigned long) _edata) - ((unsigned long) _etext));
  1509. datapages = PAGE_ALIGN(datapages) >> PAGE_SHIFT;
  1510. initpages = (((unsigned long) __init_end) - ((unsigned long) __init_begin));
  1511. initpages = PAGE_ALIGN(initpages) >> PAGE_SHIFT;
  1512. printk("Memory: %uk available (%ldk kernel code, %ldk data, %ldk init) [%016lx,%016lx]\n",
  1513. nr_free_pages() << (PAGE_SHIFT-10),
  1514. codepages << (PAGE_SHIFT-10),
  1515. datapages << (PAGE_SHIFT-10),
  1516. initpages << (PAGE_SHIFT-10),
  1517. PAGE_OFFSET, (last_valid_pfn << PAGE_SHIFT));
  1518. if (tlb_type == cheetah || tlb_type == cheetah_plus)
  1519. cheetah_ecache_flush_init();
  1520. }
  1521. void free_initmem (void)
  1522. {
  1523. unsigned long addr, initend;
  1524. /*
  1525. * The init section is aligned to 8k in vmlinux.lds. Page align for >8k pagesizes.
  1526. */
  1527. addr = PAGE_ALIGN((unsigned long)(__init_begin));
  1528. initend = (unsigned long)(__init_end) & PAGE_MASK;
  1529. for (; addr < initend; addr += PAGE_SIZE) {
  1530. unsigned long page;
  1531. struct page *p;
  1532. page = (addr +
  1533. ((unsigned long) __va(kern_base)) -
  1534. ((unsigned long) KERNBASE));
  1535. memset((void *)addr, 0xcc, PAGE_SIZE);
  1536. p = virt_to_page(page);
  1537. ClearPageReserved(p);
  1538. set_page_count(p, 1);
  1539. __free_page(p);
  1540. num_physpages++;
  1541. totalram_pages++;
  1542. }
  1543. }
  1544. #ifdef CONFIG_BLK_DEV_INITRD
  1545. void free_initrd_mem(unsigned long start, unsigned long end)
  1546. {
  1547. if (start < end)
  1548. printk ("Freeing initrd memory: %ldk freed\n", (end - start) >> 10);
  1549. for (; start < end; start += PAGE_SIZE) {
  1550. struct page *p = virt_to_page(start);
  1551. ClearPageReserved(p);
  1552. set_page_count(p, 1);
  1553. __free_page(p);
  1554. num_physpages++;
  1555. totalram_pages++;
  1556. }
  1557. }
  1558. #endif