init.c 50 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793
  1. /* $Id: init.c,v 1.209 2002/02/09 19:49:31 davem Exp $
  2. * arch/sparc64/mm/init.c
  3. *
  4. * Copyright (C) 1996-1999 David S. Miller (davem@caip.rutgers.edu)
  5. * Copyright (C) 1997-1999 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
  6. */
  7. #include <linux/config.h>
  8. #include <linux/kernel.h>
  9. #include <linux/sched.h>
  10. #include <linux/string.h>
  11. #include <linux/init.h>
  12. #include <linux/bootmem.h>
  13. #include <linux/mm.h>
  14. #include <linux/hugetlb.h>
  15. #include <linux/slab.h>
  16. #include <linux/initrd.h>
  17. #include <linux/swap.h>
  18. #include <linux/pagemap.h>
  19. #include <linux/fs.h>
  20. #include <linux/seq_file.h>
  21. #include <asm/head.h>
  22. #include <asm/system.h>
  23. #include <asm/page.h>
  24. #include <asm/pgalloc.h>
  25. #include <asm/pgtable.h>
  26. #include <asm/oplib.h>
  27. #include <asm/iommu.h>
  28. #include <asm/io.h>
  29. #include <asm/uaccess.h>
  30. #include <asm/mmu_context.h>
  31. #include <asm/tlbflush.h>
  32. #include <asm/dma.h>
  33. #include <asm/starfire.h>
  34. #include <asm/tlb.h>
  35. #include <asm/spitfire.h>
  36. #include <asm/sections.h>
  37. extern void device_scan(void);
  38. struct sparc_phys_banks sp_banks[SPARC_PHYS_BANKS];
  39. unsigned long *sparc64_valid_addr_bitmap;
  40. /* Ugly, but necessary... -DaveM */
  41. unsigned long phys_base;
  42. unsigned long kern_base;
  43. unsigned long kern_size;
  44. unsigned long pfn_base;
  45. /* This is even uglier. We have a problem where the kernel may not be
  46. * located at phys_base. However, initial __alloc_bootmem() calls need to
  47. * be adjusted to be within the 4-8Megs that the kernel is mapped to, else
  48. * those page mappings wont work. Things are ok after inherit_prom_mappings
  49. * is called though. Dave says he'll clean this up some other time.
  50. * -- BenC
  51. */
  52. static unsigned long bootmap_base;
  53. /* get_new_mmu_context() uses "cache + 1". */
  54. DEFINE_SPINLOCK(ctx_alloc_lock);
  55. unsigned long tlb_context_cache = CTX_FIRST_VERSION - 1;
  56. #define CTX_BMAP_SLOTS (1UL << (CTX_NR_BITS - 6))
  57. unsigned long mmu_context_bmap[CTX_BMAP_SLOTS];
  58. /* References to special section boundaries */
  59. extern char _start[], _end[];
  60. /* Initial ramdisk setup */
  61. extern unsigned long sparc_ramdisk_image64;
  62. extern unsigned int sparc_ramdisk_image;
  63. extern unsigned int sparc_ramdisk_size;
  64. struct page *mem_map_zero;
  65. int bigkernel = 0;
  66. /* XXX Tune this... */
  67. #define PGT_CACHE_LOW 25
  68. #define PGT_CACHE_HIGH 50
  69. void check_pgt_cache(void)
  70. {
  71. preempt_disable();
  72. if (pgtable_cache_size > PGT_CACHE_HIGH) {
  73. do {
  74. if (pgd_quicklist)
  75. free_pgd_slow(get_pgd_fast());
  76. if (pte_quicklist[0])
  77. free_pte_slow(pte_alloc_one_fast(NULL, 0));
  78. if (pte_quicklist[1])
  79. free_pte_slow(pte_alloc_one_fast(NULL, 1 << (PAGE_SHIFT + 10)));
  80. } while (pgtable_cache_size > PGT_CACHE_LOW);
  81. }
  82. preempt_enable();
  83. }
  84. #ifdef CONFIG_DEBUG_DCFLUSH
  85. atomic_t dcpage_flushes = ATOMIC_INIT(0);
  86. #ifdef CONFIG_SMP
  87. atomic_t dcpage_flushes_xcall = ATOMIC_INIT(0);
  88. #endif
  89. #endif
  90. __inline__ void flush_dcache_page_impl(struct page *page)
  91. {
  92. #ifdef CONFIG_DEBUG_DCFLUSH
  93. atomic_inc(&dcpage_flushes);
  94. #endif
  95. #ifdef DCACHE_ALIASING_POSSIBLE
  96. __flush_dcache_page(page_address(page),
  97. ((tlb_type == spitfire) &&
  98. page_mapping(page) != NULL));
  99. #else
  100. if (page_mapping(page) != NULL &&
  101. tlb_type == spitfire)
  102. __flush_icache_page(__pa(page_address(page)));
  103. #endif
  104. }
  105. #define PG_dcache_dirty PG_arch_1
  106. #define PG_dcache_cpu_shift 24
  107. #define PG_dcache_cpu_mask (256 - 1)
  108. #if NR_CPUS > 256
  109. #error D-cache dirty tracking and thread_info->cpu need fixing for > 256 cpus
  110. #endif
  111. #define dcache_dirty_cpu(page) \
  112. (((page)->flags >> PG_dcache_cpu_shift) & PG_dcache_cpu_mask)
  113. static __inline__ void set_dcache_dirty(struct page *page, int this_cpu)
  114. {
  115. unsigned long mask = this_cpu;
  116. unsigned long non_cpu_bits;
  117. non_cpu_bits = ~(PG_dcache_cpu_mask << PG_dcache_cpu_shift);
  118. mask = (mask << PG_dcache_cpu_shift) | (1UL << PG_dcache_dirty);
  119. __asm__ __volatile__("1:\n\t"
  120. "ldx [%2], %%g7\n\t"
  121. "and %%g7, %1, %%g1\n\t"
  122. "or %%g1, %0, %%g1\n\t"
  123. "casx [%2], %%g7, %%g1\n\t"
  124. "cmp %%g7, %%g1\n\t"
  125. "membar #StoreLoad | #StoreStore\n\t"
  126. "bne,pn %%xcc, 1b\n\t"
  127. " nop"
  128. : /* no outputs */
  129. : "r" (mask), "r" (non_cpu_bits), "r" (&page->flags)
  130. : "g1", "g7");
  131. }
  132. static __inline__ void clear_dcache_dirty_cpu(struct page *page, unsigned long cpu)
  133. {
  134. unsigned long mask = (1UL << PG_dcache_dirty);
  135. __asm__ __volatile__("! test_and_clear_dcache_dirty\n"
  136. "1:\n\t"
  137. "ldx [%2], %%g7\n\t"
  138. "srlx %%g7, %4, %%g1\n\t"
  139. "and %%g1, %3, %%g1\n\t"
  140. "cmp %%g1, %0\n\t"
  141. "bne,pn %%icc, 2f\n\t"
  142. " andn %%g7, %1, %%g1\n\t"
  143. "casx [%2], %%g7, %%g1\n\t"
  144. "cmp %%g7, %%g1\n\t"
  145. "membar #StoreLoad | #StoreStore\n\t"
  146. "bne,pn %%xcc, 1b\n\t"
  147. " nop\n"
  148. "2:"
  149. : /* no outputs */
  150. : "r" (cpu), "r" (mask), "r" (&page->flags),
  151. "i" (PG_dcache_cpu_mask),
  152. "i" (PG_dcache_cpu_shift)
  153. : "g1", "g7");
  154. }
  155. extern void __update_mmu_cache(unsigned long mmu_context_hw, unsigned long address, pte_t pte, int code);
  156. void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t pte)
  157. {
  158. struct page *page;
  159. unsigned long pfn;
  160. unsigned long pg_flags;
  161. pfn = pte_pfn(pte);
  162. if (pfn_valid(pfn) &&
  163. (page = pfn_to_page(pfn), page_mapping(page)) &&
  164. ((pg_flags = page->flags) & (1UL << PG_dcache_dirty))) {
  165. int cpu = ((pg_flags >> PG_dcache_cpu_shift) &
  166. PG_dcache_cpu_mask);
  167. int this_cpu = get_cpu();
  168. /* This is just to optimize away some function calls
  169. * in the SMP case.
  170. */
  171. if (cpu == this_cpu)
  172. flush_dcache_page_impl(page);
  173. else
  174. smp_flush_dcache_page_impl(page, cpu);
  175. clear_dcache_dirty_cpu(page, cpu);
  176. put_cpu();
  177. }
  178. if (get_thread_fault_code())
  179. __update_mmu_cache(CTX_NRBITS(vma->vm_mm->context),
  180. address, pte, get_thread_fault_code());
  181. }
  182. void flush_dcache_page(struct page *page)
  183. {
  184. struct address_space *mapping;
  185. int this_cpu;
  186. /* Do not bother with the expensive D-cache flush if it
  187. * is merely the zero page. The 'bigcore' testcase in GDB
  188. * causes this case to run millions of times.
  189. */
  190. if (page == ZERO_PAGE(0))
  191. return;
  192. this_cpu = get_cpu();
  193. mapping = page_mapping(page);
  194. if (mapping && !mapping_mapped(mapping)) {
  195. int dirty = test_bit(PG_dcache_dirty, &page->flags);
  196. if (dirty) {
  197. int dirty_cpu = dcache_dirty_cpu(page);
  198. if (dirty_cpu == this_cpu)
  199. goto out;
  200. smp_flush_dcache_page_impl(page, dirty_cpu);
  201. }
  202. set_dcache_dirty(page, this_cpu);
  203. } else {
  204. /* We could delay the flush for the !page_mapping
  205. * case too. But that case is for exec env/arg
  206. * pages and those are %99 certainly going to get
  207. * faulted into the tlb (and thus flushed) anyways.
  208. */
  209. flush_dcache_page_impl(page);
  210. }
  211. out:
  212. put_cpu();
  213. }
  214. void flush_icache_range(unsigned long start, unsigned long end)
  215. {
  216. /* Cheetah has coherent I-cache. */
  217. if (tlb_type == spitfire) {
  218. unsigned long kaddr;
  219. for (kaddr = start; kaddr < end; kaddr += PAGE_SIZE)
  220. __flush_icache_page(__get_phys(kaddr));
  221. }
  222. }
  223. unsigned long page_to_pfn(struct page *page)
  224. {
  225. return (unsigned long) ((page - mem_map) + pfn_base);
  226. }
  227. struct page *pfn_to_page(unsigned long pfn)
  228. {
  229. return (mem_map + (pfn - pfn_base));
  230. }
  231. void show_mem(void)
  232. {
  233. printk("Mem-info:\n");
  234. show_free_areas();
  235. printk("Free swap: %6ldkB\n",
  236. nr_swap_pages << (PAGE_SHIFT-10));
  237. printk("%ld pages of RAM\n", num_physpages);
  238. printk("%d free pages\n", nr_free_pages());
  239. printk("%d pages in page table cache\n",pgtable_cache_size);
  240. }
  241. void mmu_info(struct seq_file *m)
  242. {
  243. if (tlb_type == cheetah)
  244. seq_printf(m, "MMU Type\t: Cheetah\n");
  245. else if (tlb_type == cheetah_plus)
  246. seq_printf(m, "MMU Type\t: Cheetah+\n");
  247. else if (tlb_type == spitfire)
  248. seq_printf(m, "MMU Type\t: Spitfire\n");
  249. else
  250. seq_printf(m, "MMU Type\t: ???\n");
  251. #ifdef CONFIG_DEBUG_DCFLUSH
  252. seq_printf(m, "DCPageFlushes\t: %d\n",
  253. atomic_read(&dcpage_flushes));
  254. #ifdef CONFIG_SMP
  255. seq_printf(m, "DCPageFlushesXC\t: %d\n",
  256. atomic_read(&dcpage_flushes_xcall));
  257. #endif /* CONFIG_SMP */
  258. #endif /* CONFIG_DEBUG_DCFLUSH */
  259. }
  260. struct linux_prom_translation {
  261. unsigned long virt;
  262. unsigned long size;
  263. unsigned long data;
  264. };
  265. extern unsigned long prom_boot_page;
  266. extern void prom_remap(unsigned long physpage, unsigned long virtpage, int mmu_ihandle);
  267. extern int prom_get_mmu_ihandle(void);
  268. extern void register_prom_callbacks(void);
  269. /* Exported for SMP bootup purposes. */
  270. unsigned long kern_locked_tte_data;
  271. void __init early_pgtable_allocfail(char *type)
  272. {
  273. prom_printf("inherit_prom_mappings: Cannot alloc kernel %s.\n", type);
  274. prom_halt();
  275. }
  276. #define BASE_PAGE_SIZE 8192
  277. static pmd_t *prompmd;
  278. /*
  279. * Translate PROM's mapping we capture at boot time into physical address.
  280. * The second parameter is only set from prom_callback() invocations.
  281. */
  282. unsigned long prom_virt_to_phys(unsigned long promva, int *error)
  283. {
  284. pmd_t *pmdp = prompmd + ((promva >> 23) & 0x7ff);
  285. pte_t *ptep;
  286. unsigned long base;
  287. if (pmd_none(*pmdp)) {
  288. if (error)
  289. *error = 1;
  290. return(0);
  291. }
  292. ptep = (pte_t *)__pmd_page(*pmdp) + ((promva >> 13) & 0x3ff);
  293. if (!pte_present(*ptep)) {
  294. if (error)
  295. *error = 1;
  296. return(0);
  297. }
  298. if (error) {
  299. *error = 0;
  300. return(pte_val(*ptep));
  301. }
  302. base = pte_val(*ptep) & _PAGE_PADDR;
  303. return(base + (promva & (BASE_PAGE_SIZE - 1)));
  304. }
  305. static void inherit_prom_mappings(void)
  306. {
  307. struct linux_prom_translation *trans;
  308. unsigned long phys_page, tte_vaddr, tte_data;
  309. void (*remap_func)(unsigned long, unsigned long, int);
  310. pmd_t *pmdp;
  311. pte_t *ptep;
  312. int node, n, i, tsz;
  313. extern unsigned int obp_iaddr_patch[2], obp_daddr_patch[2];
  314. node = prom_finddevice("/virtual-memory");
  315. n = prom_getproplen(node, "translations");
  316. if (n == 0 || n == -1) {
  317. prom_printf("Couldn't get translation property\n");
  318. prom_halt();
  319. }
  320. n += 5 * sizeof(struct linux_prom_translation);
  321. for (tsz = 1; tsz < n; tsz <<= 1)
  322. /* empty */;
  323. trans = __alloc_bootmem(tsz, SMP_CACHE_BYTES, bootmap_base);
  324. if (trans == NULL) {
  325. prom_printf("inherit_prom_mappings: Cannot alloc translations.\n");
  326. prom_halt();
  327. }
  328. memset(trans, 0, tsz);
  329. if ((n = prom_getproperty(node, "translations", (char *)trans, tsz)) == -1) {
  330. prom_printf("Couldn't get translation property\n");
  331. prom_halt();
  332. }
  333. n = n / sizeof(*trans);
  334. /*
  335. * The obp translations are saved based on 8k pagesize, since obp can
  336. * use a mixture of pagesizes. Misses to the 0xf0000000 - 0x100000000,
  337. * ie obp range, are handled in entry.S and do not use the vpte scheme
  338. * (see rant in inherit_locked_prom_mappings()).
  339. */
  340. #define OBP_PMD_SIZE 2048
  341. prompmd = __alloc_bootmem(OBP_PMD_SIZE, OBP_PMD_SIZE, bootmap_base);
  342. if (prompmd == NULL)
  343. early_pgtable_allocfail("pmd");
  344. memset(prompmd, 0, OBP_PMD_SIZE);
  345. for (i = 0; i < n; i++) {
  346. unsigned long vaddr;
  347. if (trans[i].virt >= LOW_OBP_ADDRESS && trans[i].virt < HI_OBP_ADDRESS) {
  348. for (vaddr = trans[i].virt;
  349. ((vaddr < trans[i].virt + trans[i].size) &&
  350. (vaddr < HI_OBP_ADDRESS));
  351. vaddr += BASE_PAGE_SIZE) {
  352. unsigned long val;
  353. pmdp = prompmd + ((vaddr >> 23) & 0x7ff);
  354. if (pmd_none(*pmdp)) {
  355. ptep = __alloc_bootmem(BASE_PAGE_SIZE,
  356. BASE_PAGE_SIZE,
  357. bootmap_base);
  358. if (ptep == NULL)
  359. early_pgtable_allocfail("pte");
  360. memset(ptep, 0, BASE_PAGE_SIZE);
  361. pmd_set(pmdp, ptep);
  362. }
  363. ptep = (pte_t *)__pmd_page(*pmdp) +
  364. ((vaddr >> 13) & 0x3ff);
  365. val = trans[i].data;
  366. /* Clear diag TTE bits. */
  367. if (tlb_type == spitfire)
  368. val &= ~0x0003fe0000000000UL;
  369. set_pte_at(&init_mm, vaddr,
  370. ptep, __pte(val | _PAGE_MODIFIED));
  371. trans[i].data += BASE_PAGE_SIZE;
  372. }
  373. }
  374. }
  375. phys_page = __pa(prompmd);
  376. obp_iaddr_patch[0] |= (phys_page >> 10);
  377. obp_iaddr_patch[1] |= (phys_page & 0x3ff);
  378. flushi((long)&obp_iaddr_patch[0]);
  379. obp_daddr_patch[0] |= (phys_page >> 10);
  380. obp_daddr_patch[1] |= (phys_page & 0x3ff);
  381. flushi((long)&obp_daddr_patch[0]);
  382. /* Now fixup OBP's idea about where we really are mapped. */
  383. prom_printf("Remapping the kernel... ");
  384. /* Spitfire Errata #32 workaround */
  385. /* NOTE: Using plain zero for the context value is
  386. * correct here, we are not using the Linux trap
  387. * tables yet so we should not use the special
  388. * UltraSPARC-III+ page size encodings yet.
  389. */
  390. __asm__ __volatile__("stxa %0, [%1] %2\n\t"
  391. "flush %%g6"
  392. : /* No outputs */
  393. : "r" (0), "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
  394. switch (tlb_type) {
  395. default:
  396. case spitfire:
  397. phys_page = spitfire_get_dtlb_data(sparc64_highest_locked_tlbent());
  398. break;
  399. case cheetah:
  400. case cheetah_plus:
  401. phys_page = cheetah_get_litlb_data(sparc64_highest_locked_tlbent());
  402. break;
  403. };
  404. phys_page &= _PAGE_PADDR;
  405. phys_page += ((unsigned long)&prom_boot_page -
  406. (unsigned long)KERNBASE);
  407. if (tlb_type == spitfire) {
  408. /* Lock this into i/d tlb entry 59 */
  409. __asm__ __volatile__(
  410. "stxa %%g0, [%2] %3\n\t"
  411. "stxa %0, [%1] %4\n\t"
  412. "membar #Sync\n\t"
  413. "flush %%g6\n\t"
  414. "stxa %%g0, [%2] %5\n\t"
  415. "stxa %0, [%1] %6\n\t"
  416. "membar #Sync\n\t"
  417. "flush %%g6"
  418. : : "r" (phys_page | _PAGE_VALID | _PAGE_SZ8K | _PAGE_CP |
  419. _PAGE_CV | _PAGE_P | _PAGE_L | _PAGE_W),
  420. "r" (59 << 3), "r" (TLB_TAG_ACCESS),
  421. "i" (ASI_DMMU), "i" (ASI_DTLB_DATA_ACCESS),
  422. "i" (ASI_IMMU), "i" (ASI_ITLB_DATA_ACCESS)
  423. : "memory");
  424. } else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
  425. /* Lock this into i/d tlb-0 entry 11 */
  426. __asm__ __volatile__(
  427. "stxa %%g0, [%2] %3\n\t"
  428. "stxa %0, [%1] %4\n\t"
  429. "membar #Sync\n\t"
  430. "flush %%g6\n\t"
  431. "stxa %%g0, [%2] %5\n\t"
  432. "stxa %0, [%1] %6\n\t"
  433. "membar #Sync\n\t"
  434. "flush %%g6"
  435. : : "r" (phys_page | _PAGE_VALID | _PAGE_SZ8K | _PAGE_CP |
  436. _PAGE_CV | _PAGE_P | _PAGE_L | _PAGE_W),
  437. "r" ((0 << 16) | (11 << 3)), "r" (TLB_TAG_ACCESS),
  438. "i" (ASI_DMMU), "i" (ASI_DTLB_DATA_ACCESS),
  439. "i" (ASI_IMMU), "i" (ASI_ITLB_DATA_ACCESS)
  440. : "memory");
  441. } else {
  442. /* Implement me :-) */
  443. BUG();
  444. }
  445. tte_vaddr = (unsigned long) KERNBASE;
  446. /* Spitfire Errata #32 workaround */
  447. /* NOTE: Using plain zero for the context value is
  448. * correct here, we are not using the Linux trap
  449. * tables yet so we should not use the special
  450. * UltraSPARC-III+ page size encodings yet.
  451. */
  452. __asm__ __volatile__("stxa %0, [%1] %2\n\t"
  453. "flush %%g6"
  454. : /* No outputs */
  455. : "r" (0),
  456. "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
  457. if (tlb_type == spitfire)
  458. tte_data = spitfire_get_dtlb_data(sparc64_highest_locked_tlbent());
  459. else
  460. tte_data = cheetah_get_ldtlb_data(sparc64_highest_locked_tlbent());
  461. kern_locked_tte_data = tte_data;
  462. remap_func = (void *) ((unsigned long) &prom_remap -
  463. (unsigned long) &prom_boot_page);
  464. /* Spitfire Errata #32 workaround */
  465. /* NOTE: Using plain zero for the context value is
  466. * correct here, we are not using the Linux trap
  467. * tables yet so we should not use the special
  468. * UltraSPARC-III+ page size encodings yet.
  469. */
  470. __asm__ __volatile__("stxa %0, [%1] %2\n\t"
  471. "flush %%g6"
  472. : /* No outputs */
  473. : "r" (0),
  474. "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
  475. remap_func((tlb_type == spitfire ?
  476. (spitfire_get_dtlb_data(sparc64_highest_locked_tlbent()) & _PAGE_PADDR) :
  477. (cheetah_get_litlb_data(sparc64_highest_locked_tlbent()) & _PAGE_PADDR)),
  478. (unsigned long) KERNBASE,
  479. prom_get_mmu_ihandle());
  480. if (bigkernel)
  481. remap_func(((tte_data + 0x400000) & _PAGE_PADDR),
  482. (unsigned long) KERNBASE + 0x400000, prom_get_mmu_ihandle());
  483. /* Flush out that temporary mapping. */
  484. spitfire_flush_dtlb_nucleus_page(0x0);
  485. spitfire_flush_itlb_nucleus_page(0x0);
  486. /* Now lock us back into the TLBs via OBP. */
  487. prom_dtlb_load(sparc64_highest_locked_tlbent(), tte_data, tte_vaddr);
  488. prom_itlb_load(sparc64_highest_locked_tlbent(), tte_data, tte_vaddr);
  489. if (bigkernel) {
  490. prom_dtlb_load(sparc64_highest_locked_tlbent()-1, tte_data + 0x400000,
  491. tte_vaddr + 0x400000);
  492. prom_itlb_load(sparc64_highest_locked_tlbent()-1, tte_data + 0x400000,
  493. tte_vaddr + 0x400000);
  494. }
  495. /* Re-read translations property. */
  496. if ((n = prom_getproperty(node, "translations", (char *)trans, tsz)) == -1) {
  497. prom_printf("Couldn't get translation property\n");
  498. prom_halt();
  499. }
  500. n = n / sizeof(*trans);
  501. for (i = 0; i < n; i++) {
  502. unsigned long vaddr = trans[i].virt;
  503. unsigned long size = trans[i].size;
  504. if (vaddr < 0xf0000000UL) {
  505. unsigned long avoid_start = (unsigned long) KERNBASE;
  506. unsigned long avoid_end = avoid_start + (4 * 1024 * 1024);
  507. if (bigkernel)
  508. avoid_end += (4 * 1024 * 1024);
  509. if (vaddr < avoid_start) {
  510. unsigned long top = vaddr + size;
  511. if (top > avoid_start)
  512. top = avoid_start;
  513. prom_unmap(top - vaddr, vaddr);
  514. }
  515. if ((vaddr + size) > avoid_end) {
  516. unsigned long bottom = vaddr;
  517. if (bottom < avoid_end)
  518. bottom = avoid_end;
  519. prom_unmap((vaddr + size) - bottom, bottom);
  520. }
  521. }
  522. }
  523. prom_printf("done.\n");
  524. register_prom_callbacks();
  525. }
  526. /* The OBP specifications for sun4u mark 0xfffffffc00000000 and
  527. * upwards as reserved for use by the firmware (I wonder if this
  528. * will be the same on Cheetah...). We use this virtual address
  529. * range for the VPTE table mappings of the nucleus so we need
  530. * to zap them when we enter the PROM. -DaveM
  531. */
  532. static void __flush_nucleus_vptes(void)
  533. {
  534. unsigned long prom_reserved_base = 0xfffffffc00000000UL;
  535. int i;
  536. /* Only DTLB must be checked for VPTE entries. */
  537. if (tlb_type == spitfire) {
  538. for (i = 0; i < 63; i++) {
  539. unsigned long tag;
  540. /* Spitfire Errata #32 workaround */
  541. /* NOTE: Always runs on spitfire, so no cheetah+
  542. * page size encodings.
  543. */
  544. __asm__ __volatile__("stxa %0, [%1] %2\n\t"
  545. "flush %%g6"
  546. : /* No outputs */
  547. : "r" (0),
  548. "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
  549. tag = spitfire_get_dtlb_tag(i);
  550. if (((tag & ~(PAGE_MASK)) == 0) &&
  551. ((tag & (PAGE_MASK)) >= prom_reserved_base)) {
  552. __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
  553. "membar #Sync"
  554. : /* no outputs */
  555. : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
  556. spitfire_put_dtlb_data(i, 0x0UL);
  557. }
  558. }
  559. } else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
  560. for (i = 0; i < 512; i++) {
  561. unsigned long tag = cheetah_get_dtlb_tag(i, 2);
  562. if ((tag & ~PAGE_MASK) == 0 &&
  563. (tag & PAGE_MASK) >= prom_reserved_base) {
  564. __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
  565. "membar #Sync"
  566. : /* no outputs */
  567. : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
  568. cheetah_put_dtlb_data(i, 0x0UL, 2);
  569. }
  570. if (tlb_type != cheetah_plus)
  571. continue;
  572. tag = cheetah_get_dtlb_tag(i, 3);
  573. if ((tag & ~PAGE_MASK) == 0 &&
  574. (tag & PAGE_MASK) >= prom_reserved_base) {
  575. __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
  576. "membar #Sync"
  577. : /* no outputs */
  578. : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
  579. cheetah_put_dtlb_data(i, 0x0UL, 3);
  580. }
  581. }
  582. } else {
  583. /* Implement me :-) */
  584. BUG();
  585. }
  586. }
  587. static int prom_ditlb_set;
  588. struct prom_tlb_entry {
  589. int tlb_ent;
  590. unsigned long tlb_tag;
  591. unsigned long tlb_data;
  592. };
  593. struct prom_tlb_entry prom_itlb[16], prom_dtlb[16];
  594. void prom_world(int enter)
  595. {
  596. unsigned long pstate;
  597. int i;
  598. if (!enter)
  599. set_fs((mm_segment_t) { get_thread_current_ds() });
  600. if (!prom_ditlb_set)
  601. return;
  602. /* Make sure the following runs atomically. */
  603. __asm__ __volatile__("flushw\n\t"
  604. "rdpr %%pstate, %0\n\t"
  605. "wrpr %0, %1, %%pstate"
  606. : "=r" (pstate)
  607. : "i" (PSTATE_IE));
  608. if (enter) {
  609. /* Kick out nucleus VPTEs. */
  610. __flush_nucleus_vptes();
  611. /* Install PROM world. */
  612. for (i = 0; i < 16; i++) {
  613. if (prom_dtlb[i].tlb_ent != -1) {
  614. __asm__ __volatile__("stxa %0, [%1] %2\n\t"
  615. "membar #Sync"
  616. : : "r" (prom_dtlb[i].tlb_tag), "r" (TLB_TAG_ACCESS),
  617. "i" (ASI_DMMU));
  618. if (tlb_type == spitfire)
  619. spitfire_put_dtlb_data(prom_dtlb[i].tlb_ent,
  620. prom_dtlb[i].tlb_data);
  621. else if (tlb_type == cheetah || tlb_type == cheetah_plus)
  622. cheetah_put_ldtlb_data(prom_dtlb[i].tlb_ent,
  623. prom_dtlb[i].tlb_data);
  624. }
  625. if (prom_itlb[i].tlb_ent != -1) {
  626. __asm__ __volatile__("stxa %0, [%1] %2\n\t"
  627. "membar #Sync"
  628. : : "r" (prom_itlb[i].tlb_tag),
  629. "r" (TLB_TAG_ACCESS),
  630. "i" (ASI_IMMU));
  631. if (tlb_type == spitfire)
  632. spitfire_put_itlb_data(prom_itlb[i].tlb_ent,
  633. prom_itlb[i].tlb_data);
  634. else if (tlb_type == cheetah || tlb_type == cheetah_plus)
  635. cheetah_put_litlb_data(prom_itlb[i].tlb_ent,
  636. prom_itlb[i].tlb_data);
  637. }
  638. }
  639. } else {
  640. for (i = 0; i < 16; i++) {
  641. if (prom_dtlb[i].tlb_ent != -1) {
  642. __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
  643. "membar #Sync"
  644. : : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
  645. if (tlb_type == spitfire)
  646. spitfire_put_dtlb_data(prom_dtlb[i].tlb_ent, 0x0UL);
  647. else
  648. cheetah_put_ldtlb_data(prom_dtlb[i].tlb_ent, 0x0UL);
  649. }
  650. if (prom_itlb[i].tlb_ent != -1) {
  651. __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
  652. "membar #Sync"
  653. : : "r" (TLB_TAG_ACCESS),
  654. "i" (ASI_IMMU));
  655. if (tlb_type == spitfire)
  656. spitfire_put_itlb_data(prom_itlb[i].tlb_ent, 0x0UL);
  657. else
  658. cheetah_put_litlb_data(prom_itlb[i].tlb_ent, 0x0UL);
  659. }
  660. }
  661. }
  662. __asm__ __volatile__("wrpr %0, 0, %%pstate"
  663. : : "r" (pstate));
  664. }
  665. void inherit_locked_prom_mappings(int save_p)
  666. {
  667. int i;
  668. int dtlb_seen = 0;
  669. int itlb_seen = 0;
  670. /* Fucking losing PROM has more mappings in the TLB, but
  671. * it (conveniently) fails to mention any of these in the
  672. * translations property. The only ones that matter are
  673. * the locked PROM tlb entries, so we impose the following
  674. * irrecovable rule on the PROM, it is allowed 8 locked
  675. * entries in the ITLB and 8 in the DTLB.
  676. *
  677. * Supposedly the upper 16GB of the address space is
  678. * reserved for OBP, BUT I WISH THIS WAS DOCUMENTED
  679. * SOMEWHERE!!!!!!!!!!!!!!!!! Furthermore the entire interface
  680. * used between the client program and the firmware on sun5
  681. * systems to coordinate mmu mappings is also COMPLETELY
  682. * UNDOCUMENTED!!!!!! Thanks S(t)un!
  683. */
  684. if (save_p) {
  685. for (i = 0; i < 16; i++) {
  686. prom_itlb[i].tlb_ent = -1;
  687. prom_dtlb[i].tlb_ent = -1;
  688. }
  689. }
  690. if (tlb_type == spitfire) {
  691. int high = SPITFIRE_HIGHEST_LOCKED_TLBENT - bigkernel;
  692. for (i = 0; i < high; i++) {
  693. unsigned long data;
  694. /* Spitfire Errata #32 workaround */
  695. /* NOTE: Always runs on spitfire, so no cheetah+
  696. * page size encodings.
  697. */
  698. __asm__ __volatile__("stxa %0, [%1] %2\n\t"
  699. "flush %%g6"
  700. : /* No outputs */
  701. : "r" (0),
  702. "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
  703. data = spitfire_get_dtlb_data(i);
  704. if ((data & (_PAGE_L|_PAGE_VALID)) == (_PAGE_L|_PAGE_VALID)) {
  705. unsigned long tag;
  706. /* Spitfire Errata #32 workaround */
  707. /* NOTE: Always runs on spitfire, so no
  708. * cheetah+ page size encodings.
  709. */
  710. __asm__ __volatile__("stxa %0, [%1] %2\n\t"
  711. "flush %%g6"
  712. : /* No outputs */
  713. : "r" (0),
  714. "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
  715. tag = spitfire_get_dtlb_tag(i);
  716. if (save_p) {
  717. prom_dtlb[dtlb_seen].tlb_ent = i;
  718. prom_dtlb[dtlb_seen].tlb_tag = tag;
  719. prom_dtlb[dtlb_seen].tlb_data = data;
  720. }
  721. __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
  722. "membar #Sync"
  723. : : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
  724. spitfire_put_dtlb_data(i, 0x0UL);
  725. dtlb_seen++;
  726. if (dtlb_seen > 15)
  727. break;
  728. }
  729. }
  730. for (i = 0; i < high; i++) {
  731. unsigned long data;
  732. /* Spitfire Errata #32 workaround */
  733. /* NOTE: Always runs on spitfire, so no
  734. * cheetah+ page size encodings.
  735. */
  736. __asm__ __volatile__("stxa %0, [%1] %2\n\t"
  737. "flush %%g6"
  738. : /* No outputs */
  739. : "r" (0),
  740. "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
  741. data = spitfire_get_itlb_data(i);
  742. if ((data & (_PAGE_L|_PAGE_VALID)) == (_PAGE_L|_PAGE_VALID)) {
  743. unsigned long tag;
  744. /* Spitfire Errata #32 workaround */
  745. /* NOTE: Always runs on spitfire, so no
  746. * cheetah+ page size encodings.
  747. */
  748. __asm__ __volatile__("stxa %0, [%1] %2\n\t"
  749. "flush %%g6"
  750. : /* No outputs */
  751. : "r" (0),
  752. "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
  753. tag = spitfire_get_itlb_tag(i);
  754. if (save_p) {
  755. prom_itlb[itlb_seen].tlb_ent = i;
  756. prom_itlb[itlb_seen].tlb_tag = tag;
  757. prom_itlb[itlb_seen].tlb_data = data;
  758. }
  759. __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
  760. "membar #Sync"
  761. : : "r" (TLB_TAG_ACCESS), "i" (ASI_IMMU));
  762. spitfire_put_itlb_data(i, 0x0UL);
  763. itlb_seen++;
  764. if (itlb_seen > 15)
  765. break;
  766. }
  767. }
  768. } else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
  769. int high = CHEETAH_HIGHEST_LOCKED_TLBENT - bigkernel;
  770. for (i = 0; i < high; i++) {
  771. unsigned long data;
  772. data = cheetah_get_ldtlb_data(i);
  773. if ((data & (_PAGE_L|_PAGE_VALID)) == (_PAGE_L|_PAGE_VALID)) {
  774. unsigned long tag;
  775. tag = cheetah_get_ldtlb_tag(i);
  776. if (save_p) {
  777. prom_dtlb[dtlb_seen].tlb_ent = i;
  778. prom_dtlb[dtlb_seen].tlb_tag = tag;
  779. prom_dtlb[dtlb_seen].tlb_data = data;
  780. }
  781. __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
  782. "membar #Sync"
  783. : : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
  784. cheetah_put_ldtlb_data(i, 0x0UL);
  785. dtlb_seen++;
  786. if (dtlb_seen > 15)
  787. break;
  788. }
  789. }
  790. for (i = 0; i < high; i++) {
  791. unsigned long data;
  792. data = cheetah_get_litlb_data(i);
  793. if ((data & (_PAGE_L|_PAGE_VALID)) == (_PAGE_L|_PAGE_VALID)) {
  794. unsigned long tag;
  795. tag = cheetah_get_litlb_tag(i);
  796. if (save_p) {
  797. prom_itlb[itlb_seen].tlb_ent = i;
  798. prom_itlb[itlb_seen].tlb_tag = tag;
  799. prom_itlb[itlb_seen].tlb_data = data;
  800. }
  801. __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
  802. "membar #Sync"
  803. : : "r" (TLB_TAG_ACCESS), "i" (ASI_IMMU));
  804. cheetah_put_litlb_data(i, 0x0UL);
  805. itlb_seen++;
  806. if (itlb_seen > 15)
  807. break;
  808. }
  809. }
  810. } else {
  811. /* Implement me :-) */
  812. BUG();
  813. }
  814. if (save_p)
  815. prom_ditlb_set = 1;
  816. }
  817. /* Give PROM back his world, done during reboots... */
  818. void prom_reload_locked(void)
  819. {
  820. int i;
  821. for (i = 0; i < 16; i++) {
  822. if (prom_dtlb[i].tlb_ent != -1) {
  823. __asm__ __volatile__("stxa %0, [%1] %2\n\t"
  824. "membar #Sync"
  825. : : "r" (prom_dtlb[i].tlb_tag), "r" (TLB_TAG_ACCESS),
  826. "i" (ASI_DMMU));
  827. if (tlb_type == spitfire)
  828. spitfire_put_dtlb_data(prom_dtlb[i].tlb_ent,
  829. prom_dtlb[i].tlb_data);
  830. else if (tlb_type == cheetah || tlb_type == cheetah_plus)
  831. cheetah_put_ldtlb_data(prom_dtlb[i].tlb_ent,
  832. prom_dtlb[i].tlb_data);
  833. }
  834. if (prom_itlb[i].tlb_ent != -1) {
  835. __asm__ __volatile__("stxa %0, [%1] %2\n\t"
  836. "membar #Sync"
  837. : : "r" (prom_itlb[i].tlb_tag),
  838. "r" (TLB_TAG_ACCESS),
  839. "i" (ASI_IMMU));
  840. if (tlb_type == spitfire)
  841. spitfire_put_itlb_data(prom_itlb[i].tlb_ent,
  842. prom_itlb[i].tlb_data);
  843. else
  844. cheetah_put_litlb_data(prom_itlb[i].tlb_ent,
  845. prom_itlb[i].tlb_data);
  846. }
  847. }
  848. }
  849. #ifdef DCACHE_ALIASING_POSSIBLE
  850. void __flush_dcache_range(unsigned long start, unsigned long end)
  851. {
  852. unsigned long va;
  853. if (tlb_type == spitfire) {
  854. int n = 0;
  855. for (va = start; va < end; va += 32) {
  856. spitfire_put_dcache_tag(va & 0x3fe0, 0x0);
  857. if (++n >= 512)
  858. break;
  859. }
  860. } else {
  861. start = __pa(start);
  862. end = __pa(end);
  863. for (va = start; va < end; va += 32)
  864. __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
  865. "membar #Sync"
  866. : /* no outputs */
  867. : "r" (va),
  868. "i" (ASI_DCACHE_INVALIDATE));
  869. }
  870. }
  871. #endif /* DCACHE_ALIASING_POSSIBLE */
  872. /* If not locked, zap it. */
  873. void __flush_tlb_all(void)
  874. {
  875. unsigned long pstate;
  876. int i;
  877. __asm__ __volatile__("flushw\n\t"
  878. "rdpr %%pstate, %0\n\t"
  879. "wrpr %0, %1, %%pstate"
  880. : "=r" (pstate)
  881. : "i" (PSTATE_IE));
  882. if (tlb_type == spitfire) {
  883. for (i = 0; i < 64; i++) {
  884. /* Spitfire Errata #32 workaround */
  885. /* NOTE: Always runs on spitfire, so no
  886. * cheetah+ page size encodings.
  887. */
  888. __asm__ __volatile__("stxa %0, [%1] %2\n\t"
  889. "flush %%g6"
  890. : /* No outputs */
  891. : "r" (0),
  892. "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
  893. if (!(spitfire_get_dtlb_data(i) & _PAGE_L)) {
  894. __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
  895. "membar #Sync"
  896. : /* no outputs */
  897. : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
  898. spitfire_put_dtlb_data(i, 0x0UL);
  899. }
  900. /* Spitfire Errata #32 workaround */
  901. /* NOTE: Always runs on spitfire, so no
  902. * cheetah+ page size encodings.
  903. */
  904. __asm__ __volatile__("stxa %0, [%1] %2\n\t"
  905. "flush %%g6"
  906. : /* No outputs */
  907. : "r" (0),
  908. "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
  909. if (!(spitfire_get_itlb_data(i) & _PAGE_L)) {
  910. __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
  911. "membar #Sync"
  912. : /* no outputs */
  913. : "r" (TLB_TAG_ACCESS), "i" (ASI_IMMU));
  914. spitfire_put_itlb_data(i, 0x0UL);
  915. }
  916. }
  917. } else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
  918. cheetah_flush_dtlb_all();
  919. cheetah_flush_itlb_all();
  920. }
  921. __asm__ __volatile__("wrpr %0, 0, %%pstate"
  922. : : "r" (pstate));
  923. }
  924. /* Caller does TLB context flushing on local CPU if necessary.
  925. * The caller also ensures that CTX_VALID(mm->context) is false.
  926. *
  927. * We must be careful about boundary cases so that we never
  928. * let the user have CTX 0 (nucleus) or we ever use a CTX
  929. * version of zero (and thus NO_CONTEXT would not be caught
  930. * by version mis-match tests in mmu_context.h).
  931. */
  932. void get_new_mmu_context(struct mm_struct *mm)
  933. {
  934. unsigned long ctx, new_ctx;
  935. unsigned long orig_pgsz_bits;
  936. spin_lock(&ctx_alloc_lock);
  937. orig_pgsz_bits = (mm->context.sparc64_ctx_val & CTX_PGSZ_MASK);
  938. ctx = (tlb_context_cache + 1) & CTX_NR_MASK;
  939. new_ctx = find_next_zero_bit(mmu_context_bmap, 1 << CTX_NR_BITS, ctx);
  940. if (new_ctx >= (1 << CTX_NR_BITS)) {
  941. new_ctx = find_next_zero_bit(mmu_context_bmap, ctx, 1);
  942. if (new_ctx >= ctx) {
  943. int i;
  944. new_ctx = (tlb_context_cache & CTX_VERSION_MASK) +
  945. CTX_FIRST_VERSION;
  946. if (new_ctx == 1)
  947. new_ctx = CTX_FIRST_VERSION;
  948. /* Don't call memset, for 16 entries that's just
  949. * plain silly...
  950. */
  951. mmu_context_bmap[0] = 3;
  952. mmu_context_bmap[1] = 0;
  953. mmu_context_bmap[2] = 0;
  954. mmu_context_bmap[3] = 0;
  955. for (i = 4; i < CTX_BMAP_SLOTS; i += 4) {
  956. mmu_context_bmap[i + 0] = 0;
  957. mmu_context_bmap[i + 1] = 0;
  958. mmu_context_bmap[i + 2] = 0;
  959. mmu_context_bmap[i + 3] = 0;
  960. }
  961. goto out;
  962. }
  963. }
  964. mmu_context_bmap[new_ctx>>6] |= (1UL << (new_ctx & 63));
  965. new_ctx |= (tlb_context_cache & CTX_VERSION_MASK);
  966. out:
  967. tlb_context_cache = new_ctx;
  968. mm->context.sparc64_ctx_val = new_ctx | orig_pgsz_bits;
  969. spin_unlock(&ctx_alloc_lock);
  970. }
  971. #ifndef CONFIG_SMP
  972. struct pgtable_cache_struct pgt_quicklists;
  973. #endif
  974. /* OK, we have to color these pages. The page tables are accessed
  975. * by non-Dcache enabled mapping in the VPTE area by the dtlb_backend.S
  976. * code, as well as by PAGE_OFFSET range direct-mapped addresses by
  977. * other parts of the kernel. By coloring, we make sure that the tlbmiss
  978. * fast handlers do not get data from old/garbage dcache lines that
  979. * correspond to an old/stale virtual address (user/kernel) that
  980. * previously mapped the pagetable page while accessing vpte range
  981. * addresses. The idea is that if the vpte color and PAGE_OFFSET range
  982. * color is the same, then when the kernel initializes the pagetable
  983. * using the later address range, accesses with the first address
  984. * range will see the newly initialized data rather than the garbage.
  985. */
  986. #ifdef DCACHE_ALIASING_POSSIBLE
  987. #define DC_ALIAS_SHIFT 1
  988. #else
  989. #define DC_ALIAS_SHIFT 0
  990. #endif
  991. pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address)
  992. {
  993. struct page *page;
  994. unsigned long color;
  995. {
  996. pte_t *ptep = pte_alloc_one_fast(mm, address);
  997. if (ptep)
  998. return ptep;
  999. }
  1000. color = VPTE_COLOR(address);
  1001. page = alloc_pages(GFP_KERNEL|__GFP_REPEAT, DC_ALIAS_SHIFT);
  1002. if (page) {
  1003. unsigned long *to_free;
  1004. unsigned long paddr;
  1005. pte_t *pte;
  1006. #ifdef DCACHE_ALIASING_POSSIBLE
  1007. set_page_count(page, 1);
  1008. ClearPageCompound(page);
  1009. set_page_count((page + 1), 1);
  1010. ClearPageCompound(page + 1);
  1011. #endif
  1012. paddr = (unsigned long) page_address(page);
  1013. memset((char *)paddr, 0, (PAGE_SIZE << DC_ALIAS_SHIFT));
  1014. if (!color) {
  1015. pte = (pte_t *) paddr;
  1016. to_free = (unsigned long *) (paddr + PAGE_SIZE);
  1017. } else {
  1018. pte = (pte_t *) (paddr + PAGE_SIZE);
  1019. to_free = (unsigned long *) paddr;
  1020. }
  1021. #ifdef DCACHE_ALIASING_POSSIBLE
  1022. /* Now free the other one up, adjust cache size. */
  1023. preempt_disable();
  1024. *to_free = (unsigned long) pte_quicklist[color ^ 0x1];
  1025. pte_quicklist[color ^ 0x1] = to_free;
  1026. pgtable_cache_size++;
  1027. preempt_enable();
  1028. #endif
  1029. return pte;
  1030. }
  1031. return NULL;
  1032. }
  1033. void sparc_ultra_dump_itlb(void)
  1034. {
  1035. int slot;
  1036. if (tlb_type == spitfire) {
  1037. printk ("Contents of itlb: ");
  1038. for (slot = 0; slot < 14; slot++) printk (" ");
  1039. printk ("%2x:%016lx,%016lx\n",
  1040. 0,
  1041. spitfire_get_itlb_tag(0), spitfire_get_itlb_data(0));
  1042. for (slot = 1; slot < 64; slot+=3) {
  1043. printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx %2x:%016lx,%016lx\n",
  1044. slot,
  1045. spitfire_get_itlb_tag(slot), spitfire_get_itlb_data(slot),
  1046. slot+1,
  1047. spitfire_get_itlb_tag(slot+1), spitfire_get_itlb_data(slot+1),
  1048. slot+2,
  1049. spitfire_get_itlb_tag(slot+2), spitfire_get_itlb_data(slot+2));
  1050. }
  1051. } else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
  1052. printk ("Contents of itlb0:\n");
  1053. for (slot = 0; slot < 16; slot+=2) {
  1054. printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n",
  1055. slot,
  1056. cheetah_get_litlb_tag(slot), cheetah_get_litlb_data(slot),
  1057. slot+1,
  1058. cheetah_get_litlb_tag(slot+1), cheetah_get_litlb_data(slot+1));
  1059. }
  1060. printk ("Contents of itlb2:\n");
  1061. for (slot = 0; slot < 128; slot+=2) {
  1062. printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n",
  1063. slot,
  1064. cheetah_get_itlb_tag(slot), cheetah_get_itlb_data(slot),
  1065. slot+1,
  1066. cheetah_get_itlb_tag(slot+1), cheetah_get_itlb_data(slot+1));
  1067. }
  1068. }
  1069. }
  1070. void sparc_ultra_dump_dtlb(void)
  1071. {
  1072. int slot;
  1073. if (tlb_type == spitfire) {
  1074. printk ("Contents of dtlb: ");
  1075. for (slot = 0; slot < 14; slot++) printk (" ");
  1076. printk ("%2x:%016lx,%016lx\n", 0,
  1077. spitfire_get_dtlb_tag(0), spitfire_get_dtlb_data(0));
  1078. for (slot = 1; slot < 64; slot+=3) {
  1079. printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx %2x:%016lx,%016lx\n",
  1080. slot,
  1081. spitfire_get_dtlb_tag(slot), spitfire_get_dtlb_data(slot),
  1082. slot+1,
  1083. spitfire_get_dtlb_tag(slot+1), spitfire_get_dtlb_data(slot+1),
  1084. slot+2,
  1085. spitfire_get_dtlb_tag(slot+2), spitfire_get_dtlb_data(slot+2));
  1086. }
  1087. } else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
  1088. printk ("Contents of dtlb0:\n");
  1089. for (slot = 0; slot < 16; slot+=2) {
  1090. printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n",
  1091. slot,
  1092. cheetah_get_ldtlb_tag(slot), cheetah_get_ldtlb_data(slot),
  1093. slot+1,
  1094. cheetah_get_ldtlb_tag(slot+1), cheetah_get_ldtlb_data(slot+1));
  1095. }
  1096. printk ("Contents of dtlb2:\n");
  1097. for (slot = 0; slot < 512; slot+=2) {
  1098. printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n",
  1099. slot,
  1100. cheetah_get_dtlb_tag(slot, 2), cheetah_get_dtlb_data(slot, 2),
  1101. slot+1,
  1102. cheetah_get_dtlb_tag(slot+1, 2), cheetah_get_dtlb_data(slot+1, 2));
  1103. }
  1104. if (tlb_type == cheetah_plus) {
  1105. printk ("Contents of dtlb3:\n");
  1106. for (slot = 0; slot < 512; slot+=2) {
  1107. printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n",
  1108. slot,
  1109. cheetah_get_dtlb_tag(slot, 3), cheetah_get_dtlb_data(slot, 3),
  1110. slot+1,
  1111. cheetah_get_dtlb_tag(slot+1, 3), cheetah_get_dtlb_data(slot+1, 3));
  1112. }
  1113. }
  1114. }
  1115. }
  1116. extern unsigned long cmdline_memory_size;
  1117. unsigned long __init bootmem_init(unsigned long *pages_avail)
  1118. {
  1119. unsigned long bootmap_size, start_pfn, end_pfn;
  1120. unsigned long end_of_phys_memory = 0UL;
  1121. unsigned long bootmap_pfn, bytes_avail, size;
  1122. int i;
  1123. #ifdef CONFIG_DEBUG_BOOTMEM
  1124. prom_printf("bootmem_init: Scan sp_banks, ");
  1125. #endif
  1126. bytes_avail = 0UL;
  1127. for (i = 0; sp_banks[i].num_bytes != 0; i++) {
  1128. end_of_phys_memory = sp_banks[i].base_addr +
  1129. sp_banks[i].num_bytes;
  1130. bytes_avail += sp_banks[i].num_bytes;
  1131. if (cmdline_memory_size) {
  1132. if (bytes_avail > cmdline_memory_size) {
  1133. unsigned long slack = bytes_avail - cmdline_memory_size;
  1134. bytes_avail -= slack;
  1135. end_of_phys_memory -= slack;
  1136. sp_banks[i].num_bytes -= slack;
  1137. if (sp_banks[i].num_bytes == 0) {
  1138. sp_banks[i].base_addr = 0xdeadbeef;
  1139. } else {
  1140. sp_banks[i+1].num_bytes = 0;
  1141. sp_banks[i+1].base_addr = 0xdeadbeef;
  1142. }
  1143. break;
  1144. }
  1145. }
  1146. }
  1147. *pages_avail = bytes_avail >> PAGE_SHIFT;
  1148. /* Start with page aligned address of last symbol in kernel
  1149. * image. The kernel is hard mapped below PAGE_OFFSET in a
  1150. * 4MB locked TLB translation.
  1151. */
  1152. start_pfn = PAGE_ALIGN(kern_base + kern_size) >> PAGE_SHIFT;
  1153. bootmap_pfn = start_pfn;
  1154. end_pfn = end_of_phys_memory >> PAGE_SHIFT;
  1155. #ifdef CONFIG_BLK_DEV_INITRD
  1156. /* Now have to check initial ramdisk, so that bootmap does not overwrite it */
  1157. if (sparc_ramdisk_image || sparc_ramdisk_image64) {
  1158. unsigned long ramdisk_image = sparc_ramdisk_image ?
  1159. sparc_ramdisk_image : sparc_ramdisk_image64;
  1160. if (ramdisk_image >= (unsigned long)_end - 2 * PAGE_SIZE)
  1161. ramdisk_image -= KERNBASE;
  1162. initrd_start = ramdisk_image + phys_base;
  1163. initrd_end = initrd_start + sparc_ramdisk_size;
  1164. if (initrd_end > end_of_phys_memory) {
  1165. printk(KERN_CRIT "initrd extends beyond end of memory "
  1166. "(0x%016lx > 0x%016lx)\ndisabling initrd\n",
  1167. initrd_end, end_of_phys_memory);
  1168. initrd_start = 0;
  1169. }
  1170. if (initrd_start) {
  1171. if (initrd_start >= (start_pfn << PAGE_SHIFT) &&
  1172. initrd_start < (start_pfn << PAGE_SHIFT) + 2 * PAGE_SIZE)
  1173. bootmap_pfn = PAGE_ALIGN (initrd_end) >> PAGE_SHIFT;
  1174. }
  1175. }
  1176. #endif
  1177. /* Initialize the boot-time allocator. */
  1178. max_pfn = max_low_pfn = end_pfn;
  1179. min_low_pfn = pfn_base;
  1180. #ifdef CONFIG_DEBUG_BOOTMEM
  1181. prom_printf("init_bootmem(min[%lx], bootmap[%lx], max[%lx])\n",
  1182. min_low_pfn, bootmap_pfn, max_low_pfn);
  1183. #endif
  1184. bootmap_size = init_bootmem_node(NODE_DATA(0), bootmap_pfn, pfn_base, end_pfn);
  1185. bootmap_base = bootmap_pfn << PAGE_SHIFT;
  1186. /* Now register the available physical memory with the
  1187. * allocator.
  1188. */
  1189. for (i = 0; sp_banks[i].num_bytes != 0; i++) {
  1190. #ifdef CONFIG_DEBUG_BOOTMEM
  1191. prom_printf("free_bootmem(sp_banks:%d): base[%lx] size[%lx]\n",
  1192. i, sp_banks[i].base_addr, sp_banks[i].num_bytes);
  1193. #endif
  1194. free_bootmem(sp_banks[i].base_addr, sp_banks[i].num_bytes);
  1195. }
  1196. #ifdef CONFIG_BLK_DEV_INITRD
  1197. if (initrd_start) {
  1198. size = initrd_end - initrd_start;
  1199. /* Resert the initrd image area. */
  1200. #ifdef CONFIG_DEBUG_BOOTMEM
  1201. prom_printf("reserve_bootmem(initrd): base[%llx] size[%lx]\n",
  1202. initrd_start, initrd_end);
  1203. #endif
  1204. reserve_bootmem(initrd_start, size);
  1205. *pages_avail -= PAGE_ALIGN(size) >> PAGE_SHIFT;
  1206. initrd_start += PAGE_OFFSET;
  1207. initrd_end += PAGE_OFFSET;
  1208. }
  1209. #endif
  1210. /* Reserve the kernel text/data/bss. */
  1211. #ifdef CONFIG_DEBUG_BOOTMEM
  1212. prom_printf("reserve_bootmem(kernel): base[%lx] size[%lx]\n", kern_base, kern_size);
  1213. #endif
  1214. reserve_bootmem(kern_base, kern_size);
  1215. *pages_avail -= PAGE_ALIGN(kern_size) >> PAGE_SHIFT;
  1216. /* Reserve the bootmem map. We do not account for it
  1217. * in pages_avail because we will release that memory
  1218. * in free_all_bootmem.
  1219. */
  1220. size = bootmap_size;
  1221. #ifdef CONFIG_DEBUG_BOOTMEM
  1222. prom_printf("reserve_bootmem(bootmap): base[%lx] size[%lx]\n",
  1223. (bootmap_pfn << PAGE_SHIFT), size);
  1224. #endif
  1225. reserve_bootmem((bootmap_pfn << PAGE_SHIFT), size);
  1226. *pages_avail -= PAGE_ALIGN(size) >> PAGE_SHIFT;
  1227. return end_pfn;
  1228. }
  1229. /* paging_init() sets up the page tables */
  1230. extern void cheetah_ecache_flush_init(void);
  1231. static unsigned long last_valid_pfn;
  1232. void __init paging_init(void)
  1233. {
  1234. extern pmd_t swapper_pmd_dir[1024];
  1235. extern unsigned int sparc64_vpte_patchme1[1];
  1236. extern unsigned int sparc64_vpte_patchme2[1];
  1237. unsigned long alias_base = kern_base + PAGE_OFFSET;
  1238. unsigned long second_alias_page = 0;
  1239. unsigned long pt, flags, end_pfn, pages_avail;
  1240. unsigned long shift = alias_base - ((unsigned long)KERNBASE);
  1241. unsigned long real_end;
  1242. set_bit(0, mmu_context_bmap);
  1243. real_end = (unsigned long)_end;
  1244. if ((real_end > ((unsigned long)KERNBASE + 0x400000)))
  1245. bigkernel = 1;
  1246. #ifdef CONFIG_BLK_DEV_INITRD
  1247. if (sparc_ramdisk_image || sparc_ramdisk_image64)
  1248. real_end = (PAGE_ALIGN(real_end) + PAGE_ALIGN(sparc_ramdisk_size));
  1249. #endif
  1250. /* We assume physical memory starts at some 4mb multiple,
  1251. * if this were not true we wouldn't boot up to this point
  1252. * anyways.
  1253. */
  1254. pt = kern_base | _PAGE_VALID | _PAGE_SZ4MB;
  1255. pt |= _PAGE_CP | _PAGE_CV | _PAGE_P | _PAGE_L | _PAGE_W;
  1256. local_irq_save(flags);
  1257. if (tlb_type == spitfire) {
  1258. __asm__ __volatile__(
  1259. " stxa %1, [%0] %3\n"
  1260. " stxa %2, [%5] %4\n"
  1261. " membar #Sync\n"
  1262. " flush %%g6\n"
  1263. " nop\n"
  1264. " nop\n"
  1265. " nop\n"
  1266. : /* No outputs */
  1267. : "r" (TLB_TAG_ACCESS), "r" (alias_base), "r" (pt),
  1268. "i" (ASI_DMMU), "i" (ASI_DTLB_DATA_ACCESS), "r" (61 << 3)
  1269. : "memory");
  1270. if (real_end >= KERNBASE + 0x340000) {
  1271. second_alias_page = alias_base + 0x400000;
  1272. __asm__ __volatile__(
  1273. " stxa %1, [%0] %3\n"
  1274. " stxa %2, [%5] %4\n"
  1275. " membar #Sync\n"
  1276. " flush %%g6\n"
  1277. " nop\n"
  1278. " nop\n"
  1279. " nop\n"
  1280. : /* No outputs */
  1281. : "r" (TLB_TAG_ACCESS), "r" (second_alias_page), "r" (pt + 0x400000),
  1282. "i" (ASI_DMMU), "i" (ASI_DTLB_DATA_ACCESS), "r" (60 << 3)
  1283. : "memory");
  1284. }
  1285. } else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
  1286. __asm__ __volatile__(
  1287. " stxa %1, [%0] %3\n"
  1288. " stxa %2, [%5] %4\n"
  1289. " membar #Sync\n"
  1290. " flush %%g6\n"
  1291. " nop\n"
  1292. " nop\n"
  1293. " nop\n"
  1294. : /* No outputs */
  1295. : "r" (TLB_TAG_ACCESS), "r" (alias_base), "r" (pt),
  1296. "i" (ASI_DMMU), "i" (ASI_DTLB_DATA_ACCESS), "r" ((0<<16) | (13<<3))
  1297. : "memory");
  1298. if (real_end >= KERNBASE + 0x340000) {
  1299. second_alias_page = alias_base + 0x400000;
  1300. __asm__ __volatile__(
  1301. " stxa %1, [%0] %3\n"
  1302. " stxa %2, [%5] %4\n"
  1303. " membar #Sync\n"
  1304. " flush %%g6\n"
  1305. " nop\n"
  1306. " nop\n"
  1307. " nop\n"
  1308. : /* No outputs */
  1309. : "r" (TLB_TAG_ACCESS), "r" (second_alias_page), "r" (pt + 0x400000),
  1310. "i" (ASI_DMMU), "i" (ASI_DTLB_DATA_ACCESS), "r" ((0<<16) | (12<<3))
  1311. : "memory");
  1312. }
  1313. }
  1314. local_irq_restore(flags);
  1315. /* Now set kernel pgd to upper alias so physical page computations
  1316. * work.
  1317. */
  1318. init_mm.pgd += ((shift) / (sizeof(pgd_t)));
  1319. memset(swapper_pmd_dir, 0, sizeof(swapper_pmd_dir));
  1320. /* Now can init the kernel/bad page tables. */
  1321. pud_set(pud_offset(&swapper_pg_dir[0], 0),
  1322. swapper_pmd_dir + (shift / sizeof(pgd_t)));
  1323. sparc64_vpte_patchme1[0] |=
  1324. (((unsigned long)pgd_val(init_mm.pgd[0])) >> 10);
  1325. sparc64_vpte_patchme2[0] |=
  1326. (((unsigned long)pgd_val(init_mm.pgd[0])) & 0x3ff);
  1327. flushi((long)&sparc64_vpte_patchme1[0]);
  1328. /* Setup bootmem... */
  1329. pages_avail = 0;
  1330. last_valid_pfn = end_pfn = bootmem_init(&pages_avail);
  1331. /* Inherit non-locked OBP mappings. */
  1332. inherit_prom_mappings();
  1333. /* Ok, we can use our TLB miss and window trap handlers safely.
  1334. * We need to do a quick peek here to see if we are on StarFire
  1335. * or not, so setup_tba can setup the IRQ globals correctly (it
  1336. * needs to get the hard smp processor id correctly).
  1337. */
  1338. {
  1339. extern void setup_tba(int);
  1340. setup_tba(this_is_starfire);
  1341. }
  1342. inherit_locked_prom_mappings(1);
  1343. /* We only created DTLB mapping of this stuff. */
  1344. spitfire_flush_dtlb_nucleus_page(alias_base);
  1345. if (second_alias_page)
  1346. spitfire_flush_dtlb_nucleus_page(second_alias_page);
  1347. __flush_tlb_all();
  1348. {
  1349. unsigned long zones_size[MAX_NR_ZONES];
  1350. unsigned long zholes_size[MAX_NR_ZONES];
  1351. unsigned long npages;
  1352. int znum;
  1353. for (znum = 0; znum < MAX_NR_ZONES; znum++)
  1354. zones_size[znum] = zholes_size[znum] = 0;
  1355. npages = end_pfn - pfn_base;
  1356. zones_size[ZONE_DMA] = npages;
  1357. zholes_size[ZONE_DMA] = npages - pages_avail;
  1358. free_area_init_node(0, &contig_page_data, zones_size,
  1359. phys_base >> PAGE_SHIFT, zholes_size);
  1360. }
  1361. device_scan();
  1362. }
  1363. /* Ok, it seems that the prom can allocate some more memory chunks
  1364. * as a side effect of some prom calls we perform during the
  1365. * boot sequence. My most likely theory is that it is from the
  1366. * prom_set_traptable() call, and OBP is allocating a scratchpad
  1367. * for saving client program register state etc.
  1368. */
  1369. static void __init sort_memlist(struct linux_mlist_p1275 *thislist)
  1370. {
  1371. int swapi = 0;
  1372. int i, mitr;
  1373. unsigned long tmpaddr, tmpsize;
  1374. unsigned long lowest;
  1375. for (i = 0; thislist[i].theres_more != 0; i++) {
  1376. lowest = thislist[i].start_adr;
  1377. for (mitr = i+1; thislist[mitr-1].theres_more != 0; mitr++)
  1378. if (thislist[mitr].start_adr < lowest) {
  1379. lowest = thislist[mitr].start_adr;
  1380. swapi = mitr;
  1381. }
  1382. if (lowest == thislist[i].start_adr)
  1383. continue;
  1384. tmpaddr = thislist[swapi].start_adr;
  1385. tmpsize = thislist[swapi].num_bytes;
  1386. for (mitr = swapi; mitr > i; mitr--) {
  1387. thislist[mitr].start_adr = thislist[mitr-1].start_adr;
  1388. thislist[mitr].num_bytes = thislist[mitr-1].num_bytes;
  1389. }
  1390. thislist[i].start_adr = tmpaddr;
  1391. thislist[i].num_bytes = tmpsize;
  1392. }
  1393. }
  1394. void __init rescan_sp_banks(void)
  1395. {
  1396. struct linux_prom64_registers memlist[64];
  1397. struct linux_mlist_p1275 avail[64], *mlist;
  1398. unsigned long bytes, base_paddr;
  1399. int num_regs, node = prom_finddevice("/memory");
  1400. int i;
  1401. num_regs = prom_getproperty(node, "available",
  1402. (char *) memlist, sizeof(memlist));
  1403. num_regs = (num_regs / sizeof(struct linux_prom64_registers));
  1404. for (i = 0; i < num_regs; i++) {
  1405. avail[i].start_adr = memlist[i].phys_addr;
  1406. avail[i].num_bytes = memlist[i].reg_size;
  1407. avail[i].theres_more = &avail[i + 1];
  1408. }
  1409. avail[i - 1].theres_more = NULL;
  1410. sort_memlist(avail);
  1411. mlist = &avail[0];
  1412. i = 0;
  1413. bytes = mlist->num_bytes;
  1414. base_paddr = mlist->start_adr;
  1415. sp_banks[0].base_addr = base_paddr;
  1416. sp_banks[0].num_bytes = bytes;
  1417. while (mlist->theres_more != NULL){
  1418. i++;
  1419. mlist = mlist->theres_more;
  1420. bytes = mlist->num_bytes;
  1421. if (i >= SPARC_PHYS_BANKS-1) {
  1422. printk ("The machine has more banks than "
  1423. "this kernel can support\n"
  1424. "Increase the SPARC_PHYS_BANKS "
  1425. "setting (currently %d)\n",
  1426. SPARC_PHYS_BANKS);
  1427. i = SPARC_PHYS_BANKS-1;
  1428. break;
  1429. }
  1430. sp_banks[i].base_addr = mlist->start_adr;
  1431. sp_banks[i].num_bytes = mlist->num_bytes;
  1432. }
  1433. i++;
  1434. sp_banks[i].base_addr = 0xdeadbeefbeefdeadUL;
  1435. sp_banks[i].num_bytes = 0;
  1436. for (i = 0; sp_banks[i].num_bytes != 0; i++)
  1437. sp_banks[i].num_bytes &= PAGE_MASK;
  1438. }
  1439. static void __init taint_real_pages(void)
  1440. {
  1441. struct sparc_phys_banks saved_sp_banks[SPARC_PHYS_BANKS];
  1442. int i;
  1443. for (i = 0; i < SPARC_PHYS_BANKS; i++) {
  1444. saved_sp_banks[i].base_addr =
  1445. sp_banks[i].base_addr;
  1446. saved_sp_banks[i].num_bytes =
  1447. sp_banks[i].num_bytes;
  1448. }
  1449. rescan_sp_banks();
  1450. /* Find changes discovered in the sp_bank rescan and
  1451. * reserve the lost portions in the bootmem maps.
  1452. */
  1453. for (i = 0; saved_sp_banks[i].num_bytes; i++) {
  1454. unsigned long old_start, old_end;
  1455. old_start = saved_sp_banks[i].base_addr;
  1456. old_end = old_start +
  1457. saved_sp_banks[i].num_bytes;
  1458. while (old_start < old_end) {
  1459. int n;
  1460. for (n = 0; sp_banks[n].num_bytes; n++) {
  1461. unsigned long new_start, new_end;
  1462. new_start = sp_banks[n].base_addr;
  1463. new_end = new_start + sp_banks[n].num_bytes;
  1464. if (new_start <= old_start &&
  1465. new_end >= (old_start + PAGE_SIZE)) {
  1466. set_bit (old_start >> 22,
  1467. sparc64_valid_addr_bitmap);
  1468. goto do_next_page;
  1469. }
  1470. }
  1471. reserve_bootmem(old_start, PAGE_SIZE);
  1472. do_next_page:
  1473. old_start += PAGE_SIZE;
  1474. }
  1475. }
  1476. }
  1477. void __init mem_init(void)
  1478. {
  1479. unsigned long codepages, datapages, initpages;
  1480. unsigned long addr, last;
  1481. int i;
  1482. i = last_valid_pfn >> ((22 - PAGE_SHIFT) + 6);
  1483. i += 1;
  1484. sparc64_valid_addr_bitmap = (unsigned long *)
  1485. __alloc_bootmem(i << 3, SMP_CACHE_BYTES, bootmap_base);
  1486. if (sparc64_valid_addr_bitmap == NULL) {
  1487. prom_printf("mem_init: Cannot alloc valid_addr_bitmap.\n");
  1488. prom_halt();
  1489. }
  1490. memset(sparc64_valid_addr_bitmap, 0, i << 3);
  1491. addr = PAGE_OFFSET + kern_base;
  1492. last = PAGE_ALIGN(kern_size) + addr;
  1493. while (addr < last) {
  1494. set_bit(__pa(addr) >> 22, sparc64_valid_addr_bitmap);
  1495. addr += PAGE_SIZE;
  1496. }
  1497. taint_real_pages();
  1498. max_mapnr = last_valid_pfn - pfn_base;
  1499. high_memory = __va(last_valid_pfn << PAGE_SHIFT);
  1500. #ifdef CONFIG_DEBUG_BOOTMEM
  1501. prom_printf("mem_init: Calling free_all_bootmem().\n");
  1502. #endif
  1503. totalram_pages = num_physpages = free_all_bootmem() - 1;
  1504. /*
  1505. * Set up the zero page, mark it reserved, so that page count
  1506. * is not manipulated when freeing the page from user ptes.
  1507. */
  1508. mem_map_zero = alloc_pages(GFP_KERNEL|__GFP_ZERO, 0);
  1509. if (mem_map_zero == NULL) {
  1510. prom_printf("paging_init: Cannot alloc zero page.\n");
  1511. prom_halt();
  1512. }
  1513. SetPageReserved(mem_map_zero);
  1514. codepages = (((unsigned long) _etext) - ((unsigned long) _start));
  1515. codepages = PAGE_ALIGN(codepages) >> PAGE_SHIFT;
  1516. datapages = (((unsigned long) _edata) - ((unsigned long) _etext));
  1517. datapages = PAGE_ALIGN(datapages) >> PAGE_SHIFT;
  1518. initpages = (((unsigned long) __init_end) - ((unsigned long) __init_begin));
  1519. initpages = PAGE_ALIGN(initpages) >> PAGE_SHIFT;
  1520. printk("Memory: %uk available (%ldk kernel code, %ldk data, %ldk init) [%016lx,%016lx]\n",
  1521. nr_free_pages() << (PAGE_SHIFT-10),
  1522. codepages << (PAGE_SHIFT-10),
  1523. datapages << (PAGE_SHIFT-10),
  1524. initpages << (PAGE_SHIFT-10),
  1525. PAGE_OFFSET, (last_valid_pfn << PAGE_SHIFT));
  1526. if (tlb_type == cheetah || tlb_type == cheetah_plus)
  1527. cheetah_ecache_flush_init();
  1528. }
  1529. void free_initmem (void)
  1530. {
  1531. unsigned long addr, initend;
  1532. /*
  1533. * The init section is aligned to 8k in vmlinux.lds. Page align for >8k pagesizes.
  1534. */
  1535. addr = PAGE_ALIGN((unsigned long)(__init_begin));
  1536. initend = (unsigned long)(__init_end) & PAGE_MASK;
  1537. for (; addr < initend; addr += PAGE_SIZE) {
  1538. unsigned long page;
  1539. struct page *p;
  1540. page = (addr +
  1541. ((unsigned long) __va(kern_base)) -
  1542. ((unsigned long) KERNBASE));
  1543. memset((void *)addr, 0xcc, PAGE_SIZE);
  1544. p = virt_to_page(page);
  1545. ClearPageReserved(p);
  1546. set_page_count(p, 1);
  1547. __free_page(p);
  1548. num_physpages++;
  1549. totalram_pages++;
  1550. }
  1551. }
  1552. #ifdef CONFIG_BLK_DEV_INITRD
  1553. void free_initrd_mem(unsigned long start, unsigned long end)
  1554. {
  1555. if (start < end)
  1556. printk ("Freeing initrd memory: %ldk freed\n", (end - start) >> 10);
  1557. for (; start < end; start += PAGE_SIZE) {
  1558. struct page *p = virt_to_page(start);
  1559. ClearPageReserved(p);
  1560. set_page_count(p, 1);
  1561. __free_page(p);
  1562. num_physpages++;
  1563. totalram_pages++;
  1564. }
  1565. }
  1566. #endif