pgtable.c 31 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158
  1. /*
  2. * Copyright IBM Corp. 2007, 2011
  3. * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
  4. */
  5. #include <linux/sched.h>
  6. #include <linux/kernel.h>
  7. #include <linux/errno.h>
  8. #include <linux/gfp.h>
  9. #include <linux/mm.h>
  10. #include <linux/swap.h>
  11. #include <linux/smp.h>
  12. #include <linux/highmem.h>
  13. #include <linux/pagemap.h>
  14. #include <linux/spinlock.h>
  15. #include <linux/module.h>
  16. #include <linux/quicklist.h>
  17. #include <linux/rcupdate.h>
  18. #include <linux/slab.h>
  19. #include <asm/pgtable.h>
  20. #include <asm/pgalloc.h>
  21. #include <asm/tlb.h>
  22. #include <asm/tlbflush.h>
  23. #include <asm/mmu_context.h>
  24. #ifndef CONFIG_64BIT
  25. #define ALLOC_ORDER 1
  26. #define FRAG_MASK 0x0f
  27. #else
  28. #define ALLOC_ORDER 2
  29. #define FRAG_MASK 0x03
  30. #endif
  31. unsigned long *crst_table_alloc(struct mm_struct *mm)
  32. {
  33. struct page *page = alloc_pages(GFP_KERNEL, ALLOC_ORDER);
  34. if (!page)
  35. return NULL;
  36. return (unsigned long *) page_to_phys(page);
  37. }
  38. void crst_table_free(struct mm_struct *mm, unsigned long *table)
  39. {
  40. free_pages((unsigned long) table, ALLOC_ORDER);
  41. }
  42. #ifdef CONFIG_64BIT
  43. int crst_table_upgrade(struct mm_struct *mm, unsigned long limit)
  44. {
  45. unsigned long *table, *pgd;
  46. unsigned long entry;
  47. BUG_ON(limit > (1UL << 53));
  48. repeat:
  49. table = crst_table_alloc(mm);
  50. if (!table)
  51. return -ENOMEM;
  52. spin_lock_bh(&mm->page_table_lock);
  53. if (mm->context.asce_limit < limit) {
  54. pgd = (unsigned long *) mm->pgd;
  55. if (mm->context.asce_limit <= (1UL << 31)) {
  56. entry = _REGION3_ENTRY_EMPTY;
  57. mm->context.asce_limit = 1UL << 42;
  58. mm->context.asce_bits = _ASCE_TABLE_LENGTH |
  59. _ASCE_USER_BITS |
  60. _ASCE_TYPE_REGION3;
  61. } else {
  62. entry = _REGION2_ENTRY_EMPTY;
  63. mm->context.asce_limit = 1UL << 53;
  64. mm->context.asce_bits = _ASCE_TABLE_LENGTH |
  65. _ASCE_USER_BITS |
  66. _ASCE_TYPE_REGION2;
  67. }
  68. crst_table_init(table, entry);
  69. pgd_populate(mm, (pgd_t *) table, (pud_t *) pgd);
  70. mm->pgd = (pgd_t *) table;
  71. mm->task_size = mm->context.asce_limit;
  72. table = NULL;
  73. }
  74. spin_unlock_bh(&mm->page_table_lock);
  75. if (table)
  76. crst_table_free(mm, table);
  77. if (mm->context.asce_limit < limit)
  78. goto repeat;
  79. return 0;
  80. }
  81. void crst_table_downgrade(struct mm_struct *mm, unsigned long limit)
  82. {
  83. pgd_t *pgd;
  84. while (mm->context.asce_limit > limit) {
  85. pgd = mm->pgd;
  86. switch (pgd_val(*pgd) & _REGION_ENTRY_TYPE_MASK) {
  87. case _REGION_ENTRY_TYPE_R2:
  88. mm->context.asce_limit = 1UL << 42;
  89. mm->context.asce_bits = _ASCE_TABLE_LENGTH |
  90. _ASCE_USER_BITS |
  91. _ASCE_TYPE_REGION3;
  92. break;
  93. case _REGION_ENTRY_TYPE_R3:
  94. mm->context.asce_limit = 1UL << 31;
  95. mm->context.asce_bits = _ASCE_TABLE_LENGTH |
  96. _ASCE_USER_BITS |
  97. _ASCE_TYPE_SEGMENT;
  98. break;
  99. default:
  100. BUG();
  101. }
  102. mm->pgd = (pgd_t *) (pgd_val(*pgd) & _REGION_ENTRY_ORIGIN);
  103. mm->task_size = mm->context.asce_limit;
  104. crst_table_free(mm, (unsigned long *) pgd);
  105. }
  106. }
  107. #endif
  108. #ifdef CONFIG_PGSTE
  109. /**
  110. * gmap_alloc - allocate a guest address space
  111. * @mm: pointer to the parent mm_struct
  112. *
  113. * Returns a guest address space structure.
  114. */
  115. struct gmap *gmap_alloc(struct mm_struct *mm)
  116. {
  117. struct gmap *gmap;
  118. struct page *page;
  119. unsigned long *table;
  120. gmap = kzalloc(sizeof(struct gmap), GFP_KERNEL);
  121. if (!gmap)
  122. goto out;
  123. INIT_LIST_HEAD(&gmap->crst_list);
  124. gmap->mm = mm;
  125. page = alloc_pages(GFP_KERNEL, ALLOC_ORDER);
  126. if (!page)
  127. goto out_free;
  128. list_add(&page->lru, &gmap->crst_list);
  129. table = (unsigned long *) page_to_phys(page);
  130. crst_table_init(table, _REGION1_ENTRY_EMPTY);
  131. gmap->table = table;
  132. gmap->asce = _ASCE_TYPE_REGION1 | _ASCE_TABLE_LENGTH |
  133. _ASCE_USER_BITS | __pa(table);
  134. list_add(&gmap->list, &mm->context.gmap_list);
  135. return gmap;
  136. out_free:
  137. kfree(gmap);
  138. out:
  139. return NULL;
  140. }
  141. EXPORT_SYMBOL_GPL(gmap_alloc);
  142. static int gmap_unlink_segment(struct gmap *gmap, unsigned long *table)
  143. {
  144. struct gmap_pgtable *mp;
  145. struct gmap_rmap *rmap;
  146. struct page *page;
  147. if (*table & _SEGMENT_ENTRY_INV)
  148. return 0;
  149. page = pfn_to_page(*table >> PAGE_SHIFT);
  150. mp = (struct gmap_pgtable *) page->index;
  151. list_for_each_entry(rmap, &mp->mapper, list) {
  152. if (rmap->entry != table)
  153. continue;
  154. list_del(&rmap->list);
  155. kfree(rmap);
  156. break;
  157. }
  158. *table = _SEGMENT_ENTRY_INV | _SEGMENT_ENTRY_RO | mp->vmaddr;
  159. return 1;
  160. }
  161. static void gmap_flush_tlb(struct gmap *gmap)
  162. {
  163. if (MACHINE_HAS_IDTE)
  164. __tlb_flush_idte((unsigned long) gmap->table |
  165. _ASCE_TYPE_REGION1);
  166. else
  167. __tlb_flush_global();
  168. }
  169. /**
  170. * gmap_free - free a guest address space
  171. * @gmap: pointer to the guest address space structure
  172. */
  173. void gmap_free(struct gmap *gmap)
  174. {
  175. struct page *page, *next;
  176. unsigned long *table;
  177. int i;
  178. /* Flush tlb. */
  179. if (MACHINE_HAS_IDTE)
  180. __tlb_flush_idte((unsigned long) gmap->table |
  181. _ASCE_TYPE_REGION1);
  182. else
  183. __tlb_flush_global();
  184. /* Free all segment & region tables. */
  185. down_read(&gmap->mm->mmap_sem);
  186. spin_lock(&gmap->mm->page_table_lock);
  187. list_for_each_entry_safe(page, next, &gmap->crst_list, lru) {
  188. table = (unsigned long *) page_to_phys(page);
  189. if ((*table & _REGION_ENTRY_TYPE_MASK) == 0)
  190. /* Remove gmap rmap structures for segment table. */
  191. for (i = 0; i < PTRS_PER_PMD; i++, table++)
  192. gmap_unlink_segment(gmap, table);
  193. __free_pages(page, ALLOC_ORDER);
  194. }
  195. spin_unlock(&gmap->mm->page_table_lock);
  196. up_read(&gmap->mm->mmap_sem);
  197. list_del(&gmap->list);
  198. kfree(gmap);
  199. }
  200. EXPORT_SYMBOL_GPL(gmap_free);
  201. /**
  202. * gmap_enable - switch primary space to the guest address space
  203. * @gmap: pointer to the guest address space structure
  204. */
  205. void gmap_enable(struct gmap *gmap)
  206. {
  207. S390_lowcore.gmap = (unsigned long) gmap;
  208. }
  209. EXPORT_SYMBOL_GPL(gmap_enable);
  210. /**
  211. * gmap_disable - switch back to the standard primary address space
  212. * @gmap: pointer to the guest address space structure
  213. */
  214. void gmap_disable(struct gmap *gmap)
  215. {
  216. S390_lowcore.gmap = 0UL;
  217. }
  218. EXPORT_SYMBOL_GPL(gmap_disable);
  219. /*
  220. * gmap_alloc_table is assumed to be called with mmap_sem held
  221. */
  222. static int gmap_alloc_table(struct gmap *gmap,
  223. unsigned long *table, unsigned long init)
  224. {
  225. struct page *page;
  226. unsigned long *new;
  227. /* since we dont free the gmap table until gmap_free we can unlock */
  228. spin_unlock(&gmap->mm->page_table_lock);
  229. page = alloc_pages(GFP_KERNEL, ALLOC_ORDER);
  230. spin_lock(&gmap->mm->page_table_lock);
  231. if (!page)
  232. return -ENOMEM;
  233. new = (unsigned long *) page_to_phys(page);
  234. crst_table_init(new, init);
  235. if (*table & _REGION_ENTRY_INV) {
  236. list_add(&page->lru, &gmap->crst_list);
  237. *table = (unsigned long) new | _REGION_ENTRY_LENGTH |
  238. (*table & _REGION_ENTRY_TYPE_MASK);
  239. } else
  240. __free_pages(page, ALLOC_ORDER);
  241. return 0;
  242. }
  243. /**
  244. * gmap_unmap_segment - unmap segment from the guest address space
  245. * @gmap: pointer to the guest address space structure
  246. * @addr: address in the guest address space
  247. * @len: length of the memory area to unmap
  248. *
  249. * Returns 0 if the unmap succeded, -EINVAL if not.
  250. */
  251. int gmap_unmap_segment(struct gmap *gmap, unsigned long to, unsigned long len)
  252. {
  253. unsigned long *table;
  254. unsigned long off;
  255. int flush;
  256. if ((to | len) & (PMD_SIZE - 1))
  257. return -EINVAL;
  258. if (len == 0 || to + len < to)
  259. return -EINVAL;
  260. flush = 0;
  261. down_read(&gmap->mm->mmap_sem);
  262. spin_lock(&gmap->mm->page_table_lock);
  263. for (off = 0; off < len; off += PMD_SIZE) {
  264. /* Walk the guest addr space page table */
  265. table = gmap->table + (((to + off) >> 53) & 0x7ff);
  266. if (*table & _REGION_ENTRY_INV)
  267. goto out;
  268. table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
  269. table = table + (((to + off) >> 42) & 0x7ff);
  270. if (*table & _REGION_ENTRY_INV)
  271. goto out;
  272. table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
  273. table = table + (((to + off) >> 31) & 0x7ff);
  274. if (*table & _REGION_ENTRY_INV)
  275. goto out;
  276. table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
  277. table = table + (((to + off) >> 20) & 0x7ff);
  278. /* Clear segment table entry in guest address space. */
  279. flush |= gmap_unlink_segment(gmap, table);
  280. *table = _SEGMENT_ENTRY_INV;
  281. }
  282. out:
  283. spin_unlock(&gmap->mm->page_table_lock);
  284. up_read(&gmap->mm->mmap_sem);
  285. if (flush)
  286. gmap_flush_tlb(gmap);
  287. return 0;
  288. }
  289. EXPORT_SYMBOL_GPL(gmap_unmap_segment);
  290. /**
  291. * gmap_mmap_segment - map a segment to the guest address space
  292. * @gmap: pointer to the guest address space structure
  293. * @from: source address in the parent address space
  294. * @to: target address in the guest address space
  295. *
  296. * Returns 0 if the mmap succeded, -EINVAL or -ENOMEM if not.
  297. */
  298. int gmap_map_segment(struct gmap *gmap, unsigned long from,
  299. unsigned long to, unsigned long len)
  300. {
  301. unsigned long *table;
  302. unsigned long off;
  303. int flush;
  304. if ((from | to | len) & (PMD_SIZE - 1))
  305. return -EINVAL;
  306. if (len == 0 || from + len > PGDIR_SIZE ||
  307. from + len < from || to + len < to)
  308. return -EINVAL;
  309. flush = 0;
  310. down_read(&gmap->mm->mmap_sem);
  311. spin_lock(&gmap->mm->page_table_lock);
  312. for (off = 0; off < len; off += PMD_SIZE) {
  313. /* Walk the gmap address space page table */
  314. table = gmap->table + (((to + off) >> 53) & 0x7ff);
  315. if ((*table & _REGION_ENTRY_INV) &&
  316. gmap_alloc_table(gmap, table, _REGION2_ENTRY_EMPTY))
  317. goto out_unmap;
  318. table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
  319. table = table + (((to + off) >> 42) & 0x7ff);
  320. if ((*table & _REGION_ENTRY_INV) &&
  321. gmap_alloc_table(gmap, table, _REGION3_ENTRY_EMPTY))
  322. goto out_unmap;
  323. table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
  324. table = table + (((to + off) >> 31) & 0x7ff);
  325. if ((*table & _REGION_ENTRY_INV) &&
  326. gmap_alloc_table(gmap, table, _SEGMENT_ENTRY_EMPTY))
  327. goto out_unmap;
  328. table = (unsigned long *) (*table & _REGION_ENTRY_ORIGIN);
  329. table = table + (((to + off) >> 20) & 0x7ff);
  330. /* Store 'from' address in an invalid segment table entry. */
  331. flush |= gmap_unlink_segment(gmap, table);
  332. *table = _SEGMENT_ENTRY_INV | _SEGMENT_ENTRY_RO | (from + off);
  333. }
  334. spin_unlock(&gmap->mm->page_table_lock);
  335. up_read(&gmap->mm->mmap_sem);
  336. if (flush)
  337. gmap_flush_tlb(gmap);
  338. return 0;
  339. out_unmap:
  340. spin_unlock(&gmap->mm->page_table_lock);
  341. up_read(&gmap->mm->mmap_sem);
  342. gmap_unmap_segment(gmap, to, len);
  343. return -ENOMEM;
  344. }
  345. EXPORT_SYMBOL_GPL(gmap_map_segment);
  346. static unsigned long *gmap_table_walk(unsigned long address, struct gmap *gmap)
  347. {
  348. unsigned long *table;
  349. table = gmap->table + ((address >> 53) & 0x7ff);
  350. if (unlikely(*table & _REGION_ENTRY_INV))
  351. return ERR_PTR(-EFAULT);
  352. table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
  353. table = table + ((address >> 42) & 0x7ff);
  354. if (unlikely(*table & _REGION_ENTRY_INV))
  355. return ERR_PTR(-EFAULT);
  356. table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
  357. table = table + ((address >> 31) & 0x7ff);
  358. if (unlikely(*table & _REGION_ENTRY_INV))
  359. return ERR_PTR(-EFAULT);
  360. table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
  361. table = table + ((address >> 20) & 0x7ff);
  362. return table;
  363. }
  364. /**
  365. * __gmap_translate - translate a guest address to a user space address
  366. * @address: guest address
  367. * @gmap: pointer to guest mapping meta data structure
  368. *
  369. * Returns user space address which corresponds to the guest address or
  370. * -EFAULT if no such mapping exists.
  371. * This function does not establish potentially missing page table entries.
  372. * The mmap_sem of the mm that belongs to the address space must be held
  373. * when this function gets called.
  374. */
  375. unsigned long __gmap_translate(unsigned long address, struct gmap *gmap)
  376. {
  377. unsigned long *segment_ptr, vmaddr, segment;
  378. struct gmap_pgtable *mp;
  379. struct page *page;
  380. current->thread.gmap_addr = address;
  381. segment_ptr = gmap_table_walk(address, gmap);
  382. if (IS_ERR(segment_ptr))
  383. return PTR_ERR(segment_ptr);
  384. /* Convert the gmap address to an mm address. */
  385. segment = *segment_ptr;
  386. if (!(segment & _SEGMENT_ENTRY_INV)) {
  387. page = pfn_to_page(segment >> PAGE_SHIFT);
  388. mp = (struct gmap_pgtable *) page->index;
  389. return mp->vmaddr | (address & ~PMD_MASK);
  390. } else if (segment & _SEGMENT_ENTRY_RO) {
  391. vmaddr = segment & _SEGMENT_ENTRY_ORIGIN;
  392. return vmaddr | (address & ~PMD_MASK);
  393. }
  394. return -EFAULT;
  395. }
  396. EXPORT_SYMBOL_GPL(__gmap_translate);
  397. /**
  398. * gmap_translate - translate a guest address to a user space address
  399. * @address: guest address
  400. * @gmap: pointer to guest mapping meta data structure
  401. *
  402. * Returns user space address which corresponds to the guest address or
  403. * -EFAULT if no such mapping exists.
  404. * This function does not establish potentially missing page table entries.
  405. */
  406. unsigned long gmap_translate(unsigned long address, struct gmap *gmap)
  407. {
  408. unsigned long rc;
  409. down_read(&gmap->mm->mmap_sem);
  410. rc = __gmap_translate(address, gmap);
  411. up_read(&gmap->mm->mmap_sem);
  412. return rc;
  413. }
  414. EXPORT_SYMBOL_GPL(gmap_translate);
  415. static int gmap_connect_pgtable(unsigned long address, unsigned long segment,
  416. unsigned long *segment_ptr, struct gmap *gmap)
  417. {
  418. unsigned long vmaddr;
  419. struct vm_area_struct *vma;
  420. struct gmap_pgtable *mp;
  421. struct gmap_rmap *rmap;
  422. struct mm_struct *mm;
  423. struct page *page;
  424. pgd_t *pgd;
  425. pud_t *pud;
  426. pmd_t *pmd;
  427. mm = gmap->mm;
  428. vmaddr = segment & _SEGMENT_ENTRY_ORIGIN;
  429. vma = find_vma(mm, vmaddr);
  430. if (!vma || vma->vm_start > vmaddr)
  431. return -EFAULT;
  432. /* Walk the parent mm page table */
  433. pgd = pgd_offset(mm, vmaddr);
  434. pud = pud_alloc(mm, pgd, vmaddr);
  435. if (!pud)
  436. return -ENOMEM;
  437. pmd = pmd_alloc(mm, pud, vmaddr);
  438. if (!pmd)
  439. return -ENOMEM;
  440. if (!pmd_present(*pmd) &&
  441. __pte_alloc(mm, vma, pmd, vmaddr))
  442. return -ENOMEM;
  443. /* pmd now points to a valid segment table entry. */
  444. rmap = kmalloc(sizeof(*rmap), GFP_KERNEL|__GFP_REPEAT);
  445. if (!rmap)
  446. return -ENOMEM;
  447. /* Link gmap segment table entry location to page table. */
  448. page = pmd_page(*pmd);
  449. mp = (struct gmap_pgtable *) page->index;
  450. rmap->gmap = gmap;
  451. rmap->entry = segment_ptr;
  452. rmap->vmaddr = address;
  453. spin_lock(&mm->page_table_lock);
  454. if (*segment_ptr == segment) {
  455. list_add(&rmap->list, &mp->mapper);
  456. /* Set gmap segment table entry to page table. */
  457. *segment_ptr = pmd_val(*pmd) & PAGE_MASK;
  458. rmap = NULL;
  459. }
  460. spin_unlock(&mm->page_table_lock);
  461. kfree(rmap);
  462. return 0;
  463. }
  464. static void gmap_disconnect_pgtable(struct mm_struct *mm, unsigned long *table)
  465. {
  466. struct gmap_rmap *rmap, *next;
  467. struct gmap_pgtable *mp;
  468. struct page *page;
  469. int flush;
  470. flush = 0;
  471. spin_lock(&mm->page_table_lock);
  472. page = pfn_to_page(__pa(table) >> PAGE_SHIFT);
  473. mp = (struct gmap_pgtable *) page->index;
  474. list_for_each_entry_safe(rmap, next, &mp->mapper, list) {
  475. *rmap->entry =
  476. _SEGMENT_ENTRY_INV | _SEGMENT_ENTRY_RO | mp->vmaddr;
  477. list_del(&rmap->list);
  478. kfree(rmap);
  479. flush = 1;
  480. }
  481. spin_unlock(&mm->page_table_lock);
  482. if (flush)
  483. __tlb_flush_global();
  484. }
  485. /*
  486. * this function is assumed to be called with mmap_sem held
  487. */
  488. unsigned long __gmap_fault(unsigned long address, struct gmap *gmap)
  489. {
  490. unsigned long *segment_ptr, segment;
  491. struct gmap_pgtable *mp;
  492. struct page *page;
  493. int rc;
  494. current->thread.gmap_addr = address;
  495. segment_ptr = gmap_table_walk(address, gmap);
  496. if (IS_ERR(segment_ptr))
  497. return -EFAULT;
  498. /* Convert the gmap address to an mm address. */
  499. while (1) {
  500. segment = *segment_ptr;
  501. if (!(segment & _SEGMENT_ENTRY_INV)) {
  502. /* Page table is present */
  503. page = pfn_to_page(segment >> PAGE_SHIFT);
  504. mp = (struct gmap_pgtable *) page->index;
  505. return mp->vmaddr | (address & ~PMD_MASK);
  506. }
  507. if (!(segment & _SEGMENT_ENTRY_RO))
  508. /* Nothing mapped in the gmap address space. */
  509. break;
  510. rc = gmap_connect_pgtable(address, segment, segment_ptr, gmap);
  511. if (rc)
  512. return rc;
  513. }
  514. return -EFAULT;
  515. }
  516. unsigned long gmap_fault(unsigned long address, struct gmap *gmap)
  517. {
  518. unsigned long rc;
  519. down_read(&gmap->mm->mmap_sem);
  520. rc = __gmap_fault(address, gmap);
  521. up_read(&gmap->mm->mmap_sem);
  522. return rc;
  523. }
  524. EXPORT_SYMBOL_GPL(gmap_fault);
  525. void gmap_discard(unsigned long from, unsigned long to, struct gmap *gmap)
  526. {
  527. unsigned long *table, address, size;
  528. struct vm_area_struct *vma;
  529. struct gmap_pgtable *mp;
  530. struct page *page;
  531. down_read(&gmap->mm->mmap_sem);
  532. address = from;
  533. while (address < to) {
  534. /* Walk the gmap address space page table */
  535. table = gmap->table + ((address >> 53) & 0x7ff);
  536. if (unlikely(*table & _REGION_ENTRY_INV)) {
  537. address = (address + PMD_SIZE) & PMD_MASK;
  538. continue;
  539. }
  540. table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
  541. table = table + ((address >> 42) & 0x7ff);
  542. if (unlikely(*table & _REGION_ENTRY_INV)) {
  543. address = (address + PMD_SIZE) & PMD_MASK;
  544. continue;
  545. }
  546. table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
  547. table = table + ((address >> 31) & 0x7ff);
  548. if (unlikely(*table & _REGION_ENTRY_INV)) {
  549. address = (address + PMD_SIZE) & PMD_MASK;
  550. continue;
  551. }
  552. table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
  553. table = table + ((address >> 20) & 0x7ff);
  554. if (unlikely(*table & _SEGMENT_ENTRY_INV)) {
  555. address = (address + PMD_SIZE) & PMD_MASK;
  556. continue;
  557. }
  558. page = pfn_to_page(*table >> PAGE_SHIFT);
  559. mp = (struct gmap_pgtable *) page->index;
  560. vma = find_vma(gmap->mm, mp->vmaddr);
  561. size = min(to - address, PMD_SIZE - (address & ~PMD_MASK));
  562. zap_page_range(vma, mp->vmaddr | (address & ~PMD_MASK),
  563. size, NULL);
  564. address = (address + PMD_SIZE) & PMD_MASK;
  565. }
  566. up_read(&gmap->mm->mmap_sem);
  567. }
  568. EXPORT_SYMBOL_GPL(gmap_discard);
  569. static LIST_HEAD(gmap_notifier_list);
  570. static DEFINE_SPINLOCK(gmap_notifier_lock);
  571. /**
  572. * gmap_register_ipte_notifier - register a pte invalidation callback
  573. * @nb: pointer to the gmap notifier block
  574. */
  575. void gmap_register_ipte_notifier(struct gmap_notifier *nb)
  576. {
  577. spin_lock(&gmap_notifier_lock);
  578. list_add(&nb->list, &gmap_notifier_list);
  579. spin_unlock(&gmap_notifier_lock);
  580. }
  581. EXPORT_SYMBOL_GPL(gmap_register_ipte_notifier);
  582. /**
  583. * gmap_unregister_ipte_notifier - remove a pte invalidation callback
  584. * @nb: pointer to the gmap notifier block
  585. */
  586. void gmap_unregister_ipte_notifier(struct gmap_notifier *nb)
  587. {
  588. spin_lock(&gmap_notifier_lock);
  589. list_del_init(&nb->list);
  590. spin_unlock(&gmap_notifier_lock);
  591. }
  592. EXPORT_SYMBOL_GPL(gmap_unregister_ipte_notifier);
  593. /**
  594. * gmap_ipte_notify - mark a range of ptes for invalidation notification
  595. * @gmap: pointer to guest mapping meta data structure
  596. * @address: virtual address in the guest address space
  597. * @len: size of area
  598. *
  599. * Returns 0 if for each page in the given range a gmap mapping exists and
  600. * the invalidation notification could be set. If the gmap mapping is missing
  601. * for one or more pages -EFAULT is returned. If no memory could be allocated
  602. * -ENOMEM is returned. This function establishes missing page table entries.
  603. */
  604. int gmap_ipte_notify(struct gmap *gmap, unsigned long start, unsigned long len)
  605. {
  606. unsigned long addr;
  607. spinlock_t *ptl;
  608. pte_t *ptep, entry;
  609. pgste_t pgste;
  610. int rc = 0;
  611. if ((start & ~PAGE_MASK) || (len & ~PAGE_MASK))
  612. return -EINVAL;
  613. down_read(&gmap->mm->mmap_sem);
  614. while (len) {
  615. /* Convert gmap address and connect the page tables */
  616. addr = __gmap_fault(start, gmap);
  617. if (IS_ERR_VALUE(addr)) {
  618. rc = addr;
  619. break;
  620. }
  621. /* Get the page mapped */
  622. if (get_user_pages(current, gmap->mm, addr, 1, 1, 0,
  623. NULL, NULL) != 1) {
  624. rc = -EFAULT;
  625. break;
  626. }
  627. /* Walk the process page table, lock and get pte pointer */
  628. ptep = get_locked_pte(gmap->mm, addr, &ptl);
  629. if (unlikely(!ptep))
  630. continue;
  631. /* Set notification bit in the pgste of the pte */
  632. entry = *ptep;
  633. if ((pte_val(entry) & (_PAGE_INVALID | _PAGE_RO)) == 0) {
  634. pgste = pgste_get_lock(ptep);
  635. pgste_val(pgste) |= RCP_IN_BIT;
  636. pgste_set_unlock(ptep, pgste);
  637. start += PAGE_SIZE;
  638. len -= PAGE_SIZE;
  639. }
  640. spin_unlock(ptl);
  641. }
  642. up_read(&gmap->mm->mmap_sem);
  643. return rc;
  644. }
  645. EXPORT_SYMBOL_GPL(gmap_ipte_notify);
  646. /**
  647. * gmap_do_ipte_notify - call all invalidation callbacks for a specific pte.
  648. * @mm: pointer to the process mm_struct
  649. * @addr: virtual address in the process address space
  650. * @pte: pointer to the page table entry
  651. *
  652. * This function is assumed to be called with the page table lock held
  653. * for the pte to notify.
  654. */
  655. void gmap_do_ipte_notify(struct mm_struct *mm, unsigned long addr, pte_t *pte)
  656. {
  657. unsigned long segment_offset;
  658. struct gmap_notifier *nb;
  659. struct gmap_pgtable *mp;
  660. struct gmap_rmap *rmap;
  661. struct page *page;
  662. segment_offset = ((unsigned long) pte) & (255 * sizeof(pte_t));
  663. segment_offset = segment_offset * (4096 / sizeof(pte_t));
  664. page = pfn_to_page(__pa(pte) >> PAGE_SHIFT);
  665. mp = (struct gmap_pgtable *) page->index;
  666. spin_lock(&gmap_notifier_lock);
  667. list_for_each_entry(rmap, &mp->mapper, list) {
  668. list_for_each_entry(nb, &gmap_notifier_list, list)
  669. nb->notifier_call(rmap->gmap,
  670. rmap->vmaddr + segment_offset);
  671. }
  672. spin_unlock(&gmap_notifier_lock);
  673. }
  674. static inline unsigned long *page_table_alloc_pgste(struct mm_struct *mm,
  675. unsigned long vmaddr)
  676. {
  677. struct page *page;
  678. unsigned long *table;
  679. struct gmap_pgtable *mp;
  680. page = alloc_page(GFP_KERNEL|__GFP_REPEAT);
  681. if (!page)
  682. return NULL;
  683. mp = kmalloc(sizeof(*mp), GFP_KERNEL|__GFP_REPEAT);
  684. if (!mp) {
  685. __free_page(page);
  686. return NULL;
  687. }
  688. pgtable_page_ctor(page);
  689. mp->vmaddr = vmaddr & PMD_MASK;
  690. INIT_LIST_HEAD(&mp->mapper);
  691. page->index = (unsigned long) mp;
  692. atomic_set(&page->_mapcount, 3);
  693. table = (unsigned long *) page_to_phys(page);
  694. clear_table(table, _PAGE_TYPE_EMPTY, PAGE_SIZE/2);
  695. clear_table(table + PTRS_PER_PTE, 0, PAGE_SIZE/2);
  696. return table;
  697. }
  698. static inline void page_table_free_pgste(unsigned long *table)
  699. {
  700. struct page *page;
  701. struct gmap_pgtable *mp;
  702. page = pfn_to_page(__pa(table) >> PAGE_SHIFT);
  703. mp = (struct gmap_pgtable *) page->index;
  704. BUG_ON(!list_empty(&mp->mapper));
  705. pgtable_page_dtor(page);
  706. atomic_set(&page->_mapcount, -1);
  707. kfree(mp);
  708. __free_page(page);
  709. }
  710. #else /* CONFIG_PGSTE */
  711. static inline unsigned long *page_table_alloc_pgste(struct mm_struct *mm,
  712. unsigned long vmaddr)
  713. {
  714. return NULL;
  715. }
  716. static inline void page_table_free_pgste(unsigned long *table)
  717. {
  718. }
  719. static inline void gmap_disconnect_pgtable(struct mm_struct *mm,
  720. unsigned long *table)
  721. {
  722. }
  723. #endif /* CONFIG_PGSTE */
  724. static inline unsigned int atomic_xor_bits(atomic_t *v, unsigned int bits)
  725. {
  726. unsigned int old, new;
  727. do {
  728. old = atomic_read(v);
  729. new = old ^ bits;
  730. } while (atomic_cmpxchg(v, old, new) != old);
  731. return new;
  732. }
  733. /*
  734. * page table entry allocation/free routines.
  735. */
  736. unsigned long *page_table_alloc(struct mm_struct *mm, unsigned long vmaddr)
  737. {
  738. unsigned long *uninitialized_var(table);
  739. struct page *uninitialized_var(page);
  740. unsigned int mask, bit;
  741. if (mm_has_pgste(mm))
  742. return page_table_alloc_pgste(mm, vmaddr);
  743. /* Allocate fragments of a 4K page as 1K/2K page table */
  744. spin_lock_bh(&mm->context.list_lock);
  745. mask = FRAG_MASK;
  746. if (!list_empty(&mm->context.pgtable_list)) {
  747. page = list_first_entry(&mm->context.pgtable_list,
  748. struct page, lru);
  749. table = (unsigned long *) page_to_phys(page);
  750. mask = atomic_read(&page->_mapcount);
  751. mask = mask | (mask >> 4);
  752. }
  753. if ((mask & FRAG_MASK) == FRAG_MASK) {
  754. spin_unlock_bh(&mm->context.list_lock);
  755. page = alloc_page(GFP_KERNEL|__GFP_REPEAT);
  756. if (!page)
  757. return NULL;
  758. pgtable_page_ctor(page);
  759. atomic_set(&page->_mapcount, 1);
  760. table = (unsigned long *) page_to_phys(page);
  761. clear_table(table, _PAGE_TYPE_EMPTY, PAGE_SIZE);
  762. spin_lock_bh(&mm->context.list_lock);
  763. list_add(&page->lru, &mm->context.pgtable_list);
  764. } else {
  765. for (bit = 1; mask & bit; bit <<= 1)
  766. table += PTRS_PER_PTE;
  767. mask = atomic_xor_bits(&page->_mapcount, bit);
  768. if ((mask & FRAG_MASK) == FRAG_MASK)
  769. list_del(&page->lru);
  770. }
  771. spin_unlock_bh(&mm->context.list_lock);
  772. return table;
  773. }
  774. void page_table_free(struct mm_struct *mm, unsigned long *table)
  775. {
  776. struct page *page;
  777. unsigned int bit, mask;
  778. if (mm_has_pgste(mm)) {
  779. gmap_disconnect_pgtable(mm, table);
  780. return page_table_free_pgste(table);
  781. }
  782. /* Free 1K/2K page table fragment of a 4K page */
  783. page = pfn_to_page(__pa(table) >> PAGE_SHIFT);
  784. bit = 1 << ((__pa(table) & ~PAGE_MASK)/(PTRS_PER_PTE*sizeof(pte_t)));
  785. spin_lock_bh(&mm->context.list_lock);
  786. if ((atomic_read(&page->_mapcount) & FRAG_MASK) != FRAG_MASK)
  787. list_del(&page->lru);
  788. mask = atomic_xor_bits(&page->_mapcount, bit);
  789. if (mask & FRAG_MASK)
  790. list_add(&page->lru, &mm->context.pgtable_list);
  791. spin_unlock_bh(&mm->context.list_lock);
  792. if (mask == 0) {
  793. pgtable_page_dtor(page);
  794. atomic_set(&page->_mapcount, -1);
  795. __free_page(page);
  796. }
  797. }
  798. static void __page_table_free_rcu(void *table, unsigned bit)
  799. {
  800. struct page *page;
  801. if (bit == FRAG_MASK)
  802. return page_table_free_pgste(table);
  803. /* Free 1K/2K page table fragment of a 4K page */
  804. page = pfn_to_page(__pa(table) >> PAGE_SHIFT);
  805. if (atomic_xor_bits(&page->_mapcount, bit) == 0) {
  806. pgtable_page_dtor(page);
  807. atomic_set(&page->_mapcount, -1);
  808. __free_page(page);
  809. }
  810. }
  811. void page_table_free_rcu(struct mmu_gather *tlb, unsigned long *table)
  812. {
  813. struct mm_struct *mm;
  814. struct page *page;
  815. unsigned int bit, mask;
  816. mm = tlb->mm;
  817. if (mm_has_pgste(mm)) {
  818. gmap_disconnect_pgtable(mm, table);
  819. table = (unsigned long *) (__pa(table) | FRAG_MASK);
  820. tlb_remove_table(tlb, table);
  821. return;
  822. }
  823. bit = 1 << ((__pa(table) & ~PAGE_MASK) / (PTRS_PER_PTE*sizeof(pte_t)));
  824. page = pfn_to_page(__pa(table) >> PAGE_SHIFT);
  825. spin_lock_bh(&mm->context.list_lock);
  826. if ((atomic_read(&page->_mapcount) & FRAG_MASK) != FRAG_MASK)
  827. list_del(&page->lru);
  828. mask = atomic_xor_bits(&page->_mapcount, bit | (bit << 4));
  829. if (mask & FRAG_MASK)
  830. list_add_tail(&page->lru, &mm->context.pgtable_list);
  831. spin_unlock_bh(&mm->context.list_lock);
  832. table = (unsigned long *) (__pa(table) | (bit << 4));
  833. tlb_remove_table(tlb, table);
  834. }
  835. void __tlb_remove_table(void *_table)
  836. {
  837. const unsigned long mask = (FRAG_MASK << 4) | FRAG_MASK;
  838. void *table = (void *)((unsigned long) _table & ~mask);
  839. unsigned type = (unsigned long) _table & mask;
  840. if (type)
  841. __page_table_free_rcu(table, type);
  842. else
  843. free_pages((unsigned long) table, ALLOC_ORDER);
  844. }
  845. static void tlb_remove_table_smp_sync(void *arg)
  846. {
  847. /* Simply deliver the interrupt */
  848. }
  849. static void tlb_remove_table_one(void *table)
  850. {
  851. /*
  852. * This isn't an RCU grace period and hence the page-tables cannot be
  853. * assumed to be actually RCU-freed.
  854. *
  855. * It is however sufficient for software page-table walkers that rely
  856. * on IRQ disabling. See the comment near struct mmu_table_batch.
  857. */
  858. smp_call_function(tlb_remove_table_smp_sync, NULL, 1);
  859. __tlb_remove_table(table);
  860. }
  861. static void tlb_remove_table_rcu(struct rcu_head *head)
  862. {
  863. struct mmu_table_batch *batch;
  864. int i;
  865. batch = container_of(head, struct mmu_table_batch, rcu);
  866. for (i = 0; i < batch->nr; i++)
  867. __tlb_remove_table(batch->tables[i]);
  868. free_page((unsigned long)batch);
  869. }
  870. void tlb_table_flush(struct mmu_gather *tlb)
  871. {
  872. struct mmu_table_batch **batch = &tlb->batch;
  873. if (*batch) {
  874. __tlb_flush_mm(tlb->mm);
  875. call_rcu_sched(&(*batch)->rcu, tlb_remove_table_rcu);
  876. *batch = NULL;
  877. }
  878. }
  879. void tlb_remove_table(struct mmu_gather *tlb, void *table)
  880. {
  881. struct mmu_table_batch **batch = &tlb->batch;
  882. if (*batch == NULL) {
  883. *batch = (struct mmu_table_batch *)
  884. __get_free_page(GFP_NOWAIT | __GFP_NOWARN);
  885. if (*batch == NULL) {
  886. __tlb_flush_mm(tlb->mm);
  887. tlb_remove_table_one(table);
  888. return;
  889. }
  890. (*batch)->nr = 0;
  891. }
  892. (*batch)->tables[(*batch)->nr++] = table;
  893. if ((*batch)->nr == MAX_TABLE_BATCH)
  894. tlb_table_flush(tlb);
  895. }
  896. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  897. void thp_split_vma(struct vm_area_struct *vma)
  898. {
  899. unsigned long addr;
  900. struct page *page;
  901. for (addr = vma->vm_start; addr < vma->vm_end; addr += PAGE_SIZE) {
  902. page = follow_page(vma, addr, FOLL_SPLIT);
  903. }
  904. }
  905. void thp_split_mm(struct mm_struct *mm)
  906. {
  907. struct vm_area_struct *vma = mm->mmap;
  908. while (vma != NULL) {
  909. thp_split_vma(vma);
  910. vma->vm_flags &= ~VM_HUGEPAGE;
  911. vma->vm_flags |= VM_NOHUGEPAGE;
  912. vma = vma->vm_next;
  913. }
  914. }
  915. #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
  916. /*
  917. * switch on pgstes for its userspace process (for kvm)
  918. */
  919. int s390_enable_sie(void)
  920. {
  921. struct task_struct *tsk = current;
  922. struct mm_struct *mm, *old_mm;
  923. /* Do we have switched amode? If no, we cannot do sie */
  924. if (s390_user_mode == HOME_SPACE_MODE)
  925. return -EINVAL;
  926. /* Do we have pgstes? if yes, we are done */
  927. if (mm_has_pgste(tsk->mm))
  928. return 0;
  929. /* lets check if we are allowed to replace the mm */
  930. task_lock(tsk);
  931. if (!tsk->mm || atomic_read(&tsk->mm->mm_users) > 1 ||
  932. #ifdef CONFIG_AIO
  933. !hlist_empty(&tsk->mm->ioctx_list) ||
  934. #endif
  935. tsk->mm != tsk->active_mm) {
  936. task_unlock(tsk);
  937. return -EINVAL;
  938. }
  939. task_unlock(tsk);
  940. /* we copy the mm and let dup_mm create the page tables with_pgstes */
  941. tsk->mm->context.alloc_pgste = 1;
  942. /* make sure that both mms have a correct rss state */
  943. sync_mm_rss(tsk->mm);
  944. mm = dup_mm(tsk);
  945. tsk->mm->context.alloc_pgste = 0;
  946. if (!mm)
  947. return -ENOMEM;
  948. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  949. /* split thp mappings and disable thp for future mappings */
  950. thp_split_mm(mm);
  951. mm->def_flags |= VM_NOHUGEPAGE;
  952. #endif
  953. /* Now lets check again if something happened */
  954. task_lock(tsk);
  955. if (!tsk->mm || atomic_read(&tsk->mm->mm_users) > 1 ||
  956. #ifdef CONFIG_AIO
  957. !hlist_empty(&tsk->mm->ioctx_list) ||
  958. #endif
  959. tsk->mm != tsk->active_mm) {
  960. mmput(mm);
  961. task_unlock(tsk);
  962. return -EINVAL;
  963. }
  964. /* ok, we are alone. No ptrace, no threads, etc. */
  965. old_mm = tsk->mm;
  966. tsk->mm = tsk->active_mm = mm;
  967. preempt_disable();
  968. update_mm(mm, tsk);
  969. atomic_inc(&mm->context.attach_count);
  970. atomic_dec(&old_mm->context.attach_count);
  971. cpumask_set_cpu(smp_processor_id(), mm_cpumask(mm));
  972. preempt_enable();
  973. task_unlock(tsk);
  974. mmput(old_mm);
  975. return 0;
  976. }
  977. EXPORT_SYMBOL_GPL(s390_enable_sie);
  978. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  979. int pmdp_clear_flush_young(struct vm_area_struct *vma, unsigned long address,
  980. pmd_t *pmdp)
  981. {
  982. VM_BUG_ON(address & ~HPAGE_PMD_MASK);
  983. /* No need to flush TLB
  984. * On s390 reference bits are in storage key and never in TLB */
  985. return pmdp_test_and_clear_young(vma, address, pmdp);
  986. }
  987. int pmdp_set_access_flags(struct vm_area_struct *vma,
  988. unsigned long address, pmd_t *pmdp,
  989. pmd_t entry, int dirty)
  990. {
  991. VM_BUG_ON(address & ~HPAGE_PMD_MASK);
  992. if (pmd_same(*pmdp, entry))
  993. return 0;
  994. pmdp_invalidate(vma, address, pmdp);
  995. set_pmd_at(vma->vm_mm, address, pmdp, entry);
  996. return 1;
  997. }
  998. static void pmdp_splitting_flush_sync(void *arg)
  999. {
  1000. /* Simply deliver the interrupt */
  1001. }
  1002. void pmdp_splitting_flush(struct vm_area_struct *vma, unsigned long address,
  1003. pmd_t *pmdp)
  1004. {
  1005. VM_BUG_ON(address & ~HPAGE_PMD_MASK);
  1006. if (!test_and_set_bit(_SEGMENT_ENTRY_SPLIT_BIT,
  1007. (unsigned long *) pmdp)) {
  1008. /* need to serialize against gup-fast (IRQ disabled) */
  1009. smp_call_function(pmdp_splitting_flush_sync, NULL, 1);
  1010. }
  1011. }
  1012. void pgtable_trans_huge_deposit(struct mm_struct *mm, pgtable_t pgtable)
  1013. {
  1014. struct list_head *lh = (struct list_head *) pgtable;
  1015. assert_spin_locked(&mm->page_table_lock);
  1016. /* FIFO */
  1017. if (!mm->pmd_huge_pte)
  1018. INIT_LIST_HEAD(lh);
  1019. else
  1020. list_add(lh, (struct list_head *) mm->pmd_huge_pte);
  1021. mm->pmd_huge_pte = pgtable;
  1022. }
  1023. pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm)
  1024. {
  1025. struct list_head *lh;
  1026. pgtable_t pgtable;
  1027. pte_t *ptep;
  1028. assert_spin_locked(&mm->page_table_lock);
  1029. /* FIFO */
  1030. pgtable = mm->pmd_huge_pte;
  1031. lh = (struct list_head *) pgtable;
  1032. if (list_empty(lh))
  1033. mm->pmd_huge_pte = NULL;
  1034. else {
  1035. mm->pmd_huge_pte = (pgtable_t) lh->next;
  1036. list_del(lh);
  1037. }
  1038. ptep = (pte_t *) pgtable;
  1039. pte_val(*ptep) = _PAGE_TYPE_EMPTY;
  1040. ptep++;
  1041. pte_val(*ptep) = _PAGE_TYPE_EMPTY;
  1042. return pgtable;
  1043. }
  1044. #endif /* CONFIG_TRANSPARENT_HUGEPAGE */