fork.c 44 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876
  1. /*
  2. * linux/kernel/fork.c
  3. *
  4. * Copyright (C) 1991, 1992 Linus Torvalds
  5. */
  6. /*
  7. * 'fork.c' contains the help-routines for the 'fork' system call
  8. * (see also entry.S and others).
  9. * Fork is rather simple, once you get the hang of it, but the memory
  10. * management can be a bitch. See 'mm/memory.c': 'copy_page_range()'
  11. */
  12. #include <linux/slab.h>
  13. #include <linux/init.h>
  14. #include <linux/unistd.h>
  15. #include <linux/module.h>
  16. #include <linux/vmalloc.h>
  17. #include <linux/completion.h>
  18. #include <linux/personality.h>
  19. #include <linux/mempolicy.h>
  20. #include <linux/sem.h>
  21. #include <linux/file.h>
  22. #include <linux/fdtable.h>
  23. #include <linux/iocontext.h>
  24. #include <linux/key.h>
  25. #include <linux/binfmts.h>
  26. #include <linux/mman.h>
  27. #include <linux/mmu_notifier.h>
  28. #include <linux/fs.h>
  29. #include <linux/nsproxy.h>
  30. #include <linux/capability.h>
  31. #include <linux/cpu.h>
  32. #include <linux/cgroup.h>
  33. #include <linux/security.h>
  34. #include <linux/hugetlb.h>
  35. #include <linux/seccomp.h>
  36. #include <linux/swap.h>
  37. #include <linux/syscalls.h>
  38. #include <linux/jiffies.h>
  39. #include <linux/futex.h>
  40. #include <linux/compat.h>
  41. #include <linux/kthread.h>
  42. #include <linux/task_io_accounting_ops.h>
  43. #include <linux/rcupdate.h>
  44. #include <linux/ptrace.h>
  45. #include <linux/mount.h>
  46. #include <linux/audit.h>
  47. #include <linux/memcontrol.h>
  48. #include <linux/ftrace.h>
  49. #include <linux/proc_fs.h>
  50. #include <linux/profile.h>
  51. #include <linux/rmap.h>
  52. #include <linux/ksm.h>
  53. #include <linux/acct.h>
  54. #include <linux/tsacct_kern.h>
  55. #include <linux/cn_proc.h>
  56. #include <linux/freezer.h>
  57. #include <linux/delayacct.h>
  58. #include <linux/taskstats_kern.h>
  59. #include <linux/random.h>
  60. #include <linux/tty.h>
  61. #include <linux/blkdev.h>
  62. #include <linux/fs_struct.h>
  63. #include <linux/magic.h>
  64. #include <linux/perf_event.h>
  65. #include <linux/posix-timers.h>
  66. #include <linux/user-return-notifier.h>
  67. #include <linux/oom.h>
  68. #include <linux/khugepaged.h>
  69. #include <linux/signalfd.h>
  70. #include <asm/pgtable.h>
  71. #include <asm/pgalloc.h>
  72. #include <asm/uaccess.h>
  73. #include <asm/mmu_context.h>
  74. #include <asm/cacheflush.h>
  75. #include <asm/tlbflush.h>
  76. #include <trace/events/sched.h>
  77. #define CREATE_TRACE_POINTS
  78. #include <trace/events/task.h>
  79. /*
  80. * Protected counters by write_lock_irq(&tasklist_lock)
  81. */
  82. unsigned long total_forks; /* Handle normal Linux uptimes. */
  83. int nr_threads; /* The idle threads do not count.. */
  84. int max_threads; /* tunable limit on nr_threads */
  85. DEFINE_PER_CPU(unsigned long, process_counts) = 0;
  86. __cacheline_aligned DEFINE_RWLOCK(tasklist_lock); /* outer */
  87. #ifdef CONFIG_PROVE_RCU
  88. int lockdep_tasklist_lock_is_held(void)
  89. {
  90. return lockdep_is_held(&tasklist_lock);
  91. }
  92. EXPORT_SYMBOL_GPL(lockdep_tasklist_lock_is_held);
  93. #endif /* #ifdef CONFIG_PROVE_RCU */
  94. int nr_processes(void)
  95. {
  96. int cpu;
  97. int total = 0;
  98. for_each_possible_cpu(cpu)
  99. total += per_cpu(process_counts, cpu);
  100. return total;
  101. }
  102. #ifndef CONFIG_ARCH_TASK_STRUCT_ALLOCATOR
  103. static struct kmem_cache *task_struct_cachep;
  104. static inline struct task_struct *alloc_task_struct_node(int node)
  105. {
  106. return kmem_cache_alloc_node(task_struct_cachep, GFP_KERNEL, node);
  107. }
  108. void __weak arch_release_task_struct(struct task_struct *tsk) { }
  109. static inline void free_task_struct(struct task_struct *tsk)
  110. {
  111. arch_release_task_struct(tsk);
  112. kmem_cache_free(task_struct_cachep, tsk);
  113. }
  114. #endif
  115. #ifndef CONFIG_ARCH_THREAD_INFO_ALLOCATOR
  116. void __weak arch_release_thread_info(struct thread_info *ti) { }
  117. /*
  118. * Allocate pages if THREAD_SIZE is >= PAGE_SIZE, otherwise use a
  119. * kmemcache based allocator.
  120. */
  121. # if THREAD_SIZE >= PAGE_SIZE
  122. static struct thread_info *alloc_thread_info_node(struct task_struct *tsk,
  123. int node)
  124. {
  125. struct page *page = alloc_pages_node(node, THREADINFO_GFP,
  126. THREAD_SIZE_ORDER);
  127. return page ? page_address(page) : NULL;
  128. }
  129. static inline void free_thread_info(struct thread_info *ti)
  130. {
  131. arch_release_thread_info(ti);
  132. free_pages((unsigned long)ti, THREAD_SIZE_ORDER);
  133. }
  134. # else
  135. static struct kmem_cache *thread_info_cache;
  136. static struct thread_info *alloc_thread_info_node(struct task_struct *tsk,
  137. int node)
  138. {
  139. return kmem_cache_alloc_node(thread_info_cache, THREADINFO_GFP, node);
  140. }
  141. static void free_thread_info(struct thread_info *ti)
  142. {
  143. arch_release_thread_info(ti);
  144. kmem_cache_free(thread_info_cache, ti);
  145. }
  146. void thread_info_cache_init(void)
  147. {
  148. thread_info_cache = kmem_cache_create("thread_info", THREAD_SIZE,
  149. THREAD_SIZE, 0, NULL);
  150. BUG_ON(thread_info_cache == NULL);
  151. }
  152. # endif
  153. #endif
  154. /* SLAB cache for signal_struct structures (tsk->signal) */
  155. static struct kmem_cache *signal_cachep;
  156. /* SLAB cache for sighand_struct structures (tsk->sighand) */
  157. struct kmem_cache *sighand_cachep;
  158. /* SLAB cache for files_struct structures (tsk->files) */
  159. struct kmem_cache *files_cachep;
  160. /* SLAB cache for fs_struct structures (tsk->fs) */
  161. struct kmem_cache *fs_cachep;
  162. /* SLAB cache for vm_area_struct structures */
  163. struct kmem_cache *vm_area_cachep;
  164. /* SLAB cache for mm_struct structures (tsk->mm) */
  165. static struct kmem_cache *mm_cachep;
  166. static void account_kernel_stack(struct thread_info *ti, int account)
  167. {
  168. struct zone *zone = page_zone(virt_to_page(ti));
  169. mod_zone_page_state(zone, NR_KERNEL_STACK, account);
  170. }
  171. void free_task(struct task_struct *tsk)
  172. {
  173. account_kernel_stack(tsk->stack, -1);
  174. free_thread_info(tsk->stack);
  175. rt_mutex_debug_task_free(tsk);
  176. ftrace_graph_exit_task(tsk);
  177. put_seccomp_filter(tsk);
  178. free_task_struct(tsk);
  179. }
  180. EXPORT_SYMBOL(free_task);
  181. static inline void free_signal_struct(struct signal_struct *sig)
  182. {
  183. taskstats_tgid_free(sig);
  184. sched_autogroup_exit(sig);
  185. kmem_cache_free(signal_cachep, sig);
  186. }
  187. static inline void put_signal_struct(struct signal_struct *sig)
  188. {
  189. if (atomic_dec_and_test(&sig->sigcnt))
  190. free_signal_struct(sig);
  191. }
  192. void __put_task_struct(struct task_struct *tsk)
  193. {
  194. WARN_ON(!tsk->exit_state);
  195. WARN_ON(atomic_read(&tsk->usage));
  196. WARN_ON(tsk == current);
  197. security_task_free(tsk);
  198. exit_creds(tsk);
  199. delayacct_tsk_free(tsk);
  200. put_signal_struct(tsk->signal);
  201. if (!profile_handoff_task(tsk))
  202. free_task(tsk);
  203. }
  204. EXPORT_SYMBOL_GPL(__put_task_struct);
  205. void __init __weak arch_task_cache_init(void) { }
  206. void __init fork_init(unsigned long mempages)
  207. {
  208. #ifndef CONFIG_ARCH_TASK_STRUCT_ALLOCATOR
  209. #ifndef ARCH_MIN_TASKALIGN
  210. #define ARCH_MIN_TASKALIGN L1_CACHE_BYTES
  211. #endif
  212. /* create a slab on which task_structs can be allocated */
  213. task_struct_cachep =
  214. kmem_cache_create("task_struct", sizeof(struct task_struct),
  215. ARCH_MIN_TASKALIGN, SLAB_PANIC | SLAB_NOTRACK, NULL);
  216. #endif
  217. /* do the arch specific task caches init */
  218. arch_task_cache_init();
  219. /*
  220. * The default maximum number of threads is set to a safe
  221. * value: the thread structures can take up at most half
  222. * of memory.
  223. */
  224. max_threads = mempages / (8 * THREAD_SIZE / PAGE_SIZE);
  225. /*
  226. * we need to allow at least 20 threads to boot a system
  227. */
  228. if (max_threads < 20)
  229. max_threads = 20;
  230. init_task.signal->rlim[RLIMIT_NPROC].rlim_cur = max_threads/2;
  231. init_task.signal->rlim[RLIMIT_NPROC].rlim_max = max_threads/2;
  232. init_task.signal->rlim[RLIMIT_SIGPENDING] =
  233. init_task.signal->rlim[RLIMIT_NPROC];
  234. }
  235. int __attribute__((weak)) arch_dup_task_struct(struct task_struct *dst,
  236. struct task_struct *src)
  237. {
  238. *dst = *src;
  239. return 0;
  240. }
  241. static struct task_struct *dup_task_struct(struct task_struct *orig)
  242. {
  243. struct task_struct *tsk;
  244. struct thread_info *ti;
  245. unsigned long *stackend;
  246. int node = tsk_fork_get_node(orig);
  247. int err;
  248. prepare_to_copy(orig);
  249. tsk = alloc_task_struct_node(node);
  250. if (!tsk)
  251. return NULL;
  252. ti = alloc_thread_info_node(tsk, node);
  253. if (!ti) {
  254. free_task_struct(tsk);
  255. return NULL;
  256. }
  257. err = arch_dup_task_struct(tsk, orig);
  258. if (err)
  259. goto out;
  260. tsk->stack = ti;
  261. setup_thread_stack(tsk, orig);
  262. clear_user_return_notifier(tsk);
  263. clear_tsk_need_resched(tsk);
  264. stackend = end_of_stack(tsk);
  265. *stackend = STACK_END_MAGIC; /* for overflow detection */
  266. #ifdef CONFIG_CC_STACKPROTECTOR
  267. tsk->stack_canary = get_random_int();
  268. #endif
  269. /*
  270. * One for us, one for whoever does the "release_task()" (usually
  271. * parent)
  272. */
  273. atomic_set(&tsk->usage, 2);
  274. #ifdef CONFIG_BLK_DEV_IO_TRACE
  275. tsk->btrace_seq = 0;
  276. #endif
  277. tsk->splice_pipe = NULL;
  278. account_kernel_stack(ti, 1);
  279. return tsk;
  280. out:
  281. free_thread_info(ti);
  282. free_task_struct(tsk);
  283. return NULL;
  284. }
  285. #ifdef CONFIG_MMU
  286. static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)
  287. {
  288. struct vm_area_struct *mpnt, *tmp, *prev, **pprev;
  289. struct rb_node **rb_link, *rb_parent;
  290. int retval;
  291. unsigned long charge;
  292. struct mempolicy *pol;
  293. down_write(&oldmm->mmap_sem);
  294. flush_cache_dup_mm(oldmm);
  295. /*
  296. * Not linked in yet - no deadlock potential:
  297. */
  298. down_write_nested(&mm->mmap_sem, SINGLE_DEPTH_NESTING);
  299. mm->locked_vm = 0;
  300. mm->mmap = NULL;
  301. mm->mmap_cache = NULL;
  302. mm->free_area_cache = oldmm->mmap_base;
  303. mm->cached_hole_size = ~0UL;
  304. mm->map_count = 0;
  305. cpumask_clear(mm_cpumask(mm));
  306. mm->mm_rb = RB_ROOT;
  307. rb_link = &mm->mm_rb.rb_node;
  308. rb_parent = NULL;
  309. pprev = &mm->mmap;
  310. retval = ksm_fork(mm, oldmm);
  311. if (retval)
  312. goto out;
  313. retval = khugepaged_fork(mm, oldmm);
  314. if (retval)
  315. goto out;
  316. prev = NULL;
  317. for (mpnt = oldmm->mmap; mpnt; mpnt = mpnt->vm_next) {
  318. struct file *file;
  319. if (mpnt->vm_flags & VM_DONTCOPY) {
  320. long pages = vma_pages(mpnt);
  321. mm->total_vm -= pages;
  322. vm_stat_account(mm, mpnt->vm_flags, mpnt->vm_file,
  323. -pages);
  324. continue;
  325. }
  326. charge = 0;
  327. if (mpnt->vm_flags & VM_ACCOUNT) {
  328. unsigned int len = (mpnt->vm_end - mpnt->vm_start) >> PAGE_SHIFT;
  329. if (security_vm_enough_memory_mm(oldmm, len)) /* sic */
  330. goto fail_nomem;
  331. charge = len;
  332. }
  333. tmp = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
  334. if (!tmp)
  335. goto fail_nomem;
  336. *tmp = *mpnt;
  337. INIT_LIST_HEAD(&tmp->anon_vma_chain);
  338. pol = mpol_dup(vma_policy(mpnt));
  339. retval = PTR_ERR(pol);
  340. if (IS_ERR(pol))
  341. goto fail_nomem_policy;
  342. vma_set_policy(tmp, pol);
  343. tmp->vm_mm = mm;
  344. if (anon_vma_fork(tmp, mpnt))
  345. goto fail_nomem_anon_vma_fork;
  346. tmp->vm_flags &= ~VM_LOCKED;
  347. tmp->vm_next = tmp->vm_prev = NULL;
  348. file = tmp->vm_file;
  349. if (file) {
  350. struct inode *inode = file->f_path.dentry->d_inode;
  351. struct address_space *mapping = file->f_mapping;
  352. get_file(file);
  353. if (tmp->vm_flags & VM_DENYWRITE)
  354. atomic_dec(&inode->i_writecount);
  355. mutex_lock(&mapping->i_mmap_mutex);
  356. if (tmp->vm_flags & VM_SHARED)
  357. mapping->i_mmap_writable++;
  358. flush_dcache_mmap_lock(mapping);
  359. /* insert tmp into the share list, just after mpnt */
  360. vma_prio_tree_add(tmp, mpnt);
  361. flush_dcache_mmap_unlock(mapping);
  362. mutex_unlock(&mapping->i_mmap_mutex);
  363. }
  364. /*
  365. * Clear hugetlb-related page reserves for children. This only
  366. * affects MAP_PRIVATE mappings. Faults generated by the child
  367. * are not guaranteed to succeed, even if read-only
  368. */
  369. if (is_vm_hugetlb_page(tmp))
  370. reset_vma_resv_huge_pages(tmp);
  371. /*
  372. * Link in the new vma and copy the page table entries.
  373. */
  374. *pprev = tmp;
  375. pprev = &tmp->vm_next;
  376. tmp->vm_prev = prev;
  377. prev = tmp;
  378. __vma_link_rb(mm, tmp, rb_link, rb_parent);
  379. rb_link = &tmp->vm_rb.rb_right;
  380. rb_parent = &tmp->vm_rb;
  381. mm->map_count++;
  382. retval = copy_page_range(mm, oldmm, mpnt);
  383. if (tmp->vm_ops && tmp->vm_ops->open)
  384. tmp->vm_ops->open(tmp);
  385. if (retval)
  386. goto out;
  387. }
  388. /* a new mm has just been created */
  389. arch_dup_mmap(oldmm, mm);
  390. retval = 0;
  391. out:
  392. up_write(&mm->mmap_sem);
  393. flush_tlb_mm(oldmm);
  394. up_write(&oldmm->mmap_sem);
  395. return retval;
  396. fail_nomem_anon_vma_fork:
  397. mpol_put(pol);
  398. fail_nomem_policy:
  399. kmem_cache_free(vm_area_cachep, tmp);
  400. fail_nomem:
  401. retval = -ENOMEM;
  402. vm_unacct_memory(charge);
  403. goto out;
  404. }
  405. static inline int mm_alloc_pgd(struct mm_struct *mm)
  406. {
  407. mm->pgd = pgd_alloc(mm);
  408. if (unlikely(!mm->pgd))
  409. return -ENOMEM;
  410. return 0;
  411. }
  412. static inline void mm_free_pgd(struct mm_struct *mm)
  413. {
  414. pgd_free(mm, mm->pgd);
  415. }
  416. #else
  417. #define dup_mmap(mm, oldmm) (0)
  418. #define mm_alloc_pgd(mm) (0)
  419. #define mm_free_pgd(mm)
  420. #endif /* CONFIG_MMU */
  421. __cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock);
  422. #define allocate_mm() (kmem_cache_alloc(mm_cachep, GFP_KERNEL))
  423. #define free_mm(mm) (kmem_cache_free(mm_cachep, (mm)))
  424. static unsigned long default_dump_filter = MMF_DUMP_FILTER_DEFAULT;
  425. static int __init coredump_filter_setup(char *s)
  426. {
  427. default_dump_filter =
  428. (simple_strtoul(s, NULL, 0) << MMF_DUMP_FILTER_SHIFT) &
  429. MMF_DUMP_FILTER_MASK;
  430. return 1;
  431. }
  432. __setup("coredump_filter=", coredump_filter_setup);
  433. #include <linux/init_task.h>
  434. static void mm_init_aio(struct mm_struct *mm)
  435. {
  436. #ifdef CONFIG_AIO
  437. spin_lock_init(&mm->ioctx_lock);
  438. INIT_HLIST_HEAD(&mm->ioctx_list);
  439. #endif
  440. }
  441. static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p)
  442. {
  443. atomic_set(&mm->mm_users, 1);
  444. atomic_set(&mm->mm_count, 1);
  445. init_rwsem(&mm->mmap_sem);
  446. INIT_LIST_HEAD(&mm->mmlist);
  447. mm->flags = (current->mm) ?
  448. (current->mm->flags & MMF_INIT_MASK) : default_dump_filter;
  449. mm->core_state = NULL;
  450. mm->nr_ptes = 0;
  451. memset(&mm->rss_stat, 0, sizeof(mm->rss_stat));
  452. spin_lock_init(&mm->page_table_lock);
  453. mm->free_area_cache = TASK_UNMAPPED_BASE;
  454. mm->cached_hole_size = ~0UL;
  455. mm_init_aio(mm);
  456. mm_init_owner(mm, p);
  457. if (likely(!mm_alloc_pgd(mm))) {
  458. mm->def_flags = 0;
  459. mmu_notifier_mm_init(mm);
  460. return mm;
  461. }
  462. free_mm(mm);
  463. return NULL;
  464. }
  465. static void check_mm(struct mm_struct *mm)
  466. {
  467. int i;
  468. for (i = 0; i < NR_MM_COUNTERS; i++) {
  469. long x = atomic_long_read(&mm->rss_stat.count[i]);
  470. if (unlikely(x))
  471. printk(KERN_ALERT "BUG: Bad rss-counter state "
  472. "mm:%p idx:%d val:%ld\n", mm, i, x);
  473. }
  474. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  475. VM_BUG_ON(mm->pmd_huge_pte);
  476. #endif
  477. }
  478. /*
  479. * Allocate and initialize an mm_struct.
  480. */
  481. struct mm_struct *mm_alloc(void)
  482. {
  483. struct mm_struct *mm;
  484. mm = allocate_mm();
  485. if (!mm)
  486. return NULL;
  487. memset(mm, 0, sizeof(*mm));
  488. mm_init_cpumask(mm);
  489. return mm_init(mm, current);
  490. }
  491. /*
  492. * Called when the last reference to the mm
  493. * is dropped: either by a lazy thread or by
  494. * mmput. Free the page directory and the mm.
  495. */
  496. void __mmdrop(struct mm_struct *mm)
  497. {
  498. BUG_ON(mm == &init_mm);
  499. mm_free_pgd(mm);
  500. destroy_context(mm);
  501. mmu_notifier_mm_destroy(mm);
  502. check_mm(mm);
  503. free_mm(mm);
  504. }
  505. EXPORT_SYMBOL_GPL(__mmdrop);
  506. /*
  507. * Decrement the use count and release all resources for an mm.
  508. */
  509. void mmput(struct mm_struct *mm)
  510. {
  511. might_sleep();
  512. if (atomic_dec_and_test(&mm->mm_users)) {
  513. exit_aio(mm);
  514. ksm_exit(mm);
  515. khugepaged_exit(mm); /* must run before exit_mmap */
  516. exit_mmap(mm);
  517. set_mm_exe_file(mm, NULL);
  518. if (!list_empty(&mm->mmlist)) {
  519. spin_lock(&mmlist_lock);
  520. list_del(&mm->mmlist);
  521. spin_unlock(&mmlist_lock);
  522. }
  523. put_swap_token(mm);
  524. if (mm->binfmt)
  525. module_put(mm->binfmt->module);
  526. mmdrop(mm);
  527. }
  528. }
  529. EXPORT_SYMBOL_GPL(mmput);
  530. /*
  531. * We added or removed a vma mapping the executable. The vmas are only mapped
  532. * during exec and are not mapped with the mmap system call.
  533. * Callers must hold down_write() on the mm's mmap_sem for these
  534. */
  535. void added_exe_file_vma(struct mm_struct *mm)
  536. {
  537. mm->num_exe_file_vmas++;
  538. }
  539. void removed_exe_file_vma(struct mm_struct *mm)
  540. {
  541. mm->num_exe_file_vmas--;
  542. if ((mm->num_exe_file_vmas == 0) && mm->exe_file) {
  543. fput(mm->exe_file);
  544. mm->exe_file = NULL;
  545. }
  546. }
  547. void set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file)
  548. {
  549. if (new_exe_file)
  550. get_file(new_exe_file);
  551. if (mm->exe_file)
  552. fput(mm->exe_file);
  553. mm->exe_file = new_exe_file;
  554. mm->num_exe_file_vmas = 0;
  555. }
  556. struct file *get_mm_exe_file(struct mm_struct *mm)
  557. {
  558. struct file *exe_file;
  559. /* We need mmap_sem to protect against races with removal of
  560. * VM_EXECUTABLE vmas */
  561. down_read(&mm->mmap_sem);
  562. exe_file = mm->exe_file;
  563. if (exe_file)
  564. get_file(exe_file);
  565. up_read(&mm->mmap_sem);
  566. return exe_file;
  567. }
  568. static void dup_mm_exe_file(struct mm_struct *oldmm, struct mm_struct *newmm)
  569. {
  570. /* It's safe to write the exe_file pointer without exe_file_lock because
  571. * this is called during fork when the task is not yet in /proc */
  572. newmm->exe_file = get_mm_exe_file(oldmm);
  573. }
  574. /**
  575. * get_task_mm - acquire a reference to the task's mm
  576. *
  577. * Returns %NULL if the task has no mm. Checks PF_KTHREAD (meaning
  578. * this kernel workthread has transiently adopted a user mm with use_mm,
  579. * to do its AIO) is not set and if so returns a reference to it, after
  580. * bumping up the use count. User must release the mm via mmput()
  581. * after use. Typically used by /proc and ptrace.
  582. */
  583. struct mm_struct *get_task_mm(struct task_struct *task)
  584. {
  585. struct mm_struct *mm;
  586. task_lock(task);
  587. mm = task->mm;
  588. if (mm) {
  589. if (task->flags & PF_KTHREAD)
  590. mm = NULL;
  591. else
  592. atomic_inc(&mm->mm_users);
  593. }
  594. task_unlock(task);
  595. return mm;
  596. }
  597. EXPORT_SYMBOL_GPL(get_task_mm);
  598. struct mm_struct *mm_access(struct task_struct *task, unsigned int mode)
  599. {
  600. struct mm_struct *mm;
  601. int err;
  602. err = mutex_lock_killable(&task->signal->cred_guard_mutex);
  603. if (err)
  604. return ERR_PTR(err);
  605. mm = get_task_mm(task);
  606. if (mm && mm != current->mm &&
  607. !ptrace_may_access(task, mode)) {
  608. mmput(mm);
  609. mm = ERR_PTR(-EACCES);
  610. }
  611. mutex_unlock(&task->signal->cred_guard_mutex);
  612. return mm;
  613. }
  614. static void complete_vfork_done(struct task_struct *tsk)
  615. {
  616. struct completion *vfork;
  617. task_lock(tsk);
  618. vfork = tsk->vfork_done;
  619. if (likely(vfork)) {
  620. tsk->vfork_done = NULL;
  621. complete(vfork);
  622. }
  623. task_unlock(tsk);
  624. }
  625. static int wait_for_vfork_done(struct task_struct *child,
  626. struct completion *vfork)
  627. {
  628. int killed;
  629. freezer_do_not_count();
  630. killed = wait_for_completion_killable(vfork);
  631. freezer_count();
  632. if (killed) {
  633. task_lock(child);
  634. child->vfork_done = NULL;
  635. task_unlock(child);
  636. }
  637. put_task_struct(child);
  638. return killed;
  639. }
  640. /* Please note the differences between mmput and mm_release.
  641. * mmput is called whenever we stop holding onto a mm_struct,
  642. * error success whatever.
  643. *
  644. * mm_release is called after a mm_struct has been removed
  645. * from the current process.
  646. *
  647. * This difference is important for error handling, when we
  648. * only half set up a mm_struct for a new process and need to restore
  649. * the old one. Because we mmput the new mm_struct before
  650. * restoring the old one. . .
  651. * Eric Biederman 10 January 1998
  652. */
  653. void mm_release(struct task_struct *tsk, struct mm_struct *mm)
  654. {
  655. /* Get rid of any futexes when releasing the mm */
  656. #ifdef CONFIG_FUTEX
  657. if (unlikely(tsk->robust_list)) {
  658. exit_robust_list(tsk);
  659. tsk->robust_list = NULL;
  660. }
  661. #ifdef CONFIG_COMPAT
  662. if (unlikely(tsk->compat_robust_list)) {
  663. compat_exit_robust_list(tsk);
  664. tsk->compat_robust_list = NULL;
  665. }
  666. #endif
  667. if (unlikely(!list_empty(&tsk->pi_state_list)))
  668. exit_pi_state_list(tsk);
  669. #endif
  670. /* Get rid of any cached register state */
  671. deactivate_mm(tsk, mm);
  672. if (tsk->vfork_done)
  673. complete_vfork_done(tsk);
  674. /*
  675. * If we're exiting normally, clear a user-space tid field if
  676. * requested. We leave this alone when dying by signal, to leave
  677. * the value intact in a core dump, and to save the unnecessary
  678. * trouble, say, a killed vfork parent shouldn't touch this mm.
  679. * Userland only wants this done for a sys_exit.
  680. */
  681. if (tsk->clear_child_tid) {
  682. if (!(tsk->flags & PF_SIGNALED) &&
  683. atomic_read(&mm->mm_users) > 1) {
  684. /*
  685. * We don't check the error code - if userspace has
  686. * not set up a proper pointer then tough luck.
  687. */
  688. put_user(0, tsk->clear_child_tid);
  689. sys_futex(tsk->clear_child_tid, FUTEX_WAKE,
  690. 1, NULL, NULL, 0);
  691. }
  692. tsk->clear_child_tid = NULL;
  693. }
  694. }
  695. /*
  696. * Allocate a new mm structure and copy contents from the
  697. * mm structure of the passed in task structure.
  698. */
  699. struct mm_struct *dup_mm(struct task_struct *tsk)
  700. {
  701. struct mm_struct *mm, *oldmm = current->mm;
  702. int err;
  703. if (!oldmm)
  704. return NULL;
  705. mm = allocate_mm();
  706. if (!mm)
  707. goto fail_nomem;
  708. memcpy(mm, oldmm, sizeof(*mm));
  709. mm_init_cpumask(mm);
  710. /* Initializing for Swap token stuff */
  711. mm->token_priority = 0;
  712. mm->last_interval = 0;
  713. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  714. mm->pmd_huge_pte = NULL;
  715. #endif
  716. if (!mm_init(mm, tsk))
  717. goto fail_nomem;
  718. if (init_new_context(tsk, mm))
  719. goto fail_nocontext;
  720. dup_mm_exe_file(oldmm, mm);
  721. err = dup_mmap(mm, oldmm);
  722. if (err)
  723. goto free_pt;
  724. mm->hiwater_rss = get_mm_rss(mm);
  725. mm->hiwater_vm = mm->total_vm;
  726. if (mm->binfmt && !try_module_get(mm->binfmt->module))
  727. goto free_pt;
  728. return mm;
  729. free_pt:
  730. /* don't put binfmt in mmput, we haven't got module yet */
  731. mm->binfmt = NULL;
  732. mmput(mm);
  733. fail_nomem:
  734. return NULL;
  735. fail_nocontext:
  736. /*
  737. * If init_new_context() failed, we cannot use mmput() to free the mm
  738. * because it calls destroy_context()
  739. */
  740. mm_free_pgd(mm);
  741. free_mm(mm);
  742. return NULL;
  743. }
  744. static int copy_mm(unsigned long clone_flags, struct task_struct *tsk)
  745. {
  746. struct mm_struct *mm, *oldmm;
  747. int retval;
  748. tsk->min_flt = tsk->maj_flt = 0;
  749. tsk->nvcsw = tsk->nivcsw = 0;
  750. #ifdef CONFIG_DETECT_HUNG_TASK
  751. tsk->last_switch_count = tsk->nvcsw + tsk->nivcsw;
  752. #endif
  753. tsk->mm = NULL;
  754. tsk->active_mm = NULL;
  755. /*
  756. * Are we cloning a kernel thread?
  757. *
  758. * We need to steal a active VM for that..
  759. */
  760. oldmm = current->mm;
  761. if (!oldmm)
  762. return 0;
  763. if (clone_flags & CLONE_VM) {
  764. atomic_inc(&oldmm->mm_users);
  765. mm = oldmm;
  766. goto good_mm;
  767. }
  768. retval = -ENOMEM;
  769. mm = dup_mm(tsk);
  770. if (!mm)
  771. goto fail_nomem;
  772. good_mm:
  773. /* Initializing for Swap token stuff */
  774. mm->token_priority = 0;
  775. mm->last_interval = 0;
  776. tsk->mm = mm;
  777. tsk->active_mm = mm;
  778. return 0;
  779. fail_nomem:
  780. return retval;
  781. }
  782. static int copy_fs(unsigned long clone_flags, struct task_struct *tsk)
  783. {
  784. struct fs_struct *fs = current->fs;
  785. if (clone_flags & CLONE_FS) {
  786. /* tsk->fs is already what we want */
  787. spin_lock(&fs->lock);
  788. if (fs->in_exec) {
  789. spin_unlock(&fs->lock);
  790. return -EAGAIN;
  791. }
  792. fs->users++;
  793. spin_unlock(&fs->lock);
  794. return 0;
  795. }
  796. tsk->fs = copy_fs_struct(fs);
  797. if (!tsk->fs)
  798. return -ENOMEM;
  799. return 0;
  800. }
  801. static int copy_files(unsigned long clone_flags, struct task_struct *tsk)
  802. {
  803. struct files_struct *oldf, *newf;
  804. int error = 0;
  805. /*
  806. * A background process may not have any files ...
  807. */
  808. oldf = current->files;
  809. if (!oldf)
  810. goto out;
  811. if (clone_flags & CLONE_FILES) {
  812. atomic_inc(&oldf->count);
  813. goto out;
  814. }
  815. newf = dup_fd(oldf, &error);
  816. if (!newf)
  817. goto out;
  818. tsk->files = newf;
  819. error = 0;
  820. out:
  821. return error;
  822. }
  823. static int copy_io(unsigned long clone_flags, struct task_struct *tsk)
  824. {
  825. #ifdef CONFIG_BLOCK
  826. struct io_context *ioc = current->io_context;
  827. struct io_context *new_ioc;
  828. if (!ioc)
  829. return 0;
  830. /*
  831. * Share io context with parent, if CLONE_IO is set
  832. */
  833. if (clone_flags & CLONE_IO) {
  834. tsk->io_context = ioc_task_link(ioc);
  835. if (unlikely(!tsk->io_context))
  836. return -ENOMEM;
  837. } else if (ioprio_valid(ioc->ioprio)) {
  838. new_ioc = get_task_io_context(tsk, GFP_KERNEL, NUMA_NO_NODE);
  839. if (unlikely(!new_ioc))
  840. return -ENOMEM;
  841. new_ioc->ioprio = ioc->ioprio;
  842. put_io_context(new_ioc);
  843. }
  844. #endif
  845. return 0;
  846. }
  847. static int copy_sighand(unsigned long clone_flags, struct task_struct *tsk)
  848. {
  849. struct sighand_struct *sig;
  850. if (clone_flags & CLONE_SIGHAND) {
  851. atomic_inc(&current->sighand->count);
  852. return 0;
  853. }
  854. sig = kmem_cache_alloc(sighand_cachep, GFP_KERNEL);
  855. rcu_assign_pointer(tsk->sighand, sig);
  856. if (!sig)
  857. return -ENOMEM;
  858. atomic_set(&sig->count, 1);
  859. memcpy(sig->action, current->sighand->action, sizeof(sig->action));
  860. return 0;
  861. }
  862. void __cleanup_sighand(struct sighand_struct *sighand)
  863. {
  864. if (atomic_dec_and_test(&sighand->count)) {
  865. signalfd_cleanup(sighand);
  866. kmem_cache_free(sighand_cachep, sighand);
  867. }
  868. }
  869. /*
  870. * Initialize POSIX timer handling for a thread group.
  871. */
  872. static void posix_cpu_timers_init_group(struct signal_struct *sig)
  873. {
  874. unsigned long cpu_limit;
  875. /* Thread group counters. */
  876. thread_group_cputime_init(sig);
  877. cpu_limit = ACCESS_ONCE(sig->rlim[RLIMIT_CPU].rlim_cur);
  878. if (cpu_limit != RLIM_INFINITY) {
  879. sig->cputime_expires.prof_exp = secs_to_cputime(cpu_limit);
  880. sig->cputimer.running = 1;
  881. }
  882. /* The timer lists. */
  883. INIT_LIST_HEAD(&sig->cpu_timers[0]);
  884. INIT_LIST_HEAD(&sig->cpu_timers[1]);
  885. INIT_LIST_HEAD(&sig->cpu_timers[2]);
  886. }
  887. static int copy_signal(unsigned long clone_flags, struct task_struct *tsk)
  888. {
  889. struct signal_struct *sig;
  890. if (clone_flags & CLONE_THREAD)
  891. return 0;
  892. sig = kmem_cache_zalloc(signal_cachep, GFP_KERNEL);
  893. tsk->signal = sig;
  894. if (!sig)
  895. return -ENOMEM;
  896. sig->nr_threads = 1;
  897. atomic_set(&sig->live, 1);
  898. atomic_set(&sig->sigcnt, 1);
  899. init_waitqueue_head(&sig->wait_chldexit);
  900. if (clone_flags & CLONE_NEWPID)
  901. sig->flags |= SIGNAL_UNKILLABLE;
  902. sig->curr_target = tsk;
  903. init_sigpending(&sig->shared_pending);
  904. INIT_LIST_HEAD(&sig->posix_timers);
  905. hrtimer_init(&sig->real_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
  906. sig->real_timer.function = it_real_fn;
  907. task_lock(current->group_leader);
  908. memcpy(sig->rlim, current->signal->rlim, sizeof sig->rlim);
  909. task_unlock(current->group_leader);
  910. posix_cpu_timers_init_group(sig);
  911. tty_audit_fork(sig);
  912. sched_autogroup_fork(sig);
  913. #ifdef CONFIG_CGROUPS
  914. init_rwsem(&sig->group_rwsem);
  915. #endif
  916. sig->oom_adj = current->signal->oom_adj;
  917. sig->oom_score_adj = current->signal->oom_score_adj;
  918. sig->oom_score_adj_min = current->signal->oom_score_adj_min;
  919. sig->has_child_subreaper = current->signal->has_child_subreaper ||
  920. current->signal->is_child_subreaper;
  921. mutex_init(&sig->cred_guard_mutex);
  922. return 0;
  923. }
  924. static void copy_flags(unsigned long clone_flags, struct task_struct *p)
  925. {
  926. unsigned long new_flags = p->flags;
  927. new_flags &= ~(PF_SUPERPRIV | PF_WQ_WORKER);
  928. new_flags |= PF_FORKNOEXEC;
  929. p->flags = new_flags;
  930. }
  931. SYSCALL_DEFINE1(set_tid_address, int __user *, tidptr)
  932. {
  933. current->clear_child_tid = tidptr;
  934. return task_pid_vnr(current);
  935. }
  936. static void rt_mutex_init_task(struct task_struct *p)
  937. {
  938. raw_spin_lock_init(&p->pi_lock);
  939. #ifdef CONFIG_RT_MUTEXES
  940. plist_head_init(&p->pi_waiters);
  941. p->pi_blocked_on = NULL;
  942. #endif
  943. }
  944. #ifdef CONFIG_MM_OWNER
  945. void mm_init_owner(struct mm_struct *mm, struct task_struct *p)
  946. {
  947. mm->owner = p;
  948. }
  949. #endif /* CONFIG_MM_OWNER */
  950. /*
  951. * Initialize POSIX timer handling for a single task.
  952. */
  953. static void posix_cpu_timers_init(struct task_struct *tsk)
  954. {
  955. tsk->cputime_expires.prof_exp = 0;
  956. tsk->cputime_expires.virt_exp = 0;
  957. tsk->cputime_expires.sched_exp = 0;
  958. INIT_LIST_HEAD(&tsk->cpu_timers[0]);
  959. INIT_LIST_HEAD(&tsk->cpu_timers[1]);
  960. INIT_LIST_HEAD(&tsk->cpu_timers[2]);
  961. }
  962. /*
  963. * This creates a new process as a copy of the old one,
  964. * but does not actually start it yet.
  965. *
  966. * It copies the registers, and all the appropriate
  967. * parts of the process environment (as per the clone
  968. * flags). The actual kick-off is left to the caller.
  969. */
  970. static struct task_struct *copy_process(unsigned long clone_flags,
  971. unsigned long stack_start,
  972. struct pt_regs *regs,
  973. unsigned long stack_size,
  974. int __user *child_tidptr,
  975. struct pid *pid,
  976. int trace)
  977. {
  978. int retval;
  979. struct task_struct *p;
  980. int cgroup_callbacks_done = 0;
  981. if ((clone_flags & (CLONE_NEWNS|CLONE_FS)) == (CLONE_NEWNS|CLONE_FS))
  982. return ERR_PTR(-EINVAL);
  983. /*
  984. * Thread groups must share signals as well, and detached threads
  985. * can only be started up within the thread group.
  986. */
  987. if ((clone_flags & CLONE_THREAD) && !(clone_flags & CLONE_SIGHAND))
  988. return ERR_PTR(-EINVAL);
  989. /*
  990. * Shared signal handlers imply shared VM. By way of the above,
  991. * thread groups also imply shared VM. Blocking this case allows
  992. * for various simplifications in other code.
  993. */
  994. if ((clone_flags & CLONE_SIGHAND) && !(clone_flags & CLONE_VM))
  995. return ERR_PTR(-EINVAL);
  996. /*
  997. * Siblings of global init remain as zombies on exit since they are
  998. * not reaped by their parent (swapper). To solve this and to avoid
  999. * multi-rooted process trees, prevent global and container-inits
  1000. * from creating siblings.
  1001. */
  1002. if ((clone_flags & CLONE_PARENT) &&
  1003. current->signal->flags & SIGNAL_UNKILLABLE)
  1004. return ERR_PTR(-EINVAL);
  1005. retval = security_task_create(clone_flags);
  1006. if (retval)
  1007. goto fork_out;
  1008. retval = -ENOMEM;
  1009. p = dup_task_struct(current);
  1010. if (!p)
  1011. goto fork_out;
  1012. ftrace_graph_init_task(p);
  1013. get_seccomp_filter(p);
  1014. rt_mutex_init_task(p);
  1015. #ifdef CONFIG_PROVE_LOCKING
  1016. DEBUG_LOCKS_WARN_ON(!p->hardirqs_enabled);
  1017. DEBUG_LOCKS_WARN_ON(!p->softirqs_enabled);
  1018. #endif
  1019. retval = -EAGAIN;
  1020. if (atomic_read(&p->real_cred->user->processes) >=
  1021. task_rlimit(p, RLIMIT_NPROC)) {
  1022. if (!capable(CAP_SYS_ADMIN) && !capable(CAP_SYS_RESOURCE) &&
  1023. p->real_cred->user != INIT_USER)
  1024. goto bad_fork_free;
  1025. }
  1026. current->flags &= ~PF_NPROC_EXCEEDED;
  1027. retval = copy_creds(p, clone_flags);
  1028. if (retval < 0)
  1029. goto bad_fork_free;
  1030. /*
  1031. * If multiple threads are within copy_process(), then this check
  1032. * triggers too late. This doesn't hurt, the check is only there
  1033. * to stop root fork bombs.
  1034. */
  1035. retval = -EAGAIN;
  1036. if (nr_threads >= max_threads)
  1037. goto bad_fork_cleanup_count;
  1038. if (!try_module_get(task_thread_info(p)->exec_domain->module))
  1039. goto bad_fork_cleanup_count;
  1040. p->did_exec = 0;
  1041. delayacct_tsk_init(p); /* Must remain after dup_task_struct() */
  1042. copy_flags(clone_flags, p);
  1043. INIT_LIST_HEAD(&p->children);
  1044. INIT_LIST_HEAD(&p->sibling);
  1045. rcu_copy_process(p);
  1046. p->vfork_done = NULL;
  1047. spin_lock_init(&p->alloc_lock);
  1048. init_sigpending(&p->pending);
  1049. p->utime = p->stime = p->gtime = 0;
  1050. p->utimescaled = p->stimescaled = 0;
  1051. #ifndef CONFIG_VIRT_CPU_ACCOUNTING
  1052. p->prev_utime = p->prev_stime = 0;
  1053. #endif
  1054. #if defined(SPLIT_RSS_COUNTING)
  1055. memset(&p->rss_stat, 0, sizeof(p->rss_stat));
  1056. #endif
  1057. p->default_timer_slack_ns = current->timer_slack_ns;
  1058. task_io_accounting_init(&p->ioac);
  1059. acct_clear_integrals(p);
  1060. posix_cpu_timers_init(p);
  1061. do_posix_clock_monotonic_gettime(&p->start_time);
  1062. p->real_start_time = p->start_time;
  1063. monotonic_to_bootbased(&p->real_start_time);
  1064. p->io_context = NULL;
  1065. p->audit_context = NULL;
  1066. if (clone_flags & CLONE_THREAD)
  1067. threadgroup_change_begin(current);
  1068. cgroup_fork(p);
  1069. #ifdef CONFIG_NUMA
  1070. p->mempolicy = mpol_dup(p->mempolicy);
  1071. if (IS_ERR(p->mempolicy)) {
  1072. retval = PTR_ERR(p->mempolicy);
  1073. p->mempolicy = NULL;
  1074. goto bad_fork_cleanup_cgroup;
  1075. }
  1076. mpol_fix_fork_child_flag(p);
  1077. #endif
  1078. #ifdef CONFIG_CPUSETS
  1079. p->cpuset_mem_spread_rotor = NUMA_NO_NODE;
  1080. p->cpuset_slab_spread_rotor = NUMA_NO_NODE;
  1081. seqcount_init(&p->mems_allowed_seq);
  1082. #endif
  1083. #ifdef CONFIG_TRACE_IRQFLAGS
  1084. p->irq_events = 0;
  1085. #ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW
  1086. p->hardirqs_enabled = 1;
  1087. #else
  1088. p->hardirqs_enabled = 0;
  1089. #endif
  1090. p->hardirq_enable_ip = 0;
  1091. p->hardirq_enable_event = 0;
  1092. p->hardirq_disable_ip = _THIS_IP_;
  1093. p->hardirq_disable_event = 0;
  1094. p->softirqs_enabled = 1;
  1095. p->softirq_enable_ip = _THIS_IP_;
  1096. p->softirq_enable_event = 0;
  1097. p->softirq_disable_ip = 0;
  1098. p->softirq_disable_event = 0;
  1099. p->hardirq_context = 0;
  1100. p->softirq_context = 0;
  1101. #endif
  1102. #ifdef CONFIG_LOCKDEP
  1103. p->lockdep_depth = 0; /* no locks held yet */
  1104. p->curr_chain_key = 0;
  1105. p->lockdep_recursion = 0;
  1106. #endif
  1107. #ifdef CONFIG_DEBUG_MUTEXES
  1108. p->blocked_on = NULL; /* not blocked yet */
  1109. #endif
  1110. #ifdef CONFIG_CGROUP_MEM_RES_CTLR
  1111. p->memcg_batch.do_batch = 0;
  1112. p->memcg_batch.memcg = NULL;
  1113. #endif
  1114. /* Perform scheduler related setup. Assign this task to a CPU. */
  1115. sched_fork(p);
  1116. retval = perf_event_init_task(p);
  1117. if (retval)
  1118. goto bad_fork_cleanup_policy;
  1119. retval = audit_alloc(p);
  1120. if (retval)
  1121. goto bad_fork_cleanup_policy;
  1122. /* copy all the process information */
  1123. retval = copy_semundo(clone_flags, p);
  1124. if (retval)
  1125. goto bad_fork_cleanup_audit;
  1126. retval = copy_files(clone_flags, p);
  1127. if (retval)
  1128. goto bad_fork_cleanup_semundo;
  1129. retval = copy_fs(clone_flags, p);
  1130. if (retval)
  1131. goto bad_fork_cleanup_files;
  1132. retval = copy_sighand(clone_flags, p);
  1133. if (retval)
  1134. goto bad_fork_cleanup_fs;
  1135. retval = copy_signal(clone_flags, p);
  1136. if (retval)
  1137. goto bad_fork_cleanup_sighand;
  1138. retval = copy_mm(clone_flags, p);
  1139. if (retval)
  1140. goto bad_fork_cleanup_signal;
  1141. retval = copy_namespaces(clone_flags, p);
  1142. if (retval)
  1143. goto bad_fork_cleanup_mm;
  1144. retval = copy_io(clone_flags, p);
  1145. if (retval)
  1146. goto bad_fork_cleanup_namespaces;
  1147. retval = copy_thread(clone_flags, stack_start, stack_size, p, regs);
  1148. if (retval)
  1149. goto bad_fork_cleanup_io;
  1150. if (pid != &init_struct_pid) {
  1151. retval = -ENOMEM;
  1152. pid = alloc_pid(p->nsproxy->pid_ns);
  1153. if (!pid)
  1154. goto bad_fork_cleanup_io;
  1155. }
  1156. p->pid = pid_nr(pid);
  1157. p->tgid = p->pid;
  1158. if (clone_flags & CLONE_THREAD)
  1159. p->tgid = current->tgid;
  1160. p->set_child_tid = (clone_flags & CLONE_CHILD_SETTID) ? child_tidptr : NULL;
  1161. /*
  1162. * Clear TID on mm_release()?
  1163. */
  1164. p->clear_child_tid = (clone_flags & CLONE_CHILD_CLEARTID) ? child_tidptr : NULL;
  1165. #ifdef CONFIG_BLOCK
  1166. p->plug = NULL;
  1167. #endif
  1168. #ifdef CONFIG_FUTEX
  1169. p->robust_list = NULL;
  1170. #ifdef CONFIG_COMPAT
  1171. p->compat_robust_list = NULL;
  1172. #endif
  1173. INIT_LIST_HEAD(&p->pi_state_list);
  1174. p->pi_state_cache = NULL;
  1175. #endif
  1176. /*
  1177. * sigaltstack should be cleared when sharing the same VM
  1178. */
  1179. if ((clone_flags & (CLONE_VM|CLONE_VFORK)) == CLONE_VM)
  1180. p->sas_ss_sp = p->sas_ss_size = 0;
  1181. /*
  1182. * Syscall tracing and stepping should be turned off in the
  1183. * child regardless of CLONE_PTRACE.
  1184. */
  1185. user_disable_single_step(p);
  1186. clear_tsk_thread_flag(p, TIF_SYSCALL_TRACE);
  1187. #ifdef TIF_SYSCALL_EMU
  1188. clear_tsk_thread_flag(p, TIF_SYSCALL_EMU);
  1189. #endif
  1190. clear_all_latency_tracing(p);
  1191. /* ok, now we should be set up.. */
  1192. if (clone_flags & CLONE_THREAD)
  1193. p->exit_signal = -1;
  1194. else if (clone_flags & CLONE_PARENT)
  1195. p->exit_signal = current->group_leader->exit_signal;
  1196. else
  1197. p->exit_signal = (clone_flags & CSIGNAL);
  1198. p->pdeath_signal = 0;
  1199. p->exit_state = 0;
  1200. p->nr_dirtied = 0;
  1201. p->nr_dirtied_pause = 128 >> (PAGE_SHIFT - 10);
  1202. p->dirty_paused_when = 0;
  1203. /*
  1204. * Ok, make it visible to the rest of the system.
  1205. * We dont wake it up yet.
  1206. */
  1207. p->group_leader = p;
  1208. INIT_LIST_HEAD(&p->thread_group);
  1209. /* Now that the task is set up, run cgroup callbacks if
  1210. * necessary. We need to run them before the task is visible
  1211. * on the tasklist. */
  1212. cgroup_fork_callbacks(p);
  1213. cgroup_callbacks_done = 1;
  1214. /* Need tasklist lock for parent etc handling! */
  1215. write_lock_irq(&tasklist_lock);
  1216. /* CLONE_PARENT re-uses the old parent */
  1217. if (clone_flags & (CLONE_PARENT|CLONE_THREAD)) {
  1218. p->real_parent = current->real_parent;
  1219. p->parent_exec_id = current->parent_exec_id;
  1220. } else {
  1221. p->real_parent = current;
  1222. p->parent_exec_id = current->self_exec_id;
  1223. }
  1224. spin_lock(&current->sighand->siglock);
  1225. /*
  1226. * Process group and session signals need to be delivered to just the
  1227. * parent before the fork or both the parent and the child after the
  1228. * fork. Restart if a signal comes in before we add the new process to
  1229. * it's process group.
  1230. * A fatal signal pending means that current will exit, so the new
  1231. * thread can't slip out of an OOM kill (or normal SIGKILL).
  1232. */
  1233. recalc_sigpending();
  1234. if (signal_pending(current)) {
  1235. spin_unlock(&current->sighand->siglock);
  1236. write_unlock_irq(&tasklist_lock);
  1237. retval = -ERESTARTNOINTR;
  1238. goto bad_fork_free_pid;
  1239. }
  1240. if (clone_flags & CLONE_THREAD) {
  1241. current->signal->nr_threads++;
  1242. atomic_inc(&current->signal->live);
  1243. atomic_inc(&current->signal->sigcnt);
  1244. p->group_leader = current->group_leader;
  1245. list_add_tail_rcu(&p->thread_group, &p->group_leader->thread_group);
  1246. }
  1247. if (likely(p->pid)) {
  1248. ptrace_init_task(p, (clone_flags & CLONE_PTRACE) || trace);
  1249. if (thread_group_leader(p)) {
  1250. if (is_child_reaper(pid))
  1251. p->nsproxy->pid_ns->child_reaper = p;
  1252. p->signal->leader_pid = pid;
  1253. p->signal->tty = tty_kref_get(current->signal->tty);
  1254. attach_pid(p, PIDTYPE_PGID, task_pgrp(current));
  1255. attach_pid(p, PIDTYPE_SID, task_session(current));
  1256. list_add_tail(&p->sibling, &p->real_parent->children);
  1257. list_add_tail_rcu(&p->tasks, &init_task.tasks);
  1258. __this_cpu_inc(process_counts);
  1259. }
  1260. attach_pid(p, PIDTYPE_PID, pid);
  1261. nr_threads++;
  1262. }
  1263. total_forks++;
  1264. spin_unlock(&current->sighand->siglock);
  1265. write_unlock_irq(&tasklist_lock);
  1266. proc_fork_connector(p);
  1267. cgroup_post_fork(p);
  1268. if (clone_flags & CLONE_THREAD)
  1269. threadgroup_change_end(current);
  1270. perf_event_fork(p);
  1271. trace_task_newtask(p, clone_flags);
  1272. return p;
  1273. bad_fork_free_pid:
  1274. if (pid != &init_struct_pid)
  1275. free_pid(pid);
  1276. bad_fork_cleanup_io:
  1277. if (p->io_context)
  1278. exit_io_context(p);
  1279. bad_fork_cleanup_namespaces:
  1280. if (unlikely(clone_flags & CLONE_NEWPID))
  1281. pid_ns_release_proc(p->nsproxy->pid_ns);
  1282. exit_task_namespaces(p);
  1283. bad_fork_cleanup_mm:
  1284. if (p->mm)
  1285. mmput(p->mm);
  1286. bad_fork_cleanup_signal:
  1287. if (!(clone_flags & CLONE_THREAD))
  1288. free_signal_struct(p->signal);
  1289. bad_fork_cleanup_sighand:
  1290. __cleanup_sighand(p->sighand);
  1291. bad_fork_cleanup_fs:
  1292. exit_fs(p); /* blocking */
  1293. bad_fork_cleanup_files:
  1294. exit_files(p); /* blocking */
  1295. bad_fork_cleanup_semundo:
  1296. exit_sem(p);
  1297. bad_fork_cleanup_audit:
  1298. audit_free(p);
  1299. bad_fork_cleanup_policy:
  1300. perf_event_free_task(p);
  1301. #ifdef CONFIG_NUMA
  1302. mpol_put(p->mempolicy);
  1303. bad_fork_cleanup_cgroup:
  1304. #endif
  1305. if (clone_flags & CLONE_THREAD)
  1306. threadgroup_change_end(current);
  1307. cgroup_exit(p, cgroup_callbacks_done);
  1308. delayacct_tsk_free(p);
  1309. module_put(task_thread_info(p)->exec_domain->module);
  1310. bad_fork_cleanup_count:
  1311. atomic_dec(&p->cred->user->processes);
  1312. exit_creds(p);
  1313. bad_fork_free:
  1314. free_task(p);
  1315. fork_out:
  1316. return ERR_PTR(retval);
  1317. }
  1318. noinline struct pt_regs * __cpuinit __attribute__((weak)) idle_regs(struct pt_regs *regs)
  1319. {
  1320. memset(regs, 0, sizeof(struct pt_regs));
  1321. return regs;
  1322. }
  1323. static inline void init_idle_pids(struct pid_link *links)
  1324. {
  1325. enum pid_type type;
  1326. for (type = PIDTYPE_PID; type < PIDTYPE_MAX; ++type) {
  1327. INIT_HLIST_NODE(&links[type].node); /* not really needed */
  1328. links[type].pid = &init_struct_pid;
  1329. }
  1330. }
  1331. struct task_struct * __cpuinit fork_idle(int cpu)
  1332. {
  1333. struct task_struct *task;
  1334. struct pt_regs regs;
  1335. task = copy_process(CLONE_VM, 0, idle_regs(&regs), 0, NULL,
  1336. &init_struct_pid, 0);
  1337. if (!IS_ERR(task)) {
  1338. init_idle_pids(task->pids);
  1339. init_idle(task, cpu);
  1340. }
  1341. return task;
  1342. }
  1343. /*
  1344. * Ok, this is the main fork-routine.
  1345. *
  1346. * It copies the process, and if successful kick-starts
  1347. * it and waits for it to finish using the VM if required.
  1348. */
  1349. long do_fork(unsigned long clone_flags,
  1350. unsigned long stack_start,
  1351. struct pt_regs *regs,
  1352. unsigned long stack_size,
  1353. int __user *parent_tidptr,
  1354. int __user *child_tidptr)
  1355. {
  1356. struct task_struct *p;
  1357. int trace = 0;
  1358. long nr;
  1359. /*
  1360. * Do some preliminary argument and permissions checking before we
  1361. * actually start allocating stuff
  1362. */
  1363. if (clone_flags & CLONE_NEWUSER) {
  1364. if (clone_flags & CLONE_THREAD)
  1365. return -EINVAL;
  1366. /* hopefully this check will go away when userns support is
  1367. * complete
  1368. */
  1369. if (!capable(CAP_SYS_ADMIN) || !capable(CAP_SETUID) ||
  1370. !capable(CAP_SETGID))
  1371. return -EPERM;
  1372. }
  1373. /*
  1374. * Determine whether and which event to report to ptracer. When
  1375. * called from kernel_thread or CLONE_UNTRACED is explicitly
  1376. * requested, no event is reported; otherwise, report if the event
  1377. * for the type of forking is enabled.
  1378. */
  1379. if (likely(user_mode(regs)) && !(clone_flags & CLONE_UNTRACED)) {
  1380. if (clone_flags & CLONE_VFORK)
  1381. trace = PTRACE_EVENT_VFORK;
  1382. else if ((clone_flags & CSIGNAL) != SIGCHLD)
  1383. trace = PTRACE_EVENT_CLONE;
  1384. else
  1385. trace = PTRACE_EVENT_FORK;
  1386. if (likely(!ptrace_event_enabled(current, trace)))
  1387. trace = 0;
  1388. }
  1389. p = copy_process(clone_flags, stack_start, regs, stack_size,
  1390. child_tidptr, NULL, trace);
  1391. /*
  1392. * Do this prior waking up the new thread - the thread pointer
  1393. * might get invalid after that point, if the thread exits quickly.
  1394. */
  1395. if (!IS_ERR(p)) {
  1396. struct completion vfork;
  1397. trace_sched_process_fork(current, p);
  1398. nr = task_pid_vnr(p);
  1399. if (clone_flags & CLONE_PARENT_SETTID)
  1400. put_user(nr, parent_tidptr);
  1401. if (clone_flags & CLONE_VFORK) {
  1402. p->vfork_done = &vfork;
  1403. init_completion(&vfork);
  1404. get_task_struct(p);
  1405. }
  1406. wake_up_new_task(p);
  1407. /* forking complete and child started to run, tell ptracer */
  1408. if (unlikely(trace))
  1409. ptrace_event(trace, nr);
  1410. if (clone_flags & CLONE_VFORK) {
  1411. if (!wait_for_vfork_done(p, &vfork))
  1412. ptrace_event(PTRACE_EVENT_VFORK_DONE, nr);
  1413. }
  1414. } else {
  1415. nr = PTR_ERR(p);
  1416. }
  1417. return nr;
  1418. }
  1419. #ifndef ARCH_MIN_MMSTRUCT_ALIGN
  1420. #define ARCH_MIN_MMSTRUCT_ALIGN 0
  1421. #endif
  1422. static void sighand_ctor(void *data)
  1423. {
  1424. struct sighand_struct *sighand = data;
  1425. spin_lock_init(&sighand->siglock);
  1426. init_waitqueue_head(&sighand->signalfd_wqh);
  1427. }
  1428. void __init proc_caches_init(void)
  1429. {
  1430. sighand_cachep = kmem_cache_create("sighand_cache",
  1431. sizeof(struct sighand_struct), 0,
  1432. SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_DESTROY_BY_RCU|
  1433. SLAB_NOTRACK, sighand_ctor);
  1434. signal_cachep = kmem_cache_create("signal_cache",
  1435. sizeof(struct signal_struct), 0,
  1436. SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_NOTRACK, NULL);
  1437. files_cachep = kmem_cache_create("files_cache",
  1438. sizeof(struct files_struct), 0,
  1439. SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_NOTRACK, NULL);
  1440. fs_cachep = kmem_cache_create("fs_cache",
  1441. sizeof(struct fs_struct), 0,
  1442. SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_NOTRACK, NULL);
  1443. /*
  1444. * FIXME! The "sizeof(struct mm_struct)" currently includes the
  1445. * whole struct cpumask for the OFFSTACK case. We could change
  1446. * this to *only* allocate as much of it as required by the
  1447. * maximum number of CPU's we can ever have. The cpumask_allocation
  1448. * is at the end of the structure, exactly for that reason.
  1449. */
  1450. mm_cachep = kmem_cache_create("mm_struct",
  1451. sizeof(struct mm_struct), ARCH_MIN_MMSTRUCT_ALIGN,
  1452. SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_NOTRACK, NULL);
  1453. vm_area_cachep = KMEM_CACHE(vm_area_struct, SLAB_PANIC);
  1454. mmap_init();
  1455. nsproxy_cache_init();
  1456. }
  1457. /*
  1458. * Check constraints on flags passed to the unshare system call.
  1459. */
  1460. static int check_unshare_flags(unsigned long unshare_flags)
  1461. {
  1462. if (unshare_flags & ~(CLONE_THREAD|CLONE_FS|CLONE_NEWNS|CLONE_SIGHAND|
  1463. CLONE_VM|CLONE_FILES|CLONE_SYSVSEM|
  1464. CLONE_NEWUTS|CLONE_NEWIPC|CLONE_NEWNET))
  1465. return -EINVAL;
  1466. /*
  1467. * Not implemented, but pretend it works if there is nothing to
  1468. * unshare. Note that unsharing CLONE_THREAD or CLONE_SIGHAND
  1469. * needs to unshare vm.
  1470. */
  1471. if (unshare_flags & (CLONE_THREAD | CLONE_SIGHAND | CLONE_VM)) {
  1472. /* FIXME: get_task_mm() increments ->mm_users */
  1473. if (atomic_read(&current->mm->mm_users) > 1)
  1474. return -EINVAL;
  1475. }
  1476. return 0;
  1477. }
  1478. /*
  1479. * Unshare the filesystem structure if it is being shared
  1480. */
  1481. static int unshare_fs(unsigned long unshare_flags, struct fs_struct **new_fsp)
  1482. {
  1483. struct fs_struct *fs = current->fs;
  1484. if (!(unshare_flags & CLONE_FS) || !fs)
  1485. return 0;
  1486. /* don't need lock here; in the worst case we'll do useless copy */
  1487. if (fs->users == 1)
  1488. return 0;
  1489. *new_fsp = copy_fs_struct(fs);
  1490. if (!*new_fsp)
  1491. return -ENOMEM;
  1492. return 0;
  1493. }
  1494. /*
  1495. * Unshare file descriptor table if it is being shared
  1496. */
  1497. static int unshare_fd(unsigned long unshare_flags, struct files_struct **new_fdp)
  1498. {
  1499. struct files_struct *fd = current->files;
  1500. int error = 0;
  1501. if ((unshare_flags & CLONE_FILES) &&
  1502. (fd && atomic_read(&fd->count) > 1)) {
  1503. *new_fdp = dup_fd(fd, &error);
  1504. if (!*new_fdp)
  1505. return error;
  1506. }
  1507. return 0;
  1508. }
  1509. /*
  1510. * unshare allows a process to 'unshare' part of the process
  1511. * context which was originally shared using clone. copy_*
  1512. * functions used by do_fork() cannot be used here directly
  1513. * because they modify an inactive task_struct that is being
  1514. * constructed. Here we are modifying the current, active,
  1515. * task_struct.
  1516. */
  1517. SYSCALL_DEFINE1(unshare, unsigned long, unshare_flags)
  1518. {
  1519. struct fs_struct *fs, *new_fs = NULL;
  1520. struct files_struct *fd, *new_fd = NULL;
  1521. struct nsproxy *new_nsproxy = NULL;
  1522. int do_sysvsem = 0;
  1523. int err;
  1524. err = check_unshare_flags(unshare_flags);
  1525. if (err)
  1526. goto bad_unshare_out;
  1527. /*
  1528. * If unsharing namespace, must also unshare filesystem information.
  1529. */
  1530. if (unshare_flags & CLONE_NEWNS)
  1531. unshare_flags |= CLONE_FS;
  1532. /*
  1533. * CLONE_NEWIPC must also detach from the undolist: after switching
  1534. * to a new ipc namespace, the semaphore arrays from the old
  1535. * namespace are unreachable.
  1536. */
  1537. if (unshare_flags & (CLONE_NEWIPC|CLONE_SYSVSEM))
  1538. do_sysvsem = 1;
  1539. err = unshare_fs(unshare_flags, &new_fs);
  1540. if (err)
  1541. goto bad_unshare_out;
  1542. err = unshare_fd(unshare_flags, &new_fd);
  1543. if (err)
  1544. goto bad_unshare_cleanup_fs;
  1545. err = unshare_nsproxy_namespaces(unshare_flags, &new_nsproxy, new_fs);
  1546. if (err)
  1547. goto bad_unshare_cleanup_fd;
  1548. if (new_fs || new_fd || do_sysvsem || new_nsproxy) {
  1549. if (do_sysvsem) {
  1550. /*
  1551. * CLONE_SYSVSEM is equivalent to sys_exit().
  1552. */
  1553. exit_sem(current);
  1554. }
  1555. if (new_nsproxy) {
  1556. switch_task_namespaces(current, new_nsproxy);
  1557. new_nsproxy = NULL;
  1558. }
  1559. task_lock(current);
  1560. if (new_fs) {
  1561. fs = current->fs;
  1562. spin_lock(&fs->lock);
  1563. current->fs = new_fs;
  1564. if (--fs->users)
  1565. new_fs = NULL;
  1566. else
  1567. new_fs = fs;
  1568. spin_unlock(&fs->lock);
  1569. }
  1570. if (new_fd) {
  1571. fd = current->files;
  1572. current->files = new_fd;
  1573. new_fd = fd;
  1574. }
  1575. task_unlock(current);
  1576. }
  1577. if (new_nsproxy)
  1578. put_nsproxy(new_nsproxy);
  1579. bad_unshare_cleanup_fd:
  1580. if (new_fd)
  1581. put_files_struct(new_fd);
  1582. bad_unshare_cleanup_fs:
  1583. if (new_fs)
  1584. free_fs_struct(new_fs);
  1585. bad_unshare_out:
  1586. return err;
  1587. }
  1588. /*
  1589. * Helper to unshare the files of the current task.
  1590. * We don't want to expose copy_files internals to
  1591. * the exec layer of the kernel.
  1592. */
  1593. int unshare_files(struct files_struct **displaced)
  1594. {
  1595. struct task_struct *task = current;
  1596. struct files_struct *copy = NULL;
  1597. int error;
  1598. error = unshare_fd(CLONE_FILES, &copy);
  1599. if (error || !copy) {
  1600. *displaced = NULL;
  1601. return error;
  1602. }
  1603. *displaced = task->files;
  1604. task_lock(task);
  1605. task->files = copy;
  1606. task_unlock(task);
  1607. return 0;
  1608. }