aboutsummaryrefslogtreecommitdiffstats
path: root/kernel/fork.c
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/fork.c')
-rw-r--r--kernel/fork.c448
1 files changed, 236 insertions, 212 deletions
diff --git a/kernel/fork.c b/kernel/fork.c
index 2295fc69717f..5e6cf0dd031c 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -77,6 +77,7 @@
77#include <linux/blkdev.h> 77#include <linux/blkdev.h>
78#include <linux/fs_struct.h> 78#include <linux/fs_struct.h>
79#include <linux/magic.h> 79#include <linux/magic.h>
80#include <linux/sched/mm.h>
80#include <linux/perf_event.h> 81#include <linux/perf_event.h>
81#include <linux/posix-timers.h> 82#include <linux/posix-timers.h>
82#include <linux/user-return-notifier.h> 83#include <linux/user-return-notifier.h>
@@ -390,6 +391,241 @@ void free_task(struct task_struct *tsk)
390} 391}
391EXPORT_SYMBOL(free_task); 392EXPORT_SYMBOL(free_task);
392 393
394#ifdef CONFIG_MMU
395static __latent_entropy int dup_mmap(struct mm_struct *mm,
396 struct mm_struct *oldmm)
397{
398 struct vm_area_struct *mpnt, *tmp, *prev, **pprev;
399 struct rb_node **rb_link, *rb_parent;
400 int retval;
401 unsigned long charge;
402 LIST_HEAD(uf);
403
404 uprobe_start_dup_mmap();
405 if (down_write_killable(&oldmm->mmap_sem)) {
406 retval = -EINTR;
407 goto fail_uprobe_end;
408 }
409 flush_cache_dup_mm(oldmm);
410 uprobe_dup_mmap(oldmm, mm);
411 /*
412 * Not linked in yet - no deadlock potential:
413 */
414 down_write_nested(&mm->mmap_sem, SINGLE_DEPTH_NESTING);
415
416 /* No ordering required: file already has been exposed. */
417 RCU_INIT_POINTER(mm->exe_file, get_mm_exe_file(oldmm));
418
419 mm->total_vm = oldmm->total_vm;
420 mm->data_vm = oldmm->data_vm;
421 mm->exec_vm = oldmm->exec_vm;
422 mm->stack_vm = oldmm->stack_vm;
423
424 rb_link = &mm->mm_rb.rb_node;
425 rb_parent = NULL;
426 pprev = &mm->mmap;
427 retval = ksm_fork(mm, oldmm);
428 if (retval)
429 goto out;
430 retval = khugepaged_fork(mm, oldmm);
431 if (retval)
432 goto out;
433
434 prev = NULL;
435 for (mpnt = oldmm->mmap; mpnt; mpnt = mpnt->vm_next) {
436 struct file *file;
437
438 if (mpnt->vm_flags & VM_DONTCOPY) {
439 vm_stat_account(mm, mpnt->vm_flags, -vma_pages(mpnt));
440 continue;
441 }
442 charge = 0;
443 if (mpnt->vm_flags & VM_ACCOUNT) {
444 unsigned long len = vma_pages(mpnt);
445
446 if (security_vm_enough_memory_mm(oldmm, len)) /* sic */
447 goto fail_nomem;
448 charge = len;
449 }
450 tmp = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
451 if (!tmp)
452 goto fail_nomem;
453 *tmp = *mpnt;
454 INIT_LIST_HEAD(&tmp->anon_vma_chain);
455 retval = vma_dup_policy(mpnt, tmp);
456 if (retval)
457 goto fail_nomem_policy;
458 tmp->vm_mm = mm;
459 retval = dup_userfaultfd(tmp, &uf);
460 if (retval)
461 goto fail_nomem_anon_vma_fork;
462 if (tmp->vm_flags & VM_WIPEONFORK) {
463 /* VM_WIPEONFORK gets a clean slate in the child. */
464 tmp->anon_vma = NULL;
465 if (anon_vma_prepare(tmp))
466 goto fail_nomem_anon_vma_fork;
467 } else if (anon_vma_fork(tmp, mpnt))
468 goto fail_nomem_anon_vma_fork;
469 tmp->vm_flags &= ~(VM_LOCKED | VM_LOCKONFAULT);
470 tmp->vm_next = tmp->vm_prev = NULL;
471 file = tmp->vm_file;
472 if (file) {
473 struct inode *inode = file_inode(file);
474 struct address_space *mapping = file->f_mapping;
475
476 get_file(file);
477 if (tmp->vm_flags & VM_DENYWRITE)
478 atomic_dec(&inode->i_writecount);
479 i_mmap_lock_write(mapping);
480 if (tmp->vm_flags & VM_SHARED)
481 atomic_inc(&mapping->i_mmap_writable);
482 flush_dcache_mmap_lock(mapping);
483 /* insert tmp into the share list, just after mpnt */
484 vma_interval_tree_insert_after(tmp, mpnt,
485 &mapping->i_mmap);
486 flush_dcache_mmap_unlock(mapping);
487 i_mmap_unlock_write(mapping);
488 }
489
490 /*
491 * Clear hugetlb-related page reserves for children. This only
492 * affects MAP_PRIVATE mappings. Faults generated by the child
493 * are not guaranteed to succeed, even if read-only
494 */
495 if (is_vm_hugetlb_page(tmp))
496 reset_vma_resv_huge_pages(tmp);
497
498 /*
499 * Link in the new vma and copy the page table entries.
500 */
501 *pprev = tmp;
502 pprev = &tmp->vm_next;
503 tmp->vm_prev = prev;
504 prev = tmp;
505
506 __vma_link_rb(mm, tmp, rb_link, rb_parent);
507 rb_link = &tmp->vm_rb.rb_right;
508 rb_parent = &tmp->vm_rb;
509
510 mm->map_count++;
511 if (!(tmp->vm_flags & VM_WIPEONFORK))
512 retval = copy_page_range(mm, oldmm, mpnt);
513
514 if (tmp->vm_ops && tmp->vm_ops->open)
515 tmp->vm_ops->open(tmp);
516
517 if (retval)
518 goto out;
519 }
520 /* a new mm has just been created */
521 arch_dup_mmap(oldmm, mm);
522 retval = 0;
523out:
524 up_write(&mm->mmap_sem);
525 flush_tlb_mm(oldmm);
526 up_write(&oldmm->mmap_sem);
527 dup_userfaultfd_complete(&uf);
528fail_uprobe_end:
529 uprobe_end_dup_mmap();
530 return retval;
531fail_nomem_anon_vma_fork:
532 mpol_put(vma_policy(tmp));
533fail_nomem_policy:
534 kmem_cache_free(vm_area_cachep, tmp);
535fail_nomem:
536 retval = -ENOMEM;
537 vm_unacct_memory(charge);
538 goto out;
539}
540
541static inline int mm_alloc_pgd(struct mm_struct *mm)
542{
543 mm->pgd = pgd_alloc(mm);
544 if (unlikely(!mm->pgd))
545 return -ENOMEM;
546 return 0;
547}
548
549static inline void mm_free_pgd(struct mm_struct *mm)
550{
551 pgd_free(mm, mm->pgd);
552}
553#else
554static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)
555{
556 down_write(&oldmm->mmap_sem);
557 RCU_INIT_POINTER(mm->exe_file, get_mm_exe_file(oldmm));
558 up_write(&oldmm->mmap_sem);
559 return 0;
560}
561#define mm_alloc_pgd(mm) (0)
562#define mm_free_pgd(mm)
563#endif /* CONFIG_MMU */
564
565static void check_mm(struct mm_struct *mm)
566{
567 int i;
568
569 for (i = 0; i < NR_MM_COUNTERS; i++) {
570 long x = atomic_long_read(&mm->rss_stat.count[i]);
571
572 if (unlikely(x))
573 printk(KERN_ALERT "BUG: Bad rss-counter state "
574 "mm:%p idx:%d val:%ld\n", mm, i, x);
575 }
576
577 if (mm_pgtables_bytes(mm))
578 pr_alert("BUG: non-zero pgtables_bytes on freeing mm: %ld\n",
579 mm_pgtables_bytes(mm));
580
581#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS
582 VM_BUG_ON_MM(mm->pmd_huge_pte, mm);
583#endif
584}
585
586#define allocate_mm() (kmem_cache_alloc(mm_cachep, GFP_KERNEL))
587#define free_mm(mm) (kmem_cache_free(mm_cachep, (mm)))
588
589/*
590 * Called when the last reference to the mm
591 * is dropped: either by a lazy thread or by
592 * mmput. Free the page directory and the mm.
593 */
594static void __mmdrop(struct mm_struct *mm)
595{
596 BUG_ON(mm == &init_mm);
597 mm_free_pgd(mm);
598 destroy_context(mm);
599 hmm_mm_destroy(mm);
600 mmu_notifier_mm_destroy(mm);
601 check_mm(mm);
602 put_user_ns(mm->user_ns);
603 free_mm(mm);
604}
605
606void mmdrop(struct mm_struct *mm)
607{
608 if (unlikely(atomic_dec_and_test(&mm->mm_count)))
609 __mmdrop(mm);
610}
611EXPORT_SYMBOL_GPL(mmdrop);
612
613static void mmdrop_async_fn(struct work_struct *work)
614{
615 struct mm_struct *mm;
616
617 mm = container_of(work, struct mm_struct, async_put_work);
618 __mmdrop(mm);
619}
620
621static void mmdrop_async(struct mm_struct *mm)
622{
623 if (unlikely(atomic_dec_and_test(&mm->mm_count))) {
624 INIT_WORK(&mm->async_put_work, mmdrop_async_fn);
625 schedule_work(&mm->async_put_work);
626 }
627}
628
393static inline void free_signal_struct(struct signal_struct *sig) 629static inline void free_signal_struct(struct signal_struct *sig)
394{ 630{
395 taskstats_tgid_free(sig); 631 taskstats_tgid_free(sig);
@@ -594,181 +830,8 @@ free_tsk:
594 return NULL; 830 return NULL;
595} 831}
596 832
597#ifdef CONFIG_MMU
598static __latent_entropy int dup_mmap(struct mm_struct *mm,
599 struct mm_struct *oldmm)
600{
601 struct vm_area_struct *mpnt, *tmp, *prev, **pprev;
602 struct rb_node **rb_link, *rb_parent;
603 int retval;
604 unsigned long charge;
605 LIST_HEAD(uf);
606
607 uprobe_start_dup_mmap();
608 if (down_write_killable(&oldmm->mmap_sem)) {
609 retval = -EINTR;
610 goto fail_uprobe_end;
611 }
612 flush_cache_dup_mm(oldmm);
613 uprobe_dup_mmap(oldmm, mm);
614 /*
615 * Not linked in yet - no deadlock potential:
616 */
617 down_write_nested(&mm->mmap_sem, SINGLE_DEPTH_NESTING);
618
619 /* No ordering required: file already has been exposed. */
620 RCU_INIT_POINTER(mm->exe_file, get_mm_exe_file(oldmm));
621
622 mm->total_vm = oldmm->total_vm;
623 mm->data_vm = oldmm->data_vm;
624 mm->exec_vm = oldmm->exec_vm;
625 mm->stack_vm = oldmm->stack_vm;
626
627 rb_link = &mm->mm_rb.rb_node;
628 rb_parent = NULL;
629 pprev = &mm->mmap;
630 retval = ksm_fork(mm, oldmm);
631 if (retval)
632 goto out;
633 retval = khugepaged_fork(mm, oldmm);
634 if (retval)
635 goto out;
636
637 prev = NULL;
638 for (mpnt = oldmm->mmap; mpnt; mpnt = mpnt->vm_next) {
639 struct file *file;
640
641 if (mpnt->vm_flags & VM_DONTCOPY) {
642 vm_stat_account(mm, mpnt->vm_flags, -vma_pages(mpnt));
643 continue;
644 }
645 charge = 0;
646 if (mpnt->vm_flags & VM_ACCOUNT) {
647 unsigned long len = vma_pages(mpnt);
648
649 if (security_vm_enough_memory_mm(oldmm, len)) /* sic */
650 goto fail_nomem;
651 charge = len;
652 }
653 tmp = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
654 if (!tmp)
655 goto fail_nomem;
656 *tmp = *mpnt;
657 INIT_LIST_HEAD(&tmp->anon_vma_chain);
658 retval = vma_dup_policy(mpnt, tmp);
659 if (retval)
660 goto fail_nomem_policy;
661 tmp->vm_mm = mm;
662 retval = dup_userfaultfd(tmp, &uf);
663 if (retval)
664 goto fail_nomem_anon_vma_fork;
665 if (tmp->vm_flags & VM_WIPEONFORK) {
666 /* VM_WIPEONFORK gets a clean slate in the child. */
667 tmp->anon_vma = NULL;
668 if (anon_vma_prepare(tmp))
669 goto fail_nomem_anon_vma_fork;
670 } else if (anon_vma_fork(tmp, mpnt))
671 goto fail_nomem_anon_vma_fork;
672 tmp->vm_flags &= ~(VM_LOCKED | VM_LOCKONFAULT);
673 tmp->vm_next = tmp->vm_prev = NULL;
674 file = tmp->vm_file;
675 if (file) {
676 struct inode *inode = file_inode(file);
677 struct address_space *mapping = file->f_mapping;
678
679 get_file(file);
680 if (tmp->vm_flags & VM_DENYWRITE)
681 atomic_dec(&inode->i_writecount);
682 i_mmap_lock_write(mapping);
683 if (tmp->vm_flags & VM_SHARED)
684 atomic_inc(&mapping->i_mmap_writable);
685 flush_dcache_mmap_lock(mapping);
686 /* insert tmp into the share list, just after mpnt */
687 vma_interval_tree_insert_after(tmp, mpnt,
688 &mapping->i_mmap);
689 flush_dcache_mmap_unlock(mapping);
690 i_mmap_unlock_write(mapping);
691 }
692
693 /*
694 * Clear hugetlb-related page reserves for children. This only
695 * affects MAP_PRIVATE mappings. Faults generated by the child
696 * are not guaranteed to succeed, even if read-only
697 */
698 if (is_vm_hugetlb_page(tmp))
699 reset_vma_resv_huge_pages(tmp);
700
701 /*
702 * Link in the new vma and copy the page table entries.
703 */
704 *pprev = tmp;
705 pprev = &tmp->vm_next;
706 tmp->vm_prev = prev;
707 prev = tmp;
708
709 __vma_link_rb(mm, tmp, rb_link, rb_parent);
710 rb_link = &tmp->vm_rb.rb_right;
711 rb_parent = &tmp->vm_rb;
712
713 mm->map_count++;
714 if (!(tmp->vm_flags & VM_WIPEONFORK))
715 retval = copy_page_range(mm, oldmm, mpnt);
716
717 if (tmp->vm_ops && tmp->vm_ops->open)
718 tmp->vm_ops->open(tmp);
719
720 if (retval)
721 goto out;
722 }
723 /* a new mm has just been created */
724 retval = arch_dup_mmap(oldmm, mm);
725out:
726 up_write(&mm->mmap_sem);
727 flush_tlb_mm(oldmm);
728 up_write(&oldmm->mmap_sem);
729 dup_userfaultfd_complete(&uf);
730fail_uprobe_end:
731 uprobe_end_dup_mmap();
732 return retval;
733fail_nomem_anon_vma_fork:
734 mpol_put(vma_policy(tmp));
735fail_nomem_policy:
736 kmem_cache_free(vm_area_cachep, tmp);
737fail_nomem:
738 retval = -ENOMEM;
739 vm_unacct_memory(charge);
740 goto out;
741}
742
743static inline int mm_alloc_pgd(struct mm_struct *mm)
744{
745 mm->pgd = pgd_alloc(mm);
746 if (unlikely(!mm->pgd))
747 return -ENOMEM;
748 return 0;
749}
750
751static inline void mm_free_pgd(struct mm_struct *mm)
752{
753 pgd_free(mm, mm->pgd);
754}
755#else
756static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)
757{
758 down_write(&oldmm->mmap_sem);
759 RCU_INIT_POINTER(mm->exe_file, get_mm_exe_file(oldmm));
760 up_write(&oldmm->mmap_sem);
761 return 0;
762}
763#define mm_alloc_pgd(mm) (0)
764#define mm_free_pgd(mm)
765#endif /* CONFIG_MMU */
766
767__cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock); 833__cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock);
768 834
769#define allocate_mm() (kmem_cache_alloc(mm_cachep, GFP_KERNEL))
770#define free_mm(mm) (kmem_cache_free(mm_cachep, (mm)))
771
772static unsigned long default_dump_filter = MMF_DUMP_FILTER_DEFAULT; 835static unsigned long default_dump_filter = MMF_DUMP_FILTER_DEFAULT;
773 836
774static int __init coredump_filter_setup(char *s) 837static int __init coredump_filter_setup(char *s)
@@ -858,27 +921,6 @@ fail_nopgd:
858 return NULL; 921 return NULL;
859} 922}
860 923
861static void check_mm(struct mm_struct *mm)
862{
863 int i;
864
865 for (i = 0; i < NR_MM_COUNTERS; i++) {
866 long x = atomic_long_read(&mm->rss_stat.count[i]);
867
868 if (unlikely(x))
869 printk(KERN_ALERT "BUG: Bad rss-counter state "
870 "mm:%p idx:%d val:%ld\n", mm, i, x);
871 }
872
873 if (mm_pgtables_bytes(mm))
874 pr_alert("BUG: non-zero pgtables_bytes on freeing mm: %ld\n",
875 mm_pgtables_bytes(mm));
876
877#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS
878 VM_BUG_ON_MM(mm->pmd_huge_pte, mm);
879#endif
880}
881
882/* 924/*
883 * Allocate and initialize an mm_struct. 925 * Allocate and initialize an mm_struct.
884 */ 926 */
@@ -894,24 +936,6 @@ struct mm_struct *mm_alloc(void)
894 return mm_init(mm, current, current_user_ns()); 936 return mm_init(mm, current, current_user_ns());
895} 937}
896 938
897/*
898 * Called when the last reference to the mm
899 * is dropped: either by a lazy thread or by
900 * mmput. Free the page directory and the mm.
901 */
902void __mmdrop(struct mm_struct *mm)
903{
904 BUG_ON(mm == &init_mm);
905 mm_free_pgd(mm);
906 destroy_context(mm);
907 hmm_mm_destroy(mm);
908 mmu_notifier_mm_destroy(mm);
909 check_mm(mm);
910 put_user_ns(mm->user_ns);
911 free_mm(mm);
912}
913EXPORT_SYMBOL_GPL(__mmdrop);
914
915static inline void __mmput(struct mm_struct *mm) 939static inline void __mmput(struct mm_struct *mm)
916{ 940{
917 VM_BUG_ON(atomic_read(&mm->mm_users)); 941 VM_BUG_ON(atomic_read(&mm->mm_users));
generated by cgit v1.2.2 (git 2.25.0) at 2026-07-08 00:20:15 -0400