PCuABI memory reservation requires adding reservation properties while creating and modifying the VMA. reserv_vma_set_reserv() interface is used to update those reservation details. Currently, these properties are added only for mmap/mremap syscalls, and later commits will add them for other special VMA mappings.
PCuABI memory reservation also requires merging or expanding VMAs that only belong to the original reservation. Use suitable reservation interfaces to check those properties before performing such operations on the VMA.
The address parameter type is modified from unsigned long to user_uintptr_t in several do_mmap() upstream functions to determine whether a new reservation is required.
Signed-off-by: Amit Daniel Kachhap amitdaniel.kachhap@arm.com --- include/linux/mm.h | 8 ++--- kernel/fork.c | 3 ++ mm/mmap.c | 73 +++++++++++++++++++++++++++++++++++++--------- mm/mremap.c | 24 ++++++++++++--- mm/util.c | 13 ++++----- 5 files changed, 91 insertions(+), 30 deletions(-)
diff --git a/include/linux/mm.h b/include/linux/mm.h index f7f09fe0684e..44a55c3e2c06 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -3259,7 +3259,7 @@ extern int insert_vm_struct(struct mm_struct *, struct vm_area_struct *); extern void unlink_file_vma(struct vm_area_struct *); extern struct vm_area_struct *copy_vma(struct vm_area_struct **, unsigned long addr, unsigned long len, pgoff_t pgoff, - bool *need_rmap_locks); + bool *need_rmap_locks, struct reserv_struct *reserv_info); extern void exit_mmap(struct mm_struct *); struct vm_area_struct *vma_modify(struct vma_iterator *vmi, struct vm_area_struct *prev, @@ -3365,8 +3365,8 @@ extern unsigned long get_unmapped_area(struct file *, unsigned long, unsigned lo
extern unsigned long mmap_region(struct file *file, unsigned long addr, unsigned long len, vm_flags_t vm_flags, unsigned long pgoff, - struct list_head *uf); -extern unsigned long do_mmap(struct file *file, unsigned long addr, + struct list_head *uf, unsigned long prot, bool new_reserv); +extern user_uintptr_t do_mmap(struct file *file, user_uintptr_t user_ptr, unsigned long len, unsigned long prot, unsigned long flags, vm_flags_t vm_flags, unsigned long pgoff, unsigned long *populate, struct list_head *uf); @@ -3395,7 +3395,7 @@ static inline void mm_populate(unsigned long addr, unsigned long len) {} /* This takes the mm semaphore itself */ extern int __must_check vm_brk_flags(unsigned long, unsigned long, unsigned long); extern int vm_munmap(unsigned long, size_t); -extern unsigned long __must_check vm_mmap(struct file *, unsigned long, +extern user_uintptr_t __must_check vm_mmap(struct file *, user_uintptr_t, unsigned long, unsigned long, unsigned long, unsigned long);
diff --git a/kernel/fork.c b/kernel/fork.c index a460a65624d7..9ee78c76fd4a 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -99,6 +99,7 @@ #include <linux/stackprotector.h> #include <linux/user_events.h> #include <linux/iommu.h> +#include <linux/cap_addr_mgmt.h>
#include <asm/pgalloc.h> #include <linux/uaccess.h> @@ -678,6 +679,8 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm, goto out; khugepaged_fork(mm, oldmm);
+ reserv_fork(mm, oldmm); + retval = vma_iter_bulk_alloc(&vmi, oldmm->map_count); if (retval) goto out; diff --git a/mm/mmap.c b/mm/mmap.c index 305c90332424..3b072e822f99 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -912,7 +912,8 @@ static struct vm_area_struct /* Can we merge the predecessor? */ if (addr == prev->vm_end && mpol_equal(vma_policy(prev), policy) && can_vma_merge_after(prev, vm_flags, anon_vma, file, - pgoff, vm_userfaultfd_ctx, anon_name)) { + pgoff, vm_userfaultfd_ctx, anon_name) + && reserv_vma_range_within_reserv(prev, addr, end - addr)) { merge_prev = true; vma_prev(vmi); } @@ -921,7 +922,8 @@ static struct vm_area_struct /* Can we merge the successor? */ if (next && mpol_equal(policy, vma_policy(next)) && can_vma_merge_before(next, vm_flags, anon_vma, file, pgoff+pglen, - vm_userfaultfd_ctx, anon_name)) { + vm_userfaultfd_ctx, anon_name) + && reserv_vma_range_within_reserv(next, addr, end - addr)) { merge_next = true; }
@@ -1212,7 +1214,7 @@ static inline bool file_mmap_ok(struct file *file, struct inode *inode, /* * The caller must write-lock current->mm->mmap_lock. */ -unsigned long do_mmap(struct file *file, unsigned long addr, +user_uintptr_t do_mmap(struct file *file, user_uintptr_t user_ptr, unsigned long len, unsigned long prot, unsigned long flags, vm_flags_t vm_flags, unsigned long pgoff, unsigned long *populate, @@ -1220,12 +1222,18 @@ unsigned long do_mmap(struct file *file, unsigned long addr, { struct mm_struct *mm = current->mm; int pkey = 0; + unsigned long addr = (ptraddr_t)user_ptr; + bool new_reserv = true;
*populate = 0;
if (!len) return -EINVAL;
+ if (reserv_is_supported(mm)) { + if (user_ptr_is_valid((const void __user *)user_ptr)) + new_reserv = false; + } /* * Does the application expect PROT_READ to imply PROT_EXEC? * @@ -1259,9 +1267,11 @@ unsigned long do_mmap(struct file *file, unsigned long addr, /* Obtain the address to map to. we verify (or select) it and ensure * that it represents a valid section of the address space. */ - addr = get_unmapped_area(file, addr, len, pgoff, flags); - if (IS_ERR_VALUE(addr)) - return addr; + if (new_reserv) { + addr = get_unmapped_area(file, addr, len, pgoff, flags); + if (IS_ERR_VALUE(addr)) + return addr; + }
if (flags & MAP_FIXED_NOREPLACE) { if (find_vma_intersection(mm, addr, addr + len)) @@ -1383,15 +1393,19 @@ unsigned long do_mmap(struct file *file, unsigned long addr, vm_flags |= VM_NORESERVE; }
- addr = mmap_region(file, addr, len, vm_flags, pgoff, uf); + if (new_reserv) + user_ptr = addr; + + addr = mmap_region(file, user_ptr, len, vm_flags, pgoff, uf, prot, new_reserv); if (!IS_ERR_VALUE(addr) && ((vm_flags & VM_LOCKED) || (flags & (MAP_POPULATE | MAP_NONBLOCK)) == MAP_POPULATE)) *populate = len; + return addr; }
-user_uintptr_t ksys_mmap_pgoff(user_uintptr_t addr, unsigned long len, +user_uintptr_t ksys_mmap_pgoff(user_uintptr_t user_ptr, unsigned long len, unsigned long prot, unsigned long flags, unsigned long fd, unsigned long pgoff) { @@ -1429,7 +1443,7 @@ user_uintptr_t ksys_mmap_pgoff(user_uintptr_t addr, unsigned long len, return PTR_ERR(file); }
- retval = vm_mmap_pgoff(file, addr, len, prot, flags, pgoff); + retval = vm_mmap_pgoff(file, user_ptr, len, prot, flags, pgoff); out_fput: if (file) fput(file); @@ -2801,16 +2815,17 @@ int do_munmap(struct mm_struct *mm, unsigned long start, size_t len,
unsigned long mmap_region(struct file *file, unsigned long addr, unsigned long len, vm_flags_t vm_flags, unsigned long pgoff, - struct list_head *uf) + struct list_head *uf, unsigned long prot, bool new_reserv) { struct mm_struct *mm = current->mm; struct vm_area_struct *vma = NULL; - struct vm_area_struct *next, *prev, *merge; + struct vm_area_struct *next, *prev, *merge, *old_vma; pgoff_t pglen = len >> PAGE_SHIFT; unsigned long charged = 0; unsigned long end = addr + len; unsigned long merge_start = addr, merge_end = end; bool writable_file_mapping = false; + struct reserv_struct reserv_info; pgoff_t vm_pgoff; int error; VMA_ITERATOR(vmi, mm, addr); @@ -2830,6 +2845,21 @@ unsigned long mmap_region(struct file *file, unsigned long addr, return -ENOMEM; }
+ if (!new_reserv) { + old_vma = vma_find(&vmi, end); + if (!old_vma) + /* + * This error scenario may not occur as address with valid + * capability should have been verified in the upstream + * syscall functions. + */ + return -ENOMEM; +#ifdef CONFIG_CHERI_PURECAP_UABI + memcpy(&reserv_info, &old_vma->reserv_data, sizeof(reserv_info)); +#endif + vma_iter_set(&vmi, addr); + } + /* Unmap any existing mapping in the area */ if (do_vmi_munmap(&vmi, mm, addr, len, uf, false)) return -ENOMEM; @@ -2856,7 +2886,8 @@ unsigned long mmap_region(struct file *file, unsigned long addr, /* Check next */ if (next && next->vm_start == end && !vma_policy(next) && can_vma_merge_before(next, vm_flags, NULL, file, pgoff+pglen, - NULL_VM_UFFD_CTX, NULL)) { + NULL_VM_UFFD_CTX, NULL) && + reserv_vma_range_within_reserv(next, addr, len)) { merge_end = next->vm_end; vma = next; vm_pgoff = next->vm_pgoff - pglen; @@ -2867,7 +2898,8 @@ unsigned long mmap_region(struct file *file, unsigned long addr, (vma ? can_vma_merge_after(prev, vm_flags, vma->anon_vma, file, pgoff, vma->vm_userfaultfd_ctx, NULL) : can_vma_merge_after(prev, vm_flags, NULL, file, pgoff, - NULL_VM_UFFD_CTX, NULL))) { + NULL_VM_UFFD_CTX, NULL)) && + reserv_vma_range_within_reserv(prev, addr, len)) { merge_start = prev->vm_start; vma = prev; vm_pgoff = prev->vm_pgoff; @@ -2903,6 +2935,13 @@ unsigned long mmap_region(struct file *file, unsigned long addr, vm_flags_init(vma, vm_flags); vma->vm_page_prot = vm_get_page_prot(vm_flags); vma->vm_pgoff = pgoff; + if (new_reserv) + reserv_vma_set_reserv(vma, addr, len, + user_ptr_perms_from_prot(prot, (vm_flags & VM_SHARED) + ? false : true)); + else + reserv_vma_set_reserv(vma, reserv_info.reserv_start, reserv_info.reserv_len, + reserv_info.reserv_perm);
if (file) { vma->vm_file = get_file(file); @@ -3448,7 +3487,7 @@ int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma) */ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, unsigned long addr, unsigned long len, pgoff_t pgoff, - bool *need_rmap_locks) + bool *need_rmap_locks, struct reserv_struct *reserv_info) { struct vm_area_struct *vma = *vmap; unsigned long vma_start = vma->vm_start; @@ -3500,6 +3539,12 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, new_vma->vm_start = addr; new_vma->vm_end = addr + len; new_vma->vm_pgoff = pgoff; + if (reserv_info) + reserv_vma_set_reserv(new_vma, reserv_info->reserv_start, + reserv_info->reserv_len, + reserv_info->reserv_perm); + else + reserv_vma_set_reserv(new_vma, addr, len, 0); if (vma_dup_policy(vma, new_vma)) goto out_free_vma; if (anon_vma_clone(new_vma, vma)) diff --git a/mm/mremap.c b/mm/mremap.c index f014ac50d9f1..70f4031df1f4 100644 --- a/mm/mremap.c +++ b/mm/mremap.c @@ -651,7 +651,8 @@ static unsigned long move_vma(struct vm_area_struct *vma, unsigned long old_addr, unsigned long old_len, unsigned long new_len, unsigned long new_addr, bool *locked, unsigned long flags, - struct vm_userfaultfd_ctx *uf, struct list_head *uf_unmap) + struct vm_userfaultfd_ctx *uf, struct list_head *uf_unmap, + struct reserv_struct *reserv_info) { long to_account = new_len - old_len; struct mm_struct *mm = vma->vm_mm; @@ -705,7 +706,7 @@ static unsigned long move_vma(struct vm_area_struct *vma, vma_start_write(vma); new_pgoff = vma->vm_pgoff + ((old_addr - vma->vm_start) >> PAGE_SHIFT); new_vma = copy_vma(&vma, new_addr, new_len, new_pgoff, - &need_rmap_locks); + &need_rmap_locks, reserv_info); if (!new_vma) { if (vm_flags & VM_ACCOUNT) vm_unacct_memory(to_account >> PAGE_SHIFT); @@ -874,6 +875,7 @@ static unsigned long mremap_to(unsigned long addr, unsigned long old_len, struct vm_area_struct *vma; unsigned long ret = -EINVAL; unsigned long map_flags = 0; + struct reserv_struct reserv_info, *reserv_ptr = NULL;
if (offset_in_page(new_addr)) goto out; @@ -903,6 +905,20 @@ static unsigned long mremap_to(unsigned long addr, unsigned long old_len, return -ENOMEM;
if (flags & MREMAP_FIXED) { + if (reserv_is_supported(mm)) { + vma = find_vma(mm, new_addr); + if (!vma) + /* + * This error scenario may not occur as address with valid + * capability should have been verified in the upstream + * syscall functions. + */ + return -ENOMEM; +#ifdef CONFIG_CHERI_PURECAP_UABI + memcpy(&reserv_info, &vma->reserv_data, sizeof(reserv_info)); +#endif + reserv_ptr = &reserv_info; + } ret = do_munmap(mm, new_addr, new_len, uf_unmap_early); if (ret) goto out; @@ -948,7 +964,7 @@ static unsigned long mremap_to(unsigned long addr, unsigned long old_len, new_addr = ret;
ret = move_vma(vma, addr, old_len, new_len, new_addr, locked, flags, uf, - uf_unmap); + uf_unmap, reserv_ptr);
out: return ret; @@ -1169,7 +1185,7 @@ SYSCALL_DEFINE5(__retptr__(mremap), user_uintptr_t, addr, unsigned long, old_len }
ret = move_vma(vma, addr, old_len, new_len, new_addr, - &locked, flags, &uf, &uf_unmap); + &locked, flags, &uf, &uf_unmap, NULL); } out: if (offset_in_page(ret)) diff --git a/mm/util.c b/mm/util.c index afd40ed9c3c8..077c9a2592a9 100644 --- a/mm/util.c +++ b/mm/util.c @@ -540,7 +540,7 @@ int account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc) } EXPORT_SYMBOL_GPL(account_locked_vm);
-user_uintptr_t vm_mmap_pgoff(struct file *file, user_uintptr_t addr, +user_uintptr_t vm_mmap_pgoff(struct file *file, user_uintptr_t usrptr, unsigned long len, unsigned long prot, unsigned long flag, unsigned long pgoff) { @@ -553,11 +553,7 @@ user_uintptr_t vm_mmap_pgoff(struct file *file, user_uintptr_t addr, if (!ret) { if (mmap_write_lock_killable(mm)) return -EINTR; - /* - * TODO [PCuABI] - might need propagating uintcap further down - * to do_mmap to properly handle capabilities - */ - ret = do_mmap(file, addr, len, prot, flag, 0, pgoff, &populate, + ret = do_mmap(file, usrptr, len, prot, flag, 0, pgoff, &populate, &uf); mmap_write_unlock(mm); userfaultfd_unmap_complete(mm, &uf); @@ -570,7 +566,8 @@ user_uintptr_t vm_mmap_pgoff(struct file *file, user_uintptr_t addr, return ret; }
-unsigned long vm_mmap(struct file *file, unsigned long addr, +/* TODO [PCuABI] - Update the users of vm_mmap */ +user_uintptr_t vm_mmap(struct file *file, user_uintptr_t usrptr, unsigned long len, unsigned long prot, unsigned long flag, unsigned long offset) { @@ -579,7 +576,7 @@ unsigned long vm_mmap(struct file *file, unsigned long addr, if (unlikely(offset_in_page(offset))) return -EINVAL;
- return vm_mmap_pgoff(file, addr, len, prot, flag, offset >> PAGE_SHIFT); + return vm_mmap_pgoff(file, usrptr, len, prot, flag, offset >> PAGE_SHIFT); } EXPORT_SYMBOL(vm_mmap);