MIRRORS/linux-imx
1
0
Fork 0
mirror of https://github.com/nxp-imx/linux-imx.git synced 2025-07-07 18:05:21 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

ANDROID: virt: gunyah: Add gup based demand paging support

Browse Source 
This patch adds support for demand paged virtual machines
based on get_user_pages.

Bug: 338347082
Change-Id: Ifce41cfdf6450b45861bf0119e7c7ae837a886f5
Signed-off-by: Sreenad Menon <quic_sreemeno@quicinc.com>
This commit is contained in:
Sreenad Menon 2024-03-05 00:26:34 -08:00 committed by Treehugger Robot
parent 231348f984
commit 296cceed08
7 changed files with 420 additions and 1077 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
drivers/virt/gunyah/Makefile
View File

@ -1,6 +1,6 @@
# SPDX-License-Identifier: GPL-2.0 # SPDX-License-Identifier: GPL-2.0
gunyah_rsc_mgr-y += rsc_mgr.o rsc_mgr_rpc.o vm_mgr.o vm_mgr_mem.o guest_memfd.o gunyah_rsc_mgr-y += rsc_mgr.o rsc_mgr_rpc.o vm_mgr.o vm_mgr_mem.o
obj-$(CONFIG_GUNYAH) += gunyah.o gunyah_rsc_mgr.o gunyah_vcpu.o obj-$(CONFIG_GUNYAH) += gunyah.o gunyah_rsc_mgr.o gunyah_vcpu.o
obj-$(CONFIG_GUNYAH_PLATFORM_HOOKS) += gunyah_platform_hooks.o obj-$(CONFIG_GUNYAH_PLATFORM_HOOKS) += gunyah_platform_hooks.o

987
drivers/virt/gunyah/guest_memfd.c
View File

@ -1,987 +0,0 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (c) 2023-2024 Qualcomm Innovation Center, Inc. All rights reserved.
*/
#define pr_fmt(fmt) "gunyah_guest_mem: " fmt
#include <linux/anon_inodes.h>
#include <linux/types.h>
#include <linux/falloc.h>
#include <linux/file.h>
#include <linux/migrate.h>
#include <linux/pagemap.h>
#include <uapi/linux/gunyah.h>
#include "vm_mgr.h"
/**
* struct gunyah_gmem_binding - Represents a binding of guestmem to a Gunyah VM
* @gfn: Guest address to place acquired folios
* @ghvm: Pointer to Gunyah VM in this binding
* @i_off: offset into the guestmem to grab folios from
* @file: Pointer to guest_memfd
* @i_entry: list entry for inode->i_private_list
* @flags: Access flags for the binding
* @nr: Number of pages covered by this binding
*/
struct gunyah_gmem_binding {
u64 gfn;
struct gunyah_vm *ghvm;
pgoff_t i_off;
struct file *file;
struct list_head i_entry;
u32 flags;
unsigned long nr;
};
static inline pgoff_t gunyah_gfn_to_off(struct gunyah_gmem_binding *b, u64 gfn)
{
return gfn - b->gfn + b->i_off;
}
static inline u64 gunyah_off_to_gfn(struct gunyah_gmem_binding *b, pgoff_t off)
{
return off - b->i_off + b->gfn;
}
static inline bool gunyah_guest_mem_is_lend(struct gunyah_vm *ghvm, u32 flags)
{
u8 access = flags & GUNYAH_MEM_ACCESS_MASK;
if (access == GUNYAH_MEM_FORCE_LEND)
return true;
else if (access == GUNYAH_MEM_FORCE_SHARE)
return false;
/* RM requires all VMs to be protected (isolated) */
return true;
}
static struct folio *gunyah_gmem_get_huge_folio(struct inode *inode,
pgoff_t index)
{
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
unsigned long huge_index = round_down(index, HPAGE_PMD_NR);
unsigned long flags = (unsigned long)inode->i_private;
struct address_space *mapping = inode->i_mapping;
gfp_t gfp = mapping_gfp_mask(mapping);
struct folio *folio;
if (!(flags & GHMF_ALLOW_HUGEPAGE))
return NULL;
if (filemap_range_has_page(mapping, huge_index << PAGE_SHIFT,
(huge_index + HPAGE_PMD_NR - 1)
<< PAGE_SHIFT))
return NULL;
folio = filemap_alloc_folio(gfp, HPAGE_PMD_ORDER);
if (!folio)
return NULL;
if (filemap_add_folio(mapping, folio, huge_index, gfp)) {
folio_put(folio);
return NULL;
}
return folio;
#else
return NULL;
#endif
}
static struct folio *gunyah_gmem_get_folio(struct inode *inode, pgoff_t index)
{
struct folio *folio;
folio = gunyah_gmem_get_huge_folio(inode, index);
if (!folio) {
folio = filemap_grab_folio(inode->i_mapping, index);
if (IS_ERR_OR_NULL(folio))
return NULL;
}
/*
* Use the up-to-date flag to track whether or not the memory has been
* zeroed before being handed off to the guest. There is no backing
* storage for the memory, so the folio will remain up-to-date until
* it's removed.
*/
if (!folio_test_uptodate(folio)) {
unsigned long nr_pages = folio_nr_pages(folio);
unsigned long i;
for (i = 0; i < nr_pages; i++)
clear_highpage(folio_page(folio, i));
folio_mark_uptodate(folio);
}
/*
* Ignore accessed, referenced, and dirty flags. The memory is
* unevictable and there is no storage to write back to.
*/
return folio;
}
/**
* gunyah_gmem_launder_folio() - Tries to unmap one folio from virtual machine(s)
* @folio: The folio to unmap
*
* Returns - 0 if the folio has been reclaimed from any virtual machine(s) that
* folio was mapped into.
*/
static int gunyah_gmem_launder_folio(struct folio *folio)
{
struct address_space *const mapping = folio->mapping;
struct gunyah_gmem_binding *b;
pgoff_t index = folio_index(folio);
int ret = 0;
u64 gfn;
filemap_invalidate_lock_shared(mapping);
list_for_each_entry(b, &mapping->i_private_list, i_entry) {
/* if the mapping doesn't cover this folio: skip */
if (b->i_off > index || index > b->i_off + b->nr)
continue;
gfn = gunyah_off_to_gfn(b, index);
ret = gunyah_vm_reclaim_folio(b->ghvm, gfn, folio);
if (WARN_RATELIMIT(ret, "failed to reclaim gfn: %08llx %d\n",
gfn, ret))
break;
}
filemap_invalidate_unlock_shared(mapping);
return ret;
}
static vm_fault_t gunyah_gmem_host_fault(struct vm_fault *vmf)
{
struct folio *folio;
folio = gunyah_gmem_get_folio(file_inode(vmf->vma->vm_file),
vmf->pgoff);
if (!folio)
return VM_FAULT_SIGBUS;
/* If the folio is lent to a VM, try to reclaim it */
if (folio_test_private(folio) && gunyah_gmem_launder_folio(folio)) {
folio_unlock(folio);
folio_put(folio);
return VM_FAULT_SIGBUS;
}
/* gunyah_gmem_launder_folio should clear the private bit if it returns 0 */
BUG_ON(folio_test_private(folio));
vmf->page = folio_file_page(folio, vmf->pgoff);
return VM_FAULT_LOCKED;
}
static const struct vm_operations_struct gunyah_gmem_vm_ops = {
.fault = gunyah_gmem_host_fault,
};
static int gunyah_gmem_mmap(struct file *file, struct vm_area_struct *vma)
{
struct address_space *const mapping = file->f_mapping;
struct gunyah_gmem_binding *b;
pgoff_t end_off;
int ret = 0;
u64 gfn, nr;
/* No support for private mappings to avoid COW. */
if ((vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) !=
(VM_SHARED | VM_MAYSHARE)) {
return -EINVAL;
}
filemap_invalidate_lock_shared(mapping);
/**
* userspace can only mmap if the folios covered by the requested
* offset are not lent to the guest
*/
list_for_each_entry(b, &mapping->i_private_list, i_entry) {
if (!gunyah_guest_mem_is_lend(b->ghvm, b->flags))
continue;
/* if the binding doesn't cover this vma: skip */
if (vma->vm_pgoff + vma_pages(vma) < b->i_off)
continue;
if (vma->vm_pgoff > b->i_off + b->nr)
continue;
gfn = gunyah_off_to_gfn(b, vma->vm_pgoff);
end_off = max(vma->vm_pgoff + vma_pages(vma), b->i_off + b->nr);
nr = gunyah_off_to_gfn(b, end_off) - gfn;
ret = gunyah_vm_reclaim_range(b->ghvm, gfn, nr);
if (ret)
break;
}
filemap_invalidate_unlock_shared(mapping);
if (!ret) {
file_accessed(file);
vma->vm_ops = &gunyah_gmem_vm_ops;
}
return ret;
}
/**
* gunyah_gmem_punch_hole() - try to reclaim a range of pages
* @inode: guest memfd inode
* @offset: Offset into memfd to start reclaim
* @len: length to reclaim
*
* Will try to unmap from virtual machines any folios covered by
* [offset, offset+len]. If unmapped, then tries to free those folios
*
* Returns - error code if any folio in the range couldn't be freed.
*/
static long gunyah_gmem_punch_hole(struct inode *inode, loff_t offset,
loff_t len)
{
return invalidate_inode_pages2_range(inode->i_mapping, offset, offset + len - 1);
}
static long gunyah_gmem_allocate(struct inode *inode, loff_t offset, loff_t len)
{
struct address_space *mapping = inode->i_mapping;
pgoff_t start, index, end;
int r;
/* Dedicated guest is immutable by default. */
if (offset + len > i_size_read(inode))
return -EINVAL;
filemap_invalidate_lock_shared(mapping);
start = offset >> PAGE_SHIFT;
end = (offset + len) >> PAGE_SHIFT;
r = 0;
for (index = start; index < end;) {
struct folio *folio;
if (signal_pending(current)) {
r = -EINTR;
break;
}
folio = gunyah_gmem_get_folio(inode, index);
if (!folio) {
r = -ENOMEM;
break;
}
index = folio_next_index(folio);
folio_unlock(folio);
folio_put(folio);
/* 64-bit only, wrapping the index should be impossible. */
if (WARN_ON_ONCE(!index))
break;
cond_resched();
}
filemap_invalidate_unlock_shared(mapping);
return r;
}
static long gunyah_gmem_fallocate(struct file *file, int mode, loff_t offset,
loff_t len)
{
long ret;
if (!(mode & FALLOC_FL_KEEP_SIZE))
return -EOPNOTSUPP;
if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE |
FALLOC_FL_ZERO_RANGE))
return -EOPNOTSUPP;
if (!PAGE_ALIGNED(offset) || !PAGE_ALIGNED(len))
return -EINVAL;
if (mode & FALLOC_FL_PUNCH_HOLE)
ret = gunyah_gmem_punch_hole(file_inode(file), offset, len);
else
ret = gunyah_gmem_allocate(file_inode(file), offset, len);
if (!ret)
file_modified(file);
return ret;
}
static int gunyah_gmem_release(struct inode *inode, struct file *file)
{
/**
* each binding increments refcount on file, so we shouldn't be here
* if i_private_list not empty.
*/
BUG_ON(!list_empty(&inode->i_mapping->i_private_list));
return 0;
}
static const struct file_operations gunyah_gmem_fops = {
.owner = THIS_MODULE,
.llseek = generic_file_llseek,
.mmap = gunyah_gmem_mmap,
.open = generic_file_open,
.fallocate = gunyah_gmem_fallocate,
.release = gunyah_gmem_release,
};
static bool gunyah_gmem_release_folio(struct folio *folio, gfp_t gfp_flags)
{
/* should return true if released; launder folio returns 0 if freed */
return !gunyah_gmem_launder_folio(folio);
}
static int gunyah_gmem_remove_folio(struct address_space *mapping,
struct folio *folio)
{
if (mapping != folio->mapping)
return -EINVAL;
return gunyah_gmem_launder_folio(folio);
}
static const struct address_space_operations gunyah_gmem_aops = {
.dirty_folio = noop_dirty_folio,
.release_folio = gunyah_gmem_release_folio,
.launder_folio = gunyah_gmem_launder_folio,
.error_remove_folio = gunyah_gmem_remove_folio,
};
int gunyah_guest_mem_create(struct gunyah_create_mem_args *args)
{
const char *anon_name = "[gh-gmem]";
unsigned long fd_flags = 0;
struct inode *inode;
struct file *file;
int fd, err;
if (!PAGE_ALIGNED(args->size))
return -EINVAL;
if (args->flags & ~(GHMF_CLOEXEC | GHMF_ALLOW_HUGEPAGE))
return -EINVAL;
if (args->flags & GHMF_CLOEXEC)
fd_flags |= O_CLOEXEC;
fd = get_unused_fd_flags(fd_flags);
if (fd < 0)
return fd;
/*
* Use the so called "secure" variant, which creates a unique inode
* instead of reusing a single inode. Each guest_memfd instance needs
* its own inode to track the size, flags, etc.
*/
file = anon_inode_create_getfile(anon_name, &gunyah_gmem_fops, NULL,
O_RDWR, NULL);
if (IS_ERR(file)) {
err = PTR_ERR(file);
goto err_fd;
}
file->f_flags |= O_LARGEFILE;
inode = file->f_inode;
WARN_ON(file->f_mapping != inode->i_mapping);
inode->i_private = (void *)(unsigned long)args->flags;
inode->i_mapping->a_ops = &gunyah_gmem_aops;
inode->i_mode |= S_IFREG;
inode->i_size = args->size;
mapping_set_gfp_mask(inode->i_mapping, GFP_HIGHUSER);
mapping_set_large_folios(inode->i_mapping);
mapping_set_unmovable(inode->i_mapping);
mapping_set_release_always(inode->i_mapping);
/* Unmovable mappings are supposed to be marked unevictable as well. */
WARN_ON_ONCE(!mapping_unevictable(inode->i_mapping));
fd_install(fd, file);
return fd;
err_fd:
put_unused_fd(fd);
return err;
}
void gunyah_gmem_remove_binding(struct gunyah_gmem_binding *b)
{
WARN_ON(gunyah_vm_reclaim_range(b->ghvm, b->gfn, b->nr));
mtree_erase(&b->ghvm->bindings, b->gfn);
list_del(&b->i_entry);
fput(b->file);
kfree(b);
}
static inline unsigned long gunyah_gmem_page_mask(struct file *file)
{
unsigned long gmem_flags = (unsigned long)file_inode(file)->i_private;
if (gmem_flags & GHMF_ALLOW_HUGEPAGE) {
#if IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)
return HPAGE_PMD_MASK;
#else
return ULONG_MAX;
#endif
}
return PAGE_MASK;
}
static int gunyah_gmem_init_binding(struct gunyah_vm *ghvm, struct file *file,
struct gunyah_map_mem_args *args,
struct gunyah_gmem_binding *binding)
{
const unsigned long page_mask = ~gunyah_gmem_page_mask(file);
if (args->flags & ~(GUNYAH_MEM_ALLOW_RWX | GUNYAH_MEM_ACCESS_MASK))
return -EINVAL;
if (args->guest_addr & page_mask)
return -EINVAL;
if (args->offset & page_mask)
return -EINVAL;
if (args->size & page_mask)
return -EINVAL;
binding->gfn = gunyah_gpa_to_gfn(args->guest_addr);
binding->ghvm = ghvm;
binding->i_off = args->offset >> PAGE_SHIFT;
binding->file = file;
binding->flags = args->flags;
binding->nr = args->size >> PAGE_SHIFT;
return 0;
}
static int gunyah_gmem_trim_binding(struct gunyah_gmem_binding *b,
unsigned long start_delta,
unsigned long end_delta)
{
struct gunyah_vm *ghvm = b->ghvm;
int ret;
down_write(&ghvm->bindings_lock);
if (!start_delta && !end_delta) {
ret = gunyah_vm_reclaim_range(ghvm, b->gfn, b->nr);
if (ret)
goto unlock;
gunyah_gmem_remove_binding(b);
} else if (start_delta && !end_delta) {
/* keep the start */
ret = gunyah_vm_reclaim_range(ghvm, b->gfn + start_delta,
b->gfn + b->nr);
if (ret)
goto unlock;
mtree_erase(&ghvm->bindings, b->gfn);
b->nr = start_delta;
ret = mtree_insert_range(&ghvm->bindings, b->gfn,
b->gfn + b->nr - 1, b, GFP_KERNEL);
} else if (!start_delta && end_delta) {
/* keep the end */
ret = gunyah_vm_reclaim_range(ghvm, b->gfn,
b->gfn + b->nr - end_delta);
if (ret)
goto unlock;
mtree_erase(&ghvm->bindings, b->gfn);
b->gfn += b->nr - end_delta;
b->i_off += b->nr - end_delta;
b->nr = end_delta;
ret = mtree_insert_range(&ghvm->bindings, b->gfn,
b->gfn + b->nr - 1, b, GFP_KERNEL);
} else {
/* TODO: split the mapping into 2 */
ret = -EINVAL;
}
unlock:
up_write(&ghvm->bindings_lock);
return ret;
}
static int gunyah_gmem_remove_mapping(struct gunyah_vm *ghvm, struct file *file,
struct gunyah_map_mem_args *args)
{
struct inode *inode = file_inode(file);
struct gunyah_gmem_binding *b = NULL;
unsigned long start_delta, end_delta;
struct gunyah_gmem_binding remove;
int ret;
ret = gunyah_gmem_init_binding(ghvm, file, args, &remove);
if (ret)
return ret;
ret = -ENOENT;
filemap_invalidate_lock(inode->i_mapping);
list_for_each_entry(b, &inode->i_mapping->i_private_list, i_entry) {
if (b->ghvm != remove.ghvm || b->flags != remove.flags ||
WARN_ON(b->file != remove.file))
continue;
/**
* Test if the binding to remove is within this binding
* [gfn b nr]
* [gfn remove nr]
*/
if (b->gfn > remove.gfn)
continue;
if (b->gfn + b->nr < remove.gfn + remove.nr)
continue;
/**
* We found the binding!
* Compute the delta in gfn start and make sure the offset
* into guest memfd matches.
*/
start_delta = remove.gfn - b->gfn;
if (remove.i_off - b->i_off != start_delta)
break;
end_delta = b->gfn + b->nr - remove.gfn - remove.nr;
ret = gunyah_gmem_trim_binding(b, start_delta, end_delta);
break;
}
filemap_invalidate_unlock(inode->i_mapping);
return ret;
}
static bool gunyah_gmem_binding_overlaps(struct gunyah_gmem_binding *a,
struct gunyah_gmem_binding *b)
{
/* assumes we are operating on the same file, check to be sure */
BUG_ON(a->file != b->file);
/**
* Gunyah only guarantees we can share a page with one VM and
* doesn't (currently) allow us to share same page with multiple VMs,
* regardless whether host can also access.
* Gunyah supports, but Linux hasn't implemented mapping same page
* into 2 separate addresses in guest's address space. This doesn't
* seem reasonable today, but we could do it later.
* All this to justify: check that the `a` region doesn't overlap with
* `b` region w.r.t. file offsets.
*/
if (a->i_off + a->nr <= b->i_off)
return false;
if (a->i_off >= b->i_off + b->nr)
return false;
return true;
}
static int gunyah_gmem_add_mapping(struct gunyah_vm *ghvm, struct file *file,
struct gunyah_map_mem_args *args)
{
struct gunyah_gmem_binding *b, *tmp = NULL;
struct inode *inode = file_inode(file);
int ret;
b = kzalloc(sizeof(*b), GFP_KERNEL);
if (!b)
return -ENOMEM;
ret = gunyah_gmem_init_binding(ghvm, file, args, b);
if (ret)
return ret;
/**
* When lending memory, we need to unmap single page from kernel's
* logical map. To do that, we need can_set_direct_map().
* arm64 doesn't map at page granularity without rodata=full.
*/
if (gunyah_guest_mem_is_lend(ghvm, b->flags) && !can_set_direct_map()) {
kfree(b);
pr_warn_once("Cannot lend memory without rodata=full");
return -EINVAL;
}
/**
* First, check that the region of guets memfd user is binding isn't
* already bound to some other guest region.
*/
filemap_invalidate_lock(inode->i_mapping);
list_for_each_entry(tmp, &inode->i_mapping->i_private_list, i_entry) {
if (gunyah_gmem_binding_overlaps(b, tmp)) {
ret = -EEXIST;
goto unlock;
}
}
/**
* mtree_insert_range will check that user hasn't mapped some other guest
* memfd region to the same addresses.
*/
ret = mtree_insert_range(&ghvm->bindings, b->gfn, b->gfn + b->nr - 1, b,
GFP_KERNEL);
if (ret)
goto unlock;
list_add(&b->i_entry, &inode->i_mapping->i_private_list);
unlock:
filemap_invalidate_unlock(inode->i_mapping);
return ret;
}
int gunyah_gmem_modify_mapping(struct gunyah_vm *ghvm,
struct gunyah_map_mem_args *args)
{
u8 access = args->flags & GUNYAH_MEM_ACCESS_MASK;
struct file *file;
int ret = -EINVAL;
file = fget(args->guest_mem_fd);
if (!file)
return -EINVAL;
if (file->f_op != &gunyah_gmem_fops)
goto err_file;
if (args->flags & ~(GUNYAH_MEM_ALLOW_RWX | GUNYAH_MEM_UNMAP | GUNYAH_MEM_ACCESS_MASK))
goto err_file;
/* VM needs to have some permissions to the memory */
if (!(args->flags & GUNYAH_MEM_ALLOW_RWX))
goto err_file;
if (access != GUNYAH_MEM_DEFAULT_ACCESS &&
access != GUNYAH_MEM_FORCE_LEND && access != GUNYAH_MEM_FORCE_SHARE)
goto err_file;
if (!PAGE_ALIGNED(args->guest_addr) || !PAGE_ALIGNED(args->offset) ||
!PAGE_ALIGNED(args->size))
goto err_file;
if (args->flags & GUNYAH_MEM_UNMAP) {
args->flags &= ~GUNYAH_MEM_UNMAP;
ret = gunyah_gmem_remove_mapping(ghvm, file, args);
} else {
ret = gunyah_gmem_add_mapping(ghvm, file, args);
}
err_file:
if (ret)
fput(file);
return ret;
}
int gunyah_gmem_share_parcel(struct gunyah_vm *ghvm, struct gunyah_rm_mem_parcel *parcel,
u64 *gfn, u64 *nr)
{
struct folio *folio, *prev_folio;
unsigned long nr_entries, i, j, start, end;
struct gunyah_gmem_binding *b;
bool lend;
int ret;
parcel->mem_handle = GUNYAH_MEM_HANDLE_INVAL;
if (!*nr)
return -EINVAL;
down_read(&ghvm->bindings_lock);
b = mtree_load(&ghvm->bindings, *gfn);
if (!b || *gfn > b->gfn + b->nr || *gfn < b->gfn) {
ret = -ENOENT;
goto unlock;
}
/**
* Generally, indices can be based on gfn, guest_memfd offset, or
* offset into binding. start and end are based on offset into binding.
*/
start = *gfn - b->gfn;
if (start + *nr > b->nr) {
ret = -ENOENT;
goto unlock;
}
end = start + *nr;
lend = gunyah_guest_mem_is_lend(ghvm, b->flags);
/**
* First, calculate the number of physically discontiguous regions
* the parcel covers. Each memory entry corresponds to one folio.
* In future, each memory entry could correspond to contiguous
* folios that are also adjacent in guest_memfd, but parcels
* are only being used for small amounts of memory for now, so
* this optimization is premature.
*/
nr_entries = 0;
prev_folio = NULL;
for (i = start + b->i_off; i < end + b->i_off;) {
folio = gunyah_gmem_get_folio(file_inode(b->file), i); /* A */
if (!folio) {
ret = -ENOMEM;
goto out;
}
if (lend) {
/* don't lend a folio that is mapped by host */
if (!gunyah_folio_lend_safe(folio)) {
folio_unlock(folio);
folio_put(folio);
ret = -EPERM;
goto out;
}
folio_set_private(folio);
}
nr_entries++;
i = folio_index(folio) + folio_nr_pages(folio);
}
end = i - b->i_off;
parcel->mem_entries =
kcalloc(nr_entries, sizeof(*parcel->mem_entries), GFP_KERNEL);
if (!parcel->mem_entries) {
ret = -ENOMEM;
goto out;
}
/**
* Walk through all the folios again, now filling the mem_entries array.
*/
j = 0;
prev_folio = NULL;
for (i = start + b->i_off; i < end + b->i_off; j++) {
folio = filemap_get_folio(file_inode(b->file)->i_mapping, i); /* B */
if (WARN_ON(IS_ERR(folio))) {
ret = PTR_ERR(folio);
i = end + b->i_off;
goto out;
}
parcel->mem_entries[j].size = cpu_to_le64(folio_size(folio));
parcel->mem_entries[j].phys_addr = cpu_to_le64(PFN_PHYS(folio_pfn(folio)));
i = folio_index(folio) + folio_nr_pages(folio);
folio_put(folio); /* B */
}
BUG_ON(j != nr_entries);
parcel->n_mem_entries = nr_entries;
if (lend)
parcel->n_acl_entries = 1;
parcel->acl_entries = kcalloc(parcel->n_acl_entries,
sizeof(*parcel->acl_entries), GFP_KERNEL);
if (!parcel->n_acl_entries) {
ret = -ENOMEM;
goto free_entries;
}
parcel->acl_entries[0].vmid = cpu_to_le16(ghvm->vmid);
if (b->flags & GUNYAH_MEM_ALLOW_READ)
parcel->acl_entries[0].perms |= GUNYAH_RM_ACL_R;
if (b->flags & GUNYAH_MEM_ALLOW_WRITE)
parcel->acl_entries[0].perms |= GUNYAH_RM_ACL_W;
if (b->flags & GUNYAH_MEM_ALLOW_EXEC)
parcel->acl_entries[0].perms |= GUNYAH_RM_ACL_X;
if (!lend) {
u16 host_vmid;
ret = gunyah_rm_get_vmid(ghvm->rm, &host_vmid);
if (ret)
goto free_acl;
parcel->acl_entries[1].vmid = cpu_to_le16(host_vmid);
parcel->acl_entries[1].perms = GUNYAH_RM_ACL_R | GUNYAH_RM_ACL_W | GUNYAH_RM_ACL_X;
}
parcel->mem_handle = GUNYAH_MEM_HANDLE_INVAL;
folio = filemap_get_folio(file_inode(b->file)->i_mapping, start); /* C */
*gfn = folio_index(folio) - b->i_off + b->gfn;
*nr = end - (folio_index(folio) - b->i_off);
folio_put(folio); /* C */
ret = gunyah_rm_mem_share(ghvm->rm, parcel);
goto out;
free_acl:
kfree(parcel->acl_entries);
parcel->acl_entries = NULL;
free_entries:
kfree(parcel->mem_entries);
parcel->mem_entries = NULL;
parcel->n_mem_entries = 0;
out:
/* unlock the folios */
for (j = start + b->i_off; j < i;) {
folio = filemap_get_folio(file_inode(b->file)->i_mapping, j); /* D */
if (WARN_ON(IS_ERR(folio)))
continue;
j = folio_index(folio) + folio_nr_pages(folio);
folio_unlock(folio); /* A */
if (ret) {
if (folio_test_private(folio)) {
gunyah_folio_host_reclaim(folio);
folio_clear_private(folio);
}
folio_put(folio); /* A */
}
folio_put(folio); /* D */
/* matching folio_put for A is done at
* (1) gunyah_gmem_reclaim_parcel or
* (2) after gunyah_gmem_parcel_to_paged, gunyah_vm_reclaim_folio
*/
}
unlock:
up_read(&ghvm->bindings_lock);
return ret;
}
int gunyah_gmem_reclaim_parcel(struct gunyah_vm *ghvm,
struct gunyah_rm_mem_parcel *parcel, u64 gfn,
u64 nr)
{
struct gunyah_rm_mem_entry *entry;
struct folio *folio;
pgoff_t i;
int ret;
if (parcel->mem_handle != GUNYAH_MEM_HANDLE_INVAL) {
ret = gunyah_rm_mem_reclaim(ghvm->rm, parcel);
if (ret) {
dev_err(ghvm->parent, "Failed to reclaim parcel: %d\n",
ret);
/* We can't reclaim the pages -- hold onto the pages
* forever because we don't know what state the memory
* is in
*/
return ret;
}
parcel->mem_handle = GUNYAH_MEM_HANDLE_INVAL;
for (i = 0; i < parcel->n_mem_entries; i++) {
entry = &parcel->mem_entries[i];
folio = pfn_folio(PHYS_PFN(le64_to_cpu(entry->phys_addr)));
if (folio_test_private(folio))
gunyah_folio_host_reclaim(folio);
folio_clear_private(folio);
folio_put(folio); /* A */
}
kfree(parcel->mem_entries);
kfree(parcel->acl_entries);
}
return 0;
}
int gunyah_gmem_setup_demand_paging(struct gunyah_vm *ghvm)
{
struct gunyah_rm_mem_entry *entries;
struct gunyah_gmem_binding *b;
unsigned long index = 0;
u32 count = 0, i;
int ret = 0;
down_read(&ghvm->bindings_lock);
mt_for_each(&ghvm->bindings, b, index, ULONG_MAX)
if (gunyah_guest_mem_is_lend(ghvm, b->flags))
count++;
if (!count)
goto out;
entries = kcalloc(count, sizeof(*entries), GFP_KERNEL);
if (!entries) {
ret = -ENOMEM;
goto out;
}
index = i = 0;
mt_for_each(&ghvm->bindings, b, index, ULONG_MAX) {
if (!gunyah_guest_mem_is_lend(ghvm, b->flags))
continue;
entries[i].phys_addr = cpu_to_le64(gunyah_gfn_to_gpa(b->gfn));
entries[i].size = cpu_to_le64(b->nr << PAGE_SHIFT);
if (++i == count)
break;
}
ret = gunyah_rm_vm_set_demand_paging(ghvm->rm, ghvm->vmid, i, entries);
kfree(entries);
out:
up_read(&ghvm->bindings_lock);
return ret;
}
int gunyah_gmem_demand_page(struct gunyah_vm *ghvm, u64 gpa, bool write)
{
unsigned long gfn = gunyah_gpa_to_gfn(gpa);
struct gunyah_gmem_binding *b;
struct folio *folio;
int ret;
down_read(&ghvm->bindings_lock);
b = mtree_load(&ghvm->bindings, gfn);
if (!b) {
ret = -ENOENT;
goto unlock;
}
if (write && !(b->flags & GUNYAH_MEM_ALLOW_WRITE)) {
ret = -EPERM;
goto unlock;
}
folio = gunyah_gmem_get_folio(file_inode(b->file),
gunyah_gfn_to_off(b, gfn));
if (!folio) {
ret = -ENOMEM;
pr_err_ratelimited("Failed to obtain memory for guest addr %016llx\n", gpa);
goto unlock;
}
/**
* the folio covers the requested guest address, but the folio may not
* start at the requested guest address. recompute the gfn based on the
* folio itself.
*/
gfn = gunyah_off_to_gfn(b, folio_index(folio));
filemap_invalidate_lock_shared(b->file->f_mapping);
ret = gunyah_vm_provide_folio(ghvm, folio, gfn,
!gunyah_guest_mem_is_lend(ghvm, b->flags),
!!(b->flags & GUNYAH_MEM_ALLOW_WRITE));
filemap_invalidate_unlock_shared(b->file->f_mapping);
if (ret) {
if (ret != -EAGAIN)
pr_err_ratelimited(
"Failed to provide folio for guest addr: %016llx: %d\n",
gpa, ret);
goto out;
}
out:
folio_unlock(folio);
folio_put(folio);
unlock:
up_read(&ghvm->bindings_lock);
return ret;
}
EXPORT_SYMBOL_GPL(gunyah_gmem_demand_page);

18
drivers/virt/gunyah/gunyah_vcpu.c
View File

@ -97,7 +97,7 @@ static bool gunyah_handle_page_fault(
bool write = !!vcpu_run_resp->state_data[1]; bool write = !!vcpu_run_resp->state_data[1];
int ret = 0; int ret = 0;
ret = gunyah_gmem_demand_page(vcpu->ghvm, addr, write); ret = gunyah_gup_demand_page(vcpu->ghvm, addr, write);
if (!ret || ret == -EAGAIN) if (!ret || ret == -EAGAIN)
return true; return true;
@ -120,8 +120,8 @@ gunyah_handle_mmio(struct gunyah_vcpu *vcpu, unsigned long resume_data[3],
if (WARN_ON(len > sizeof(u64))) if (WARN_ON(len > sizeof(u64)))
len = sizeof(u64); len = sizeof(u64);
ret = gunyah_gmem_demand_page(vcpu->ghvm, addr, ret = gunyah_gup_demand_page(vcpu->ghvm, addr,
vcpu->vcpu_run->mmio.is_write); vcpu->vcpu_run->mmio.is_write);
if (!ret || ret == -EAGAIN) { if (!ret || ret == -EAGAIN) {
resume_data[1] = GUNYAH_ADDRSPACE_VMMIO_ACTION_RETRY; resume_data[1] = GUNYAH_ADDRSPACE_VMMIO_ACTION_RETRY;
return true; return true;
@ -154,8 +154,6 @@ gunyah_handle_mmio(struct gunyah_vcpu *vcpu, unsigned long resume_data[3],
static int gunyah_handle_mmio_resume(struct gunyah_vcpu *vcpu, static int gunyah_handle_mmio_resume(struct gunyah_vcpu *vcpu,
unsigned long resume_data[3]) unsigned long resume_data[3])
{ {
bool write = vcpu->state == GUNYAH_VCPU_RUN_STATE_MMIO_WRITE;
switch (vcpu->vcpu_run->mmio.resume_action) { switch (vcpu->vcpu_run->mmio.resume_action) {
case GUNYAH_VCPU_RESUME_HANDLED: case GUNYAH_VCPU_RESUME_HANDLED:
if (vcpu->state == GUNYAH_VCPU_RUN_STATE_MMIO_READ) { if (vcpu->state == GUNYAH_VCPU_RUN_STATE_MMIO_READ) {
@ -170,11 +168,6 @@ static int gunyah_handle_mmio_resume(struct gunyah_vcpu *vcpu,
case GUNYAH_VCPU_RESUME_FAULT: case GUNYAH_VCPU_RESUME_FAULT:
resume_data[1] = GUNYAH_ADDRSPACE_VMMIO_ACTION_FAULT; resume_data[1] = GUNYAH_ADDRSPACE_VMMIO_ACTION_FAULT;
break; break;
case GUNYAH_VCPU_RESUME_RETRY:
/* userspace probably added a memory binding */
gunyah_gmem_demand_page(vcpu->ghvm, vcpu->mmio_addr, write);
resume_data[1] = GUNYAH_ADDRSPACE_VMMIO_ACTION_RETRY;
break;
default: default:
return -EINVAL; return -EINVAL;
} }
@ -283,6 +276,11 @@ static int gunyah_vcpu_run(struct gunyah_vcpu *vcpu)
break; break;
} }
if (current->mm != vcpu->ghvm->mm_s) {
ret = -EPERM;
goto out;
}
while (!ret && !signal_pending(current)) { while (!ret && !signal_pending(current)) {
if (vcpu->vcpu_run->immediate_exit) { if (vcpu->vcpu_run->immediate_exit) {
ret = -EINTR; ret = -EINTR;

132
drivers/virt/gunyah/vm_mgr.c
View File

@ -504,6 +504,8 @@ static __must_check struct gunyah_vm *gunyah_vm_alloc(struct gunyah_rm *rm)
ghvm->vmid = GUNYAH_VMID_INVAL; ghvm->vmid = GUNYAH_VMID_INVAL;
ghvm->rm = rm; ghvm->rm = rm;
mmgrab(current->mm);
ghvm->mm_s = current->mm;
init_rwsem(&ghvm->status_lock); init_rwsem(&ghvm->status_lock);
init_waitqueue_head(&ghvm->vm_status_wait); init_waitqueue_head(&ghvm->vm_status_wait);
kref_init(&ghvm->kref); kref_init(&ghvm->kref);
@ -603,6 +605,45 @@ static inline int gunyah_vm_fill_boot_context(struct gunyah_vm *ghvm)
return 0; return 0;
} }
int gunyah_gup_setup_demand_paging(struct gunyah_vm *ghvm)
{
struct gunyah_rm_mem_entry *entries;
struct gunyah_vm_gup_binding *b;
unsigned long index = 0;
u32 count = 0, i;
int ret = 0;
down_read(&ghvm->bindings_lock);
mt_for_each(&ghvm->bindings, b, index, ULONG_MAX)
if (b->share_type == VM_MEM_LEND)
count++;
if (!count)
goto out;
entries = kcalloc(count, sizeof(*entries), GFP_KERNEL);
if (!entries) {
ret = -ENOMEM;
goto out;
}
index = i = 0;
mt_for_each(&ghvm->bindings, b, index, ULONG_MAX) {
if (b->share_type != VM_MEM_LEND)
continue;
entries[i].phys_addr = cpu_to_le64(b->guest_phys_addr);
entries[i].size = cpu_to_le64(b->size);
if (++i == count)
break;
}
ret = gunyah_rm_vm_set_demand_paging(ghvm->rm, ghvm->vmid, i, entries);
kfree(entries);
out:
up_read(&ghvm->bindings_lock);
return ret;
}
static int gunyah_vm_start(struct gunyah_vm *ghvm) static int gunyah_vm_start(struct gunyah_vm *ghvm)
{ {
struct gunyah_rm_hyp_resources *resources; struct gunyah_rm_hyp_resources *resources;
@ -630,9 +671,9 @@ static int gunyah_vm_start(struct gunyah_vm *ghvm)
ghvm->dtb.parcel_start = ghvm->dtb.config.guest_phys_addr >> PAGE_SHIFT; ghvm->dtb.parcel_start = ghvm->dtb.config.guest_phys_addr >> PAGE_SHIFT;
ghvm->dtb.parcel_pages = ghvm->dtb.config.size >> PAGE_SHIFT; ghvm->dtb.parcel_pages = ghvm->dtb.config.size >> PAGE_SHIFT;
ret = gunyah_gmem_share_parcel(ghvm, &ghvm->dtb.parcel, ret = gunyah_gup_share_parcel(ghvm, &ghvm->dtb.parcel,
&ghvm->dtb.parcel_start, &ghvm->dtb.parcel_start,
&ghvm->dtb.parcel_pages); &ghvm->dtb.parcel_pages);
if (ret) { if (ret) {
dev_warn(ghvm->parent, dev_warn(ghvm->parent,
"Failed to allocate parcel for DTB: %d\n", ret); "Failed to allocate parcel for DTB: %d\n", ret);
@ -650,7 +691,7 @@ static int gunyah_vm_start(struct gunyah_vm *ghvm)
goto err; goto err;
} }
ret = gunyah_gmem_setup_demand_paging(ghvm); ret = gunyah_gup_setup_demand_paging(ghvm);
if (ret) { if (ret) {
dev_warn(ghvm->parent, dev_warn(ghvm->parent,
"Failed to set up gmem demand paging: %d\n", ret); "Failed to set up gmem demand paging: %d\n", ret);
@ -761,6 +802,7 @@ static long gunyah_vm_ioctl(struct file *filp, unsigned int cmd,
struct gunyah_vm *ghvm = filp->private_data; struct gunyah_vm *ghvm = filp->private_data;
void __user *argp = (void __user *)arg; void __user *argp = (void __user *)arg;
long r; long r;
bool lend = false;
switch (cmd) { switch (cmd) {
case GUNYAH_VM_SET_DTB_CONFIG: { case GUNYAH_VM_SET_DTB_CONFIG: {
@ -800,13 +842,25 @@ static long gunyah_vm_ioctl(struct file *filp, unsigned int cmd,
r = gunyah_vm_rm_function_instance(ghvm, &f); r = gunyah_vm_rm_function_instance(ghvm, &f);
break; break;
} }
case GUNYAH_VM_MAP_MEM: { case GH_VM_ANDROID_LEND_USER_MEM:
struct gunyah_map_mem_args args; lend = true;
fallthrough;
case GH_VM_SET_USER_MEM_REGION: {
struct gunyah_userspace_memory_region region;
if (copy_from_user(&args, argp, sizeof(args))) /* only allow owner task to add memory */
if (ghvm->mm_s != current->mm)
return -EPERM;
if (copy_from_user(&region, argp, sizeof(region)))
return -EFAULT; return -EFAULT;
return gunyah_gmem_modify_mapping(ghvm, &args); if (region.flags & ~(GUNYAH_MEM_ALLOW_READ |
GUNYAH_MEM_ALLOW_WRITE |
GUNYAH_MEM_ALLOW_EXEC))
return -EINVAL;
r = gunyah_vm_binding_alloc(ghvm, &region, lend);
break;
} }
case GUNYAH_VM_SET_BOOT_CONTEXT: { case GUNYAH_VM_SET_BOOT_CONTEXT: {
struct gunyah_vm_boot_context boot_ctx; struct gunyah_vm_boot_context boot_ctx;
@ -830,11 +884,51 @@ int __must_check gunyah_vm_get(struct gunyah_vm *ghvm)
} }
EXPORT_SYMBOL_GPL(gunyah_vm_get); EXPORT_SYMBOL_GPL(gunyah_vm_get);
int gunyah_gup_reclaim_parcel(struct gunyah_vm *ghvm,
struct gunyah_rm_mem_parcel *parcel, u64 gfn,
u64 nr)
{
struct gunyah_rm_mem_entry *entry;
struct folio *folio;
pgoff_t i;
int ret;
if (parcel->mem_handle != GUNYAH_MEM_HANDLE_INVAL) {
ret = gunyah_rm_mem_reclaim(ghvm->rm, parcel);
if (ret) {
dev_err(ghvm->parent, "Failed to reclaim parcel: %d\n",
ret);
/* We can't reclaim the pages -- hold onto the pages
* forever because we don't know what state the memory
* is in
*/
return ret;
}
parcel->mem_handle = GUNYAH_MEM_HANDLE_INVAL;
for (i = 0; i < parcel->n_mem_entries; i++) {
entry = &parcel->mem_entries[i];
folio = pfn_folio(PHYS_PFN(le64_to_cpu(entry->phys_addr)));
if (folio_test_private(folio))
gunyah_folio_host_reclaim(folio);
folio_put(folio);
}
kfree(parcel->mem_entries);
kfree(parcel->acl_entries);
}
return 0;
}
static void _gunyah_vm_put(struct kref *kref) static void _gunyah_vm_put(struct kref *kref)
{ {
struct gunyah_vm *ghvm = container_of(kref, struct gunyah_vm, kref); struct gunyah_vm *ghvm = container_of(kref, struct gunyah_vm, kref);
struct gunyah_gmem_binding *b; struct gunyah_vm_gup_binding *b;
unsigned long idx = 0; unsigned long index = 0;
int ret; int ret;
/** /**
@ -847,7 +941,7 @@ static void _gunyah_vm_put(struct kref *kref)
if (ghvm->vm_status == GUNYAH_RM_VM_STATUS_LOAD || if (ghvm->vm_status == GUNYAH_RM_VM_STATUS_LOAD ||
ghvm->vm_status == GUNYAH_RM_VM_STATUS_READY || ghvm->vm_status == GUNYAH_RM_VM_STATUS_READY ||
ghvm->vm_status == GUNYAH_RM_VM_STATUS_INIT_FAILED) { ghvm->vm_status == GUNYAH_RM_VM_STATUS_INIT_FAILED) {
ret = gunyah_gmem_reclaim_parcel(ghvm, &ghvm->dtb.parcel, ret = gunyah_gup_reclaim_parcel(ghvm, &ghvm->dtb.parcel,
ghvm->dtb.parcel_start, ghvm->dtb.parcel_start,
ghvm->dtb.parcel_pages); ghvm->dtb.parcel_pages);
if (ret) if (ret)
@ -858,12 +952,14 @@ static void _gunyah_vm_put(struct kref *kref)
gunyah_vm_remove_functions(ghvm); gunyah_vm_remove_functions(ghvm);
down_write(&ghvm->bindings_lock); down_write(&ghvm->bindings_lock);
mt_for_each(&ghvm->bindings, b, idx, ULONG_MAX) { mt_for_each(&ghvm->bindings, b, index, ULONG_MAX) {
gunyah_gmem_remove_binding(b); mtree_erase(&ghvm->bindings, gunyah_gpa_to_gfn(b->guest_phys_addr));
kfree(b);
} }
up_write(&ghvm->bindings_lock); up_write(&ghvm->bindings_lock);
WARN_ON(!mtree_empty(&ghvm->bindings)); WARN_ON(!mtree_empty(&ghvm->bindings));
mtree_destroy(&ghvm->bindings); mtree_destroy(&ghvm->bindings);
/** /**
* If this fails, we're going to lose the memory for good and is * If this fails, we're going to lose the memory for good and is
* BUG_ON-worthy, but not unrecoverable (we just lose memory). * BUG_ON-worthy, but not unrecoverable (we just lose memory).
@ -909,6 +1005,7 @@ static void _gunyah_vm_put(struct kref *kref)
xa_destroy(&ghvm->boot_context); xa_destroy(&ghvm->boot_context);
gunyah_rm_put(ghvm->rm); gunyah_rm_put(ghvm->rm);
mmdrop(ghvm->mm_s);
kfree(ghvm); kfree(ghvm);
} }
@ -978,15 +1075,6 @@ long gunyah_dev_vm_mgr_ioctl(struct gunyah_rm *rm, unsigned int cmd,
switch (cmd) { switch (cmd) {
case GUNYAH_CREATE_VM: case GUNYAH_CREATE_VM:
return gunyah_dev_ioctl_create_vm(rm, arg); return gunyah_dev_ioctl_create_vm(rm, arg);
case GUNYAH_CREATE_GUEST_MEM: {
struct gunyah_create_mem_args args;
if (copy_from_user(&args, (const void __user *)arg,
sizeof(args)))
return -EFAULT;
return gunyah_guest_mem_create(&args);
}
default: default:
return -ENOTTY; return -ENOTTY;
} }

37
drivers/virt/gunyah/vm_mgr.h
View File

@ -20,6 +20,19 @@
#include "rsc_mgr.h" #include "rsc_mgr.h"
enum gunyah_vm_mem_share_type {
VM_MEM_SHARE,
VM_MEM_LEND,
};
struct gunyah_vm_gup_binding {
enum gunyah_vm_mem_share_type share_type;
u64 guest_phys_addr;
u64 userspace_addr;
u64 size;
u32 flags;
};
static inline u64 gunyah_gpa_to_gfn(u64 gpa) static inline u64 gunyah_gpa_to_gfn(u64 gpa)
{ {
return gpa >> PAGE_SHIFT; return gpa >> PAGE_SHIFT;
@ -42,6 +55,7 @@ long gunyah_dev_vm_mgr_ioctl(struct gunyah_rm *rm, unsigned int cmd,
* @bindings: A maple tree of guest memfd bindings. Indices are guest frame * @bindings: A maple tree of guest memfd bindings. Indices are guest frame
* numbers; entries are &struct gunyah_gmem_binding * numbers; entries are &struct gunyah_gmem_binding
* @bindings_lock: For serialization to @bindings * @bindings_lock: For serialization to @bindings
* @mm_s: Userspace tied to this vm
* @addrspace_ticket: Resource ticket to the capability for guest VM's * @addrspace_ticket: Resource ticket to the capability for guest VM's
* address space * address space
* @host_private_extent_ticket: Resource ticket to the capability for our * @host_private_extent_ticket: Resource ticket to the capability for our
@ -94,6 +108,7 @@ struct gunyah_vm {
struct maple_tree mm; struct maple_tree mm;
struct maple_tree bindings; struct maple_tree bindings;
struct rw_semaphore bindings_lock; struct rw_semaphore bindings_lock;
struct mm_struct *mm_s;
struct gunyah_vm_resource_ticket addrspace_ticket, struct gunyah_vm_resource_ticket addrspace_ticket,
host_private_extent_ticket, host_shared_extent_ticket, host_private_extent_ticket, host_shared_extent_ticket,
guest_private_extent_ticket, guest_shared_extent_ticket; guest_private_extent_ticket, guest_shared_extent_ticket;
@ -197,19 +212,13 @@ int gunyah_vm_provide_folio(struct gunyah_vm *ghvm, struct folio *folio,
int gunyah_vm_reclaim_folio(struct gunyah_vm *ghvm, u64 gfn, struct folio *folio); int gunyah_vm_reclaim_folio(struct gunyah_vm *ghvm, u64 gfn, struct folio *folio);
int gunyah_vm_reclaim_range(struct gunyah_vm *ghvm, u64 gfn, u64 nr); int gunyah_vm_reclaim_range(struct gunyah_vm *ghvm, u64 gfn, u64 nr);
int gunyah_guest_mem_create(struct gunyah_create_mem_args *args); int gunyah_vm_binding_alloc(struct gunyah_vm *ghvm,
int gunyah_gmem_modify_mapping(struct gunyah_vm *ghvm, struct gunyah_userspace_memory_region *region,
struct gunyah_map_mem_args *args); bool lend);
struct gunyah_gmem_binding; int gunyah_gup_setup_demand_paging(struct gunyah_vm *ghvm);
void gunyah_gmem_remove_binding(struct gunyah_gmem_binding *binding); int gunyah_gup_share_parcel(struct gunyah_vm *ghvm,
int gunyah_gmem_share_parcel(struct gunyah_vm *ghvm, struct gunyah_rm_mem_parcel *parcel,
struct gunyah_rm_mem_parcel *parcel, u64 *gfn, u64 *gfn, u64 *nr);
u64 *nr); int gunyah_gup_demand_page(struct gunyah_vm *ghvm, u64 gpa, bool write);
int gunyah_gmem_reclaim_parcel(struct gunyah_vm *ghvm,
struct gunyah_rm_mem_parcel *parcel, u64 gfn,
u64 nr);
int gunyah_gmem_setup_demand_paging(struct gunyah_vm *ghvm);
int gunyah_gmem_demand_page(struct gunyah_vm *ghvm, u64 gpa, bool write);
#endif #endif

262
drivers/virt/gunyah/vm_mgr_mem.c
View File

@ -52,6 +52,7 @@ int gunyah_vm_parcel_to_paged(struct gunyah_vm *ghvm,
} }
off += folio_nr_pages(folio); off += folio_nr_pages(folio);
} }
BUG_ON(off != nr);
return 0; return 0;
} }
@ -122,7 +123,6 @@ int gunyah_vm_provide_folio(struct gunyah_vm *ghvm, struct folio *folio,
addrspace = __first_resource(&ghvm->addrspace_ticket); addrspace = __first_resource(&ghvm->addrspace_ticket);
if (!addrspace || !guest_extent || !host_extent) { if (!addrspace || !guest_extent || !host_extent) {
folio_unlock(folio);
return -ENODEV; return -ENODEV;
} }
@ -144,12 +144,6 @@ int gunyah_vm_provide_folio(struct gunyah_vm *ghvm, struct folio *folio,
if (ret) if (ret)
return ret; return ret;
/* don't lend a folio that is (or could be) mapped by Linux */
if (!share && !gunyah_folio_lend_safe(folio)) {
ret = -EPERM;
goto remove;
}
if (share && write) if (share && write)
access = GUNYAH_PAGETABLE_ACCESS_RW; access = GUNYAH_PAGETABLE_ACCESS_RW;
else if (share && !write) else if (share && !write)
@ -192,8 +186,6 @@ int gunyah_vm_provide_folio(struct gunyah_vm *ghvm, struct folio *folio,
} }
folio_get(folio); folio_get(folio);
if (!share)
folio_set_private(folio);
return 0; return 0;
memextent_reclaim: memextent_reclaim:
gunyah_error = gunyah_hypercall_memextent_donate(reclaim_flags(share), gunyah_error = gunyah_hypercall_memextent_donate(reclaim_flags(share),
@ -213,7 +205,6 @@ platform_release:
} }
reclaim_host: reclaim_host:
gunyah_folio_host_reclaim(folio); gunyah_folio_host_reclaim(folio);
remove:
mtree_erase(&ghvm->mm, gfn); mtree_erase(&ghvm->mm, gfn);
return ret; return ret;
} }
@ -301,12 +292,8 @@ static int __gunyah_vm_reclaim_folio_locked(struct gunyah_vm *ghvm, void *entry,
BUG_ON(mtree_erase(&ghvm->mm, gfn) != entry); BUG_ON(mtree_erase(&ghvm->mm, gfn) != entry);
if (folio_test_private(folio)) { unpin_user_page(folio_page(folio, 0));
gunyah_folio_host_reclaim(folio); account_locked_vm(current->mm, 1, false);
folio_clear_private(folio);
}
folio_put(folio);
return 0; return 0;
err: err:
return ret; return ret;
@ -354,3 +341,246 @@ int gunyah_vm_reclaim_range(struct gunyah_vm *ghvm, u64 gfn, u64 nr)
return ret2; return ret2;
} }
int gunyah_vm_binding_alloc(struct gunyah_vm *ghvm,
struct gunyah_userspace_memory_region *region,
bool lend)
{
struct gunyah_vm_gup_binding *binding;
int ret = 0;
if (!region->memory_size || !PAGE_ALIGNED(region->memory_size) ||
!PAGE_ALIGNED(region->userspace_addr) ||
!PAGE_ALIGNED(region->guest_phys_addr))
return -EINVAL;
if (overflows_type(region->guest_phys_addr + region->memory_size, u64))
return -EOVERFLOW;
binding = kzalloc(sizeof(*binding), GFP_KERNEL_ACCOUNT);
if (!binding) {
return -ENOMEM;
}
binding->userspace_addr = region->userspace_addr;
binding->guest_phys_addr = region->guest_phys_addr;
binding->size = region->memory_size;
binding->flags = region->flags;
if (lend) {
binding->share_type = VM_MEM_LEND;
} else {
binding->share_type = VM_MEM_SHARE;
}
down_write(&ghvm->bindings_lock);
ret = mtree_insert_range(&ghvm->bindings,
gunyah_gpa_to_gfn(binding->guest_phys_addr),
gunyah_gpa_to_gfn(binding->guest_phys_addr + region->memory_size - 1),
binding, GFP_KERNEL);
if(ret != 0)
kfree(binding);
up_write(&ghvm->bindings_lock);
return ret;
}
int gunyah_gup_demand_page(struct gunyah_vm *ghvm, u64 gpa, bool write)
{
unsigned long gfn = gunyah_gpa_to_gfn(gpa);
struct gunyah_vm_gup_binding *b;
unsigned int gup_flags;
u64 offset;
int pinned, ret;
struct page *page;
struct folio *folio;
down_read(&ghvm->bindings_lock);
b = mtree_load(&ghvm->bindings, gfn);
if (!b) {
ret = -ENOENT;
goto unlock;
}
if (write && !(b->flags & GUNYAH_MEM_ALLOW_WRITE)) {
ret = -EPERM;
goto unlock;
}
gup_flags = FOLL_LONGTERM;
if (b->flags & GUNYAH_MEM_ALLOW_WRITE)
gup_flags |= FOLL_WRITE;
offset = (gunyah_gfn_to_gpa(gfn) - b->guest_phys_addr);
ret = account_locked_vm(current->mm, 1, true);
if (ret)
goto unlock;
pinned = pin_user_pages_fast(b->userspace_addr + offset, 1,
gup_flags, &page);
if (pinned != 1) {
ret = pinned;
goto unlock_page;
}
folio = page_folio(page);
if (!PageSwapBacked(page)) {
ret = -EIO;
goto unpin_page;
}
folio_lock(folio);
ret = gunyah_vm_provide_folio(ghvm, folio, gfn - folio_page_idx(folio, page),
!(b->share_type == VM_MEM_LEND),
!!(b->flags & GUNYAH_MEM_ALLOW_WRITE));
folio_unlock(folio);
if (ret) {
if (ret != -EAGAIN)
pr_err_ratelimited(
"Failed to provide folio for guest addr: %016llx: %d\n",
gpa, ret);
goto unpin_page;
}
goto unlock;
unpin_page:
unpin_user_page(page);
unlock_page:
account_locked_vm(current->mm, 1, false);
unlock:
up_read(&ghvm->bindings_lock);
return ret;
}
EXPORT_SYMBOL_GPL(gunyah_gup_demand_page);
int gunyah_gup_share_parcel(struct gunyah_vm *ghvm, struct gunyah_rm_mem_parcel *parcel,
u64 *gfn, u64 *nr)
{
struct gunyah_vm_gup_binding *b;
bool lend = false;
struct page **pages;
int pinned, ret;
u16 vmid;
struct folio *folio;
unsigned int gup_flags;
unsigned long i, offset;
parcel->mem_handle = GUNYAH_MEM_HANDLE_INVAL;
if (!*nr)
return -EINVAL;
down_read(&ghvm->bindings_lock);
b = mtree_load(&ghvm->bindings, *gfn);
if (!b) {
ret = -ENOENT;
goto unlock;
}
offset = gunyah_gfn_to_gpa(*gfn) - b->guest_phys_addr;
pages = kcalloc(*nr, sizeof(*pages), GFP_KERNEL_ACCOUNT);
if (!pages) {
ret = -ENOMEM;
goto unlock;
}
gup_flags = FOLL_LONGTERM;
if (b->flags & GUNYAH_MEM_ALLOW_WRITE)
gup_flags |= FOLL_WRITE;
pinned = pin_user_pages_fast(b->userspace_addr + offset, *nr,
gup_flags, pages);
if (pinned < 0) {
ret = pinned;
goto free_pages;
} else if (pinned != *nr) {
ret = -EFAULT;
goto unpin_pages;
}
ret = account_locked_vm(current->mm, pinned, true);
if (ret)
goto unpin_pages;
if (b->share_type == VM_MEM_LEND) {
parcel->n_acl_entries = 1;
lend = true;
} else {
lend = false;
parcel->n_acl_entries = 2;
}
parcel->acl_entries = kcalloc(parcel->n_acl_entries,
sizeof(*parcel->acl_entries), GFP_KERNEL);
if (!parcel->acl_entries) {
ret = -ENOMEM;
goto unaccount_pages;
}
/* acl_entries[0].vmid will be this VM's vmid. We'll fill it when the
* VM is starting and we know the VM's vmid.
*/
parcel->acl_entries[0].vmid = cpu_to_le16(ghvm->vmid);
if (b->flags & GUNYAH_MEM_ALLOW_READ)
parcel->acl_entries[0].perms |= GUNYAH_RM_ACL_R;
if (b->flags & GUNYAH_MEM_ALLOW_WRITE)
parcel->acl_entries[0].perms |= GUNYAH_RM_ACL_W;
if (b->flags & GUNYAH_MEM_ALLOW_EXEC)
parcel->acl_entries[0].perms |= GUNYAH_RM_ACL_X;
if (!lend) {
ret = gunyah_rm_get_vmid(ghvm->rm, &vmid);
if (ret)
goto free_acl;
parcel->acl_entries[1].vmid = cpu_to_le16(vmid);
/* Host assumed to have all these permissions. Gunyah will not
* grant new permissions if host actually had less than RWX
*/
parcel->acl_entries[1].perms = GUNYAH_RM_ACL_R | GUNYAH_RM_ACL_W | GUNYAH_RM_ACL_X;
}
parcel->mem_entries = kcalloc(pinned, sizeof(parcel->mem_entries[0]),
GFP_KERNEL_ACCOUNT);
if (!parcel->mem_entries) {
ret = -ENOMEM;
goto free_acl;
}
folio = page_folio(pages[0]);
*gfn -= folio_page_idx(folio, pages[0]);
parcel->mem_entries[0].size = cpu_to_le64(folio_size(folio));
parcel->mem_entries[0].phys_addr = cpu_to_le64(PFN_PHYS(folio_pfn(folio)));
for (i = 1; i < pinned; i++) {
folio = page_folio(pages[i]);
if (pages[i] == folio_page(folio, 0)) {
parcel->mem_entries[i].size = cpu_to_le64(folio_size(folio));
parcel->mem_entries[i].phys_addr = cpu_to_le64(PFN_PHYS(folio_pfn(folio)));
} else {
unpin_user_page(pages[i]);
account_locked_vm(current->mm, 1, false);
}
}
parcel->n_mem_entries = i;
ret = gunyah_rm_mem_share(ghvm->rm, parcel);
goto free_pages;
free_acl:
kfree(parcel->acl_entries);
parcel->acl_entries = NULL;
kfree(parcel->mem_entries);
parcel->mem_entries = NULL;
parcel->n_mem_entries = 0;
unaccount_pages:
account_locked_vm(current->mm, pinned, false);
unpin_pages:
unpin_user_pages(pages, pinned);
free_pages:
kfree(pages);
unlock:
up_read(&ghvm->bindings_lock);
return ret;
}

59
include/uapi/linux/gunyah.h
View File

@ -20,25 +20,6 @@
*/ */
#define GUNYAH_CREATE_VM _IO(GUNYAH_IOCTL_TYPE, 0x0) /* Returns a Gunyah VM fd */ #define GUNYAH_CREATE_VM _IO(GUNYAH_IOCTL_TYPE, 0x0) /* Returns a Gunyah VM fd */
enum gunyah_mem_flags {
GHMF_CLOEXEC = (1UL << 0),
GHMF_ALLOW_HUGEPAGE = (1UL << 1),
};
/**
* struct gunyah_create_mem_args - Description of guest memory to create
* @flags: See GHMF_*.
*/
struct gunyah_create_mem_args {
__u64 flags;
__u64 size;
__u64 reserved[6];
};
#define GUNYAH_CREATE_GUEST_MEM \
_IOW(GUNYAH_IOCTL_TYPE, 0x8, \
struct gunyah_create_mem_args) /* Returns a Gunyah memory fd */
/* /*
* ioctls for gunyah-vm fds (returned by GUNYAH_CREATE_VM) * ioctls for gunyah-vm fds (returned by GUNYAH_CREATE_VM)
*/ */
@ -183,16 +164,15 @@ struct gunyah_fn_desc {
* @GUNYAH_MEM_ALLOW_EXEC: Allow guest to execute instructions in the memory * @GUNYAH_MEM_ALLOW_EXEC: Allow guest to execute instructions in the memory
*/ */
enum gunyah_map_flags { enum gunyah_map_flags {
GUNYAH_MEM_DEFAULT_ACCESS = 0, GUNYAH_MEM_ALLOW_READ = 1UL << 0,
GUNYAH_MEM_FORCE_LEND = 1, GUNYAH_MEM_ALLOW_WRITE = 1UL << 1,
GUNYAH_MEM_FORCE_SHARE = 2, GUNYAH_MEM_ALLOW_EXEC = 1UL << 2,
#define GUNYAH_MEM_ACCESS_MASK 0x7
GUNYAH_MEM_ALLOW_READ = 1UL << 4,
GUNYAH_MEM_ALLOW_WRITE = 1UL << 5,
GUNYAH_MEM_ALLOW_EXEC = 1UL << 6,
GUNYAH_MEM_ALLOW_RWX = GUNYAH_MEM_ALLOW_RWX =
(GUNYAH_MEM_ALLOW_READ | GUNYAH_MEM_ALLOW_WRITE | GUNYAH_MEM_ALLOW_EXEC), (GUNYAH_MEM_ALLOW_READ | GUNYAH_MEM_ALLOW_WRITE | GUNYAH_MEM_ALLOW_EXEC),
GUNYAH_MEM_DEFAULT_ACCESS = 0x00,
GUNYAH_MEM_FORCE_LEND = 0x10,
GUNYAH_MEM_FORCE_SHARE = 0x20,
#define GUNYAH_MEM_ACCESS_MASK 0x70
GUNYAH_MEM_UNMAP = 1UL << 8, GUNYAH_MEM_UNMAP = 1UL << 8,
}; };
@ -375,4 +355,29 @@ struct gunyah_vcpu_run {
#define GUNYAH_VCPU_RUN _IO(GUNYAH_IOCTL_TYPE, 0x5) #define GUNYAH_VCPU_RUN _IO(GUNYAH_IOCTL_TYPE, 0x5)
#define GUNYAH_VCPU_MMAP_SIZE _IO(GUNYAH_IOCTL_TYPE, 0x6) #define GUNYAH_VCPU_MMAP_SIZE _IO(GUNYAH_IOCTL_TYPE, 0x6)
/**
* struct gunyah_userspace_memory_region - Userspace memory descripion for GH_VM_SET_USER_MEM_REGION
* @label: Identifer to the region which is unique to the VM.
* @flags: Flags for memory parcel behavior. See &enum gh_mem_flags.
* @guest_phys_addr: Location of the memory region in guest's memory space (page-aligned)
* @memory_size: Size of the region (page-aligned)
* @userspace_addr: Location of the memory region in caller (userspace)'s memory
*
* See Documentation/virt/gunyah/vm-manager.rst for further details.
*/
struct gunyah_userspace_memory_region {
__u32 label;
__u32 flags;
__u64 guest_phys_addr;
__u64 memory_size;
__u64 userspace_addr;
};
#define GH_VM_SET_USER_MEM_REGION _IOW(GUNYAH_IOCTL_TYPE, 0x1, \
struct gunyah_userspace_memory_region)
#define GH_ANDROID_IOCTL_TYPE 'A'
#define GH_VM_ANDROID_LEND_USER_MEM _IOW(GH_ANDROID_IOCTL_TYPE, 0x11, \
struct gunyah_userspace_memory_region)
#endif #endif