vfio/type1: Reduce repetitive calls in vfio_pin_pages_remote()
vfio_pin_pages_remote() is typically called to iterate over a range of memory. Testing CAP_IPC_LOCK is relatively expensive, so it makes sense to push it up to the caller, which can then repeatedly call vfio_pin_pages_remote() using that value. This can show nearly a 20% improvement on the worst case path through VFIO_IOMMU_MAP_DMA with contiguous page mapping disabled. Testing RLIMIT_MEMLOCK is much more lightweight, but we bring it along on the same principle and it does seem to show a marginal improvement. Reviewed-by: Peter Xu <peterx@redhat.com> Reviewed-by: Kirti Wankhede <kwankhede@nvidia.com> Signed-off-by: Alex Williamson <alex.williamson@redhat.com>
This commit is contained in:
parent
80dbe1fbaf
commit
7cb671e7a3
|
@ -380,10 +380,10 @@ static int vaddr_get_pfn(struct mm_struct *mm, unsigned long vaddr,
|
||||||
* first page and all consecutive pages with the same locking.
|
* first page and all consecutive pages with the same locking.
|
||||||
*/
|
*/
|
||||||
static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr,
|
static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr,
|
||||||
long npage, unsigned long *pfn_base)
|
long npage, unsigned long *pfn_base,
|
||||||
|
bool lock_cap, unsigned long limit)
|
||||||
{
|
{
|
||||||
unsigned long pfn = 0, limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
|
unsigned long pfn = 0;
|
||||||
bool lock_cap = capable(CAP_IPC_LOCK);
|
|
||||||
long ret, pinned = 0, lock_acct = 0;
|
long ret, pinned = 0, lock_acct = 0;
|
||||||
bool rsvd;
|
bool rsvd;
|
||||||
dma_addr_t iova = vaddr - dma->vaddr + dma->iova;
|
dma_addr_t iova = vaddr - dma->vaddr + dma->iova;
|
||||||
|
@ -924,13 +924,15 @@ static int vfio_pin_map_dma(struct vfio_iommu *iommu, struct vfio_dma *dma,
|
||||||
unsigned long vaddr = dma->vaddr;
|
unsigned long vaddr = dma->vaddr;
|
||||||
size_t size = map_size;
|
size_t size = map_size;
|
||||||
long npage;
|
long npage;
|
||||||
unsigned long pfn;
|
unsigned long pfn, limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
|
||||||
|
bool lock_cap = capable(CAP_IPC_LOCK);
|
||||||
int ret = 0;
|
int ret = 0;
|
||||||
|
|
||||||
while (size) {
|
while (size) {
|
||||||
/* Pin a contiguous chunk of memory */
|
/* Pin a contiguous chunk of memory */
|
||||||
npage = vfio_pin_pages_remote(dma, vaddr + dma->size,
|
npage = vfio_pin_pages_remote(dma, vaddr + dma->size,
|
||||||
size >> PAGE_SHIFT, &pfn);
|
size >> PAGE_SHIFT, &pfn,
|
||||||
|
lock_cap, limit);
|
||||||
if (npage <= 0) {
|
if (npage <= 0) {
|
||||||
WARN_ON(!npage);
|
WARN_ON(!npage);
|
||||||
ret = (int)npage;
|
ret = (int)npage;
|
||||||
|
@ -1040,6 +1042,8 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu,
|
||||||
{
|
{
|
||||||
struct vfio_domain *d;
|
struct vfio_domain *d;
|
||||||
struct rb_node *n;
|
struct rb_node *n;
|
||||||
|
unsigned long limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
|
||||||
|
bool lock_cap = capable(CAP_IPC_LOCK);
|
||||||
int ret;
|
int ret;
|
||||||
|
|
||||||
/* Arbitrarily pick the first domain in the list for lookups */
|
/* Arbitrarily pick the first domain in the list for lookups */
|
||||||
|
@ -1086,7 +1090,8 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu,
|
||||||
|
|
||||||
npage = vfio_pin_pages_remote(dma, vaddr,
|
npage = vfio_pin_pages_remote(dma, vaddr,
|
||||||
n >> PAGE_SHIFT,
|
n >> PAGE_SHIFT,
|
||||||
&pfn);
|
&pfn, lock_cap,
|
||||||
|
limit);
|
||||||
if (npage <= 0) {
|
if (npage <= 0) {
|
||||||
WARN_ON(!npage);
|
WARN_ON(!npage);
|
||||||
ret = (int)npage;
|
ret = (int)npage;
|
||||||
|
|
Loading…
Reference in New Issue