@@ -525,9 +525,53 @@ static int pid_maps_open(struct inode *inode, struct file *file)
PROCMAP_QUERY_VMA_FLAGS \
)
-static int query_vma_setup(struct mm_struct *mm)
+#ifdef CONFIG_PER_VMA_LOCK
+
+static int query_vma_setup(struct proc_maps_private *priv)
{
- return mmap_read_lock_killable(mm);
+ if (!mmap_lock_speculate_try_begin(priv->mm, &priv->mm_wr_seq)) {
+ int ret = mmap_read_lock_killable(priv->mm);
+
+ if (ret)
+ return ret;
+
+ /* mmap_lock_speculate_try_begin() succeeds when holding mmap_read_lock */
+ mmap_lock_speculate_try_begin(priv->mm, &priv->mm_wr_seq);
+ mmap_read_unlock(priv->mm);
+ }
+
+ memset(&priv->vma_copy, 0, sizeof(priv->vma_copy));
+ rcu_read_lock();
+
+ return 0;
+}
+
+static void query_vma_teardown(struct mm_struct *mm, struct vm_area_struct *vma)
+{
+ rcu_read_unlock();
+}
+
+static struct vm_area_struct *query_vma_find_by_addr(struct proc_maps_private *priv,
+ unsigned long addr)
+{
+ struct vm_area_struct *vma;
+
+ vma_iter_init(&priv->iter, priv->mm, addr);
+ vma = vma_next(&priv->iter);
+ if (!vma)
+ return NULL;
+
+ vma = get_stable_vma(vma, priv, addr);
+
+ /* The only possible error is EINTR, just pretend we found nothing */
+ return IS_ERR(vma) ? NULL : vma;
+}
+
+#else /* CONFIG_PER_VMA_LOCK */
+
+static int query_vma_setup(struct proc_maps_private *priv)
+{
+ return mmap_read_lock_killable(priv->mm);
}
static void query_vma_teardown(struct mm_struct *mm, struct vm_area_struct *vma)
@@ -535,18 +579,21 @@ static void query_vma_teardown(struct mm_struct *mm, struct vm_area_struct *vma)
mmap_read_unlock(mm);
}
-static struct vm_area_struct *query_vma_find_by_addr(struct mm_struct *mm, unsigned long addr)
+static struct vm_area_struct *query_vma_find_by_addr(struct proc_maps_private *priv,
+ unsigned long addr)
{
- return find_vma(mm, addr);
+ return find_vma(priv->mm, addr);
}
-static struct vm_area_struct *query_matching_vma(struct mm_struct *mm,
+#endif /* CONFIG_PER_VMA_LOCK */
+
+static struct vm_area_struct *query_matching_vma(struct proc_maps_private *priv,
unsigned long addr, u32 flags)
{
struct vm_area_struct *vma;
next_vma:
- vma = query_vma_find_by_addr(mm, addr);
+ vma = query_vma_find_by_addr(priv, addr);
if (!vma)
goto no_vma;
@@ -622,13 +669,13 @@ static int do_procmap_query(struct proc_maps_private *priv, void __user *uarg)
if (!mm || !mmget_not_zero(mm))
return -ESRCH;
- err = query_vma_setup(mm);
+ err = query_vma_setup(priv);
if (err) {
mmput(mm);
return err;
}
- vma = query_matching_vma(mm, karg.query_addr, karg.query_flags);
+ vma = query_matching_vma(priv, karg.query_addr, karg.query_flags);
if (IS_ERR(vma)) {
err = PTR_ERR(vma);
vma = NULL;
Utilize speculative vma lookup to find and snapshot a vma without taking mmap_lock during PROCMAP_QUERY ioctl execution. Concurrent address space modifications are detected and the lookup is retried. While we take the mmap_lock for reading during such contention, we do that momentarily only to record new mm_wr_seq counter. This change is designed to reduce mmap_lock contention and prevent PROCMAP_QUERY ioctl calls (often a low priority task, such as monitoring/data collection services) from blocking address space updates. Signed-off-by: Suren Baghdasaryan <surenb@google.com> --- fs/proc/task_mmu.c | 63 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 55 insertions(+), 8 deletions(-)