criu: page-pipe: introduce page_pipe_split

This will allow to split a ppb so that data residing at specified address
will be immediately available

Signed-off-by: Mike Rapoport <[email protected]>
Signed-off-by: Pavel Emelyanov <[email protected]>

criu/include/page-pipe.h

@@ -109,4 +109,7 @@ extern int page_pipe_add_hole(struct page_pipe *p, unsigned long addr);
 extern void debug_show_page_pipe(struct page_pipe *pp);
 void page_pipe_reinit(struct page_pipe *pp);
 
+extern int page_pipe_split(struct page_pipe *pp, unsigned long addr,
+			   unsigned int *nr_pages);
+
 #endif /* __CR_PAGE_PIPE_H__ */

criu/page-pipe.c

@@ -81,6 +81,25 @@ static int ppb_resize_pipe(struct page_pipe_buf *ppb, unsigned long new_size)
 	return 0;
 }
 
+static struct page_pipe_buf *ppb_alloc_resize(struct page_pipe *pp, int size)
+{
+	struct page_pipe_buf *ppb;
+	int nr_pages = size / PAGE_SIZE;
+
+	ppb = ppb_alloc(pp);
+	if (!ppb)
+		return NULL;
+
+	if (ppb->pipe_size < nr_pages) {
+		if (ppb_resize_pipe(ppb, nr_pages)) {
+			ppb_destroy(ppb);
+			return NULL;
+		}
+	}
+
+	return ppb;
+}
+
 static int page_pipe_grow(struct page_pipe *pp, unsigned int flags)
 {
 	struct page_pipe_buf *ppb;
@@ -266,6 +285,161 @@ int page_pipe_add_hole(struct page_pipe *pp, unsigned long addr)
 	return 0;
 }
 
+/*
+ * Get ppb and iov that contain addr and count amount of data between
+ * beginning of the pipe belonging to the ppb and addr
+ */
+static struct page_pipe_buf *get_ppb(struct page_pipe *pp, unsigned long addr,
+				     struct iovec **iov_ret,
+				     unsigned long *len)
+{
+	struct page_pipe_buf *ppb;
+	int i;
+
+	list_for_each_entry(ppb, &pp->bufs, l) {
+		for (i = 0, *len = 0; i < ppb->nr_segs; i++) {
+			struct iovec *iov = &ppb->iov[i];
+			unsigned long base = (unsigned long)iov->iov_base;
+
+			if (addr < base || addr >= base + iov->iov_len) {
+				*len += iov->iov_len;
+				continue;
+			}
+
+			/* got iov that contains the addr */
+			*len += (addr - base);
+			*iov_ret = iov;
+
+			list_move(&ppb->l, &pp->bufs);
+			return ppb;
+		}
+	}
+
+	return NULL;
+}
+
+static int page_pipe_split_iov(struct page_pipe *pp, struct page_pipe_buf *ppb,
+			       struct iovec *iov, unsigned long addr,
+			       bool popup_new)
+{
+	unsigned long len = addr - (unsigned long)iov->iov_base;
+	struct page_pipe_buf *ppb_new;
+	struct iovec *iov_new;
+	int ret;
+
+	if (len == iov->iov_len)
+		return 0;
+
+	ppb_new = ppb_alloc_resize(pp, len);
+	if (!ppb_new)
+		return -1;
+
+	ret = splice(ppb->p[0], NULL, ppb_new->p[1], NULL, len, SPLICE_F_MOVE);
+	if (ret != len)
+		return -1;
+
+	iov_new = &pp->iovs[pp->free_iov++];
+	BUG_ON(pp->free_iov > pp->nr_iovs);
+	iov_new->iov_base = iov->iov_base;
+	iov_new->iov_len = len;
+
+	ppb_init(ppb_new, len / PAGE_SIZE, 1, ppb->flags, iov_new);
+
+	ppb->pages_in -= len / PAGE_SIZE;
+
+	iov->iov_base += len;
+	iov->iov_len -= len;
+
+	if (popup_new)
+		ppb = ppb_new;
+	list_move(&ppb->l, &pp->bufs);
+
+	return 0;
+}
+
+static int page_pipe_split_ppb(struct page_pipe *pp, struct page_pipe_buf *ppb,
+			       struct iovec *iov, unsigned long len)
+{
+	struct page_pipe_buf *ppb_new;
+	int ret;
+
+	ppb_new = ppb_alloc_resize(pp, len);
+	if (!ppb_new)
+		return -1;
+
+	ret = splice(ppb->p[0], NULL, ppb_new->p[1], NULL, len, SPLICE_F_MOVE);
+	if (ret != len)
+		return -1;
+
+	ppb_init(ppb_new, len / PAGE_SIZE, iov - ppb->iov, ppb->flags, ppb->iov);
+
+	ppb->iov += ppb_new->nr_segs;
+	ppb->nr_segs -= ppb_new->nr_segs;
+	ppb->pages_in -= len / PAGE_SIZE;
+
+	list_move(&ppb->l, &pp->bufs);
+
+	return 0;
+}
+
+/*
+ * Find the ppb containing addr and split so that we can splice
+ * nr_pages starting from addr. Make the ppb containing relavant pages
+ * the first entry in bb->bufs list
+ */
+int page_pipe_split(struct page_pipe *pp, unsigned long addr,
+		    unsigned int *nr_pages)
+{
+	struct page_pipe_buf *ppb;
+	struct iovec *iov = NULL;
+	unsigned long len = 0;
+	int ret;
+
+	/*
+	 * Get ppb that contains addr and count length of data between
+	 * the beginning of the pipe and addr. If no ppb is found, the
+	 * requested page is mapped to zero pfn
+	 */
+	ppb = get_ppb(pp, addr, &iov, &len);
+	if (!ppb) {
+		*nr_pages = 0;
+		return 0;
+	}
+
+	/* split origingal ppb on boundary of iov that contains addr */
+	if (iov != ppb->iov) {
+		len -= (addr - (unsigned long)iov->iov_base);
+		ret = page_pipe_split_ppb(pp, ppb, iov, len);
+		if (ret)
+			return -1;
+	}
+
+
+	/*
+	 * if address does not match iov base, split the iov and
+	 * create a new ppb pointing to the new iov
+	 */
+	if (addr != (unsigned long)iov->iov_base) {
+		ret = page_pipe_split_iov(pp, ppb, iov, addr, false);
+		if (ret)
+			return -1;
+	}
+
+	/*
+	 * at this point iov_base points to addr, so we need to split
+	 * the part after addr + requested pages to a separate iov
+	 */
+	len = min((unsigned long)iov->iov_base + iov->iov_len - addr,
+		  *nr_pages * PAGE_SIZE);
+	ret = page_pipe_split_iov(pp, ppb, iov, addr + len, true);
+	if (ret)
+		return -1;
+
+	*nr_pages = len / PAGE_SIZE;
+
+	return 0;
+}
+
 void debug_show_page_pipe(struct page_pipe *pp)
 {
 	struct page_pipe_buf *ppb;