@@ -22,22 +22,27 @@ void swap_write_unplug(struct swap_iocb *sio);
int swap_writepage(struct page *page, struct writeback_control *wbc);
void __swap_writepage(struct folio *folio, struct writeback_control *wbc);
-/* linux/mm/swap_state.c */
-/* One swap address space for each 64M swap space */
+/* Return the swap device position of the swap slot. */
+static inline loff_t swap_slot_pos(swp_slot_t slot)
+{
+ return ((loff_t)swp_slot_offset(slot)) << PAGE_SHIFT;
+}
+
#define SWAP_ADDRESS_SPACE_SHIFT 14
#define SWAP_ADDRESS_SPACE_PAGES (1 << SWAP_ADDRESS_SPACE_SHIFT)
#define SWAP_ADDRESS_SPACE_MASK (SWAP_ADDRESS_SPACE_PAGES - 1)
+
+/* linux/mm/swap_state.c */
+#ifdef CONFIG_VIRTUAL_SWAP
+extern struct address_space *swap_address_space(swp_entry_t entry);
+#define swap_cache_index(entry) entry.val
+#else
+/* One swap address space for each 64M swap space */
extern struct address_space *swapper_spaces[];
#define swap_address_space(entry) \
(&swapper_spaces[swp_type(entry)][swp_offset(entry) \
>> SWAP_ADDRESS_SPACE_SHIFT])
-/* Return the swap device position of the swap slot. */
-static inline loff_t swap_slot_pos(swp_slot_t slot)
-{
- return ((loff_t)swp_slot_offset(slot)) << PAGE_SHIFT;
-}
-
/*
* Return the swap cache index of the swap entry.
*/
@@ -46,6 +51,7 @@ static inline pgoff_t swap_cache_index(swp_entry_t entry)
BUILD_BUG_ON((SWP_OFFSET_MASK | SWAP_ADDRESS_SPACE_MASK) != SWP_OFFSET_MASK);
return swp_offset(entry) & SWAP_ADDRESS_SPACE_MASK;
}
+#endif
void show_swap_cache_info(void);
bool add_to_swap(struct folio *folio);
@@ -38,8 +38,18 @@ static const struct address_space_operations swap_aops = {
#endif
};
+#ifdef CONFIG_VIRTUAL_SWAP
+static struct address_space swapper_space __read_mostly;
+
+struct address_space *swap_address_space(swp_entry_t entry)
+{
+ return &swapper_space;
+}
+#else
struct address_space *swapper_spaces[MAX_SWAPFILES] __read_mostly;
static unsigned int nr_swapper_spaces[MAX_SWAPFILES] __read_mostly;
+#endif
+
static bool enable_vma_readahead __read_mostly = true;
#define SWAP_RA_ORDER_CEILING 5
@@ -718,23 +728,34 @@ struct folio *swap_cluster_readahead(swp_entry_t entry, gfp_t gfp_mask,
return folio;
}
+static void init_swapper_space(struct address_space *space)
+{
+ xa_init_flags(&space->i_pages, XA_FLAGS_LOCK_IRQ);
+ atomic_set(&space->i_mmap_writable, 0);
+ space->a_ops = &swap_aops;
+ /* swap cache doesn't use writeback related tags */
+ mapping_set_no_writeback_tags(space);
+}
+
+#ifdef CONFIG_VIRTUAL_SWAP
+int init_swap_address_space(unsigned int type, unsigned long nr_pages)
+{
+ return 0;
+}
+
+void exit_swap_address_space(unsigned int type) {}
+#else
int init_swap_address_space(unsigned int type, unsigned long nr_pages)
{
- struct address_space *spaces, *space;
+ struct address_space *spaces;
unsigned int i, nr;
nr = DIV_ROUND_UP(nr_pages, SWAP_ADDRESS_SPACE_PAGES);
spaces = kvcalloc(nr, sizeof(struct address_space), GFP_KERNEL);
if (!spaces)
return -ENOMEM;
- for (i = 0; i < nr; i++) {
- space = spaces + i;
- xa_init_flags(&space->i_pages, XA_FLAGS_LOCK_IRQ);
- atomic_set(&space->i_mmap_writable, 0);
- space->a_ops = &swap_aops;
- /* swap cache doesn't use writeback related tags */
- mapping_set_no_writeback_tags(space);
- }
+ for (i = 0; i < nr; i++)
+ init_swapper_space(spaces + i);
nr_swapper_spaces[type] = nr;
swapper_spaces[type] = spaces;
@@ -752,6 +773,7 @@ void exit_swap_address_space(unsigned int type)
nr_swapper_spaces[type] = 0;
swapper_spaces[type] = NULL;
}
+#endif
static int swap_vma_ra_win(struct vm_fault *vmf, unsigned long *start,
unsigned long *end)
@@ -930,6 +952,10 @@ static int __init swap_init_sysfs(void)
int err;
struct kobject *swap_kobj;
+#ifdef CONFIG_VIRTUAL_SWAP
+ init_swapper_space(&swapper_space);
+#endif
+
err = vswap_init();
if (err) {
pr_err("failed to initialize virtual swap space\n");
@@ -203,8 +203,6 @@ struct zswap_entry {
struct list_head lru;
};
-static struct xarray *zswap_trees[MAX_SWAPFILES];
-static unsigned int nr_zswap_trees[MAX_SWAPFILES];
/* RCU-protected iteration */
static LIST_HEAD(zswap_pools);
@@ -231,12 +229,28 @@ static bool zswap_has_pool;
* helpers and fwd declarations
**********************************/
+#ifdef CONFIG_VIRTUAL_SWAP
+static DEFINE_XARRAY(zswap_tree);
+
+static inline struct xarray *swap_zswap_tree(swp_entry_t swp)
+{
+ return &zswap_tree;
+}
+
+#define zswap_tree_index(entry) entry.val
+#else
+static struct xarray *zswap_trees[MAX_SWAPFILES];
+static unsigned int nr_zswap_trees[MAX_SWAPFILES];
+
static inline struct xarray *swap_zswap_tree(swp_entry_t swp)
{
return &zswap_trees[swp_type(swp)][swp_offset(swp)
>> SWAP_ADDRESS_SPACE_SHIFT];
}
+#define zswap_tree_index(entry) swp_offset(entry)
+#endif
+
#define zswap_pool_debug(msg, p) \
pr_debug("%s pool %s/%s\n", msg, (p)->tfm_name, \
zpool_get_type((p)->zpool))
@@ -1047,7 +1061,7 @@ static int zswap_writeback_entry(struct zswap_entry *entry,
swp_entry_t swpentry)
{
struct xarray *tree;
- pgoff_t offset = swp_offset(swpentry);
+ pgoff_t offset = zswap_tree_index(swpentry);
struct folio *folio;
struct mempolicy *mpol;
bool folio_was_allocated;
@@ -1463,7 +1477,7 @@ static bool zswap_store_page(struct page *page,
goto compress_failed;
old = xa_store(swap_zswap_tree(page_swpentry),
- swp_offset(page_swpentry),
+ zswap_tree_index(page_swpentry),
entry, GFP_KERNEL);
if (xa_is_err(old)) {
int err = xa_err(old);
@@ -1612,7 +1626,7 @@ bool zswap_store(struct folio *folio)
bool zswap_load(struct folio *folio)
{
swp_entry_t swp = folio->swap;
- pgoff_t offset = swp_offset(swp);
+ pgoff_t offset = zswap_tree_index(swp);
bool swapcache = folio_test_swapcache(folio);
struct xarray *tree = swap_zswap_tree(swp);
struct zswap_entry *entry;
@@ -1670,7 +1684,7 @@ bool zswap_load(struct folio *folio)
void zswap_invalidate(swp_entry_t swp)
{
- pgoff_t offset = swp_offset(swp);
+ pgoff_t offset = zswap_tree_index(swp);
struct xarray *tree = swap_zswap_tree(swp);
struct zswap_entry *entry;
@@ -1682,6 +1696,16 @@ void zswap_invalidate(swp_entry_t swp)
zswap_entry_free(entry);
}
+#ifdef CONFIG_VIRTUAL_SWAP
+int zswap_swapon(int type, unsigned long nr_pages)
+{
+ return 0;
+}
+
+void zswap_swapoff(int type)
+{
+}
+#else
int zswap_swapon(int type, unsigned long nr_pages)
{
struct xarray *trees, *tree;
@@ -1718,6 +1742,8 @@ void zswap_swapoff(int type)
nr_zswap_trees[type] = 0;
zswap_trees[type] = NULL;
}
+#endif /* CONFIG_VIRTUAL_SWAP */
+
/*********************************
* debugfs functions
Currently, the swap cache code assumes that the swap space is of a fixed size. The virtual swap space is dynamically sized, so the existing partitioning code cannot be easily reused. A dynamic partitioning is planned, but for now keep the design simple and just use a flat swapcache for vswap. Similar to swap cache, the zswap tree code, specifically the range partition logic, can no longer easily be reused for the new virtual swap space design. Use a simple unified zswap tree in the new implementation for now. As in the case of swap cache, range partitioning is planned as a follow up work. Since the vswap's implementation has begun to diverge from the old implementation, we also introduce a new build config (CONFIG_VIRTUAL_SWAP). Users who do not select this config will get the old implementation, with no behavioral change. Signed-off-by: Nhat Pham <nphamcs@gmail.com> --- mm/swap.h | 22 ++++++++++++++-------- mm/swap_state.c | 44 +++++++++++++++++++++++++++++++++++--------- mm/zswap.c | 38 ++++++++++++++++++++++++++++++++------ 3 files changed, 81 insertions(+), 23 deletions(-)