4141#include <linux/workqueue.h>
4242#include <linux/slab.h>
4343#include <linux/spinlock.h>
44+ #include <linux/wait.h>
4445#include <linux/zpool.h>
4546#include <linux/magic.h>
4647
@@ -145,6 +146,8 @@ struct z3fold_header {
145146 * @release_wq: workqueue for safe page release
146147 * @work: work_struct for safe page release
147148 * @inode: inode for z3fold pseudo filesystem
149+ * @destroying: bool to stop migration once we start destruction
150+ * @isolated: int to count the number of pages currently in isolation
148151 *
149152 * This structure is allocated at pool creation time and maintains metadata
150153 * pertaining to a particular z3fold pool.
@@ -163,8 +166,11 @@ struct z3fold_pool {
163166 const struct zpool_ops * zpool_ops ;
164167 struct workqueue_struct * compact_wq ;
165168 struct workqueue_struct * release_wq ;
169+ struct wait_queue_head isolate_wait ;
166170 struct work_struct work ;
167171 struct inode * inode ;
172+ bool destroying ;
173+ int isolated ;
168174};
169175
170176/*
@@ -769,6 +775,7 @@ static struct z3fold_pool *z3fold_create_pool(const char *name, gfp_t gfp,
769775 goto out_c ;
770776 spin_lock_init (& pool -> lock );
771777 spin_lock_init (& pool -> stale_lock );
778+ init_waitqueue_head (& pool -> isolate_wait );
772779 pool -> unbuddied = __alloc_percpu (sizeof (struct list_head )* NCHUNKS , 2 );
773780 if (!pool -> unbuddied )
774781 goto out_pool ;
@@ -808,6 +815,15 @@ static struct z3fold_pool *z3fold_create_pool(const char *name, gfp_t gfp,
808815 return NULL ;
809816}
810817
818+ static bool pool_isolated_are_drained (struct z3fold_pool * pool )
819+ {
820+ bool ret ;
821+
822+ spin_lock (& pool -> lock );
823+ ret = pool -> isolated == 0 ;
824+ spin_unlock (& pool -> lock );
825+ return ret ;
826+ }
811827/**
812828 * z3fold_destroy_pool() - destroys an existing z3fold pool
813829 * @pool: the z3fold pool to be destroyed
@@ -817,6 +833,22 @@ static struct z3fold_pool *z3fold_create_pool(const char *name, gfp_t gfp,
817833static void z3fold_destroy_pool (struct z3fold_pool * pool )
818834{
819835 kmem_cache_destroy (pool -> c_handle );
836+ /*
837+ * We set pool-> destroying under lock to ensure that
838+ * z3fold_page_isolate() sees any changes to destroying. This way we
839+ * avoid the need for any memory barriers.
840+ */
841+
842+ spin_lock (& pool -> lock );
843+ pool -> destroying = true;
844+ spin_unlock (& pool -> lock );
845+
846+ /*
847+ * We need to ensure that no pages are being migrated while we destroy
848+ * these workqueues, as migration can queue work on either of the
849+ * workqueues.
850+ */
851+ wait_event (pool -> isolate_wait , !pool_isolated_are_drained (pool ));
820852
821853 /*
822854 * We need to destroy pool->compact_wq before pool->release_wq,
@@ -1307,6 +1339,28 @@ static u64 z3fold_get_pool_size(struct z3fold_pool *pool)
13071339 return atomic64_read (& pool -> pages_nr );
13081340}
13091341
1342+ /*
1343+ * z3fold_dec_isolated() expects to be called while pool->lock is held.
1344+ */
1345+ static void z3fold_dec_isolated (struct z3fold_pool * pool )
1346+ {
1347+ assert_spin_locked (& pool -> lock );
1348+ VM_BUG_ON (pool -> isolated <= 0 );
1349+ pool -> isolated -- ;
1350+
1351+ /*
1352+ * If we have no more isolated pages, we have to see if
1353+ * z3fold_destroy_pool() is waiting for a signal.
1354+ */
1355+ if (pool -> isolated == 0 && waitqueue_active (& pool -> isolate_wait ))
1356+ wake_up_all (& pool -> isolate_wait );
1357+ }
1358+
1359+ static void z3fold_inc_isolated (struct z3fold_pool * pool )
1360+ {
1361+ pool -> isolated ++ ;
1362+ }
1363+
13101364static bool z3fold_page_isolate (struct page * page , isolate_mode_t mode )
13111365{
13121366 struct z3fold_header * zhdr ;
@@ -1333,6 +1387,33 @@ static bool z3fold_page_isolate(struct page *page, isolate_mode_t mode)
13331387 spin_lock (& pool -> lock );
13341388 if (!list_empty (& page -> lru ))
13351389 list_del (& page -> lru );
1390+ /*
1391+ * We need to check for destruction while holding pool->lock, as
1392+ * otherwise destruction could see 0 isolated pages, and
1393+ * proceed.
1394+ */
1395+ if (unlikely (pool -> destroying )) {
1396+ spin_unlock (& pool -> lock );
1397+ /*
1398+ * If this page isn't stale, somebody else holds a
1399+ * reference to it. Let't drop our refcount so that they
1400+ * can call the release logic.
1401+ */
1402+ if (unlikely (kref_put (& zhdr -> refcount ,
1403+ release_z3fold_page_locked ))) {
1404+ /*
1405+ * If we get here we have kref problems, so we
1406+ * should freak out.
1407+ */
1408+ WARN (1 , "Z3fold is experiencing kref problems\n" );
1409+ return false;
1410+ }
1411+ z3fold_page_unlock (zhdr );
1412+ return false;
1413+ }
1414+
1415+
1416+ z3fold_inc_isolated (pool );
13361417 spin_unlock (& pool -> lock );
13371418 z3fold_page_unlock (zhdr );
13381419 return true;
@@ -1401,6 +1482,10 @@ static int z3fold_page_migrate(struct address_space *mapping, struct page *newpa
14011482
14021483 queue_work_on (new_zhdr -> cpu , pool -> compact_wq , & new_zhdr -> work );
14031484
1485+ spin_lock (& pool -> lock );
1486+ z3fold_dec_isolated (pool );
1487+ spin_unlock (& pool -> lock );
1488+
14041489 page_mapcount_reset (page );
14051490 put_page (page );
14061491 return 0 ;
@@ -1420,10 +1505,14 @@ static void z3fold_page_putback(struct page *page)
14201505 INIT_LIST_HEAD (& page -> lru );
14211506 if (kref_put (& zhdr -> refcount , release_z3fold_page_locked )) {
14221507 atomic64_dec (& pool -> pages_nr );
1508+ spin_lock (& pool -> lock );
1509+ z3fold_dec_isolated (pool );
1510+ spin_unlock (& pool -> lock );
14231511 return ;
14241512 }
14251513 spin_lock (& pool -> lock );
14261514 list_add (& page -> lru , & pool -> lru );
1515+ z3fold_dec_isolated (pool );
14271516 spin_unlock (& pool -> lock );
14281517 z3fold_page_unlock (zhdr );
14291518}
0 commit comments