diff mbox series

[RFC,42/48] target: cluster: sync SPC-2 reservations

Message ID 20220803162857.27770-43-d.bogdanov@yadro.com
State New
Headers show
Series Target cluster implementation over DLM | expand

Commit Message

Dmitry Bogdanov Dec. 16, 2021, 10:20 a.m. UTC
Reservation status is stored in DLM_CKV as part of PR data.
If a reservation holder node is left the cluster other nodes will clear
reservation status locally.

Signed-off-by: Dmitry Bogdanov <d.bogdanov@yadro.com>
---
 drivers/target/dlm_ckv.c               | 16 ++++++-
 drivers/target/dlm_ckv.h               |  4 +-
 drivers/target/target_cluster_dlm.c    | 59 +++++++++++++++++++++++++-
 drivers/target/target_core_device.c    | 10 +++++
 drivers/target/target_core_pr.c        | 22 +++++++---
 drivers/target/target_core_pr.h        |  1 +
 drivers/target/target_core_tmr.c       |  4 +-
 drivers/target/target_core_transport.c |  2 +-
 include/target/target_core_base.h      |  1 +
 9 files changed, 104 insertions(+), 15 deletions(-)
diff mbox series

Patch

diff --git a/drivers/target/dlm_ckv.c b/drivers/target/dlm_ckv.c
index 417159f18fc6..f4e7bc410a08 100644
--- a/drivers/target/dlm_ckv.c
+++ b/drivers/target/dlm_ckv.c
@@ -49,6 +49,7 @@  struct dlm_ckv_bucket {
 	size_t num_nodes;
 	int nodeid[64];
 	void *userarg;
+	dlm_ckv_nodeleft_cb nodeleft_cb;
 	struct completion sync_compl;
 	struct workqueue_struct *notify_wq;
 };
@@ -79,7 +80,12 @@  static void dlm_ckv_recover_prep(void *arg)
 
 static void dlm_ckv_recover_slot(void *arg, struct dlm_slot *slot)
 {
-	pr_info("nodeid %d left the cluster\n", slot->nodeid);
+	struct dlm_ckv_bucket *bucket = arg;
+
+	pr_debug("nodeid %d left the cluster\n", slot->nodeid);
+
+	if (bucket->nodeleft_cb)
+		bucket->nodeleft_cb(bucket->userarg, slot->nodeid);
 }
 
 /* dlm calls after recover_slot and after it completes lock recovery */
@@ -667,7 +673,8 @@  static void bucket_release(struct kref *ref)
 }
 
 struct dlm_ckv_bucket *
-dlm_ckv_open_bucket(const char *name, const char *cluster_name, void *userarg)
+dlm_ckv_open_bucket(const char *name, const char *cluster_name, void *userarg,
+		    dlm_ckv_nodeleft_cb nodeleft_cb, int *local_nodeid)
 {
 	struct dlm_ckv_bucket *bucket;
 	int name_len = strlen(name);
@@ -677,6 +684,9 @@  dlm_ckv_open_bucket(const char *name, const char *cluster_name, void *userarg)
 	if (!name)
 		return ERR_PTR(-EINVAL);
 
+	if (!local_nodeid)
+		return ERR_PTR(-EINVAL);
+
 	if (name_len > DLM_LOCKSPACE_LEN)
 		return ERR_PTR(-EINVAL);
 
@@ -684,6 +694,7 @@  dlm_ckv_open_bucket(const char *name, const char *cluster_name, void *userarg)
 	kref_init(&bucket->refcount);
 
 	bucket->userarg = userarg;
+	bucket->nodeleft_cb = nodeleft_cb;
 	init_completion(&bucket->sync_compl);
 	bucket->notify_wq = alloc_ordered_workqueue("notify_wq-%s", 0, name);
 
@@ -708,6 +719,7 @@  dlm_ckv_open_bucket(const char *name, const char *cluster_name, void *userarg)
 		goto fail_init;
 	}
 
+	*local_nodeid = bucket->local_nodeid;
 	return bucket;
 
 fail_init:
diff --git a/drivers/target/dlm_ckv.h b/drivers/target/dlm_ckv.h
index e8045917067e..c5a69486a3bf 100644
--- a/drivers/target/dlm_ckv.h
+++ b/drivers/target/dlm_ckv.h
@@ -14,7 +14,9 @@  typedef void (*dlm_ckv_nodeleft_cb)(void *arg, int nodeid);
 
 struct dlm_ckv_bucket *dlm_ckv_open_bucket(const char *name,
 					   const char *cluster_name,
-					   void *userarg);
+					   void *userarg,
+					   dlm_ckv_nodeleft_cb nodeleft_cb,
+					   int *local_nodeid);
 int dlm_ckv_close_bucket(struct dlm_ckv_bucket *bucket);
 
 struct dlm_ckv_lock *
diff --git a/drivers/target/target_cluster_dlm.c b/drivers/target/target_cluster_dlm.c
index 16391a8661fa..4c4d337e6a1a 100644
--- a/drivers/target/target_cluster_dlm.c
+++ b/drivers/target/target_cluster_dlm.c
@@ -98,6 +98,7 @@  struct async_group {
 };
 
 static void target_pr_sync_cb(void *arg);
+static void target_nodeleft_cb(void *arg, int nodeid);
 static int pr_reg_realloc(struct target_cluster_data *cluster_data,
 			  size_t nr_registrants);
 
@@ -128,7 +129,8 @@  static int target_init_dlm(struct se_device *dev)
 
 	cluster_data->bucket = dlm_ckv_open_bucket(ls_name,
 						cluster_cfg.cluster_name,
-						dev);
+						dev, target_nodeleft_cb,
+						&cluster_data->local_nodeid);
 	if (!cluster_data->bucket) {
 		err = -EIO;
 		kfree(cluster_data);
@@ -679,6 +681,27 @@  static void target_pr_sync_cb(void *arg)
 	atomic_set(&dev->t10_pr.pr_generation, pr_data.pr_generation);
 	dev->t10_pr.pr_aptpl_active = pr_data.pr_aptpl;
 
+	/* update SPC-2 reservation */
+	cluster_data->reserved_node_id = pr_data.reserved_by_nodeid;
+	spin_lock(&dev->dev_reservation_lock);
+	if (cluster_data->reserved_node_id == cluster_data->local_nodeid &&
+	    dev->reservation_holder == NULL) {
+		pr_err("TARGET_CORE[%d]:SCP-2 reservation is out of sync. Cluster thinks that we are reservation holder\n",
+			dev->dev_index);
+	}
+	if (cluster_data->reserved_node_id != cluster_data->local_nodeid &&
+	    dev->reservation_holder != NULL) {
+		pr_err("TARGET_CORE[%d]: SPC-2 reservation is out of sync. Possible split-brain. Cluster thinks that reservation holder is other node\n",
+			dev->dev_index);
+		dev->reservation_holder = NULL;
+	}
+	if (cluster_data->reserved_node_id != 0)
+		dev->dev_reservation_flags |= DRF_SPC2_RESERVATIONS;
+	else
+		target_release_reservation(dev);
+
+	spin_unlock(&dev->dev_reservation_lock);
+
 	/* update reservation */
 	spin_lock(&dev->dev_reservation_lock);
 	if (pr_prev_res_holder &&
@@ -703,6 +726,39 @@  static void target_pr_sync_cb(void *arg)
 	kfree(pr_reg_data);
 }
 
+static void
+target_spc2_reserve(struct se_device *dev, struct se_session *sess)
+{
+	struct target_cluster_data *cluster_data = dev->cluster_data;
+
+	target_pr_lock_dlm(dev);
+
+	if (sess) {
+		target_get_reservation(dev, sess);
+		cluster_data->reserved_node_id = cluster_data->local_nodeid;
+	} else {
+		target_release_reservation(dev);
+		cluster_data->reserved_node_id = 0;
+	}
+
+	target_pr_sync_dlm(dev);
+	target_pr_unlock_dlm(dev);
+}
+
+static void target_nodeleft_cb(void *arg, int nodeid)
+{
+	struct se_device *dev = arg;
+	struct target_cluster_data *cluster_data = dev->cluster_data;
+
+	if (cluster_data->reserved_node_id == nodeid &&
+	    dev->dev_reservation_flags & DRF_SPC2_RESERVATIONS) {
+		pr_info("TARGET_CORE[%d]: SPC-2 reservation holder left the cluster\n",
+			dev->dev_index);
+		dev->reservation_holder = NULL;
+		cluster_data->reserved_node_id = 0;
+	}
+}
+
 const struct target_cluster_ops dlm_cluster_ops = {
 	.name = "dlm",
 	.owner = THIS_MODULE,
@@ -715,6 +771,7 @@  const struct target_cluster_ops dlm_cluster_ops = {
 	.pr_lock = target_pr_lock_dlm,
 	.pr_unlock = target_pr_unlock_dlm,
 	.pr_sync = target_pr_sync_dlm,
+	.reserve = target_spc2_reserve,
 };
 
 static int __init target_cluster_dlm_module_init(void)
diff --git a/drivers/target/target_core_device.c b/drivers/target/target_core_device.c
index 297bf5985f48..9f01a28ef72f 100644
--- a/drivers/target/target_core_device.c
+++ b/drivers/target/target_core_device.c
@@ -788,6 +788,15 @@  int target_dummy_nodlm(struct se_device *dev)
 	return 0;
 }
 
+static void target_reserve2_nodlm(struct se_device *dev, struct se_session *sess)
+{
+	if (sess) {
+		target_get_reservation(dev, sess);
+	} else {
+		target_release_reservation(dev);
+	}
+}
+
 const struct target_cluster_ops nodlm_cluster_ops = {
 	.name = "single",
 	.owner = NULL,
@@ -798,6 +807,7 @@  const struct target_cluster_ops nodlm_cluster_ops = {
 	.pr_lock = target_dummy_nodlm,
 	.pr_unlock = target_dummy_nodlm,
 	.pr_sync = target_dummy_nodlm,
+	.reserve = target_reserve2_nodlm,
 };
 
 struct se_device *target_alloc_device(struct se_hba *hba, const char *name)
diff --git a/drivers/target/target_core_pr.c b/drivers/target/target_core_pr.c
index 22f2aa91cf2e..930f0bebd75d 100644
--- a/drivers/target/target_core_pr.c
+++ b/drivers/target/target_core_pr.c
@@ -192,6 +192,7 @@  void target_release_reservation(struct se_device *dev)
 		dev->dev_reservation_flags &= ~DRF_SPC2_RESERVATIONS_WITH_ISID;
 	}
 }
+EXPORT_SYMBOL(target_release_reservation);
 
 sense_reason_t
 target_scsi2_reservation_release(struct se_cmd *cmd)
@@ -222,7 +223,7 @@  target_scsi2_reservation_release(struct se_cmd *cmd)
 	if (dev->dev_res_bin_isid != sess->sess_bin_isid)
 		goto out_unlock;
 
-	target_release_reservation(dev);
+	dev->cl_ops->reserve(dev, NULL);
 	tpg = sess->se_tpg;
 	pr_debug("SCSI-2 Released reservation for %s LUN: %llu ->"
 		" MAPPED LUN: %llu for %s\n",
@@ -237,6 +238,17 @@  target_scsi2_reservation_release(struct se_cmd *cmd)
 	return 0;
 }
 
+void target_get_reservation(struct se_device *dev, struct se_session *sess)
+{
+	dev->reservation_holder = sess;
+	dev->dev_reservation_flags |= DRF_SPC2_RESERVATIONS;
+	if (sess->sess_bin_isid != 0) {
+		dev->dev_res_bin_isid = sess->sess_bin_isid;
+		dev->dev_reservation_flags |= DRF_SPC2_RESERVATIONS_WITH_ISID;
+	}
+}
+EXPORT_SYMBOL(target_get_reservation);
+
 sense_reason_t
 target_scsi2_reservation_reserve(struct se_cmd *cmd)
 {
@@ -284,12 +296,8 @@  target_scsi2_reservation_reserve(struct se_cmd *cmd)
 		goto out_unlock;
 	}
 
-	dev->reservation_holder = sess;
-	dev->dev_reservation_flags |= DRF_SPC2_RESERVATIONS;
-	if (sess->sess_bin_isid != 0) {
-		dev->dev_res_bin_isid = sess->sess_bin_isid;
-		dev->dev_reservation_flags |= DRF_SPC2_RESERVATIONS_WITH_ISID;
-	}
+	dev->cl_ops->reserve(dev, sess);
+
 	pr_debug("SCSI-2 Reserved %s LUN: %llu -> MAPPED LUN: %llu"
 		" for %s\n", tpg->se_tpg_tfo->fabric_name,
 		cmd->se_lun->unpacked_lun, cmd->orig_fe_lun,
diff --git a/drivers/target/target_core_pr.h b/drivers/target/target_core_pr.h
index e7eff50a01f2..a3a6013dc530 100644
--- a/drivers/target/target_core_pr.h
+++ b/drivers/target/target_core_pr.h
@@ -59,6 +59,7 @@  extern struct kmem_cache *t10_pr_reg_cache;
 extern void core_pr_dump_initiator_port(struct t10_pr_registration *,
 			char *, u32);
 extern void target_release_reservation(struct se_device *dev);
+extern void target_get_reservation(struct se_device *dev, struct se_session *sess);
 extern sense_reason_t target_scsi2_reservation_release(struct se_cmd *);
 extern sense_reason_t target_scsi2_reservation_reserve(struct se_cmd *);
 extern int core_scsi3_alloc_aptpl_registration(
diff --git a/drivers/target/target_core_tmr.c b/drivers/target/target_core_tmr.c
index 2db45efedd97..26df0f685499 100644
--- a/drivers/target/target_core_tmr.c
+++ b/drivers/target/target_core_tmr.c
@@ -406,9 +406,7 @@  int core_tmr_lun_reset(
 	 * LOGICAL UNIT RESET
 	 */
 	if (dev->dev_reservation_flags & DRF_SPC2_RESERVATIONS) {
-		spin_lock(&dev->dev_reservation_lock);
-		target_release_reservation(dev);
-		spin_unlock(&dev->dev_reservation_lock);
+		dev->cl_ops->reserve(dev, NULL);
 		pr_debug("LUN_RESET: SCSI-2 Released reservation\n");
 	}
 
diff --git a/drivers/target/target_core_transport.c b/drivers/target/target_core_transport.c
index d3cbba294510..47b44a1e6925 100644
--- a/drivers/target/target_core_transport.c
+++ b/drivers/target/target_core_transport.c
@@ -612,7 +612,7 @@  static int target_release_res(struct se_device *dev, void *data)
 	struct se_session *sess = data;
 
 	if (dev->reservation_holder == sess)
-		target_release_reservation(dev);
+		dev->cl_ops->reserve(dev, NULL);
 	return 0;
 }
 
diff --git a/include/target/target_core_base.h b/include/target/target_core_base.h
index b0f0a6b93f6e..c49ddf0828f6 100644
--- a/include/target/target_core_base.h
+++ b/include/target/target_core_base.h
@@ -790,6 +790,7 @@  struct target_cluster_ops {
 	int (*pr_lock)(struct se_device *dev);
 	int (*pr_unlock)(struct se_device *dev);
 	int (*pr_sync)(struct se_device *dev);
+	void (*reserve)(struct se_device *dev, struct se_session *sess);
 };
 
 struct se_device {