Avoid taking global lock to destroy zfsdev state

We have exclusive access to our zfsdev state object in this section
until it is invalidated by setting zs_minor to -1, so we can destroy
the state without taking a lock if we do the invalidation last, after
a member to ensure correct ordering.

While here, strengthen the assertions that zs_minor is valid when we
enter.

Reviewed-by: Brian Behlendorf <[email protected]>
Reviewed-by: Alexander Motin <[email protected]>
Signed-off-by: Ryan Moeller <[email protected]>
Closes #11751

module/os/freebsd/zfs/kmod_core.c

@@ -185,18 +185,14 @@ zfsdev_close(void *data)
 	zfsdev_state_t *zs = data;
 
 	ASSERT(zs != NULL);
+	ASSERT3S(zs->zs_minor, >, 0);
 
-	mutex_enter(&zfsdev_state_lock);
-
-	ASSERT(zs->zs_minor != 0);
-
-	zs->zs_minor = -1;
 	zfs_onexit_destroy(zs->zs_onexit);
 	zfs_zevent_destroy(zs->zs_zevent);
 	zs->zs_onexit = NULL;
 	zs->zs_zevent = NULL;
-
-	mutex_exit(&zfsdev_state_lock);
+	membar_producer();
+	zs->zs_minor = -1;
 }
 
 static int

module/os/linux/zfs/zfs_ioctl_os.c

@@ -136,22 +136,20 @@ zfsdev_state_init(struct file *filp)
 	return (0);
 }
 
-static int
+static void
 zfsdev_state_destroy(struct file *filp)
 {
-	zfsdev_state_t *zs;
+	zfsdev_state_t *zs = filp->private_data;
 
-	ASSERT(MUTEX_HELD(&zfsdev_state_lock));
-	ASSERT(filp->private_data != NULL);
+	ASSERT(zs != NULL);
+	ASSERT3S(zs->zs_minor, >, 0);
 
-	zs = filp->private_data;
-	zs->zs_minor = -1;
 	zfs_onexit_destroy(zs->zs_onexit);
 	zfs_zevent_destroy(zs->zs_zevent);
 	zs->zs_onexit = NULL;
 	zs->zs_zevent = NULL;
-
-	return (0);
+	membar_producer();
+	zs->zs_minor = -1;
 }
 
 static int
@@ -169,13 +167,9 @@ zfsdev_open(struct inode *ino, struct file *filp)
 static int
 zfsdev_release(struct inode *ino, struct file *filp)
 {
-	int error;
-
-	mutex_enter(&zfsdev_state_lock);
-	error = zfsdev_state_destroy(filp);
-	mutex_exit(&zfsdev_state_lock);
+	zfsdev_state_destroy(filp);
 
-	return (-error);
+	return (0);
 }
 
 static long