2014-03-07 18:21:15 +01:00
|
|
|
/*
|
|
|
|
* Copyright (C) 2015, SUSE
|
|
|
|
*
|
|
|
|
* This program is free software; you can redistribute it and/or modify
|
|
|
|
* it under the terms of the GNU General Public License as published by
|
|
|
|
* the Free Software Foundation; either version 2, or (at your option)
|
|
|
|
* any later version.
|
|
|
|
*
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
|
|
#include <linux/module.h>
|
2014-03-07 20:49:26 +01:00
|
|
|
#include <linux/dlm.h>
|
|
|
|
#include <linux/sched.h>
|
|
|
|
#include "md.h"
|
2014-03-29 16:01:53 +01:00
|
|
|
#include "md-cluster.h"
|
2014-03-07 20:49:26 +01:00
|
|
|
|
|
|
|
#define LVB_SIZE 64
|
|
|
|
|
|
|
|
struct dlm_lock_resource {
|
|
|
|
dlm_lockspace_t *ls;
|
|
|
|
struct dlm_lksb lksb;
|
|
|
|
char *name; /* lock name. */
|
|
|
|
uint32_t flags; /* flags to pass to dlm_lock() */
|
|
|
|
struct completion completion; /* completion for synchronized locking */
|
2014-03-29 16:20:02 +01:00
|
|
|
void (*bast)(void *arg, int mode); /* blocking AST function pointer*/
|
|
|
|
struct mddev *mddev; /* pointing back to mddev. */
|
|
|
|
};
|
|
|
|
|
|
|
|
struct md_cluster_info {
|
|
|
|
/* dlm lock space and resources for clustered raid. */
|
|
|
|
dlm_lockspace_t *lockspace;
|
2014-03-30 07:42:49 +02:00
|
|
|
int slot_number;
|
|
|
|
struct completion completion;
|
2014-03-29 16:20:02 +01:00
|
|
|
struct dlm_lock_resource *sb_lock;
|
|
|
|
struct mutex sb_mutex;
|
2014-03-07 20:49:26 +01:00
|
|
|
};
|
|
|
|
|
|
|
|
static void sync_ast(void *arg)
|
|
|
|
{
|
|
|
|
struct dlm_lock_resource *res;
|
|
|
|
|
|
|
|
res = (struct dlm_lock_resource *) arg;
|
|
|
|
complete(&res->completion);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int dlm_lock_sync(struct dlm_lock_resource *res, int mode)
|
|
|
|
{
|
|
|
|
int ret = 0;
|
|
|
|
|
|
|
|
init_completion(&res->completion);
|
|
|
|
ret = dlm_lock(res->ls, mode, &res->lksb,
|
|
|
|
res->flags, res->name, strlen(res->name),
|
|
|
|
0, sync_ast, res, res->bast);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
wait_for_completion(&res->completion);
|
|
|
|
return res->lksb.sb_status;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int dlm_unlock_sync(struct dlm_lock_resource *res)
|
|
|
|
{
|
|
|
|
return dlm_lock_sync(res, DLM_LOCK_NL);
|
|
|
|
}
|
|
|
|
|
2014-03-29 16:20:02 +01:00
|
|
|
static struct dlm_lock_resource *lockres_init(struct mddev *mddev,
|
2014-03-07 20:49:26 +01:00
|
|
|
char *name, void (*bastfn)(void *arg, int mode), int with_lvb)
|
|
|
|
{
|
|
|
|
struct dlm_lock_resource *res = NULL;
|
|
|
|
int ret, namelen;
|
2014-03-29 16:20:02 +01:00
|
|
|
struct md_cluster_info *cinfo = mddev->cluster_info;
|
2014-03-07 20:49:26 +01:00
|
|
|
|
|
|
|
res = kzalloc(sizeof(struct dlm_lock_resource), GFP_KERNEL);
|
|
|
|
if (!res)
|
|
|
|
return NULL;
|
2014-03-29 16:20:02 +01:00
|
|
|
res->ls = cinfo->lockspace;
|
|
|
|
res->mddev = mddev;
|
2014-03-07 20:49:26 +01:00
|
|
|
namelen = strlen(name);
|
|
|
|
res->name = kzalloc(namelen + 1, GFP_KERNEL);
|
|
|
|
if (!res->name) {
|
|
|
|
pr_err("md-cluster: Unable to allocate resource name for resource %s\n", name);
|
|
|
|
goto out_err;
|
|
|
|
}
|
|
|
|
strlcpy(res->name, name, namelen + 1);
|
|
|
|
if (with_lvb) {
|
|
|
|
res->lksb.sb_lvbptr = kzalloc(LVB_SIZE, GFP_KERNEL);
|
|
|
|
if (!res->lksb.sb_lvbptr) {
|
|
|
|
pr_err("md-cluster: Unable to allocate LVB for resource %s\n", name);
|
|
|
|
goto out_err;
|
|
|
|
}
|
|
|
|
res->flags = DLM_LKF_VALBLK;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (bastfn)
|
|
|
|
res->bast = bastfn;
|
|
|
|
|
|
|
|
res->flags |= DLM_LKF_EXPEDITE;
|
|
|
|
|
|
|
|
ret = dlm_lock_sync(res, DLM_LOCK_NL);
|
|
|
|
if (ret) {
|
|
|
|
pr_err("md-cluster: Unable to lock NL on new lock resource %s\n", name);
|
|
|
|
goto out_err;
|
|
|
|
}
|
|
|
|
res->flags &= ~DLM_LKF_EXPEDITE;
|
|
|
|
res->flags |= DLM_LKF_CONVERT;
|
|
|
|
|
|
|
|
return res;
|
|
|
|
out_err:
|
|
|
|
kfree(res->lksb.sb_lvbptr);
|
|
|
|
kfree(res->name);
|
|
|
|
kfree(res);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void lockres_free(struct dlm_lock_resource *res)
|
|
|
|
{
|
|
|
|
if (!res)
|
|
|
|
return;
|
|
|
|
|
|
|
|
init_completion(&res->completion);
|
|
|
|
dlm_unlock(res->ls, res->lksb.sb_lkid, 0, &res->lksb, res);
|
|
|
|
wait_for_completion(&res->completion);
|
|
|
|
|
|
|
|
kfree(res->name);
|
|
|
|
kfree(res->lksb.sb_lvbptr);
|
|
|
|
kfree(res);
|
|
|
|
}
|
2014-03-07 18:21:15 +01:00
|
|
|
|
2014-03-29 16:20:02 +01:00
|
|
|
static char *pretty_uuid(char *dest, char *src)
|
|
|
|
{
|
|
|
|
int i, len = 0;
|
|
|
|
|
|
|
|
for (i = 0; i < 16; i++) {
|
|
|
|
if (i == 4 || i == 6 || i == 8 || i == 10)
|
|
|
|
len += sprintf(dest + len, "-");
|
|
|
|
len += sprintf(dest + len, "%02x", (__u8)src[i]);
|
|
|
|
}
|
|
|
|
return dest;
|
|
|
|
}
|
|
|
|
|
2014-03-30 07:42:49 +02:00
|
|
|
static void recover_prep(void *arg)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
|
|
|
static void recover_slot(void *arg, struct dlm_slot *slot)
|
|
|
|
{
|
|
|
|
struct mddev *mddev = arg;
|
|
|
|
struct md_cluster_info *cinfo = mddev->cluster_info;
|
|
|
|
|
|
|
|
pr_info("md-cluster: %s Node %d/%d down. My slot: %d. Initiating recovery.\n",
|
|
|
|
mddev->bitmap_info.cluster_name,
|
|
|
|
slot->nodeid, slot->slot,
|
|
|
|
cinfo->slot_number);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void recover_done(void *arg, struct dlm_slot *slots,
|
|
|
|
int num_slots, int our_slot,
|
|
|
|
uint32_t generation)
|
|
|
|
{
|
|
|
|
struct mddev *mddev = arg;
|
|
|
|
struct md_cluster_info *cinfo = mddev->cluster_info;
|
|
|
|
|
|
|
|
cinfo->slot_number = our_slot;
|
|
|
|
complete(&cinfo->completion);
|
|
|
|
}
|
|
|
|
|
|
|
|
static const struct dlm_lockspace_ops md_ls_ops = {
|
|
|
|
.recover_prep = recover_prep,
|
|
|
|
.recover_slot = recover_slot,
|
|
|
|
.recover_done = recover_done,
|
|
|
|
};
|
|
|
|
|
2014-03-29 16:01:53 +01:00
|
|
|
static int join(struct mddev *mddev, int nodes)
|
|
|
|
{
|
2014-03-29 16:20:02 +01:00
|
|
|
struct md_cluster_info *cinfo;
|
2014-03-30 07:42:49 +02:00
|
|
|
int ret, ops_rv;
|
2014-03-29 16:20:02 +01:00
|
|
|
char str[64];
|
|
|
|
|
|
|
|
if (!try_module_get(THIS_MODULE))
|
|
|
|
return -ENOENT;
|
|
|
|
|
|
|
|
cinfo = kzalloc(sizeof(struct md_cluster_info), GFP_KERNEL);
|
|
|
|
if (!cinfo)
|
|
|
|
return -ENOMEM;
|
|
|
|
|
2014-03-30 07:42:49 +02:00
|
|
|
init_completion(&cinfo->completion);
|
|
|
|
|
|
|
|
mutex_init(&cinfo->sb_mutex);
|
|
|
|
mddev->cluster_info = cinfo;
|
|
|
|
|
2014-03-29 16:20:02 +01:00
|
|
|
memset(str, 0, 64);
|
|
|
|
pretty_uuid(str, mddev->uuid);
|
2014-03-30 07:42:49 +02:00
|
|
|
ret = dlm_new_lockspace(str, mddev->bitmap_info.cluster_name,
|
|
|
|
DLM_LSFL_FS, LVB_SIZE,
|
|
|
|
&md_ls_ops, mddev, &ops_rv, &cinfo->lockspace);
|
2014-03-29 16:20:02 +01:00
|
|
|
if (ret)
|
|
|
|
goto err;
|
2014-03-30 07:42:49 +02:00
|
|
|
wait_for_completion(&cinfo->completion);
|
2014-03-29 16:20:02 +01:00
|
|
|
cinfo->sb_lock = lockres_init(mddev, "cmd-super",
|
|
|
|
NULL, 0);
|
|
|
|
if (!cinfo->sb_lock) {
|
|
|
|
ret = -ENOMEM;
|
|
|
|
goto err;
|
|
|
|
}
|
2014-03-29 16:01:53 +01:00
|
|
|
return 0;
|
2014-03-29 16:20:02 +01:00
|
|
|
err:
|
|
|
|
if (cinfo->lockspace)
|
|
|
|
dlm_release_lockspace(cinfo->lockspace, 2);
|
2014-03-30 07:42:49 +02:00
|
|
|
mddev->cluster_info = NULL;
|
2014-03-29 16:20:02 +01:00
|
|
|
kfree(cinfo);
|
|
|
|
module_put(THIS_MODULE);
|
|
|
|
return ret;
|
2014-03-29 16:01:53 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
static int leave(struct mddev *mddev)
|
|
|
|
{
|
2014-03-29 16:20:02 +01:00
|
|
|
struct md_cluster_info *cinfo = mddev->cluster_info;
|
|
|
|
|
|
|
|
if (!cinfo)
|
|
|
|
return 0;
|
|
|
|
lockres_free(cinfo->sb_lock);
|
|
|
|
dlm_release_lockspace(cinfo->lockspace, 2);
|
2014-03-29 16:01:53 +01:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2014-03-30 07:42:49 +02:00
|
|
|
/* slot_number(): Returns the MD slot number to use
|
|
|
|
* DLM starts the slot numbers from 1, wheras cluster-md
|
|
|
|
* wants the number to be from zero, so we deduct one
|
|
|
|
*/
|
|
|
|
static int slot_number(struct mddev *mddev)
|
|
|
|
{
|
|
|
|
struct md_cluster_info *cinfo = mddev->cluster_info;
|
|
|
|
|
|
|
|
return cinfo->slot_number - 1;
|
|
|
|
}
|
|
|
|
|
2014-03-29 16:01:53 +01:00
|
|
|
static struct md_cluster_operations cluster_ops = {
|
|
|
|
.join = join,
|
|
|
|
.leave = leave,
|
2014-03-30 07:42:49 +02:00
|
|
|
.slot_number = slot_number,
|
2014-03-29 16:01:53 +01:00
|
|
|
};
|
|
|
|
|
2014-03-07 18:21:15 +01:00
|
|
|
static int __init cluster_init(void)
|
|
|
|
{
|
|
|
|
pr_warn("md-cluster: EXPERIMENTAL. Use with caution\n");
|
|
|
|
pr_info("Registering Cluster MD functions\n");
|
2014-03-29 16:01:53 +01:00
|
|
|
register_md_cluster_operations(&cluster_ops, THIS_MODULE);
|
2014-03-07 18:21:15 +01:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void cluster_exit(void)
|
|
|
|
{
|
2014-03-29 16:01:53 +01:00
|
|
|
unregister_md_cluster_operations();
|
2014-03-07 18:21:15 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
module_init(cluster_init);
|
|
|
|
module_exit(cluster_exit);
|
|
|
|
MODULE_LICENSE("GPL");
|
|
|
|
MODULE_DESCRIPTION("Clustering support for MD");
|