2 commits - src/client.c src/cmd.c src/direct.c src/main.c src/paxos_lease.c src/paxos_lease.h src/resource.c src/resource.h src/sanlock_internal.h src/sanlock_resource.h src/sanlock_sock.h tests/sanlk_lvb.c tests/sanlk_rename.c
src/client.c | 104 +++++++++++++++++++++++++++++ src/cmd.c | 138 +++++++++++++++++++++++++++++++++++++-- src/direct.c | 2 src/main.c | 21 ++++- src/paxos_lease.c | 4 + src/paxos_lease.h | 1 src/resource.c | 173 +++++++++++++++++++++++++++++++++++++++++++++---- src/resource.h | 8 +- src/sanlock_internal.h | 2 src/sanlock_resource.h | 28 +++++++ src/sanlock_sock.h | 2 tests/sanlk_lvb.c | 125 +++++++++++++++++++++++++++++++++++ tests/sanlk_rename.c | 77 +++++++++++++++++++++ 13 files changed, 659 insertions(+), 26 deletions(-)
New commits: commit 513ed1a335be9c2803786e83d27ce24accc1c4cc Author: David Teigland teigland@redhat.com Date: Wed Jul 10 16:32:47 2013 -0500
sanlock: add SANLK_REL_RENAME
Used to rename a resource when releasing the lease.
Signed-off-by: David Teigland teigland@redhat.com
diff --git a/src/cmd.c b/src/cmd.c index 4ca87f2..da56d9d 100644 --- a/src/cmd.c +++ b/src/cmd.c @@ -65,7 +65,7 @@ static void release_cl_tokens(struct task *task, struct client *cl) token = cl->tokens[j]; if (!token) continue; - release_token(task, token); + release_token(task, token, NULL); free(token); } } @@ -76,7 +76,7 @@ static void release_new_tokens(struct task *task, struct token *new_tokens[], int i;
for (i = 0; i < acquire_count; i++) - release_token(task, new_tokens[i]); + release_token(task, new_tokens[i], NULL);
for (i = 0; i < alloc_count; i++) free(new_tokens[i]); @@ -477,6 +477,8 @@ static void cmd_release(struct task *task, struct cmd_args *ca) struct token *token; struct token *rem_tokens[SANLK_MAX_RESOURCES]; struct sanlk_resource res; + struct sanlk_resource new; + struct sanlk_resource *resrename = NULL; int fd, rv, i, j, found, pid_dead; int rem_tokens_count = 0; int result = 0; @@ -506,6 +508,54 @@ static void cmd_release(struct task *task, struct cmd_args *ca) goto do_remove; }
+ if (ca->header.cmd_flags & SANLK_REL_RENAME) { + rv = recv(fd, &res, sizeof(struct sanlk_resource), MSG_WAITALL); + if (rv != sizeof(struct sanlk_resource)) { + log_error("cmd_release %d,%d,%d recv res %d %d", + cl_ci, cl_fd, cl_pid, rv, errno); + result = -ENOTCONN; + goto do_remove; + } + + /* second res struct has new name for first res */ + rv = recv(fd, &new, sizeof(struct sanlk_resource), MSG_WAITALL); + if (rv != sizeof(struct sanlk_resource)) { + log_error("cmd_release %d,%d,%d recv new %d %d", + cl_ci, cl_fd, cl_pid, rv, errno); + result = -ENOTCONN; + goto do_remove; + } + + found = 0; + + pthread_mutex_lock(&cl->mutex); + for (j = 0; j < SANLK_MAX_RESOURCES; j++) { + token = cl->tokens[j]; + if (!token) + continue; + + if (memcmp(token->r.lockspace_name, res.lockspace_name, NAME_ID_SIZE)) + continue; + if (memcmp(token->r.name, res.name, NAME_ID_SIZE)) + continue; + + rem_tokens[rem_tokens_count++] = token; + cl->tokens[j] = NULL; + found = 1; + break; + } + pthread_mutex_unlock(&cl->mutex); + + if (!found) { + log_error("cmd_release %d,%d,%d no resource %.48s", + cl_ci, cl_fd, cl_pid, res.name); + result = -1; + } + + resrename = &new; + goto do_remove; + } + /* caller is specifying specific resources to release */
for (i = 0; i < ca->header.data; i++) { @@ -548,7 +598,7 @@ static void cmd_release(struct task *task, struct cmd_args *ca)
for (i = 0; i < rem_tokens_count; i++) { token = rem_tokens[i]; - rv = release_token(task, token); + rv = release_token(task, token, resrename); if (rv < 0) result = rv; free(token); diff --git a/src/direct.c b/src/direct.c index 63684b8..ed6cca4 100644 --- a/src/direct.c +++ b/src/direct.c @@ -132,7 +132,7 @@ static int do_paxos_action(int action, struct task *task, int io_timeout, rv = paxos_lease_leader_read(task, token, &leader, "direct_release"); if (rv < 0) break; - rv = paxos_lease_release(task, token, &leader, leader_ret); + rv = paxos_lease_release(task, token, NULL, &leader, leader_ret); break;
case ACT_READ_LEADER: diff --git a/src/paxos_lease.c b/src/paxos_lease.c index 69a4ed1..3b21017 100644 --- a/src/paxos_lease.c +++ b/src/paxos_lease.c @@ -1669,6 +1669,7 @@ int paxos_lease_renew(struct task *task,
int paxos_lease_release(struct task *task, struct token *token, + struct sanlk_resource *resrename, struct leader_record *leader_last, struct leader_record *leader_ret) { @@ -1725,6 +1726,9 @@ int paxos_lease_release(struct task *task, */ }
+ if (resrename) + memcpy(leader.resource_name, resrename->name, NAME_ID_SIZE); + leader.timestamp = LEASE_FREE; leader.write_id = token->host_id; leader.write_generation = token->host_generation; diff --git a/src/paxos_lease.h b/src/paxos_lease.h index d924923..588cb39 100644 --- a/src/paxos_lease.h +++ b/src/paxos_lease.h @@ -29,6 +29,7 @@ int paxos_lease_acquire(struct task *task,
int paxos_lease_release(struct task *task, struct token *token, + struct sanlk_resource *resrename, struct leader_record *leader_last, struct leader_record *leader_ret);
diff --git a/src/resource.c b/src/resource.c index 2fa0752..b2a7ba1 100644 --- a/src/resource.c +++ b/src/resource.c @@ -556,12 +556,13 @@ static int acquire_disk(struct task *task, struct token *token, /* return < 0 on error, 1 on success */
static int release_disk(struct task *task, struct token *token, - struct leader_record *leader) + struct sanlk_resource *resrename, + struct leader_record *leader) { struct leader_record leader_tmp; int rv;
- rv = paxos_lease_release(task, token, leader, &leader_tmp); + rv = paxos_lease_release(task, token, resrename, leader, &leader_tmp);
log_token(token, "release_disk rv %d", rv);
@@ -572,7 +573,9 @@ static int release_disk(struct task *task, struct token *token, return rv; /* SANLK_OK */ }
-static int _release_token(struct task *task, struct token *token, int opened, int nodisk) +static int _release_token(struct task *task, struct token *token, + struct sanlk_resource *resrename, + int opened, int nodisk) { struct resource *r = token->resource; uint64_t lver; @@ -642,7 +645,7 @@ static int _release_token(struct task *task, struct token *token, int opened, in if (r->flags & R_LVB_WRITE_RELEASE) write_lvb_block(task, r, token);
- rv = release_disk(task, token, &r->leader); + rv = release_disk(task, token, resrename, &r->leader); }
close_disks(token->disks, token->r.num_disks); @@ -664,17 +667,18 @@ static int _release_token(struct task *task, struct token *token, int opened, in
static int release_token_nodisk(struct task *task, struct token *token) { - return _release_token(task, token, 0, 1); + return _release_token(task, token, NULL, 0, 1); }
static int release_token_opened(struct task *task, struct token *token) { - return _release_token(task, token, 1, 0); + return _release_token(task, token, NULL, 1, 0); }
-int release_token(struct task *task, struct token *token) +int release_token(struct task *task, struct token *token, + struct sanlk_resource *resrename) { - return _release_token(task, token, 0, 0); + return _release_token(task, token, resrename, 0, 0); }
/* We're releasing a token from the main thread, in which we don't want to block, @@ -931,7 +935,7 @@ int acquire_token(struct task *task, struct token *token, uint32_t cmd_flags, release_token_opened(task, token); return rv; } else { - release_disk(task, token, &leader); + release_disk(task, token, NULL, &leader); /* the token is kept, the paxos lease is released but with shared set */ goto out; } @@ -1195,7 +1199,7 @@ static void resource_thread_release(struct task *task, struct resource *r, struc if (r->flags & R_LVB_WRITE_RELEASE) write_lvb_block(task, r, tt);
- release_disk(task, tt, &r->leader); + release_disk(task, tt, NULL, &r->leader); }
close_disks(tt->disks, tt->r.num_disks); diff --git a/src/resource.h b/src/resource.h index 25e9c58..11ea4c0 100644 --- a/src/resource.h +++ b/src/resource.h @@ -17,7 +17,8 @@ void check_mode_block(struct token *token, int q, char *dblock);
int acquire_token(struct task *task, struct token *token, uint32_t cmd_flags, char *killpath, char *killargs); -int release_token(struct task *task, struct token *token); +int release_token(struct task *task, struct token *token, + struct sanlk_resource *resrename); void release_token_async(struct token *token);
int request_token(struct task *task, struct token *token, uint32_t force_mode, diff --git a/src/sanlock_resource.h b/src/sanlock_resource.h index 9c6112f..f691032 100644 --- a/src/sanlock_resource.h +++ b/src/sanlock_resource.h @@ -31,8 +31,25 @@ /* acquire flags */ #define SANLK_ACQUIRE_LVB 0x00000001
-/* release flags */ +/* + * release flags + * + * SANLK_REL_ALL + * Release all resources held by the client. + * The res args are ignored. + * + * SANLK_REL_RENAME + * Rename the resource lease on disk when it + * is released. The resource is freed and + * renamed in a single disk operation (write + * to the leader record.) The first res + * arg is the resource to release, and the + * second resource arg contains the new name + * for the first resource. + */ + #define SANLK_REL_ALL 0x00000001 +#define SANLK_REL_RENAME 0x00000002
/* * request flags diff --git a/tests/sanlk_rename.c b/tests/sanlk_rename.c new file mode 100644 index 0000000..4043c3e --- /dev/null +++ b/tests/sanlk_rename.c @@ -0,0 +1,77 @@ +#include <sys/types.h> +#include <sys/wait.h> +#include <sys/un.h> +#include <sys/mount.h> +#include <sys/signalfd.h> +#include <inttypes.h> +#include <unistd.h> +#include <stdio.h> +#include <stdlib.h> +#include <stdint.h> +#include <stddef.h> +#include <fcntl.h> +#include <string.h> +#include <errno.h> +#include <limits.h> +#include <time.h> +#include <signal.h> + +#include "sanlock.h" +#include "sanlock_resource.h" + +/* gcc with -lsanlock */ + +int main(int argc, char *argv[]) +{ + char rd[sizeof(struct sanlk_resource) + sizeof(struct sanlk_disk)]; + char rd2[sizeof(struct sanlk_resource) + sizeof(struct sanlk_disk)]; + struct sanlk_resource *res; + struct sanlk_resource *res2; + struct sanlk_resource **res_args; + int fd, rv; + + if (argc < 6) { + printf("acquire with old name, release with new name\n"); + printf("sanlk_rename <lockspace_name> <resource_name_old> <resource_name_new> <path> <resource_offset>\n"); + return -1; + } + + res_args = malloc(2 * sizeof(struct sanlk_resource *)); + + memset(rd, 0, sizeof(rd)); + memset(rd2, 0, sizeof(rd2)); + + res = (struct sanlk_resource *)&rd; + res2 = (struct sanlk_resource *)&rd2; + + res_args[0] = res; + res_args[1] = res2; + + strcpy(res->lockspace_name, argv[1]); + strcpy(res->name, argv[2]); + strcpy(res2->name, argv[3]); + res->num_disks = 1; + strcpy(res->disks[0].path, argv[4]); + res->disks[0].offset = atoi(argv[5]); + + fd = sanlock_register(); + if (fd < 0) { + fprintf(stderr, "register error %d\n", fd); + return -1; + } + + rv = sanlock_acquire(fd, -1, 0, 1, &res, NULL); + if (rv < 0) { + fprintf(stderr, "acquire error %d\n", rv); + return -1; + } + + rv = sanlock_release(fd, -1, SANLK_REL_RENAME, 2, res_args); + if (rv < 0) { + fprintf(stderr, "release error %d\n", rv); + return -1; + } + + return 0; +} +
commit a309b0af1f5598fdea56d738c86944aa4a751df1 Author: David Teigland teigland@redhat.com Date: Wed Jul 10 11:19:32 2013 -0500
sanlock: add lvb feature
set_lvb and get_lvb allow up to a sector size of application specific data to be stored with a resource lease. They are similar to the lvb feature found in the dlm.
host 1: sanlock_acquire(ACQUIRE_LVB, r) application data written to lvb sanlock_set_lvb(r, lvb) sanlock_release(r)
The lvb data is copied to the sanlock daemon in set_lvb, and written to the resource lease on disk in release.
host 2: sanlock_acquire(ACQUIRE_LVB, r) sanlock_get_lvb(r, lvb) application data read from lvb sanlock_release(r)
The lvb data is read by the sanlock daemon from disk in acquire, and copied to the application in get_lvb.
Signed-off-by: David Teigland teigland@redhat.com
diff --git a/src/client.c b/src/client.c index 6ecf431..3c2f5b7 100644 --- a/src/client.c +++ b/src/client.c @@ -1115,6 +1115,110 @@ int sanlock_examine(uint32_t flags, struct sanlk_lockspace *ls, return rv; }
+int sanlock_set_lvb(uint32_t flags, struct sanlk_resource *res, char *lvb, int lvblen) +{ + int datalen = 0; + int rv, fd; + + if (!res || !lvb || !lvblen) + return -EINVAL; + + datalen = sizeof(struct sanlk_resource) + lvblen; + + rv = connect_socket(&fd); + if (rv < 0) + return rv; + + rv = send_header(fd, SM_CMD_SET_LVB, flags, datalen, 0, 0); + if (rv < 0) + return rv; + + rv = send(fd, res, sizeof(struct sanlk_resource), 0); + if (rv < 0) { + rv = -1; + goto out; + } + + rv = send(fd, lvb, lvblen, 0); + if (rv < 0) { + rv = -1; + goto out; + } + + rv = recv_result(fd); + out: + close(fd); + return rv; +} + +int sanlock_get_lvb(uint32_t flags, struct sanlk_resource *res, char *lvb, int lvblen) +{ + struct sm_header h; + char *reply_data = NULL; + int datalen = 0; + int rv, fd, len; + + if (!res || !lvb || !lvblen) + return -EINVAL; + + datalen = sizeof(struct sanlk_resource); + + rv = connect_socket(&fd); + if (rv < 0) + return rv; + + rv = send_header(fd, SM_CMD_GET_LVB, flags, datalen, 0, 0); + if (rv < 0) + return rv; + + rv = send(fd, res, sizeof(struct sanlk_resource), 0); + if (rv < 0) { + rv = -1; + goto out; + } + + /* get result */ + + memset(&h, 0, sizeof(h)); + + rv = recv(fd, &h, sizeof(h), MSG_WAITALL); + if (rv != sizeof(h)) { + rv = -1; + goto out; + } + + len = h.length - sizeof(h); + if (!len) { + rv = (int)h.data; + goto out; + } + + reply_data = malloc(len); + if (!reply_data) { + rv = -ENOMEM; + goto out; + } + + rv = recv(fd, reply_data, len, MSG_WAITALL); + if (rv != len) { + free(reply_data); + rv = -1; + goto out; + } + + if (lvblen < len) + len = lvblen; + + memcpy(lvb, reply_data, len); + + free(reply_data); + + rv = (int)h.data; + out: + close(fd); + return rv; +} + /* * convert from struct sanlk_resource to string with format: * <lockspace_name>:<resource_name>:<path>:<offset>[:<path>:<offset>...]:<lver> diff --git a/src/cmd.c b/src/cmd.c index 3c11055..4ca87f2 100644 --- a/src/cmd.c +++ b/src/cmd.c @@ -327,7 +327,7 @@ static void cmd_acquire(struct task *task, struct cmd_args *ca) for (i = 0; i < new_tokens_count; i++) { token = new_tokens[i];
- rv = acquire_token(task, token, killpath, killargs); + rv = acquire_token(task, token, ca->header.cmd_flags, killpath, killargs); if (rv < 0) { switch (rv) { case -EEXIST: @@ -888,6 +888,80 @@ static void cmd_examine(struct task *task GNUC_UNUSED, struct cmd_args *ca) client_resume(ca->ci_in); }
+static void cmd_set_lvb(struct task *task GNUC_UNUSED, struct cmd_args *ca) +{ + struct sanlk_resource res; + char *lvb = NULL; + int lvblen, rv, fd, result; + + fd = client[ca->ci_in].fd; + + rv = recv(fd, &res, sizeof(struct sanlk_resource), MSG_WAITALL); + if (rv != sizeof(struct sanlk_resource)) { + log_error("cmd_set_lvb %d,%d recv %d %d", ca->ci_in, fd, rv, errno); + result = -ENOTCONN; + goto reply; + } + + lvblen = ca->header.length - sizeof(struct sm_header) - sizeof(struct sanlk_resource); + + lvb = malloc(lvblen); + if (!lvb) { + result = -ENOMEM; + goto reply; + } + + rv = recv(fd, lvb, lvblen, MSG_WAITALL); + if (rv != lvblen) { + result = -ENOTCONN; + goto reply; + } + + result = res_set_lvb(&res, lvb, lvblen); + reply: + if (lvb) + free(lvb); + + send_result(fd, &ca->header, result); + client_resume(ca->ci_in); +} + +static void cmd_get_lvb(struct task *task GNUC_UNUSED, struct cmd_args *ca) +{ + struct sm_header h; + struct sanlk_resource res; + char *lvb = NULL; + int lvblen = 0, rv, fd, result; + + fd = client[ca->ci_in].fd; + + rv = recv(fd, &res, sizeof(struct sanlk_resource), MSG_WAITALL); + if (rv != sizeof(struct sanlk_resource)) { + log_error("cmd_get_lvb %d,%d recv %d %d", ca->ci_in, fd, rv, errno); + result = -ENOTCONN; + goto reply; + } + + /* if 0 then we use the sector size as lvb len */ + lvblen = ca->header.data2; + + result = res_get_lvb(&res, &lvb, &lvblen); + reply: + memcpy(&h, &ca->header, sizeof(struct sm_header)); + h.data = result; + h.data2 = 0; + h.length = sizeof(h) + lvblen; + + send(fd, &h, sizeof(h), MSG_NOSIGNAL); + + if (lvb) { + send(fd, lvb, lvblen, MSG_NOSIGNAL); + free(lvb); + } + + client_resume(ca->ci_in); +} + static void cmd_add_lockspace(struct cmd_args *ca) { struct sanlk_lockspace lockspace; @@ -1625,6 +1699,12 @@ void call_cmd_thread(struct task *task, struct cmd_args *ca) case SM_CMD_KILLPATH: cmd_killpath(task, ca); break; + case SM_CMD_SET_LVB: + cmd_set_lvb(task, ca); + break; + case SM_CMD_GET_LVB: + cmd_get_lvb(task, ca); + break; }; }
diff --git a/src/main.c b/src/main.c index 5c5fb2d..cac66ff 100644 --- a/src/main.c +++ b/src/main.c @@ -945,8 +945,12 @@ static int thread_pool_create(int min_workers, int max_workers) return rv; }
-/* cmd comes from a transient client/fd set up just to pass the cmd, - and is not being done on behalf of another registered client/fd */ +/* + * cmd comes from a transient client/fd set up just to pass the cmd, + * and is not being done on behalf of another registered client/fd. + * The command is processed independently of the lifetime of a specific + * client or the tokens held by a specific client. + */
static void process_cmd_thread_unregistered(int ci_in, struct sm_header *h_recv) { @@ -975,8 +979,15 @@ static void process_cmd_thread_unregistered(int ci_in, struct sm_header *h_recv) close(client[ci_in].fd); }
-/* cmd either comes from a registered client/fd, - or is targeting a registered client/fd */ +/* + * cmd either comes from a registered client/fd, or is targeting a registered + * client/fd. The processing of the cmd is closely coordinated with the + * lifetime of a specific client and to tokens held by that client. Handling + * of the client's death or changing of the client's tokens will be serialized + * with the processing of this command. This means that the end of processing + * this command needs to check if the client failed during the command + * processing and handle the cleanup of the client if so. + */
static void process_cmd_thread_registered(int ci_in, struct sm_header *h_recv) { @@ -1168,6 +1179,8 @@ static void process_connection(int ci) case SM_CMD_READ_LOCKSPACE: case SM_CMD_READ_RESOURCE: case SM_CMD_READ_RESOURCE_OWNERS: + case SM_CMD_SET_LVB: + case SM_CMD_GET_LVB: rv = client_suspend(ci); if (rv < 0) return; diff --git a/src/resource.c b/src/resource.c index bb6c643..2fa0752 100644 --- a/src/resource.c +++ b/src/resource.c @@ -50,6 +50,13 @@ static pthread_mutex_t resource_mutex; static pthread_cond_t resource_cond;
+static void free_resource(struct resource *r) +{ + if (r->lvb) + free(r->lvb); + free(r); +} + void send_state_resources(int fd) { struct resource *r; @@ -405,6 +412,117 @@ static int clear_dead_shared(struct task *task, struct token *token, return rv; }
+/* the lvb is the sector after the dblock for host_id 2000, i.e. 2002 */ + +#define LVB_SECTOR 2002 + +static int read_lvb_block(struct task *task, struct token *token) +{ + struct sync_disk *disk; + struct resource *r; + char *iobuf; + uint64_t offset; + int iobuf_len, rv; + + r = token->resource; + disk = &token->disks[0]; + iobuf_len = disk->sector_size; + iobuf = r->lvb; + offset = disk->offset + (LVB_SECTOR * disk->sector_size); + + rv = read_iobuf(disk->fd, offset, iobuf, iobuf_len, task, token->io_timeout); + + return rv; +} + +static int write_lvb_block(struct task *task, struct resource *r, struct token *token) +{ + struct sync_disk *disk; + char *iobuf; + uint64_t offset; + int iobuf_len, rv; + + disk = &token->disks[0]; + iobuf_len = disk->sector_size; + iobuf = r->lvb; + offset = disk->offset + (LVB_SECTOR * disk->sector_size); + + rv = write_iobuf(disk->fd, offset, iobuf, iobuf_len, task, token->io_timeout); + + return rv; +} + +int res_set_lvb(struct sanlk_resource *res, char *lvb, int lvblen) +{ + struct resource *r; + int rv = -ENOENT; + + pthread_mutex_lock(&resource_mutex); + list_for_each_entry(r, &resources_held, list) { + if (strncmp(r->r.lockspace_name, res->lockspace_name, NAME_ID_SIZE)) + continue; + if (strncmp(r->r.name, res->name, NAME_ID_SIZE)) + continue; + + if (!r->lvb) { + rv = -EINVAL; + break; + } + + if (lvblen > r->leader.sector_size) { + rv = -E2BIG; + break; + } + + memcpy(r->lvb, lvb, lvblen); + r->flags |= R_LVB_WRITE_RELEASE; + rv = 0; + break; + } + pthread_mutex_unlock(&resource_mutex); + + return rv; +} + +int res_get_lvb(struct sanlk_resource *res, char **lvb_out, int *lvblen) +{ + struct resource *r; + char *lvb; + int rv = -ENOENT; + int len = *lvblen; + + pthread_mutex_lock(&resource_mutex); + list_for_each_entry(r, &resources_held, list) { + if (strncmp(r->r.lockspace_name, res->lockspace_name, NAME_ID_SIZE)) + continue; + if (strncmp(r->r.name, res->name, NAME_ID_SIZE)) + continue; + + if (!r->lvb) { + rv = -EINVAL; + break; + } + + if (!len) + len = r->leader.sector_size; + + lvb = malloc(len); + if (!lvb) { + rv = -ENOMEM; + break; + } + + memcpy(lvb, r->lvb, len); + *lvb_out = lvb; + *lvblen = len; + rv = 0; + break; + } + pthread_mutex_unlock(&resource_mutex); + + return rv; +} + /* return < 0 on error, 1 on success */
static int acquire_disk(struct task *task, struct token *token, @@ -521,6 +639,9 @@ static int _release_token(struct task *task, struct token *token, int opened, in if (r->flags & R_SHARED) { rv = set_mode_block(task, token, token->host_id, 0, 0); } else { + if (r->flags & R_LVB_WRITE_RELEASE) + write_lvb_block(task, r, token); + rv = release_disk(task, token, &r->leader); }
@@ -536,7 +657,7 @@ static int _release_token(struct task *task, struct token *token, int opened, in pthread_mutex_lock(&resource_mutex); list_del(&r->list); pthread_mutex_unlock(&resource_mutex); - free(r); + free_resource(r);
return rv; } @@ -572,7 +693,7 @@ void release_token_async(struct token *token) is dead (release will probably fail), or the lease wasn't never acquired */ list_del(&r->list); - free(r); + free_resource(r); } else { r->flags |= R_THREAD_RELEASE; r->release_token_id = token->token_id; @@ -682,13 +803,14 @@ static struct resource *new_resource(struct token *token) return r; }
-int acquire_token(struct task *task, struct token *token, +int acquire_token(struct task *task, struct token *token, uint32_t cmd_flags, char *killpath, char *killargs) { struct leader_record leader; struct resource *r; uint64_t acquire_lver = 0; uint32_t new_num_hosts = 0; + int sector_size; int sh_retries = 0; int live_count = 0; int rv; @@ -756,6 +878,19 @@ int acquire_token(struct task *task, struct token *token,
copy_disks(&r->r.disks, &token->r.disks, token->r.num_disks);
+ sector_size = token->disks[0].sector_size; + + if (cmd_flags & SANLK_ACQUIRE_LVB) { + char *iobuf, **p_iobuf; + p_iobuf = &iobuf; + + rv = posix_memalign((void *)p_iobuf, getpagesize(), sector_size); + if (rv) + log_error("acquire_token cannot allocate lvb"); + else + r->lvb = iobuf; + } + retry: memset(&leader, 0, sizeof(struct leader_record));
@@ -818,6 +953,9 @@ int acquire_token(struct task *task, struct token *token, }
out: + if (cmd_flags & SANLK_ACQUIRE_LVB) + read_lvb_block(task, token); + close_disks(token->disks, token->r.num_disks);
pthread_mutex_lock(&resource_mutex); @@ -1054,6 +1192,9 @@ static void resource_thread_release(struct task *task, struct resource *r, struc if (r->flags & R_SHARED) { set_mode_block(task, tt, tt->host_id, 0, 0); } else { + if (r->flags & R_LVB_WRITE_RELEASE) + write_lvb_block(task, r, tt); + release_disk(task, tt, &r->leader); }
@@ -1062,7 +1203,7 @@ static void resource_thread_release(struct task *task, struct resource *r, struc pthread_mutex_lock(&resource_mutex); list_del(&r->list); pthread_mutex_unlock(&resource_mutex); - free(r); + free_resource(r); }
static void resource_thread_examine(struct task *task, struct token *tt, int pid, uint64_t lver) diff --git a/src/resource.h b/src/resource.h index aa7b615..25e9c58 100644 --- a/src/resource.h +++ b/src/resource.h @@ -15,7 +15,7 @@ int lockspace_is_used(struct sanlk_lockspace *ls);
void check_mode_block(struct token *token, int q, char *dblock);
-int acquire_token(struct task *task, struct token *token, +int acquire_token(struct task *task, struct token *token, uint32_t cmd_flags, char *killpath, char *killargs); int release_token(struct task *task, struct token *token); void release_token_async(struct token *token); @@ -25,6 +25,9 @@ int request_token(struct task *task, struct token *token, uint32_t force_mode,
int set_resource_examine(char *space_name, char *res_name);
+int res_set_lvb(struct sanlk_resource *res, char *lvb, int lvblen); +int res_get_lvb(struct sanlk_resource *res, char **lvb_out, int *lvblen); + int read_resource_owners(struct task *task, struct token *token, struct sanlk_resource *res, char **send_buf, int *send_len, int *count); diff --git a/src/sanlock_internal.h b/src/sanlock_internal.h index b3e34ee..f3e5877 100644 --- a/src/sanlock_internal.h +++ b/src/sanlock_internal.h @@ -101,6 +101,7 @@ struct token { #define R_THREAD_RELEASE 0x00000004 #define R_RESTRICT_SIGKILL 0x00000008 /* inherited from token */ #define R_RESTRICT_SIGTERM 0x00000010 /* inherited from token */ +#define R_LVB_WRITE_RELEASE 0x00000020
struct resource { struct list_head list; @@ -111,6 +112,7 @@ struct resource { int pid; /* copied from token when ex */ uint32_t flags; uint32_t release_token_id; /* copy to temp token (tt) for log messages */ + char *lvb; char killpath[SANLK_HELPER_PATH_LEN]; /* copied from client */ char killargs[SANLK_HELPER_ARGS_LEN]; /* copied from client */ struct leader_record leader; /* copy of last leader_record we wrote */ diff --git a/src/sanlock_resource.h b/src/sanlock_resource.h index 2b95d3e..9c6112f 100644 --- a/src/sanlock_resource.h +++ b/src/sanlock_resource.h @@ -28,6 +28,9 @@ /* killpath flags */ #define SANLK_KILLPATH_PID 0x00000001
+/* acquire flags */ +#define SANLK_ACQUIRE_LVB 0x00000001 + /* release flags */ #define SANLK_REL_ALL 0x00000001
@@ -86,6 +89,12 @@ int sanlock_request(uint32_t flags, uint32_t force_mode, int sanlock_examine(uint32_t flags, struct sanlk_lockspace *ls, struct sanlk_resource *res);
+int sanlock_set_lvb(uint32_t flags, struct sanlk_resource *res, + char *lvb, int lvblen); + +int sanlock_get_lvb(uint32_t flags, struct sanlk_resource *res, + char *lvb, int lvblen); + /* * Functions to convert between string and struct resource formats. * All allocate space for returned data that the caller must free. diff --git a/src/sanlock_sock.h b/src/sanlock_sock.h index bf26b40..952eab4 100644 --- a/src/sanlock_sock.h +++ b/src/sanlock_sock.h @@ -42,6 +42,8 @@ enum { SM_CMD_GET_LOCKSPACES = 22, SM_CMD_GET_HOSTS = 23, SM_CMD_READ_RESOURCE_OWNERS = 24, + SM_CMD_SET_LVB = 25, + SM_CMD_GET_LVB = 26, };
struct sm_header { diff --git a/tests/sanlk_lvb.c b/tests/sanlk_lvb.c new file mode 100644 index 0000000..1b4eacf --- /dev/null +++ b/tests/sanlk_lvb.c @@ -0,0 +1,125 @@ +#include <sys/types.h> +#include <sys/wait.h> +#include <sys/un.h> +#include <sys/mount.h> +#include <sys/signalfd.h> +#include <inttypes.h> +#include <unistd.h> +#include <stdio.h> +#include <stdlib.h> +#include <stdint.h> +#include <stddef.h> +#include <fcntl.h> +#include <string.h> +#include <errno.h> +#include <limits.h> +#include <time.h> +#include <signal.h> + +#include "sanlock.h" +#include "sanlock_resource.h" + +/* gcc with -lsanlock */ + +int main(int argc, char *argv[]) +{ + char rd[sizeof(struct sanlk_resource) + sizeof(struct sanlk_disk)]; + struct sanlk_resource *res; + char lvb[512]; + char *filename; + char *act; + FILE *fp; + int fd, rv; + int set = 0, get = 0; + + memset(lvb, 0, sizeof(lvb)); + + if (argc < 7) { + printf("read file, write it to lvb:\n"); + printf("sanlk_lvb set <lockspace_name> <resource_name> <path> <resource_offset> <file>\n"); + printf("\n"); + printf("read lvb, write it to file:\n"); + printf("sanlk_lvb get <lockspace_name> <resource_name> <path> <resource_offset> <file>\n"); + return -1; + } + + memset(rd, 0, sizeof(rd)); + + res = (struct sanlk_resource *)&rd; + + act = argv[1]; + strcpy(res->lockspace_name, argv[2]); + strcpy(res->name, argv[3]); + res->num_disks = 1; + strcpy(res->disks[0].path, argv[4]); + res->disks[0].offset = atoi(argv[5]); + filename = argv[6]; + + if (!strcmp(act, "set")) { + set = 1; + fp = fopen(filename, "r"); + } else if (!strcmp(act, "get")) { + get = 1; + fp = fopen(filename, "w"); + } else { + printf("bad action %s\n", act); + return -1; + } + + if (!fp) { + printf("fopen failed %s\n", strerror(errno)); + return -1; + } + + fd = sanlock_register(); + if (fd < 0) { + printf("register error %d\n", fd); + return -1; + } + + rv = sanlock_acquire(fd, -1, SANLK_ACQUIRE_LVB, 1, &res, NULL); + if (rv < 0) { + printf("acquire error %d\n", rv); + return -1; + } + + if (get) { + rv = sanlock_get_lvb(0, res, lvb, sizeof(lvb)); + if (rv < 0) { + printf("get_lvb error %d\n", rv); + return -1; + } + + fwrite(lvb, sizeof(lvb), 1, fp); + if (ferror(fp)) { + printf("fwrite error\n"); + return -1; + } + + } + + if (set) { + fread(lvb, sizeof(lvb), 1, fp); + if (ferror(fp)) { + printf("fread error\n"); + return -1; + } + + rv = sanlock_set_lvb(0, res, lvb, sizeof(lvb)); + if (rv < 0) { + printf("set_lvb error %d\n", rv); + return -1; + } + } + + fclose(fp); + + rv = sanlock_release(fd, -1, 0, 1, &res); + if (rv < 0) { + fprintf(stderr, "release error %d\n", rv); + return -1; + } + + return 0; +} +
sanlock-devel@lists.fedorahosted.org
-
David Teigland