[sanlock] branch master updated (b8a885c -> 90b2ffa)

[git repository hosting]

This is an automated email from the git hooks/post-receive script.

teigland pushed a change to branch master
in repository sanlock.

from  b8a885c   sanlock: add resource index
       new  129e0d0   Fix rindex_delete
       new  ec204f8   More verbose and detailed pytest output
       new  5f3f7d9   Kill test sanlock daemon
       new  90b2ffa   Start rindex tests

The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "adds" were already present in the repository and have only
been added to this reference.

Summary of changes:
 pytest.ini           |   8 ++-
 src/rindex.c         |  14 +++--
 tests/conftest.py    |   4 +-
 tests/constants.py   |  10 ++++
 tests/daemon_test.py | 158 ++++++++++++++++++++++++++++++++++++++++++++++++++-
 tests/util.py        |  15 +++++
 6 files changed, 199 insertions(+), 10 deletions(-)

[sanlock] 01/04: Fix rindex_delete

[git repository hosting]

This is an automated email from the git hooks/post-receive script.

teigland pushed a commit to branch master
in repository sanlock.

commit 129e0d023fedd2ddd21ca8c115b00e4134fcdba7
Author: Nir Soffer 
Date:   Sat Mar 3 03:17:40 2018 +0200

Fix rindex_delete

rindex_delete() did not search for the entry, and was deleting the wrong
    entry (at offset 320) and clearing the wrong lease (0).

Signed-off-by: Nir Soffer 
---
 src/rindex.c | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/src/rindex.c b/src/rindex.c
index 601c8bf..4e8e3b7 100644
--- a/src/rindex.c
+++ b/src/rindex.c
@@ -584,7 +584,6 @@ int rindex_delete(struct task *task, struct sanlk_rindex *ri,
    char *rindex_iobuf = NULL;
    uint64_t res_offset = re->offset;
    uint64_t ent_offset;
-	int entry_num;
    int sector_size, align_size;
    int rv;

@@ -620,8 +619,6 @@ int rindex_delete(struct task *task, struct sanlk_rindex *ri,

sector_size = rx.header.sector_size;
    align_size = sector_size_to_align_size(sector_size);
-	entry_num = (res_offset - rx.disk->offset - (2 * align_size)) / align_size;
-	ent_offset = sector_size + (entry_num * sizeof(struct rindex_entry));

if (re->offset && (re->offset % align_size)) {
    	rv = SANLK_RINDEX_OFFSET;
@@ -642,7 +639,6 @@ int rindex_delete(struct task *task, struct sanlk_rindex *ri,
    	rv = -ENOMEM;
    	goto out_clear;
    }
-	res_token->disks[0].offset = res_offset;

rv = paxos_lease_acquire(task, rx_token,
    		         PAXOS_ACQUIRE_OWNER_NOWAIT | PAXOS_ACQUIRE_QUIET_FAIL,
@@ -659,6 +655,14 @@ int rindex_delete(struct task *task, struct sanlk_rindex *ri,
    	goto out_lease;
    }

+	/* find the entry */
+
+	rv = search_entries(&rx, rindex_iobuf, &ent_offset, &res_offset, 0, re->name);
+	if (rv < 0) {
+		log_error("rindex_delete failed to find entry '%s': %d", re->name, rv);
+		goto out_iobuf;
+	}
+
    rv = update_rindex(task, &spi, &rx, rindex_iobuf, re, ent_offset, res_offset, 1);
    if (rv < 0) {
    	log_error("rindex_delete failed to update rindex %d", rv);
@@ -667,6 +671,8 @@ int rindex_delete(struct task *task, struct sanlk_rindex *ri,

/* clear the paxos lease */

+	res_token->disks[0].offset = res_offset;
+
    rv = paxos_lease_init(task, res_token, 0, 0, 1);
    if (rv < 0) {
    	log_error("rindex_delete failed to init new lease %d", rv);

[sanlock] 02/04: More verbose and detailed pytest output

[git repository hosting]

This is an automated email from the git hooks/post-receive script.

teigland pushed a commit to branch master
in repository sanlock.

commit ec204f851648023d9831d73676b3b4b718b83869
Author: Nir Soffer 
Date:   Fri Mar 2 20:21:40 2018 +0200

More verbose and detailed pytest output

Tune pytest options:
    - more verbose output that may help to debug failures
    - show slowest tests duration, we would like to optimize these
    - document all options

Signed-off-by: Nir Soffer 
---
 pytest.ini | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 73176d5..bb17bd5 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -1,3 +1,7 @@
 [pytest]
-# Note: we must use /var/tmp as sanlock uses direct I/O.
-addopts = -rxs --basetemp=/var/tmp/sanlock
+# Notes:
+# --basetemp: we must use /var/tmp as sanlock uses direct I/O.
+# -vv: increasing verbosify twice shows more detailed failures tracebacks.
+# -rxs: show extra test summary: (s)skipped, (x)failed
+# --durations: show slowest test duration
+addopts = -rxs -vv --basetemp=/var/tmp/sanlock --durations=10

[sanlock] 03/04: Kill test sanlock daemon

[git repository hosting]

This is an automated email from the git hooks/post-receive script.

teigland pushed a commit to branch master
in repository sanlock.

commit 5f3f7d9071c48dd347655c3ad2165470dde3d6b5
Author: Nir Soffer 
Date:   Sat Mar 3 01:38:58 2018 +0200

Kill test sanlock daemon

Killing the test daemon allows skipping of rem_lockspace in the tests,
    saving about 3 seconds per test. This also ensures that bugs in sanlock
    or the tests do not hang the tests forever.

Signed-off-by: Nir Soffer 
---
 tests/conftest.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/conftest.py b/tests/conftest.py
index 8055c88..6b8b0e8 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -17,5 +17,7 @@ def sanlock_daemon():
         util.wait_for_daemon(0.5)
         yield
     finally:
-        p.terminate()
+        # Killing sanlock allows terminating without reomving the lockspace,
+        # which takes about 3 seconds, slowing down the tests.
+        p.kill()
         p.wait()

[sanlock] 04/04: Start rindex tests

[git repository hosting]

This is an automated email from the git hooks/post-receive script.

teigland pushed a commit to branch master
in repository sanlock.

commit 90b2ffa77edd46bea007b7bb39bfd4d2db2ff7af
Author: Nir Soffer 
Date:   Sun Feb 25 22:35:24 2018 +0200

Start rindex tests

Add couple of tests for the new lookup command using the daemon.

To make the tests faster, we use 1 second io timeout. This is much
    faster then real usage, but still takes about 3 seconds per test when we
    add a lockspace. Need to check how to make this faster.

Unfinished: cover all commands.

Signed-off-by: Nir Soffer 
---
 tests/constants.py   |  10 ++++
 tests/daemon_test.py | 158 ++++++++++++++++++++++++++++++++++++++++++++++++++-
 tests/util.py        |  15 +++++
 3 files changed, 180 insertions(+), 3 deletions(-)

diff --git a/tests/constants.py b/tests/constants.py
index 70f4226..a7eb16b 100644
--- a/tests/constants.py
+++ b/tests/constants.py
@@ -7,3 +7,13 @@ Constants copied from sanlock source.
 PAXOS_DISK_MAGIC = 0x06152010
 PAXOS_DISK_CLEAR = 0x11282016
 DELTA_DISK_MAGIC = 0x12212010
+
+# src/rindex_disk.h
+
+RINDEX_DISK_MAGIC = 0x01042018
+
+# src/rindex_disk.h
+# Copied from the docs module comment.
+
+RINDEX_ENTRY_SIZE = 64
+RINDEX_ENTRIES_SECTORS = 2000
diff --git a/tests/daemon_test.py b/tests/daemon_test.py
index dc5aadd..8896fb2 100644
--- a/tests/daemon_test.py
+++ b/tests/daemon_test.py
@@ -8,7 +8,7 @@ import struct

import pytest

-from . import constants
+from . constants import *
 from . import util

@@ -52,7 +52,7 @@ def test_init_lockspace(tmpdir, sanlock_daemon):

with io.open(str(path), "rb") as f:
         magic, = struct.unpack("< I", f.read(4))
-        assert magic == constants.DELTA_DISK_MAGIC
+        assert magic == DELTA_DISK_MAGIC

# TODO: check more stuff here...

@@ -69,8 +69,160 @@ def test_init_resource(tmpdir, sanlock_daemon):

with io.open(str(path), "rb") as f:
         magic, = struct.unpack("< I", f.read(4))
-        assert magic == constants.PAXOS_DISK_MAGIC
+        assert magic == PAXOS_DISK_MAGIC

# TODO: check more stuff here...

util.check_guard(str(path), size)
+
+
+def test_format(tmpdir, sanlock_daemon):
+    path = tmpdir.join("rindex")
+    size = 1024**2 * 3
+    util.create_file(str(path), size)
+
+    rindex = "ls_name:%s:1M" % path
+    util.sanlock("client", "format", "-x", rindex)
+
+    with io.open(str(path), "rb") as f:
+        # The first slot should contain the rindex header sector.
+        f.seek(1024**2)
+        magic, = struct.unpack("< I", f.read(4))
+        assert magic == RINDEX_DISK_MAGIC
+
+        # The rindex entries starts at the second rindex slot sector. All
+        # entries should be zeroed.
+        f.seek(1024**2 + 512)
+        entries_size = 512 * RINDEX_ENTRIES_SECTORS
+        assert f.read(entries_size) == b"\0" * entries_size
+
+        # The next slot should contain the internal lease.
+        f.seek(1024**2 * 2)
+        magic, = struct.unpack("< I", f.read(4))
+        assert magic == PAXOS_DISK_MAGIC
+
+    util.check_guard(str(path), size)
+
+
+def test_create(tmpdir, sanlock_daemon):
+    path = tmpdir.join("rindex")
+    # Slots: lockspace rindex master-lease user-lease-1
+    size = 1024**2 * 4
+    util.create_file(str(path), size)
+
+    # Note: using 1 second io timeout (-o 1) for quicker tests.
+    lockspace = "ls_name:1:%s:0" % path
+    util.sanlock("client", "init", "-s", lockspace, "-o", "1")
+
+    rindex = "ls_name:%s:1M" % path
+    util.sanlock("client", "format", "-x", rindex)
+
+    util.sanlock("client", "add_lockspace", "-s", lockspace, "-o", "1")
+    util.sanlock("client", "create", "-x", rindex, "-e", "res")
+
+    with io.open(str(path), "rb") as f:
+        # New entry should be created at the first slot
+        # The first rindex sector is used by the rindex header.
+        f.seek(1024**2 + 512)
+        util.check_rindex_entry(f.read(RINDEX_ENTRY_SIZE),
+                                b"res", 1024**2 * 3, 0)
+
+        # The rest of the entries should not be modified.
+        rest = 512 * RINDEX_ENTRIES_SECTORS - RINDEX_ENTRY_SIZE
+        assert f.read(rest) == b"\0" * rest
+
+        # The next slot should contain the internal lease.
+        f.seek(1024**2 * 3)
+        magic, = struct.unpack("< I", f.read(4))
+        assert magic == PAXOS_DISK_MAGIC
+
+    util.check_guard(str(path), size)
+
+
+def test_delete(tmpdir, sanlock_daemon):
+    path = tmpdir.join("rindex")
+    # Slots: lockspace rindex master-lease user-lease-1
+    size = 1024**2 * 4
+    util.create_file(str(path), size)
+
+    # Note: using 1 second io timeout (-o 1) for quicker tests.
+    lockspace = "ls_name:1:%s:0" % path
+    util.sanlock("client", "init", "-s", lockspace, "-o", "1")
+
+    rindex = "ls_name:%s:1M" % path
+    util.sanlock("client", "format", "-x", rindex)
+
+    util.sanlock("client", "add_lockspace", "-s", lockspace, "-o", "1")
+    util.sanlock("client", "create", "-x", rindex, "-e", "res")
+    util.sanlock("client", "delete", "-x", rindex, "-e", "res")
+
+    with io.open(str(path), "rb") as f:
+        # First entry should be cleared.
+        f.seek(1024**2 + 512)
+        util.check_rindex_entry(f.read(RINDEX_ENTRY_SIZE), b"", 0, 0)
+
+        # Rest of entires should not be modified.
+        rest = 512 * RINDEX_ENTRIES_SECTORS - RINDEX_ENTRY_SIZE
+        assert f.read(rest) == b"\0" * rest
+
+        # The next slot should contain a cleared lease.
+        f.seek(1024**2 * 3)
+        magic, = struct.unpack("< I", f.read(4))
+        assert magic == PAXOS_DISK_CLEAR
+
+    util.check_guard(str(path), size)
+
+
+def test_lookup(tmpdir, sanlock_daemon):
+    path = tmpdir.join("rindex")
+    # Slots: lockspace rindex master-lease user-lease-1 ... user-lease-7
+    size = 1024**2 * 10
+    util.create_file(str(path), size)
+
+    # Note: using 1 second io timeout (-o 1) for quicker tests.
+    lockspace = "ls_name:1:%s:0" % path
+    util.sanlock("client", "init", "-s", lockspace, "-o", "1")
+
+    rindex = "ls_name:%s:1M" % path
+    util.sanlock("client", "format", "-x", rindex)
+
+    util.sanlock("client", "add_lockspace", "-s", lockspace, "-o", "1")
+    util.sanlock("client", "create", "-x", rindex, "-e", "res")
+    lookup = util.sanlock("client", "lookup", "-x", rindex, "-e", "res")
+
+    assert lookup == "lookup done 0\nname res offset 3145728\n"
+
+
+def test_lookup_uninitialized(tmpdir, sanlock_daemon):
+    path = tmpdir.join("rindex")
+    util.create_file(str(path), 1024**2)
+    rindex = "ls_name:%s:1M" % path
+
+    with pytest.raises(util.CommandError) as e:
+        util.sanlock("client", "lookup", "-x", rindex, "-e", "res")
+
+    assert e.value.returncode == 1
+    assert e.value.stdout == "lookup done -2\n"
+    assert e.value.stderr == ""
+
+
+def test_lookup_missing(tmpdir, sanlock_daemon):
+    path = tmpdir.join("rindex")
+    # Slots: lockspace rindex master-lease user-lease-1 ... user-lease-7
+    size = 1024**2 * 10
+    util.create_file(str(path), size)
+
+    # Note: using 1 second io timeout (-o 1) for quicker tests.
+    lockspace = "ls_name:1:%s:0" % path
+    util.sanlock("client", "init", "-s", lockspace, "-o", "1")
+
+    rindex = "ls_name:%s:1M" % path
+    util.sanlock("client", "format", "-x", rindex)
+
+    util.sanlock("client", "add_lockspace", "-s", lockspace, "-o", "1")
+    with pytest.raises(util.CommandError) as e:
+        util.sanlock("client", "lookup", "-x", rindex, "-e", "res")
+
+    assert e.value.returncode == 1
+    assert e.value.stdout == "lookup done -2\n"
+    assert e.value.stderr == ""
diff --git a/tests/util.py b/tests/util.py
index 98f1d14..2259473 100644
--- a/tests/util.py
+++ b/tests/util.py
@@ -6,6 +6,7 @@ import errno
 import io
 import os
 import socket
+import struct
 import subprocess
 import time

@@ -122,3 +123,17 @@ def check_guard(path, size, guard=b"X", guard_size=4096):
     with io.open(path, "rb") as f:
         f.seek(size)
         assert f.read() == guard * guard_size
+
+
+def check_rindex_entry(entry, name, offset=None, flags=None):
+    # See src/ondisk.c rindex_entry_in()
+    e_offset, e_flags, e_unused, e_name = struct.unpack("<Q L L 48s", entry)
+
+    padding = b"\0" * (48 - len(name))
+    assert e_name == name + padding
+
+    if offset is not None:
+        assert e_offset == offset
+
+    if flags is not None:
+        assert e_flags == flags