[lib389] branch master updated (46ef071 -> 4b97c75)

[git repository hosting]

This is an automated email from the git hooks/post-receive script.

firstyear pushed a change to branch master
in repository lib389.

      from  46ef071   Ticket 65 - Add m2c2 topology
       new  4b97c75   Ticket 50 - Add db2* tasks to dsctl

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "adds" were already present in the repository and have only
been added to this reference.


Summary of changes:
 lib389/__init__.py                   | 159 ++++++++++++++++++++---------------
 lib389/cli_ctl/dbtasks.py            |  40 ++++++++-
 lib389/instance/setup.py             |   4 +
 lib389/tests/cli/__init__.py         |  33 +++++++-
 lib389/tests/cli/ctl_dbtasks_test.py |  71 ++++++++++++++++
 5 files changed, 237 insertions(+), 70 deletions(-)
 create mode 100644 lib389/tests/cli/ctl_dbtasks_test.py

-- 
To stop receiving notification emails like this one, please contact
the administrator of this repository.

[lib389] 01/01: Ticket 50 - Add db2* tasks to dsctl

[git repository hosting]

This is an automated email from the git hooks/post-receive script.

firstyear pushed a commit to branch master
in repository lib389.

commit 4b97c75b04270f1554e548befbecdb3fa2f214f4
Author: William Brown <firstyear(a)redhat.com&gt;
Date:   Tue Jun 6 16:58:42 2017 +1000

    Ticket 50 - Add db2* tasks to dsctl
    
    Bug Description:  To make dsctl complete, we need to add the
    various db2* tasks to allow backup and restore.
    
    Fix Description:  Add the tasks, along with their tests and
    some minor fixes to lib389 to support there.
    
    https://pagure.io/lib389/issue/50
    
    Author: wibrown
    
    Review by: tbordaz, ilias95 (Thanks!)
---
 lib389/__init__.py                   | 159 ++++++++++++++++++++---------------
 lib389/cli_ctl/dbtasks.py            |  40 ++++++++-
 lib389/instance/setup.py             |   4 +
 lib389/tests/cli/__init__.py         |  33 +++++++-
 lib389/tests/cli/ctl_dbtasks_test.py |  71 ++++++++++++++++
 5 files changed, 237 insertions(+), 70 deletions(-)

diff --git a/lib389/__init__.py b/lib389/__init__.py
index 9e2465f..ae3e6f7 100644
--- a/lib389/__init__.py
+++ b/lib389/__init__.py
@@ -2587,52 +2587,60 @@ class DirSrv(SimpleLDAPObject, object):
     # server is stopped)
     #
     def ldif2db(self, bename, suffixes, excludeSuffixes, encrypt,
-                *import_files):
+                import_file):
         """
         @param bename - The backend name of the database to import
         @param suffixes - List/tuple of suffixes to import
         @param excludeSuffixes - List/tuple of suffixes to exclude from import
         @param encrypt - Perform attribute encryption
-        @param input_files - Files to import: file, file, file
+        @param input_file - File to import: file
         @return - True if import succeeded
         """
         DirSrvTools.lib389User(user=DEFAULT_USER)
         prog = os.path.join(self.ds_paths.sbin_dir, 'ns-slapd')
 
+        if self.status():
+            log.error("ldif2db: Can not operate while directory server is
running")
+            return False
+
         if not bename and not suffixes:
             log.error("ldif2db: backend name or suffix missing")
             return False
 
-        for ldif in import_files:
-            if not os.path.isfile(ldif):
-                log.error("ldif2db: Can't find file: %s" % ldif)
-                return False
+        if not os.path.isfile(import_file):
+            log.error("ldif2db: Can't find file: %s" % import_file)
+            return False
 
-        cmd = '%s ldif2db -D %s' % (prog, self.get_config_dir())
+        cmd = [
+            prog,
+            'ldif2db',
+            '-D', self.get_config_dir(),
+            '-i', import_file,
+        ]
         if bename:
-            cmd = cmd + ' -n ' + bename
+            cmd.append('-n')
+            cmd.append(bename)
         if suffixes:
             for suffix in suffixes:
-                cmd = cmd + ' -s ' + suffix
+                cmd.append('-s')
+                cmd.append(suffix)
         if excludeSuffixes:
             for excludeSuffix in excludeSuffixes:
                 cmd = cmd + ' -x ' + excludeSuffix
+                cmd.append('-x')
+                cmd.append(excludeSuffix)
         if encrypt:
-            cmd = cmd + ' -E'
-        for ldif in import_files:
-            cmd = cmd + ' -i ' + ldif
+            cmd.append('-E')
 
-        self.stop(timeout=10)
-        log.info('Running script: %s' % cmd)
-        result = True
-        try:
-            os.system(cmd)
-        except:
-            log.error("ldif2db: error executing %s" % cmd)
-            result = False
-        self.start(timeout=10)
+        result = subprocess.check_output(cmd)
+        u_result = ensure_str(result)
 
-        return result
+        log.debug("ldif2db output: BEGIN")
+        for line in u_result.split("\n"):
+            log.debug(line)
+        log.debug("ldif2db output: END")
+
+        return True
 
     def db2ldif(self, bename, suffixes, excludeSuffixes, encrypt, repl_data,
                 outputfile):
@@ -2648,39 +2656,48 @@ class DirSrv(SimpleLDAPObject, object):
         DirSrvTools.lib389User(user=DEFAULT_USER)
         prog = os.path.join(self.ds_paths.sbin_dir, 'ns-slapd')
 
+        if self.status():
+            log.error("db2ldif: Can not operate while directory server is
running")
+            return False
+
         if not bename and not suffixes:
             log.error("db2ldif: backend name or suffix missing")
             return False
 
-        # The shell wrapper is not always reliable, so bypass it. We want to
-        # kill it off anyway!
-        cmd = '%s db2ldif -D %s' % (prog, self.get_config_dir())
+        cmd = [
+            prog,
+            'db2ldif',
+            '-D', self.get_config_dir()
+        ]
         if bename:
-            cmd = cmd + ' -n ' + bename
+            cmd.append('-n')
+            cmd.append(bename)
         if suffixes:
             for suffix in suffixes:
-                cmd = cmd + ' -s ' + suffix
+                cmd.append('-s')
+                cmd.append(suffix)
         if excludeSuffixes:
             for excludeSuffix in excludeSuffixes:
                 cmd = cmd + ' -x ' + excludeSuffix
+                cmd.append('-x')
+                cmd.append(excludeSuffix)
         if encrypt:
-            cmd = cmd + ' -E'
+            cmd.append('-E')
         if repl_data:
-            cmd = cmd + ' -r'
+            cmd.append('-r')
         if outputfile:
-            cmd = cmd + ' -a ' + outputfile
+            cmd.append('-a')
+            cmd.append(outputfile)
 
-        self.stop(timeout=10)
-        log.info('Running script: %s' % cmd)
-        result = True
-        try:
-            os.system(cmd)
-        except:
-            log.error("db2ldif: error executing %s" % cmd)
-            result = False
-        self.start(timeout=10)
+        result = subprocess.check_output(cmd)
+        u_result = ensure_str(result)
 
-        return result
+        log.debug("db2ldif output: BEGIN")
+        for line in u_result.split("\n"):
+            log.debug(line)
+        log.debug("db2ldif output: END")
+
+        return True
 
     def bak2db(self, archive_dir, bename=None):
         """
@@ -2689,27 +2706,30 @@ class DirSrv(SimpleLDAPObject, object):
         @return - True if the restore succeeded
         """
         DirSrvTools.lib389User(user=DEFAULT_USER)
-        prog = os.path.join(self.ds_paths.sbin_dir, BAK2DB)
+        prog = os.path.join(self.ds_paths.sbin_dir, 'ns-slapd')
+
+        if self.status():
+            log.error("bak2db: Can not operate while directory server is
running")
+            return False
 
         if not archive_dir:
             log.error("bak2db: backup directory missing")
             return False
 
-        cmd = '%s %s -Z %s' % (prog, archive_dir, self.serverid)
-        if bename:
-            cmd = cmd + ' -n ' + bename
+        result = subprocess.check_output([
+            prog,
+            'archive2db',
+            '-a', archive_dir,
+            '-D', self.get_config_dir()
+        ])
+        u_result = ensure_str(result)
 
-        self.stop(timeout=10)
-        log.info('Running script: %s' % cmd)
-        result = True
-        try:
-            os.system(cmd)
-        except:
-            log.error("bak2db: error executing %s" % cmd)
-            result = False
-        self.start(timeout=10)
+        log.debug("bak2db output: BEGIN")
+        for line in u_result.split("\n"):
+            log.debug(line)
+        log.debug("bak2db output: END")
 
-        return result
+        return True
 
     def db2bak(self, archive_dir):
         """
@@ -2717,25 +2737,30 @@ class DirSrv(SimpleLDAPObject, object):
         @return - True if the backup succeeded
         """
         DirSrvTools.lib389User(user=DEFAULT_USER)
-        prog = os.path.join(self.ds_paths.sbin_dir, DB2BAK)
+        prog = os.path.join(self.ds_paths.sbin_dir, 'ns-slapd')
+
+        if self.status():
+            log.error("db2bak: Can not operate while directory server is
running")
+            return False
 
         if not archive_dir:
-            log.error("db2bak: backup directory missing")
+            log.error("db2bak: archive directory missing")
             return False
 
-        cmd = '%s %s -Z %s' % (prog, archive_dir, self.serverid)
+        result = subprocess.check_output([
+            prog,
+            'db2archive',
+            '-a', archive_dir,
+            '-D', self.get_config_dir()
+        ])
+        u_result = ensure_str(result)
 
-        self.stop(timeout=10)
-        log.info('Running script: %s' % cmd)
-        result = True
-        try:
-            os.system(cmd)
-        except:
-            log.error("db2bak: error executing %s" % cmd)
-            result = False
-        self.start(timeout=10)
+        log.debug("db2bak output: BEGIN")
+        for line in u_result.split("\n"):
+            log.debug(line)
+        log.debug("db2bak output: END")
 
-        return result
+        return True
 
     def db2index(self, bename=None, suffixes=None, attrs=None, vlvTag=None):
         """
diff --git a/lib389/cli_ctl/dbtasks.py b/lib389/cli_ctl/dbtasks.py
index 276f478..f2fd53c 100644
--- a/lib389/cli_ctl/dbtasks.py
+++ b/lib389/cli_ctl/dbtasks.py
@@ -7,12 +7,50 @@
 # --- END COPYRIGHT BLOCK ---
 
 def dbtasks_db2index(inst, log, args):
-    # inst.db2index(suffixes=[args.suffix,])
     inst.db2index(bename=args.backend)
 
+def dbtasks_db2bak(inst, log, args):
+    # Needs an output name?
+    inst.db2bak(args.archive)
+    log.info("db2bak successful")
+
+def dbtasks_bak2db(inst, log, args):
+    # Needs the archive to restore.
+    inst.bak2db(args.archive)
+    log.info("bak2db successful")
+
+def dbtasks_db2ldif(inst, log, args):
+    inst.db2ldif(bename=args.backend, encrypt=args.encrypt, repl_data=args.replication,
outputfile=args.ldif, suffixes=None, excludeSuffixes=None)
+    log.info("db2ldif successful")
+
+def dbtasks_ldif2db(inst, log, args):
+    inst.ldif2db(bename=args.backend, encrypt=args.encrypt, import_file=args.ldif,
suffixes=None, excludeSuffixes=None)
+    log.info("ldif2db successful")
+
 def create_parser(subcommands):
     db2index_parser = subcommands.add_parser('db2index', help="Initialise a
reindex of the server database. The server must be stopped for this to proceed.")
     # db2index_parser.add_argument('suffix', help="The suffix to reindex. IE
dc=example,dc=com.")
     db2index_parser.add_argument('backend', help="The backend to reindex. IE
userRoot")
     db2index_parser.set_defaults(func=dbtasks_db2index)
 
+    db2bak_parser = subcommands.add_parser('db2bak', help="Initialise a BDB
backup of the database. The server must be stopped for this to proceed.")
+    db2bak_parser.add_argument('archive', help="The destination for the
archive. This will be created during the db2bak process.")
+    db2bak_parser.set_defaults(func=dbtasks_db2bak)
+
+    db2ldif_parser = subcommands.add_parser('db2ldif', help="Initialise an
LDIF dump of the database. The server must be stopped for this to proceed.")
+    db2ldif_parser.add_argument('backend', help="The backend to output as an
LDIF. IE userRoot")
+    db2ldif_parser.add_argument('ldif', help="The path to the ldif output
location.")
+    db2ldif_parser.add_argument('--replication', help="Export replication
information, suitable for importing on a new consumer or backups.", default=False,
action='store_true')
+    db2ldif_parser.add_argument('--encrypted', help="Export encrypted
attributes", default=False, action='store_true')
+    db2ldif_parser.set_defaults(func=dbtasks_db2ldif)
+
+    bak2db_parser = subcommands.add_parser('bak2db', help="Restore a BDB
backup of the database. The server must be stopped for this to proceed.")
+    bak2db_parser.add_argument('archive', help="The archive to restore. This
will erase all current server databases.")
+    bak2db_parser.set_defaults(func=dbtasks_bak2db)
+
+    ldif2db_parser = subcommands.add_parser('ldif2db', help="Restore an LDIF
dump of the database. The server must be stopped for this to proceed.")
+    ldif2db_parser.add_argument('backend', help="The backend to restore from
an LDIF. IE userRoot")
+    db2ldif_parser.add_argument('ldif', help="The path to the ldif to
import")
+    ldif2db_parser.add_argument('--encrypted', help="Import encrypted
attributes", default=False, action='store_true')
+    ldif2db_parser.set_defaults(func=dbtasks_ldif2db)
+
diff --git a/lib389/instance/setup.py b/lib389/instance/setup.py
index 996c5df..f7468d1 100644
--- a/lib389/instance/setup.py
+++ b/lib389/instance/setup.py
@@ -336,6 +336,10 @@ class SetupDs(object):
             except OSError:
                 pass
             os.chown(slapd[path], slapd['user_uid'], slapd['group_gid'])
+        ### Warning! We need to down the directory under db too for .restore to work.
+        # See dblayer.c for more!
+        db_parent = os.path.join(slapd['db_dir'], '..')
+        os.chown(db_parent, slapd['user_uid'], slapd['group_gid'])
 
         # Copy correct data to the paths.
         # Copy in the schema
diff --git a/lib389/tests/cli/__init__.py b/lib389/tests/cli/__init__.py
index 9a57230..02cfc51 100644
--- a/lib389/tests/cli/__init__.py
+++ b/lib389/tests/cli/__init__.py
@@ -15,6 +15,8 @@ from lib389.instance.setup import SetupDs
 from lib389.instance.options import General2Base, Slapd2Base
 from lib389._constants import *
 
+from lib389.configurations import get_sample_entries
+
 INSTANCE_PORT = 54321
 INSTANCE_SERVERID = 'standalone'
 
@@ -28,9 +30,8 @@ class TopologyInstance(object):
         self.logcap = logcap
 
 # Need a teardown to destroy the instance.
-(a)pytest.fixture
+(a)pytest.fixture(scope=&quot;module&quot;)
 def topology(request):
-
     lc = LogCapture()
     instance = DirSrv(verbose=DEBUGGING)
     instance.log.debug("Instance allocated")
@@ -73,3 +74,31 @@ def topology(request):
     request.addfinalizer(fin)
 
     return TopologyInstance(instance, lc)
+
+
+(a)pytest.fixture(scope=&quot;module&quot;)
+def topology_be_latest(topology):
+    be = topology.standalone.backends.create(properties={
+        'cn': 'userRoot',
+        'suffix' : DEFAULT_SUFFIX,
+    })
+    # Now apply sample entries
+    centries = get_sample_entries(INSTALL_LATEST_CONFIG)
+    cent = centries(topology.standalone, DEFAULT_SUFFIX)
+    cent.apply()
+    return topology
+
+
+(a)pytest.fixture(scope=&quot;module&quot;)
+def topology_be_001003006(topology):
+    be = topology.standalone.backends.create(properties={
+        'cn': 'userRoot',
+        'suffix' : DEFAULT_SUFFIX,
+    })
+    # Now apply sample entries
+    centries = get_sample_entries('001003006')
+    cent = centries(topology.standalone, DEFAULT_SUFFIX)
+    cent.apply()
+    return topology
+
+
diff --git a/lib389/tests/cli/ctl_dbtasks_test.py b/lib389/tests/cli/ctl_dbtasks_test.py
new file mode 100644
index 0000000..f8abb53
--- /dev/null
+++ b/lib389/tests/cli/ctl_dbtasks_test.py
@@ -0,0 +1,71 @@
+# --- BEGIN COPYRIGHT BLOCK ---
+# Copyright (C) 2016 Red Hat, Inc.
+# All rights reserved.
+#
+# License: GPL (version 3 or any later version).
+# See LICENSE for details.
+# --- END COPYRIGHT BLOCK ---
+
+# Test the cli tools from the dsctl command for correct behaviour.
+
+import os
+import pytest
+from lib389.cli_ctl.dbtasks import dbtasks_db2index, dbtasks_db2bak, dbtasks_db2ldif,
dbtasks_ldif2db, dbtasks_bak2db
+
+from lib389.cli_base import LogCapture, FakeArgs
+from lib389.tests.cli import topology, topology_be_latest
+
+def test_db2index(topology):
+    pass
+
+def test_db2bak_bak2db(topology_be_latest):
+    standalone = topology_be_latest.standalone
+    standalone.stop()
+    args = FakeArgs()
+    args.archive = os.path.join(standalone.get_bak_dir(), "testdb2bak")
+    # Stop the instance
+    dbtasks_db2bak(standalone, topology_be_latest.logcap.log, args)
+    # Assert none.
+    assert topology_be_latest.logcap.contains("db2bak successful")
+    topology_be_latest.logcap.flush()
+    # We can re-use the same arguments
+    dbtasks_bak2db(standalone, topology_be_latest.logcap.log, args)
+    # Assert none.
+    assert topology_be_latest.logcap.contains("bak2db successful")
+
+def test_ldif2db_db2ldif_no_repl(topology_be_latest):
+    standalone = topology_be_latest.standalone
+    standalone.stop()
+    args = FakeArgs()
+    args.backend = 'userRoot'
+    args.ldif = os.path.join(standalone.get_ldif_dir(), "test.ldif")
+    args.encrypt = False
+    args.replication = False
+    # Stop the instance
+    dbtasks_db2ldif(standalone, topology_be_latest.logcap.log, args)
+    # Assert none.
+    assert topology_be_latest.logcap.contains("db2ldif successful")
+    topology_be_latest.logcap.flush()
+    # We can re-use the same arguments
+    dbtasks_ldif2db(standalone, topology_be_latest.logcap.log, args)
+    # Assert none.
+    assert topology_be_latest.logcap.contains("ldif2db successful")
+
+def test_ldif2db_db2ldif_repl(topology_be_latest):
+    standalone = topology_be_latest.standalone
+    standalone.stop()
+    args = FakeArgs()
+    args.backend = 'userRoot'
+    args.ldif = os.path.join(standalone.get_ldif_dir(), "test.ldif")
+    args.encrypt = False
+    args.replication = False
+    args.archive = os.path.join(standalone.get_ldif_dir(), "test.ldif")
+    # Stop the instance
+    dbtasks_db2ldif(standalone, topology_be_latest.logcap.log, args)
+    # Assert none.
+    assert topology_be_latest.logcap.contains("db2ldif successful")
+    topology_be_latest.logcap.flush()
+    # We can re-use the same arguments
+    dbtasks_ldif2db(standalone, topology_be_latest.logcap.log, args)
+    # Assert none.
+    assert topology_be_latest.logcap.contains("ldif2db successful")

-- 
To stop receiving notification emails like this one, please contact
the administrator of this repository.