This is an automated email from the git hooks/post-receive script.
firstyear pushed a change to branch master in repository lib389.
from 46ef071 Ticket 65 - Add m2c2 topology new 4b97c75 Ticket 50 - Add db2* tasks to dsctl
The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "adds" were already present in the repository and have only been added to this reference.
Summary of changes: lib389/__init__.py | 159 ++++++++++++++++++++--------------- lib389/cli_ctl/dbtasks.py | 40 ++++++++- lib389/instance/setup.py | 4 + lib389/tests/cli/__init__.py | 33 +++++++- lib389/tests/cli/ctl_dbtasks_test.py | 71 ++++++++++++++++ 5 files changed, 237 insertions(+), 70 deletions(-) create mode 100644 lib389/tests/cli/ctl_dbtasks_test.py
This is an automated email from the git hooks/post-receive script.
firstyear pushed a commit to branch master in repository lib389.
commit 4b97c75b04270f1554e548befbecdb3fa2f214f4 Author: William Brown firstyear@redhat.com Date: Tue Jun 6 16:58:42 2017 +1000
Ticket 50 - Add db2* tasks to dsctl
Bug Description: To make dsctl complete, we need to add the various db2* tasks to allow backup and restore.
Fix Description: Add the tasks, along with their tests and some minor fixes to lib389 to support there.
https://pagure.io/lib389/issue/50
Author: wibrown
Review by: tbordaz, ilias95 (Thanks!) --- lib389/__init__.py | 159 ++++++++++++++++++++--------------- lib389/cli_ctl/dbtasks.py | 40 ++++++++- lib389/instance/setup.py | 4 + lib389/tests/cli/__init__.py | 33 +++++++- lib389/tests/cli/ctl_dbtasks_test.py | 71 ++++++++++++++++ 5 files changed, 237 insertions(+), 70 deletions(-)
diff --git a/lib389/__init__.py b/lib389/__init__.py index 9e2465f..ae3e6f7 100644 --- a/lib389/__init__.py +++ b/lib389/__init__.py @@ -2587,52 +2587,60 @@ class DirSrv(SimpleLDAPObject, object): # server is stopped) # def ldif2db(self, bename, suffixes, excludeSuffixes, encrypt, - *import_files): + import_file): """ @param bename - The backend name of the database to import @param suffixes - List/tuple of suffixes to import @param excludeSuffixes - List/tuple of suffixes to exclude from import @param encrypt - Perform attribute encryption - @param input_files - Files to import: file, file, file + @param input_file - File to import: file @return - True if import succeeded """ DirSrvTools.lib389User(user=DEFAULT_USER) prog = os.path.join(self.ds_paths.sbin_dir, 'ns-slapd')
+ if self.status(): + log.error("ldif2db: Can not operate while directory server is running") + return False + if not bename and not suffixes: log.error("ldif2db: backend name or suffix missing") return False
- for ldif in import_files: - if not os.path.isfile(ldif): - log.error("ldif2db: Can't find file: %s" % ldif) - return False + if not os.path.isfile(import_file): + log.error("ldif2db: Can't find file: %s" % import_file) + return False
- cmd = '%s ldif2db -D %s' % (prog, self.get_config_dir()) + cmd = [ + prog, + 'ldif2db', + '-D', self.get_config_dir(), + '-i', import_file, + ] if bename: - cmd = cmd + ' -n ' + bename + cmd.append('-n') + cmd.append(bename) if suffixes: for suffix in suffixes: - cmd = cmd + ' -s ' + suffix + cmd.append('-s') + cmd.append(suffix) if excludeSuffixes: for excludeSuffix in excludeSuffixes: cmd = cmd + ' -x ' + excludeSuffix + cmd.append('-x') + cmd.append(excludeSuffix) if encrypt: - cmd = cmd + ' -E' - for ldif in import_files: - cmd = cmd + ' -i ' + ldif + cmd.append('-E')
- self.stop(timeout=10) - log.info('Running script: %s' % cmd) - result = True - try: - os.system(cmd) - except: - log.error("ldif2db: error executing %s" % cmd) - result = False - self.start(timeout=10) + result = subprocess.check_output(cmd) + u_result = ensure_str(result)
- return result + log.debug("ldif2db output: BEGIN") + for line in u_result.split("\n"): + log.debug(line) + log.debug("ldif2db output: END") + + return True
def db2ldif(self, bename, suffixes, excludeSuffixes, encrypt, repl_data, outputfile): @@ -2648,39 +2656,48 @@ class DirSrv(SimpleLDAPObject, object): DirSrvTools.lib389User(user=DEFAULT_USER) prog = os.path.join(self.ds_paths.sbin_dir, 'ns-slapd')
+ if self.status(): + log.error("db2ldif: Can not operate while directory server is running") + return False + if not bename and not suffixes: log.error("db2ldif: backend name or suffix missing") return False
- # The shell wrapper is not always reliable, so bypass it. We want to - # kill it off anyway! - cmd = '%s db2ldif -D %s' % (prog, self.get_config_dir()) + cmd = [ + prog, + 'db2ldif', + '-D', self.get_config_dir() + ] if bename: - cmd = cmd + ' -n ' + bename + cmd.append('-n') + cmd.append(bename) if suffixes: for suffix in suffixes: - cmd = cmd + ' -s ' + suffix + cmd.append('-s') + cmd.append(suffix) if excludeSuffixes: for excludeSuffix in excludeSuffixes: cmd = cmd + ' -x ' + excludeSuffix + cmd.append('-x') + cmd.append(excludeSuffix) if encrypt: - cmd = cmd + ' -E' + cmd.append('-E') if repl_data: - cmd = cmd + ' -r' + cmd.append('-r') if outputfile: - cmd = cmd + ' -a ' + outputfile + cmd.append('-a') + cmd.append(outputfile)
- self.stop(timeout=10) - log.info('Running script: %s' % cmd) - result = True - try: - os.system(cmd) - except: - log.error("db2ldif: error executing %s" % cmd) - result = False - self.start(timeout=10) + result = subprocess.check_output(cmd) + u_result = ensure_str(result)
- return result + log.debug("db2ldif output: BEGIN") + for line in u_result.split("\n"): + log.debug(line) + log.debug("db2ldif output: END") + + return True
def bak2db(self, archive_dir, bename=None): """ @@ -2689,27 +2706,30 @@ class DirSrv(SimpleLDAPObject, object): @return - True if the restore succeeded """ DirSrvTools.lib389User(user=DEFAULT_USER) - prog = os.path.join(self.ds_paths.sbin_dir, BAK2DB) + prog = os.path.join(self.ds_paths.sbin_dir, 'ns-slapd') + + if self.status(): + log.error("bak2db: Can not operate while directory server is running") + return False
if not archive_dir: log.error("bak2db: backup directory missing") return False
- cmd = '%s %s -Z %s' % (prog, archive_dir, self.serverid) - if bename: - cmd = cmd + ' -n ' + bename + result = subprocess.check_output([ + prog, + 'archive2db', + '-a', archive_dir, + '-D', self.get_config_dir() + ]) + u_result = ensure_str(result)
- self.stop(timeout=10) - log.info('Running script: %s' % cmd) - result = True - try: - os.system(cmd) - except: - log.error("bak2db: error executing %s" % cmd) - result = False - self.start(timeout=10) + log.debug("bak2db output: BEGIN") + for line in u_result.split("\n"): + log.debug(line) + log.debug("bak2db output: END")
- return result + return True
def db2bak(self, archive_dir): """ @@ -2717,25 +2737,30 @@ class DirSrv(SimpleLDAPObject, object): @return - True if the backup succeeded """ DirSrvTools.lib389User(user=DEFAULT_USER) - prog = os.path.join(self.ds_paths.sbin_dir, DB2BAK) + prog = os.path.join(self.ds_paths.sbin_dir, 'ns-slapd') + + if self.status(): + log.error("db2bak: Can not operate while directory server is running") + return False
if not archive_dir: - log.error("db2bak: backup directory missing") + log.error("db2bak: archive directory missing") return False
- cmd = '%s %s -Z %s' % (prog, archive_dir, self.serverid) + result = subprocess.check_output([ + prog, + 'db2archive', + '-a', archive_dir, + '-D', self.get_config_dir() + ]) + u_result = ensure_str(result)
- self.stop(timeout=10) - log.info('Running script: %s' % cmd) - result = True - try: - os.system(cmd) - except: - log.error("db2bak: error executing %s" % cmd) - result = False - self.start(timeout=10) + log.debug("db2bak output: BEGIN") + for line in u_result.split("\n"): + log.debug(line) + log.debug("db2bak output: END")
- return result + return True
def db2index(self, bename=None, suffixes=None, attrs=None, vlvTag=None): """ diff --git a/lib389/cli_ctl/dbtasks.py b/lib389/cli_ctl/dbtasks.py index 276f478..f2fd53c 100644 --- a/lib389/cli_ctl/dbtasks.py +++ b/lib389/cli_ctl/dbtasks.py @@ -7,12 +7,50 @@ # --- END COPYRIGHT BLOCK ---
def dbtasks_db2index(inst, log, args): - # inst.db2index(suffixes=[args.suffix,]) inst.db2index(bename=args.backend)
+def dbtasks_db2bak(inst, log, args): + # Needs an output name? + inst.db2bak(args.archive) + log.info("db2bak successful") + +def dbtasks_bak2db(inst, log, args): + # Needs the archive to restore. + inst.bak2db(args.archive) + log.info("bak2db successful") + +def dbtasks_db2ldif(inst, log, args): + inst.db2ldif(bename=args.backend, encrypt=args.encrypt, repl_data=args.replication, outputfile=args.ldif, suffixes=None, excludeSuffixes=None) + log.info("db2ldif successful") + +def dbtasks_ldif2db(inst, log, args): + inst.ldif2db(bename=args.backend, encrypt=args.encrypt, import_file=args.ldif, suffixes=None, excludeSuffixes=None) + log.info("ldif2db successful") + def create_parser(subcommands): db2index_parser = subcommands.add_parser('db2index', help="Initialise a reindex of the server database. The server must be stopped for this to proceed.") # db2index_parser.add_argument('suffix', help="The suffix to reindex. IE dc=example,dc=com.") db2index_parser.add_argument('backend', help="The backend to reindex. IE userRoot") db2index_parser.set_defaults(func=dbtasks_db2index)
+ db2bak_parser = subcommands.add_parser('db2bak', help="Initialise a BDB backup of the database. The server must be stopped for this to proceed.") + db2bak_parser.add_argument('archive', help="The destination for the archive. This will be created during the db2bak process.") + db2bak_parser.set_defaults(func=dbtasks_db2bak) + + db2ldif_parser = subcommands.add_parser('db2ldif', help="Initialise an LDIF dump of the database. The server must be stopped for this to proceed.") + db2ldif_parser.add_argument('backend', help="The backend to output as an LDIF. IE userRoot") + db2ldif_parser.add_argument('ldif', help="The path to the ldif output location.") + db2ldif_parser.add_argument('--replication', help="Export replication information, suitable for importing on a new consumer or backups.", default=False, action='store_true') + db2ldif_parser.add_argument('--encrypted', help="Export encrypted attributes", default=False, action='store_true') + db2ldif_parser.set_defaults(func=dbtasks_db2ldif) + + bak2db_parser = subcommands.add_parser('bak2db', help="Restore a BDB backup of the database. The server must be stopped for this to proceed.") + bak2db_parser.add_argument('archive', help="The archive to restore. This will erase all current server databases.") + bak2db_parser.set_defaults(func=dbtasks_bak2db) + + ldif2db_parser = subcommands.add_parser('ldif2db', help="Restore an LDIF dump of the database. The server must be stopped for this to proceed.") + ldif2db_parser.add_argument('backend', help="The backend to restore from an LDIF. IE userRoot") + db2ldif_parser.add_argument('ldif', help="The path to the ldif to import") + ldif2db_parser.add_argument('--encrypted', help="Import encrypted attributes", default=False, action='store_true') + ldif2db_parser.set_defaults(func=dbtasks_ldif2db) + diff --git a/lib389/instance/setup.py b/lib389/instance/setup.py index 996c5df..f7468d1 100644 --- a/lib389/instance/setup.py +++ b/lib389/instance/setup.py @@ -336,6 +336,10 @@ class SetupDs(object): except OSError: pass os.chown(slapd[path], slapd['user_uid'], slapd['group_gid']) + ### Warning! We need to down the directory under db too for .restore to work. + # See dblayer.c for more! + db_parent = os.path.join(slapd['db_dir'], '..') + os.chown(db_parent, slapd['user_uid'], slapd['group_gid'])
# Copy correct data to the paths. # Copy in the schema diff --git a/lib389/tests/cli/__init__.py b/lib389/tests/cli/__init__.py index 9a57230..02cfc51 100644 --- a/lib389/tests/cli/__init__.py +++ b/lib389/tests/cli/__init__.py @@ -15,6 +15,8 @@ from lib389.instance.setup import SetupDs from lib389.instance.options import General2Base, Slapd2Base from lib389._constants import *
+from lib389.configurations import get_sample_entries + INSTANCE_PORT = 54321 INSTANCE_SERVERID = 'standalone'
@@ -28,9 +30,8 @@ class TopologyInstance(object): self.logcap = logcap
# Need a teardown to destroy the instance. -@pytest.fixture +@pytest.fixture(scope="module") def topology(request): - lc = LogCapture() instance = DirSrv(verbose=DEBUGGING) instance.log.debug("Instance allocated") @@ -73,3 +74,31 @@ def topology(request): request.addfinalizer(fin)
return TopologyInstance(instance, lc) + + +@pytest.fixture(scope="module") +def topology_be_latest(topology): + be = topology.standalone.backends.create(properties={ + 'cn': 'userRoot', + 'suffix' : DEFAULT_SUFFIX, + }) + # Now apply sample entries + centries = get_sample_entries(INSTALL_LATEST_CONFIG) + cent = centries(topology.standalone, DEFAULT_SUFFIX) + cent.apply() + return topology + + +@pytest.fixture(scope="module") +def topology_be_001003006(topology): + be = topology.standalone.backends.create(properties={ + 'cn': 'userRoot', + 'suffix' : DEFAULT_SUFFIX, + }) + # Now apply sample entries + centries = get_sample_entries('001003006') + cent = centries(topology.standalone, DEFAULT_SUFFIX) + cent.apply() + return topology + + diff --git a/lib389/tests/cli/ctl_dbtasks_test.py b/lib389/tests/cli/ctl_dbtasks_test.py new file mode 100644 index 0000000..f8abb53 --- /dev/null +++ b/lib389/tests/cli/ctl_dbtasks_test.py @@ -0,0 +1,71 @@ +# --- BEGIN COPYRIGHT BLOCK --- +# Copyright (C) 2016 Red Hat, Inc. +# All rights reserved. +# +# License: GPL (version 3 or any later version). +# See LICENSE for details. +# --- END COPYRIGHT BLOCK --- + +# Test the cli tools from the dsctl command for correct behaviour. + +import os +import pytest +from lib389.cli_ctl.dbtasks import dbtasks_db2index, dbtasks_db2bak, dbtasks_db2ldif, dbtasks_ldif2db, dbtasks_bak2db + +from lib389.cli_base import LogCapture, FakeArgs +from lib389.tests.cli import topology, topology_be_latest + +def test_db2index(topology): + pass + +def test_db2bak_bak2db(topology_be_latest): + standalone = topology_be_latest.standalone + standalone.stop() + args = FakeArgs() + args.archive = os.path.join(standalone.get_bak_dir(), "testdb2bak") + # Stop the instance + dbtasks_db2bak(standalone, topology_be_latest.logcap.log, args) + # Assert none. + assert topology_be_latest.logcap.contains("db2bak successful") + topology_be_latest.logcap.flush() + # We can re-use the same arguments + dbtasks_bak2db(standalone, topology_be_latest.logcap.log, args) + # Assert none. + assert topology_be_latest.logcap.contains("bak2db successful") + +def test_ldif2db_db2ldif_no_repl(topology_be_latest): + standalone = topology_be_latest.standalone + standalone.stop() + args = FakeArgs() + args.backend = 'userRoot' + args.ldif = os.path.join(standalone.get_ldif_dir(), "test.ldif") + args.encrypt = False + args.replication = False + # Stop the instance + dbtasks_db2ldif(standalone, topology_be_latest.logcap.log, args) + # Assert none. + assert topology_be_latest.logcap.contains("db2ldif successful") + topology_be_latest.logcap.flush() + # We can re-use the same arguments + dbtasks_ldif2db(standalone, topology_be_latest.logcap.log, args) + # Assert none. + assert topology_be_latest.logcap.contains("ldif2db successful") + +def test_ldif2db_db2ldif_repl(topology_be_latest): + standalone = topology_be_latest.standalone + standalone.stop() + args = FakeArgs() + args.backend = 'userRoot' + args.ldif = os.path.join(standalone.get_ldif_dir(), "test.ldif") + args.encrypt = False + args.replication = False + args.archive = os.path.join(standalone.get_ldif_dir(), "test.ldif") + # Stop the instance + dbtasks_db2ldif(standalone, topology_be_latest.logcap.log, args) + # Assert none. + assert topology_be_latest.logcap.contains("db2ldif successful") + topology_be_latest.logcap.flush() + # We can re-use the same arguments + dbtasks_ldif2db(standalone, topology_be_latest.logcap.log, args) + # Assert none. + assert topology_be_latest.logcap.contains("ldif2db successful")
389-commits@lists.fedoraproject.org
-
git repository hosting