This is an automated email from the git hooks/post-receive script.
spichugi pushed a commit to branch 389-ds-base-1.4.0 in repository 389-ds-base.
commit dbb89923719224d3e808ef1c0e6ec0c4967c1148 Author: Simon Pichugin spichugi@redhat.com AuthorDate: Wed Sep 11 20:21:17 2019 +0200
Issue 50545 - Port repl-monitor.pl to lib389 CLI
Description: Add a new command to 'dsconf replication' CLI. 'dsconf replication monitor' generates a report which shows the replication topology to which the instance does belong.
Additional arguments: -c or --connections [CONNECTION [CONNECTION ...]] The connection values for monitoring other not connected topologies. The format: 'host:port:binddn:bindpwd'. You can use regex for host and port. You can set bindpwd to * and it will be requested at the runtime or you can include the path to the password file in square brackets - [~/pwd.txt] -a or --aliases [ALIAS [ALIAS ...]] If a host:port is assigned an alias, then the alias instead of host:port will be displayed in the output. The format: alias=host:port
Also, ~/.dsrc can be used for specifying the connections and aliases.
[repl-monitor-connections] connection1 = server1.example.com:38901:cn=Directory manager:* connection2 = server2.example.com:38902:cn=Directory manager:[~/pwd.txt] connection3 = hub1.example.com:.*:cn=Directory manager:password
[repl-monitor-aliases] M1 = server1.example.com:38901 M2 = server2.example.com:38902
https://pagure.io/389-ds-base/issue/50545
Reviewed by: mreynolds (Thanks!) --- src/lib389/cli/dsconf | 2 +- src/lib389/cli/dsidm | 4 +- src/lib389/lib389/_constants.py | 3 + src/lib389/lib389/agreement.py | 2 +- src/lib389/lib389/cli_base/dsrc.py | 75 ++++++++++-- src/lib389/lib389/cli_conf/replication.py | 105 ++++++++++++++++- src/lib389/lib389/replica.py | 186 +++++++++++++++++++++++++++++- 7 files changed, 355 insertions(+), 22 deletions(-)
diff --git a/src/lib389/cli/dsconf b/src/lib389/cli/dsconf index 7486dbe..2059a06 100755 --- a/src/lib389/cli/dsconf +++ b/src/lib389/cli/dsconf @@ -110,7 +110,7 @@ if __name__ == '__main__': log.debug("Inspired by works of: ITS, The University of Adelaide")
# Now that we have our args, see how they relate with our instance. - dsrc_inst = dsrc_to_ldap("~/.dsrc", args.instance, log.getChild('dsrc')) + dsrc_inst = dsrc_to_ldap(DSRC_HOME, args.instance, log.getChild('dsrc'))
# Now combine this with our arguments dsrc_inst = dsrc_arg_concat(args, dsrc_inst) diff --git a/src/lib389/cli/dsidm b/src/lib389/cli/dsidm index 51ffb56..05fcb64 100755 --- a/src/lib389/cli/dsidm +++ b/src/lib389/cli/dsidm @@ -15,7 +15,7 @@ import argparse, argcomplete import logging import sys import signal -from lib389._constants import DN_DM +from lib389._constants import DSRC_HOME from lib389.cli_idm import account as cli_account from lib389.cli_idm import initialise as cli_init from lib389.cli_idm import organizationalunit as cli_ou @@ -96,7 +96,7 @@ if __name__ == '__main__': log.debug("Inspired by works of: ITS, The University of Adelaide")
# Now that we have our args, see how they relate with our instance. - dsrc_inst = dsrc_to_ldap("~/.dsrc", args.instance, log.getChild('dsrc')) + dsrc_inst = dsrc_to_ldap(DSRC_HOME, args.instance, log.getChild('dsrc'))
# Now combine this with our arguments
diff --git a/src/lib389/lib389/_constants.py b/src/lib389/lib389/_constants.py index e656131..1ad72b5 100644 --- a/src/lib389/lib389/_constants.py +++ b/src/lib389/lib389/_constants.py @@ -345,3 +345,6 @@ args_instance = {SER_DEPLOYED_DIR: os.environ.get('PREFIX', None),
# Helper for linking dse.ldif values to the parse_config function args_dse_keys = SER_PROPNAME_TO_ATTRNAME + +DSRC_HOME = '~/.dsrc' +DSRC_CONTAINER = '/data/config/container.inf' diff --git a/src/lib389/lib389/agreement.py b/src/lib389/lib389/agreement.py index 84e2f8c..a9ee68c 100644 --- a/src/lib389/lib389/agreement.py +++ b/src/lib389/lib389/agreement.py @@ -366,7 +366,7 @@ class Agreement(DSLdapObject): return (json.dumps(result)) else: retstr = ( - "Status for agreement: "%(cn)s" (%(nsDS5ReplicaHost)s:" + "Status For Agreement: "%(cn)s" (%(nsDS5ReplicaHost)s:" "%(nsDS5ReplicaPort)s)" "\n" "Replica Enabled: %(nsds5ReplicaEnabled)s" "\n" "Update In Progress: %(nsds5replicaUpdateInProgress)s" "\n" diff --git a/src/lib389/lib389/cli_base/dsrc.py b/src/lib389/lib389/cli_base/dsrc.py index 8fd8364..bbd160e 100644 --- a/src/lib389/lib389/cli_base/dsrc.py +++ b/src/lib389/lib389/cli_base/dsrc.py @@ -8,14 +8,18 @@
import sys import os +import json import ldap -from lib389.properties import * +from lib389.properties import (SER_LDAP_URL, SER_ROOT_DN, SER_LDAPI_ENABLED, + SER_LDAPI_SOCKET, SER_LDAPI_AUTOBIND) +from lib389._constants import DSRC_CONTAINER
MAJOR, MINOR, _, _, _ = sys.version_info
if MAJOR >= 3: import configparser
+ def dsrc_arg_concat(args, dsrc_inst): """ Given a set of argparse args containing: @@ -64,6 +68,23 @@ def dsrc_arg_concat(args, dsrc_inst): dsrc_inst['starttls'] = True return dsrc_inst
+ +def _read_dsrc(path, log, case_sensetive=False): + path = os.path.expanduser(path) + log.debug("dsrc path: %s" % path) + log.debug("dsrc container path: %s" % DSRC_CONTAINER) + config = configparser.ConfigParser() + if case_sensetive: + config.optionxform = str + # First read our container config if it exists + # Then overlap the user config. + config.read([DSRC_CONTAINER, path]) + + log.debug("dsrc instances: %s" % config.sections()) + + return config + + def dsrc_to_ldap(path, instance_name, log): """ Given a path to a file, return the required details for an instance. @@ -83,14 +104,7 @@ def dsrc_to_ldap(path, instance_name, log): tls_reqcert = [never, hard, allow] starttls = [true, false] """ - path = os.path.expanduser(path) - log.debug("dsrc path: %s" % path) - # First read our config - # No such file? - config = configparser.ConfigParser() - config.read([path]) - - log.debug("dsrc instances: %s" % config.sections()) + config = _read_dsrc(path, log)
# Does our section exist? if not config.has_section(instance_name): @@ -107,14 +121,15 @@ def dsrc_to_ldap(path, instance_name, log): dsrc_inst['binddn'] = config.get(instance_name, 'binddn', fallback=None) dsrc_inst['saslmech'] = config.get(instance_name, 'saslmech', fallback=None) if dsrc_inst['saslmech'] is not None and dsrc_inst['saslmech'] not in ['EXTERNAL', 'PLAIN']: - raise Exception("~/.dsrc [%s] saslmech must be one of EXTERNAL or PLAIN" % instance_name) + raise Exception("%s [%s] saslmech must be one of EXTERNAL or PLAIN" % (path, instance_name))
dsrc_inst['tls_cacertdir'] = config.get(instance_name, 'tls_cacertdir', fallback=None) dsrc_inst['tls_cert'] = config.get(instance_name, 'tls_cert', fallback=None) dsrc_inst['tls_key'] = config.get(instance_name, 'tls_key', fallback=None) dsrc_inst['tls_reqcert'] = config.get(instance_name, 'tls_reqcert', fallback='hard') if dsrc_inst['tls_reqcert'] not in ['never', 'allow', 'hard']: - raise Exception("dsrc tls_reqcert value invalid. ~/.dsrc [%s] tls_reqcert should be one of never, allow or hard" % instance_name) + raise Exception("dsrc tls_reqcert value invalid. %s [%s] tls_reqcert should be one of never, allow or hard" % (instance_name, + path)) if dsrc_inst['tls_reqcert'] == 'never': dsrc_inst['tls_reqcert'] = ldap.OPT_X_TLS_NEVER elif dsrc_inst['tls_reqcert'] == 'allow': @@ -131,9 +146,45 @@ def dsrc_to_ldap(path, instance_name, log): dsrc_inst['args'][SER_LDAPI_SOCKET] = dsrc_inst['uri'][9:] dsrc_inst['args'][SER_LDAPI_AUTOBIND] = "on"
- # Return the dict. log.debug("dsrc completed with %s" % dsrc_inst) return dsrc_inst
+def dsrc_to_repl_monitor(path, log): + """ + Given a path to a file, return the required details for an instance. + + The connection values for monitoring other not connected topologies. The format: + 'host:port:binddn:bindpwd'. You can use regex for host and port. You can set bindpwd + to * and it will be requested at the runtime. + + If a host:port is assigned an alias, then the alias instead of host:port will be + displayed in the output. The format: "alias=host:port". + + The file should be an ini file, and instance should identify a section. + + The ini fileshould have the content: + + [repl-monitor-connections] + connection1 = server1.example.com:38901:cn=Directory manager:* + connection2 = server2.example.com:38902:cn=Directory manager:[~/pwd.txt] + connection3 = hub1.example.com:.*:cn=Directory manager:password + + [repl-monitor-aliases] + M1 = server1.example.com:38901 + M2 = server2.example.com:38902 + """ + + config = _read_dsrc(path, log, case_sensetive=True) + dsrc_repl_monitor = {"connections": None, + "aliases": None} + + if config.has_section("repl-monitor-connections"): + dsrc_repl_monitor["connections"] = [conn for _, conn in config.items("repl-monitor-connections")] + + if config.has_section("repl-monitor-aliases"): + dsrc_repl_monitor["aliases"] = {alias: inst for alias, inst in config.items("repl-monitor-aliases")} + + log.debug(f"dsrc completed with {dsrc_repl_monitor}") + return dsrc_repl_monitor diff --git a/src/lib389/lib389/cli_conf/replication.py b/src/lib389/lib389/cli_conf/replication.py index 3e147e9..282fe91 100644 --- a/src/lib389/lib389/cli_conf/replication.py +++ b/src/lib389/lib389/cli_conf/replication.py @@ -6,16 +6,16 @@ # See LICENSE for details. # --- END COPYRIGHT BLOCK ---
+import re import logging -import time -import base64 import os import json import ldap from getpass import getpass -from lib389._constants import * -from lib389.utils import is_a_dn, ensure_str -from lib389.replica import Replicas, BootstrapReplicationManager, RUV, Changelog5, ChangelogLDIF +from lib389._constants import ReplicaRole, DSRC_HOME +from lib389.cli_base.dsrc import dsrc_to_repl_monitor +from lib389.utils import is_a_dn +from lib389.replica import Replicas, ReplicationMonitor, BootstrapReplicationManager, Changelog5, ChangelogLDIF from lib389.tasks import CleanAllRUVTask, AbortCleanAllRUVTask from lib389._mapped_object import DSLdapObjects
@@ -338,6 +338,90 @@ def set_repl_config(inst, basedn, log, args): log.info("Successfully updated replication configuration")
+def get_repl_monitor_info(inst, basedn, log, args): + connection_data = dsrc_to_repl_monitor(DSRC_HOME, log) + + # Additional details for the connections to the topology + def get_credentials(host, port): + found = False + if args.connections: + connections = args.connections + elif connection_data["connections"]: + connections = connection_data["connections"] + else: + connections = [] + + if connections: + for connection_str in connections: + if len(connection_str.split(":")) != 4: + raise ValueError(f"Connection string {connection_str} is in wrong format." + "It should be host:port:binddn:bindpw") + host_regex = connection_str.split(":")[0] + port_regex = connection_str.split(":")[1] + if re.match(host_regex, host) and re.match(port_regex, port): + found = True + binddn = connection_str.split(":")[2] + bindpw = connection_str.split(":")[3] + # Search for the password file or ask the user to write it + if bindpw.startswith("[") and bindpw.endswith("]"): + pwd_file_path = os.path.expanduser(bindpw[1:][:-1]) + try: + with open(pwd_file_path) as f: + bindpw = f.readline().strip() + except FileNotFoundError: + bindpw = getpass(f"File '{pwd_file_path}' was not found. Please, enter " + f"a password for {binddn} on {host}:{port}: ").rstrip() + if bindpw == "*": + bindpw = getpass(f"Enter a password for {binddn} on {host}:{port}: ").rstrip() + if not found: + binddn = input(f'\nEnter a bind DN for {host}:{port}: ').rstrip() + bindpw = getpass(f"Enter a password for {binddn} on {host}:{port}: ").rstrip() + + return {"binddn": binddn, + "bindpw": bindpw} + + repl_monitor = ReplicationMonitor(inst) + report_dict = repl_monitor.generate_report(get_credentials) + + if args.json: + log.info(json.dumps({"type": "list", "items": report_dict})) + else: + for instance, report_data in report_dict.items(): + found_alias = False + if args.aliases: + aliases = {al.split("=")[0]: al.split("=")[1] for al in args.aliases} + elif connection_data["aliases"]: + aliases = connection_data["aliases"] + else: + aliases = {} + if aliases: + for alias_name, alias_host_port in aliases.items(): + if alias_host_port.lower() == instance.lower(): + supplier_header = f"Supplier: {alias_name} ({instance})" + found_alias = True + break + if not found_alias: + supplier_header = f"Supplier: {instance}" + log.info(supplier_header) + # Draw a line with the same length as the header + log.info("-".join(["" for _ in range(0, len(supplier_header)+1)])) + if "status" in report_data and report_data["status"] == "Unavailable": + status = report_data["status"] + reason = report_data["reason"] + log.info(f"Status: {status}") + log.info(f"Reason: {reason}\n") + else: + for replica in report_data: + replica_root = replica["replica_root"] + replica_id = replica["replica_id"] + maxcsn = replica["maxcsn"] + log.info(f"Replica Root: {replica_root}") + log.info(f"Replica ID: {replica_id}") + log.info(f"Max CSN: {maxcsn}\n") + for agreement_status in replica["agmts_status"]: + log.info(agreement_status) + + def create_cl(inst, basedn, log, args): cl = Changelog5(inst) try: @@ -1029,6 +1113,17 @@ def create_parser(subparsers): repl_set_parser.add_argument('--repl-release-timeout', help="A timeout in seconds a replication master should send " "updates before it yields its replication session")
+ repl_monitor_parser = repl_subcommands.add_parser('monitor', help='Get the full replication topology report') + repl_monitor_parser.set_defaults(func=get_repl_monitor_info) + repl_monitor_parser.add_argument('-c', '--connections', nargs="*", + help="The connection values for monitoring other not connected topologies. " + "The format: 'host:port:binddn:bindpwd'. You can use regex for host and port. " + "You can set bindpwd to * and it will be requested at the runtime or " + "you can include the path to the password file in square brackets - [~/pwd.txt]") + repl_monitor_parser.add_argument('-a', '--aliases', nargs="*", + help="If a host:port is assigned an alias, then the alias instead of " + "host:port will be displayed in the output. The format: alias=host:port") +# ############################################ # Replication Agmts ############################################ diff --git a/src/lib389/lib389/replica.py b/src/lib389/lib389/replica.py index a67a829..e0b41a9 100644 --- a/src/lib389/lib389/replica.py +++ b/src/lib389/lib389/replica.py @@ -1381,10 +1381,62 @@ class Replica(DSLdapObject): """Return the set of agreements related to this suffix replica :param: winsync: If True then return winsync replication agreements, otherwise return teh standard replication agreements. - :returns: Agreements object + :returns: A list Replicas objects """ return Agreements(self._instance, self.dn, winsync=winsync)
+ def get_consumer_replicas(self, get_credentials): + """Return the set of consumer replicas related to this suffix replica through its agreements + + :param get_credentials: A user-defined callback function which returns the binding credentials + using given host and port data - {"binddn": "cn=Directory Manager", + "bindpw": "password"} + :returns: Replicas object + """ + + agmts = self.get_agreements() + result_replicas = [] + connections = [] + + try: + for agmt in agmts: + host = agmt.get_attr_val_utf8("nsDS5ReplicaHost") + port = agmt.get_attr_val_utf8("nsDS5ReplicaPort") + protocol = agmt.get_attr_val_utf8("nsDS5ReplicaTransportInfo").lower() + + # The function should be defined outside and + # it should have all the logic for figuring out the credentials + credentials = get_credentials(host, port) + if not credentials["binddn"]: + report_data[supplier] = {"status": "Unavailable", + "reason": "Bind DN was not specified"} + continue + + # Open a connection to the consumer + consumer = DirSrv(verbose=self._instance.verbose) + args_instance[SER_HOST] = host + if protocol == "ssl" or protocol == "ldaps": + args_instance[SER_SECURE_PORT] = int(port) + else: + args_instance[SER_PORT] = int(port) + args_instance[SER_ROOT_DN] = credentials["binddn"] + args_instance[SER_ROOT_PW] = credentials["bindpw"] + args_standalone = args_instance.copy() + consumer.allocate(args_standalone) + try: + consumer.open() + except ldap.LDAPError as e: + self._log.debug(f"Connection to consumer ({host}:{port}) failed, error: {e}") + raise + connections.append(consumer) + result_replicas.append(Replicas(consumer)) + except: + for conn in connections: + conn.close() + raise + + return result_replicas + def get_rid(self): """Return the current replicas RID for this suffix
@@ -1412,6 +1464,15 @@ class Replica(DSLdapObject):
return RUV(data)
+ def get_maxcsn(self): + """Return the current replica's maxcsn for this suffix + + :returns: str + """ + replica_id = self.get_rid() + replica_ruvs = self.get_ruv() + return replica_ruvs._rid_maxcsn.get(replica_id, '00000000000000000000') + def get_ruv_agmt_maxcsns(self): """Return the in memory ruv of this replica suffix.
@@ -2227,3 +2288,126 @@ class ReplicationManager(object): replicas = Replicas(instance) replica = replicas.get(self._suffix) return replica.get_rid() + + +class ReplicationMonitor(object): + """The lib389 replication monitor. This is used to check the status + of many instances at once. + It also allows to monitor independent topologies and get them into + the one combined report. + + :param instance: A supplier or hub for replication topology monitoring + :type instance: list of DirSrv objects + :param logger: A logging interface + :type logger: python logging + """ + + def __init__(self, instance, logger=None): + self._instance = instance + if logger is not None: + self._log = logger + else: + self._log = logging.getLogger(__name__) + + def _get_replica_status(self, instance, report_data, use_json): + """Load all of the status data to report + and add new hostname:port pairs for future processing + """ + + replicas_status = [] + replicas = Replicas(instance) + for replica in replicas.list(): + replica_id = replica.get_rid() + replica_root = replica.get_suffix() + replica_maxcsn = replica.get_maxcsn() + agmts_status = [] + agmts = replica.get_agreements() + for agmt in agmts.list(): + host = agmt.get_attr_val_utf8_l("nsds5replicahost") + port = agmt.get_attr_val_utf8_l("nsds5replicaport") + protocol = agmt.get_attr_val_utf8_l('nsds5replicatransportinfo') + # Supply protocol here because we need it only for connection + # and agreement status is already preformatted for the user output + consumer = f"{host}:{port}:{protocol}" + if consumer not in report_data: + report_data[consumer] = None + agmts_status.append(agmt.status(use_json)) + replicas_status.append({"replica_id": replica_id, + "replica_root": replica_root, + "maxcsn": replica_maxcsn, + "agmts_status": agmts_status}) + return replicas_status + + def generate_report(self, get_credentials, use_json=False): + """Generate a replication report for each supplier or hub and the instances + that are connected with it by agreements. + + :param get_credentials: A user-defined callback function with parameters (host, port) which returns + a dictionary with binddn and bindpw keys - + example values "cn=Directory Manager" and "password" + :type get_credentials: function + :returns: dict + """ + report_data = {} + + initial_inst_key = f"{self._instance.host.lower()}:{str(self._instance.port).lower()}" + # Do this on an initial instance to get the agreements to other instances + report_data[initial_inst_key] = self._get_replica_status(self._instance, report_data, use_json) + + # Check if at least some replica report on other instances was generated + repl_exists = False + + # While we have unprocessed instances - continue + while True: + try: + supplier = [host_port for host_port, processed_data in report_data.items() if processed_data is None][0] + except IndexError: + break + + s_splitted = supplier.split(":") + supplier_hostname = s_splitted[0] + supplier_port = s_splitted[1] + supplier_protocol = s_splitted[2] + + # The function should be defined outside and + # it should have all the logic for figuring out the credentials. + # It is done for flexibility purpuses between CLI, WebUI and lib389 API applications + credentials = get_credentials(supplier_hostname, supplier_port) + if not credentials["binddn"]: + report_data[supplier] = {"status": "Unavailable", + "reason": "Bind DN was not specified"} + continue + + # Open a connection to the consumer + supplier_inst = DirSrv(verbose=self._instance.verbose) + args_instance[SER_HOST] = supplier_hostname + if supplier_protocol == "ssl" or supplier_protocol == "ldaps": + args_instance[SER_SECURE_PORT] = int(supplier_port) + else: + args_instance[SER_PORT] = int(supplier_port) + args_instance[SER_ROOT_DN] = credentials["binddn"] + args_instance[SER_ROOT_PW] = credentials["bindpw"] + args_standalone = args_instance.copy() + supplier_inst.allocate(args_standalone) + try: + supplier_inst.open() + except ldap.LDAPError as e: + self._log.debug(f"Connection to consumer ({supplier_hostname}:{supplier_port}) failed, error: {e}") + report_data[supplier] = {"status": "Unavailable", + "reason": e.args[0]['desc']} + continue + + report_data[supplier] = self._get_replica_status(supplier_inst, report_data, use_json) + repl_exists = True + + # Now remove the protocol from the name + report_data_final = {} + for key, value in report_data.items(): + # We take the initial instance only if it is the only existing part of the report + if key != initial_inst_key or not repl_exists: + if not value: + value = {"status": "Unavailable", + "reason": "No replicas were found"} + report_data_final[":".join(key.split(":")[:2])] = value + + return report_data_final
389-commits@lists.fedoraproject.org
-
pagure＠pagure.io